
    ^jU                    n   U d dl mZ d dlZd dlmZ d dlmZ d dlmZ ddl	m
Z
mZmZmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZmZmZmZ  ej@                  d      Z! ejD                         Z#e#jI                   ejJ                  d              e&ed       Z'de(d<   	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ)	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ*	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ+	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ,y)    )annotationsN)	lru_cache)PathLike)BinaryIO   )coherence_ratioencoding_languagesmb_encoding_languagesmerge_coherence_ratios)IANA_SUPPORTEDIANA_SUPPORTED_SIMILARTOO_BIG_SEQUENCETOO_SMALL_SEQUENCETRACE)
mess_ratio)CharsetMatchCharsetMatches)any_specified_encodingcut_sequence_chunks	iana_nameidentify_sig_or_bomis_multi_byte_encodingshould_strip_sig_or_bomcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sc                    t        |        S N)r   encodings    O/opt/ringagent/.cad-venv/lib/python3.12/site-packages/charset_normalizer/api.py<lambda>r    .   s    -CH-M)M     )keyz	list[str]IANA_SUPPORTED_MB_FIRSTc
                    t        | t        t        f      s#t        dj	                  t        |                   |rBt        j                  }
t        j                  t               t        j                  t               t        |       }|dk(  r_t        j                  d       |r.t        j                  t               t        j                  
       t        t!        | dddg d      g      S |Dt        j#                  t        d	d
j%                  |             |D cg c]  }t'        |d       }}ng }|Dt        j#                  t        dd
j%                  |             |D cg c]  }t'        |d       }}ng }|||z  k  r!t        j#                  t        d|||       d}|}|dkD  r||z  |k  rt)        ||z        }t        |       t*        k  }t        |       t,        k\  }|r*t        j#                  t        dj	                  |             n+|r)t        j#                  t        dj	                  |             g }|rt/        |       nd}|,|j1                  |       t        j#                  t        d|       t3               }g }g }t3               }t3               }i } t5        d      t6              } t5        d      t8              }d}t3               }d}d}d}d}d}d} t               }!t               }"t;        |       \  }#}$|#6|j1                  |#       t        j#                  t        dt        |$      |#       |j1                  d       d|vr|j1                  d       |t<        z   D 	]  }%|r|%|vr|r|%|v r|%|v r|j?                  |%       d}&|#|%k(  }'|'xr tA        |%      }(|%dv r|'st        j#                  t        d|%       `|%dv r|'st        j#                  t        d|%       |%|v rt        j#                  t        d|%       |%|v rt        j#                  t        d|%       	 tC        |%      })|r[|)st3        tI        |%            }*nt3        tK        |%            }*|*jM                  |      st        j#                  t        d|%|*|       +|r&|)s$||k\  rt        j#                  t        d|%||       S|r|)st        j#                  t        d|%       t|) xr | }+	 |r5|)s3tO        |(s| dt)        d       n| t        |$      t)        d       |%        nC|+sA|%d!k(  r|'rtO        | |%       }&|&r+|&d   d"k(  r#|&dd }&ntO        |(s| n| t        |$      d |%       }&tU        |'sdn
t        |$      |t)        ||z              }-|)xr |&duxr t        |&      |k  }.|.rt        j#                  t        d$|%       t)        t        |-      d%z        }/tW        |/d&      }/d}0d}1g }2g }3	 tY        | |%|-||'|(|$|)|&|+
      D ]`  }4|2j1                  |4       |3j1                   ||4||xr dt        |      cxk  xr d&k  nc              |3d'   |k\  r|0dz  }0|0|/k\  s|'s]|(r` n |1s$|r"|)s 	 | t)        d*      d j[                  |%d+,       |3rt]        |3      t        |3      z  nd}5|5|k\  s|0|/k\  r|j1                  |%       |%t^        v r|ja                  t^        |%          |& |)s|jc                  te        |&      |5g df       t        j#                  t        d.|%|0tg        |5d/z  d01             |	rS|%dd|d2d3fv rJ|1sH|&"	 tO        |(s| n| t        |$      d |%       }&|rd}&t!        | |%||'g |&|5      }6|%|k(  r|6} n
|%dk(  r|6}n|6}|+r	 tO        |(s| n| t        |$      d |%       }&|&|)ste        |&      }7|ji                  |7      }8|8|8\  }9}:};|;rrt!        | |%|9|'|:|r|%|ddfv r|&nd|5      }<|!j1                  |<       |j?                  |%       t        j#                  t        d6|%tg        |9d/z  d01             |%|ddfv ry|9d7k  rt|9dk(  r^t        j                  d8|<jj                         |r.t        j                  t               t        j                  
       t        |<g      c S |"j1                  |<       t        |"      r||||v rvd|v rrd|v rn|"jm                         }=t        j                  d8|=jj                         |r.t        j                  t               t        j                  
       t        |=g      c S |j1                  |%       t        j#                  t        d9|%       |	r-|%dd|d2d3fv r$t!        | |%||'g |&|5      }6|%|k(  r|6} n
|%dk(  r|6}n|6}2t        j#                  t        d:|%tg        |5d/z  d01             |)stI        |%      }>ntK        |%      }>|>r3t        j#                  t        d;j	                  |%tO        |>                   g }?|%dk7  r7|>rd<j%                  |>      nd}@|2D ]  }4 ||4|@      }A|?j1                  |A        to        |?      }B|Br*t        j#                  t        d=j	                  B|%             t!        | |%|5|'B|r|%|ddfv r|&nd|5      }C|!j1                  |C       |& |)s|jc                  te        |&      |5Bd)f       |r|)s
|5d>k  r|dz  }|%|ddfv ry|5d7k  rt|5dk(  r^t        j                  d8Cjj                         |r.t        j                  t               t        j                  
       t        Cg      c S |"j1                  C       t        |"      r||||v rvd|v rrd|v rn|"jm                         }=t        j                  d8|=jj                         |r.t        j                  t               t        j                  
       t        |=g      c S |sW|)sUBrtW        d? BD        d@      nd}D|DdAk\  r8d|v r4d|v r0d)}|ja                  |>       t        j#                  t        dB|%|5D       |s\|)rZ|.rX|&Vt        |&      |dCz  k  rE|%dDvrAd|v r=d|v r9d)}t        j#                  t        dE|%|5t        |&      |t        |&      |z  d/z         |%|#k(  s	Bt        j                  dF|%       |r.t        j                  t               t        j                  
       t        |!|%   g      c S  t        |!      dk(  r|s|s| rt        j#                  t        dG       | r2t        j                  dH| jj                         |!j1                  |        nr|r||r|r|jp                  |jp                  k7  s|'t        j                  dI       |!j1                  |       n(|r&t        j                  dJ       |!j1                  |       |!r<t        j                  dK|!jm                         jj                  t        |!      dz
         nt        j                  dL       |r.t        j                  t               t        j                  
       |!S c c}w c c}w # tD        tF        f$ r t        j#                  t        d|%       Y *w xY w# tP        tR        f$ rQ},t        |,tR              s%t        j#                  t        d#|%tO        |,             |j1                  |%       Y d},~,d},~,ww xY w# tP        tR        f$ rr},|+r<t        j#                  t        d#|%tO        |,             |j1                  |%       Y d},~,t        j#                  t        d(|%tO        |,             |/}0d)}1Y d},~,	d},~,ww xY w# tP        $ rA},t        j#                  t        d-|%tO        |,             |j1                  |%       Y d},~,Zd},~,ww xY w# tP        tR        f$ r t        j#                  t        d4|%       Y w xY w# tP        tR        f$ rA},t        j#                  t        d#|%tO        |,             |j1                  |%       Y d},~,d},~,ww xY w)Maf  
    Given a raw bytes sequence, return the best possibles charset usable to render str objects.
    If there is no results, it is a strong indicator that the source is binary/not text.
    By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence.
    And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will.

    The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page
    but never take it for granted. Can improve the performance.

    You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that
    purpose.

    This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32.
    By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain'
    toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging.
    Custom logging format and handler can be set manually.
    z3Expected object of type bytes or bytearray, got: {}r   z<Encoding detection on empty bytes, assuming utf_8 intention.utf_8g        F Nz`cp_isolation is set. use this flag for debugging purpose. limited list of encoding allowed : %s.z, zacp_exclusion is set. use this flag for debugging purpose. limited list of encoding excluded : %s.z^override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.r   z>Trying to detect encoding from a tiny portion of ({}) byte(s).zIUsing lazy str decoding because the payload is quite large, ({}) byte(s).z@Detected declarative mark in sequence. Priority +1 given for %s.)maxsize   zIDetected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.ascii>   utf_16utf_32z\Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.>   utf_7zREncoding %s won't be tested as-is because detection is unreliable without BOM/SIG.zY%s is deemed too similar to a code page that was already considered unsuited. Continuing!zESkipping %s: already fast-tracked from a similar successful encoding.z2Encoding %s does not provide an IncrementalDecoderzbSkipping %s: definitive match already found, this encoding targets different languages (%s vs %s).zXSkipping %s: already accumulated %d same-family results after definitive match (cap=%d).zCSkipping single-byte %s: multi-byte definitive match already found.g    Ar   r,   u   ﻿z9Code page %s does not fit given bytes sequence at ALL. %szpCode page %s is a multi byte encoding table and it appear that at least one character was encoded using n-bytes.      zaLazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %sTg     j@strict)errorsz^LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %szc%s was excluded because of initial chaos probing. Gave up %i time(s). Computed mean chaos is %f %%.d      )ndigitsr*   r+   z>%s does not decode the whole payload: fallback entry withheld.)preemptive_declarationzM%s fast-tracked (identical decoded payload to a prior encoding, chaos=%f %%).皙?z.Encoding detection: %s is most likely the one.zZ%s fast-skipped (identical decoded payload to a prior encoding that failed chaos probing).z=%s passed initial chaos probing. Mean measured chaos is %f %%z&{} should target any language(s) of {},z We detected language {} using {}g{Gz?c              3  &   K   | ]	  \  }}|  y wr    ).0_vs      r   	<genexpr>zfrom_bytes.<locals>.<genexpr>C  s     441aQ4s   )defaultg      ?zyDefinitive match found: %s (chaos=%.3f, coherence=%.2f). Encodings targeting different language families will be skipped.g\(\?>	   r,   r%   r*   r+   	utf_16_be	utf_16_le	utf_32_be	utf_32_le	utf_8_sigzjMulti-byte definitive match: %s (chaos=%.3f, decoded=%d/%d=%.1f%%). Single-byte encodings will be skipped.zoEncoding detection: %s is most likely the one as we detected a BOM or SIG within the beginning of the sequence.zONothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.z7Encoding detection: %s will be used as a fallback matchz:Encoding detection: utf_8 will be used as a fallback matchz:Encoding detection: ascii will be used as a fallback matchz]Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.z=Encoding detection: Unable to determine any suitable charset.)9
isinstance	bytearraybytes	TypeErrorformattypeloggerlevel
addHandlerexplain_handlersetLevelr   lendebugremoveHandlerr   r   logjoinr   intr   r   r   appendsetr   r   r   r   r#   addr   r   ModuleNotFoundErrorImportErrorr	   r
   intersectionstrUnicodeDecodeErrorLookupErrorrangemaxr   decodesumr   update
setdefaulthashroundgetr   bestr   fingerprint)E	sequencessteps
chunk_size	thresholdcp_isolationcp_exclusionpreemptive_behaviourexplainlanguage_thresholdenable_fallbackprevious_logger_levellengthcpis_too_small_sequenceis_too_large_sequenceprioritized_encodingsspecified_encodingtestedtested_but_hard_failuretested_but_soft_failuresoft_failure_skipsuccess_fast_trackedpayload_result_cachecached_mess_ratiocached_coherence_ratiodefinitive_match_founddefinitive_target_languages post_definitive_sb_success_countPOST_DEFINITIVE_SB_CAPmb_definitive_match_foundfallback_asciifallback_u8fallback_specifiedresultsearly_stop_resultssig_encodingsig_payloadencoding_ianadecoded_payloadbom_or_sig_availablestrip_sig_or_bomis_multi_byte_decoderenc_languagesdeferred_decodinger_multi_byte_bonusmax_chunk_gave_upearly_stop_countlazy_str_hard_failure	md_chunks	md_ratioschunkmean_mess_ratiofallback_entrypayload_hashcachedcached_mess	cached_cdcached_passed
fast_matchprobable_resulttarget_languages	cd_ratioslg_inclusionchunk_languagescd_ratios_mergedcurrent_matchbest_coherencesE                                                                        r   
from_bytesr   2   s   < i)U!34AHHY
 	
 %+\\/*i.F{ST  1OO12|IwUBPRSTUU

5IIl#		
 8DD	"e,DD

6IIl#		
 8DD	"e,DD*u$%

l	
 
qyVe^j0%(
"%i.3E"E"%i.4D"D

LSS	
 


W^^	
 (* .By)t  %$$%78

N	
 uF)+)+"%%%(U
 TV 0	$/
;4Yt4_E $),/E -.$"# ',*.N'+K.2,.G)7)9 3I >L+$$\2

W		
   )++$$W-.1HH Q
<M=M\9F"

=!&*%1]%B!5 "
:Q;
 009MJJn
 I%.BJJd
  --JJk
  00JJW
 	*@*O! "( #$6}$E F #$9-$H I --.IJ

x!!/  #)04JJJJj0& 
 %-BJJU
  &%C.C*C 	*	$-B  0 "+CI.&s;'7#d)D* ' !G+0D&)!!.'O '?1+=+I*9!"*=&) $4 &!*3{+;+=!>!.'O& )As;/?
 " .t+.O$v- 	 JJ-	 "%SWq[!1 115 ! %!		5	),$ %!    '  %!?A\):$?a$? R=I-$)$$(99(1A9r &%)
#d)+&--mH-M ENY#i.!@SVi'+;?P+P#**=9 66!(()?)NO *3H$//)OR+G JJ0 o+Q7  !W&8(HMN- #*!*- (8 !*%.s;/?/A%B%2+ -*.!-!(#+=" !$66)7&"g-%3N"0K"%  0 "&s;'7'9:*#, &/D $_ 5L)--l;F!8>5Y !-!%#,! %:#0$6#I$J , "&/A"J" NN:.(,,];JJg%kC/;	 &*<gw)OO'#-&#-"LL P * 3 3  ' & 4 4_ E &0E F#1:,#??*11*= .//7;MQW;W#v-#v-8J8O8O8QL+44 #"00A"OO,AB-.?@@ ,22=AJJt% '=*  = , *6%)%0+3E* ),>>1?.*g5-;N*8K

K/C'3		
 %*<]*K4]CJJ8??!3'7#8 	 G# /?)*D  # 2"8& #   12 2)<JJ299$m %  .$);Wg(NN  
 #5
" 	}% &/D ++_% "2D9 #)$&,1, 0'7CC#% #%D!** ((9OO$9:%}o66%%m4 "##+/AV/K6!6!0557OLL@(( $$_5 56!?"344 &.C $ 4#34cB 
 $F):w&?P)-&+223CD

 P!#" *% +O$v}4
 6!6!(,%JJ|O$O$v-3 L(LL1
 $$_5 56!7=#9":;;cQ
<f 7|q.,>JJa
 LLI"++ NN-.^3"++~/I/II'LLUVNN;'LLUVNN>*kLLN##L1	
 	TU_--.N} E EZ $[1 	JJD
 	H #K0 		a-

O!F	 $**=9		N 
 	) ! 

O!F	 (..}=JJsA	  1$(!/	)D & 

t!F	 (..}=d /< !

!\)
 !!P '4 

O!F	 (..}=s   y%y$/y))A:z*A0{={={=(~:?)*zz{:)A{55{:=}>8}9
)}99}>	
6*<;<?AA@6AA
A
AAc
                F    t        | j                         |||||||||	
      S )z
    Same thing than the function from_bytes but using a file pointer that is already ready.
    Will not close the file pointer.
    )r   read)
fprj   rk   rl   rm   rn   ro   rp   rq   rr   s
             r   from_fpr     s5      
	 r!   c
                n    t        | d      5 }
t        |
|||||||||	
      cddd       S # 1 sw Y   yxY w)z
    Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode.
    Can raise IOError.
    rbN)openr   )pathrj   rk   rl   rm   rn   ro   rp   rq   rr   r   s              r   	from_pathr     sK      
dD	 
R 

 
 
s   +4c
                    t        | t        t        f      rt        | |||||||||	
      }
|
 S t        | t        t
        f      rt        | |||||||||	
      }
|
 S t        | |||||||||	
      }
|
 S )a)  
    Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string.
    Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match
    are disabled to be stricter around ASCII-compatible but unlikely to be a string.
    )	rj   rk   rl   rm   rn   ro   rp   rq   rr   )rD   r[   r   r   rF   rE   r   r   )fp_or_path_or_payloadrj   rk   rl   rm   rn   ro   rp   rq   rr   guessess              r   	is_binaryr     s    " '#x9!!%%!51+
Z ;C 
	

 !!%%!51+
4 ; !!%%!51+
 ;r!   )	      皙?NNTFr6   T)ri   zbytes | bytearrayrj   rT   rk   rT   rl   floatrm   list[str] | Nonern   r   ro   boolrp   r   rq   r   rr   r   returnr   )r   r   rj   rT   rk   rT   rl   r   rm   r   rn   r   ro   r   rp   r   rq   r   rr   r   r   r   )r   zstr | bytes | PathLikerj   rT   rk   rT   rl   r   rm   r   rn   r   ro   r   rp   r   rq   r   rr   r   r   r   )	r   r   r   NNTFr6   F)r   z!PathLike | str | BinaryIO | bytesrj   rT   rk   rT   rl   r   rm   r   rn   r   ro   r   rp   r   rq   r   rr   r   r   r   )-
__future__r   logging	functoolsr   osr   typingr   cdr   r	   r
   r   constantr   r   r   r   r   mdr   modelsr   r   utilsr   r   r   r   r   r   	getLoggerrJ   StreamHandlerrM   setFormatter	Formattersortedr#   __annotations__r   r   r   r   r9   r!   r   <module>r      s   "        0  
		/	0'''')   GAB &,M&   %)%)!% # x xx x 	x
 #x #x x x x x xz %)%)!% #   	
 # #     @ %)%)!% # 

 

 
 	

 #
 #
 
 
 
 
 
B %)%)!% #!?<?? ? 	?
 #? #? ? ? ? ? 
?r!   