o n~b3@sddlZddlmZddlmZddlmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZdd lmZmZdd lmZdd lmZmZmZGd d d ZGdddZeeefZ ee Z!GdddZ"dS)N)Counter)aliases)sha256)dumps)sub)AnyDictIteratorListOptionalTupleUnion)NOT_PRINTABLE_PATTERNTOO_BIG_SEQUENCE) mess_ratio) iana_nameis_multi_byte_encoding unicode_rangec@s$eZdZ dGdededededddeef d d Zd e d efd dZ d e d efddZ e d efddZ e d efddZe d efddZe d efddZd efddZd efddZdHddZe d efdd Ze d eefd!d"Ze d efd#d$Ze d efd%d&Ze d eefd'd(Ze d efd)d*Ze d efd+d,Ze d efd-d.Ze d efd/d0Ze d efd1d2Ze d efd3d4Z e d edfd5d6Z!e d efd7d8Z"e d eefd9d:Z#e d eefd;d<Z$dId=d>Z%dId?d@Z&dJdBed efdCdDZ'e d efdEdFZ(dS)K CharsetMatchNpayloadguessed_encodingmean_mess_ratiohas_sig_or_bom languagesCoherenceMatchesdecoded_payloadcCsF||_||_||_||_||_d|_g|_d|_d|_d|_ ||_ dS)N) _payload _encoding_mean_mess_ratio _languages_has_sig_or_bom_unicode_ranges_leavesZ_mean_coherence_ratio_output_payload_output_encoding_string)selfrrrrrrr)D/usr/local/lib/python3.10/dist-packages/charset_normalizer/models.py__init__s  zCharsetMatch.__init__otherreturncCs>t|tstdt|jt|j|j|jko|j|jkS)Nz&__eq__ cannot be invoked on {} and {}.) isinstancer TypeErrorformatstr __class__encoding fingerprintr(r,r)r)r*__eq__(s zCharsetMatch.__eq__cCsvt|tstt|j|j}t|j|j}|dkr5|dkr5|dkr/|j|jkr/|j|jkS|j|jkS|j|jkS)zQ Implemented to make sorted available upon CharsetMatches items. g{Gz?g{Gz?r)r.r ValueErrorabschaos coherencemulti_byte_usage)r(r,Zchaos_differenceZcoherence_differencer)r)r*__lt__1s    zCharsetMatch.__lt__cCsdtt|t|jS)N?)lenr1rawr(r)r)r*r;DszCharsetMatch.multi_byte_usagecCstdttt|dS)z Check once again chaos in decoded text, except this time, with full content. Use with caution, this can be very slow. Notice: Will be removed in 3.0 z=chaos_secondary_pass is deprecated and will be removed in 3.0r=)warningswarnDeprecationWarningrr1r@r)r)r*chaos_secondary_passHs z!CharsetMatch.chaos_secondary_passcCstdtdS)zy Coherence ratio on the first non-latin language detected if ANY. Notice: Will be removed in 3.0 z)r0r3r4r@r)r)r*__repr__uzCharsetMatch.__repr__cCs8t|tr ||krtd|jd|_|j|dS)Nz;Unable to add instance <{}> as a submatch of a CharsetMatch)r.rr7r0r2r'r$appendr5r)r)r* add_submatchxszCharsetMatch.add_submatchcC|jSN)rr@r)r)r*r3zCharsetMatch.encodingcCsDg}tD]\}}|j|kr||q|j|kr||q|S)z Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. )ritemsr3rN)r(Z also_known_asupr)r)r*encoding_aliasess    zCharsetMatch.encoding_aliasescCrPrQr"r@r)r)r*bomrRzCharsetMatch.bomcCrPrQrWr@r)r)r*byte_order_markrRzCharsetMatch.byte_order_markcCsdd|jDS)z Return the complete list of possible languages found in decoded sequence. Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. cSsg|]}|dqS)rr)).0er)r)r* z*CharsetMatch.languages..r!r@r)r)r*rszCharsetMatch.languagescCsp|js1d|jvr dSddlm}m}t|jr||jn||j}t|dks+d|vr-dS|dS|jddS)z Most probable language found in decoded sequence. If none were detected or inferred, the property will return "Unknown". asciiZEnglishr)encoding_languagesmb_encoding_languagesz Latin BasedUnknown)r!could_be_from_charsetZcharset_normalizer.cdr`rarr3r>)r(r`rarr)r)r*languages  zCharsetMatch.languagecCrPrQ)r r@r)r)r*r9rRzCharsetMatch.chaoscCs|jsdS|jddS)Nrrrr^r@r)r)r*r:szCharsetMatch.coherencecCt|jdddSNd)ndigits)roundr9r@r)r)r* percent_chaoszCharsetMatch.percent_chaoscCrerf)rjr:r@r)r)r*percent_coherencerlzCharsetMatch.percent_coherencecCrP)z+ Original untouched bytes. )rr@r)r)r*r?szCharsetMatch.rawcCrPrQ)r$r@r)r)r*submatchrRzCharsetMatch.submatchcCt|jdkSNr)r>r$r@r)r)r* has_submatchszCharsetMatch.has_submatchcCs@|jdur|jSddt|D}ttdd|D|_|jS)NcSsg|]}t|qSr))r)rZcharr)r)r*r\sz*CharsetMatch.alphabets..cSsh|]}|r|qSr)r))rZrr)r)r* r]z)CharsetMatch.alphabets..)r#r1sortedlist)r(Zdetected_rangesr)r)r* alphabetss zCharsetMatch.alphabetscCs|jgdd|jDS)z The complete list of encoding that output the exact SAME str result and therefore could be the originating encoding. This list does include the encoding available in property 'encoding'. cSsg|]}|jqSr))r3)rZmr)r)r*r\sz6CharsetMatch.could_be_from_charset..)rr$r@r)r)r*rcsz"CharsetMatch.could_be_from_charsetcC|Sz> Kept for BC reasons. Will be removed in 3.0. r)r@r)r)r*firstzCharsetMatch.firstcCryrzr)r@r)r)r*bestr|zCharsetMatch.bestutf_8r3cCs2|jdus |j|kr||_t||d|_|jS)z Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. Any errors will be simply ignored by the encoder NOT replaced. Nreplace)r&r1encoder%)r(r3r)r)r*outputszCharsetMatch.outputcCst|S)zw Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one. )rr hexdigestr@r)r)r*r4 szCharsetMatch.fingerprintrQ)r,rr-N)r-r)r~))__name__ __module__ __qualname__bytesr1floatboolr r+objectr6r<propertyr;rDrErrIrKrLrOr3r rVrXrYrrdr9r:rkrmr?rnrqrwrcr{r}rr4r)r)r)r*rs~            rc@seZdZdZddeefddZdeefddZd e e e fdefd d Z de fd d Z defddZd eddfddZdedfddZdedfddZdS)CharsetMatchesz Container with every CharsetMatch items ordered by default from most probable to the less one. Act like a list(iterable) but does not implements all related methods. NresultscCs|r t||_dSg|_dSrQ)ru_results)r(rr)r)r*r+szCharsetMatches.__init__r-ccs|jEdHdSrQrr@r)r)r*__iter__szCharsetMatches.__iter__itemcCsJt|tr |j|St|tr#t|d}|jD] }||jvr"|Sqt)z Retrieve a single item either by its position or encoding name (alias may be used here). Raise KeyError upon invalid index or encoding not present in results. F)r.intrr1rrcKeyError)r(rresultr)r)r* __getitem__!s      zCharsetMatches.__getitem__cCs t|jSrQr>rr@r)r)r*__len__/s zCharsetMatches.__len__cCrorprr@r)r)r*__bool__2szCharsetMatches.__bool__cCs|t|tstdt|jt|jtkr0|j D]}|j |j kr/|j |j kr/| |dSq|j |t|j |_ dS)z~ Insert a single match. Will be inserted accordingly to preserve sort. Can be inserted as a submatch. z-Cannot append instance '{}' to CharsetMatchesN)r.rr7r0r1r2r>r?rrr4r9rOrNru)r(rmatchr)r)r*rN5s    zCharsetMatches.appendrcCs|jsdS|jdS)zQ Simply return the first match. Strict equivalent to matches[0]. Nrrr@r)r)r*r}Is zCharsetMatches.bestcCs|S)zP Redundant method, call the method best(). Kept for BC reasons. )r}r@r)r)r*r{QszCharsetMatches.firstrQ)rrr__doc__r rr+r rr rr1rrrrrNr r}r{r)r)r)r*rsrc@s~eZdZdedeedeedeededeededed ed eed efd d Ze de ee ffddZ defddZ dS)CliDetectionResultpathr3rValternative_encodingsrdrwrr9r: unicode_path is_preferredc CsF||_| |_||_||_||_||_||_||_||_| |_ | |_ dSrQ) rrr3rVrrdrwrr9r:r) r(rr3rVrrdrwrr9r:rrr)r)r*r+]s zCliDetectionResult.__init__r-c Cs2|j|j|j|j|j|j|j|j|j|j |j d S)N rr3rVrrdrwrr9r:rrrr@r)r)r*__dict__wszCliDetectionResult.__dict__cCst|jdddS)NT) ensure_asciiindent)rrr@r)r)r*to_jsonrMzCliDetectionResult.to_jsonN)rrrr1r r rrr+rrrrrr)r)r)r*r\s6    r)#rA collectionsrZencodings.aliasesrhashlibrjsonrrertypingrrr r r r r Zconstantrrmdrutilsrrrrrr1rZCoherenceMatchrrr)r)r)r*s"     $  C