ó
    ¹ïõi*;  ã                   óæ   • S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKJ	r	  S SK
r
S\4S jrS rS\4S jrS\S	\4S
 jrS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rg)é    N)ÚloggerÚreturnc                  óL  •  [         R                  " SSS0SS9R                  5       S   n [         R                  " S[        U 5      -   S-   S	S
9R                  n/ nUR                  5       nU H{  nUR                  S5      S   nUR                  S5      S   nUR                  S5      S   nUR                  S5      S   n	SU-   S-   U	-   S-   U-   S-   U-   n
UR                  U
5        M}     U$ ! [
         a+  n[        R                  " [        U5      5        / s S nA$ S nAff = f)Nz.https://proxy.webshare.io/api/v2/proxy/config/ÚAuthorizationÚ(wsvwkkpdl6e2zirlyimm1nbkxbrcxf0zr07myh61é
   )ÚheadersÚtimeoutÚproxy_list_download_tokenz5https://proxy.webshare.io/api/v2/proxy/list/download/z/-/any/username/direct/é   )r
   Ú:r   é   é   é   zhttp://Ú@)ÚrequestsÚgetÚjsonÚstrÚtextÚ	Exceptionr   ÚerrorÚ
splitlinesÚsplitÚappend)Údl_tokenÚ
proxy_listÚeÚall_proxiesÚproxiesÚproxyÚhostÚportÚusernameÚpasswordÚfinal_proxys              ÚQ/var/www/highfloat_scraper_backup_02_05_2026/highfloat_scraper/HelperFunctions.pyÚgetWebshareProxiesr(      s<  € ðÜ—<’<Ø<Ø$Ð&PÐQØñ
÷ ‰$‹&Ð,ñ	.ˆô —\’\ØCÜ�(‹mñà'ñ(ð ñ	
÷
 ‰$ð 	ð €Kà×#Ñ#Ó%€GÛˆØ�{‰{˜3Ó Ñ"ˆØ�{‰{˜3Ó Ñ"ˆØ—;‘;˜sÓ# AÑ&ˆØ—;‘;˜sÓ# AÑ&ˆØ (Ñ*¨SÑ0°8Ñ;¸cÑAÀDÑHÈ3ÑNÐQUÑUˆØ×Ñ˜;Ö'ñ ð Ðøô ó Ü�Š”S˜“VÔØ�	ûðús   ‚AC. Ã.
D#Ã8 DÄD#ÄD#c                 óŒ   • [        SS5       nXR                  5       ;   a
   S S S 5        g S S S 5        g! , (       d  f       g = f)Nzexcluded_skins.txtÚrTF)ÚopenÚread)Úskin_name_with_conditionÚfs     r'   ÚcheckIfSkinExcludedr/   -   s7   € Ü	Ð" CÔ	(¨AØ#§v¡v£xÓ/Ø÷ 
)Ð	(ð ÷	 
)×	(Ö	(ús   �5«5µ
AÚjs_callc                 ó,  • Sn[         R                  " X5      nU(       d  gUR                  S5      n[         R                  " SU5      n/ nU H.  nUR	                  US   S:w  a  US   O[        US   5      5        M0     US   US   US   US   US	   S
.$ )zE
Parses:
BuyMarketListing('listing', '6424...', 730, '2', '4613...')
zBuyMarketListing\((.*?)\)Nr   z'([^']*)'|(\d+)r   Ú r   r   é   )ÚtypeÚ
listing_idÚapp_idÚ
context_idÚasset_id)ÚreÚsearchÚgroupÚfindallr   Úint)r0   ÚpatternÚmatchÚargs_strÚargsÚparsedÚas          r'   Úparse_buy_market_listingrD   5   s¡   € ð +€GÜ�IŠI�gÓ'€EÞØà�{‰{˜1‹~€Hô �:Š:Ð(¨(Ó3€Dð €FÛˆØ�‰˜a ™d b›j�a˜’d¬c°!°A±$«iÖ8ñ ð �q‘	Ø˜Q‘iØ˜‘)Ø˜Q‘iØ˜1‘Iñð ó    Úraw_textÚhigh_lowc                 ó  • [         R                  " U 5      nU R                  5       R                  S5      (       a(  [         R                  " U 5      nUR	                  SS5      n [
        R                  " U 5      n/ n[        R                  " SU5      nU H¾  n[        U5      nU(       d  M  US   [        US   5         n	SU	;   a#  [        U	S   5      S-  n
[        U	S	   5      S-  nO"[        U	S
   5      S-  n
[        U	S   5      S-  nX«-   n[        US5      nUR                  [        US   5      [        U	S   S   5      [        U5      /5        MÀ     [        R                  " SU5      nSnU HÍ  nU[        U5      :¼  a    O½SU;   d  M  [        R                   R#                  U5      S:X  aF  [%        SS5       nUR'                  S[        U5      -   S-   [        U5      -   S-   5        S S S 5        M…  X^   R                  [        R                   R#                  U5      5        X^   R                  U5        US-  nMÏ     [        U5      S:X  a"  [(        R*                  " S[        U 5      -   5        U$ ! , (       d  f       N�= f)NÚ{Úresults_htmlr2   zBuyMarketListing\([^)]*\)Úlistinginfor5   Úconverted_priceéd   Úconverted_feeÚpriceÚfeer   ÚassetÚidzhref="([^"]+)"r   zsteam://run/730z+steam://run/730//+csgo_econ_action_preview zFATERROR.txtrC   zFAT ERROR WITH HREF: z AND RAW_TEXT: Ú
r   z9No listings found in getNewInspectLinks() with raw_text: )r   ÚloadsÚstripÚ
startswithr   ÚhtmlÚunescaper9   r<   rD   r   ÚfloatÚroundr   ÚlenÚurllibÚparseÚunquoter+   Úwriter   Úcritical)rF   rG   Úraw_text_full_jsonÚdataÚdecoded_htmlÚlistinginfo_listÚjs_callsÚcallrB   Úlisting_inforO   rP   Ú
full_priceÚhrefsÚcounterÚhrefr.   s                    r'   ÚgetNewInspectLinksrl   S   sL  € ÜŸš HÓ-ÐØ‡~�~Ó×"Ñ" 3×'Ñ'Ü�zŠz˜(Ó#ˆØ—8‘8˜N¨BÓ/ˆä—=’= Ó*€LàÐô �zŠzÐ6¸ÓE€HÛˆÜ)¨$Ó/ˆßˆ6à-¨mÑ<¼SÀÈÑAUÓ=VÑWˆLà  LÓ0Ü˜lÐ+<Ñ=Ó>ÀÑD�Ü˜L¨Ñ9Ó:¸SÑ@‘ô ˜l¨7Ñ3Ó4°sÑ:�Ü˜L¨Ñ/Ó0°3Ñ6�à™ˆJÜ˜z¨1Ó-ˆJØ×#Ñ#¤S¨°Ñ)=Ó%>ÄÀLÐQXÑDYÐZ^ÑD_Ó@`ÔbeÐfpÓbqÐ$rÖsñ! ô( �JŠJÐ(¨,Ó7€EØ€GÛˆØ”cÐ*Ó+Ó+ÙØ Õ$Ü�|‰|×#Ñ# DÓ)Ð-ZÓZä˜.¨#Ô.°!Ø—G‘GÐ3´c¸$³iÑ?ÐBSÑSÔVYÐZfÓVgÑgÐjnÑnÔo÷ /áØÑ%×,Ñ,¬V¯\©\×-AÑ-AÀ$Ó-GÔHØÑ%×,Ñ,¨XÔ6Ø�q‰LŠGñ ô ÐÓ Ó!Ü�ŠÐSÔVYÐZbÓVcÑcÔdØÐ÷ /Õ.ús   Æ<0I5É5
J	c           
      óŒ   • [         R                  " 5        nUR                  SXX#XE45        S S S 5        g ! , (       d  f       g = f)NzpINSERT INTO ownskinsdb (full_item_name, inspect_link, iteminfo, rank, a, floaty) VALUES (%s, %s, %s, %s, %s, %s))Ú
topfloatdbÚ	db_cursorÚexecute)Úfull_item_nameÚinspect_linkÚiteminfoÚrankrC   ÚfloatyÚcurs          r'   ÚinsertItemIntoDBrw   ‘   s5   € Ü	×	Ò	Ô	 3Ø�‰Ø~Ø¨8¸1ÐEô	
÷ 
 ×	Ö	ús	   –5µ
Ac                 óÎ   • [         R                  " 5        nUR                  SU 45        UR                  5       nU(       a  UsS S S 5        $  S S S 5        g ! , (       d  f       g = f)Nz%SELECT * FROM ownskinsdb WHERE a = %s)rn   ro   rp   Úfetchone)rC   rv   Úresults      r'   ÚcheckIfItemInDBr{   ˜   sK   € Ü	×	Ò	Ô	 3Ø�‰Ð;¸a¸TÔBØ—‘“ˆÞØ÷	 
 Ñ	ð ÷ 
 ×	Ö	ús   –,AÁAÁ
A$c                 ó*  • SU S   ;  a$  [        U S   5      n[        U S   S   5      nSnSnO$[        U S   5      nU S   S   nU S   S   S   S   n[        U5      R                  S	5      n[        US
   5      SS  nUS:X  a  SnS[        U5      -   S-   [        U5      -   S-   [        U5      -   nSU ;   a#  [        U S   5      S-  n[        U S   5      S-  n	O"[        U S   5      S-  n[        U S   5      S-  n	X‰-   n
[        U
S5      n
UUUUU
US.$ )NÚmarket_actionsrQ   Ú	listingidrR   r2   zZsteam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D0r   ÚlinkÚ%éÿÿÿÿr   Ú1úCsteam://rungame/730/76561202255233023/+csgo_econ_action_preview%20MÚAÚDrL   rM   rN   rO   rP   r   )r   ÚmrC   ÚdrO   rG   )r   r   rY   rZ   )rg   rG   Úm_paramÚa_paramÚd_paramÚfull_inspect_linkÚspliced_inspect_linkÚfinal_inspect_linkrO   rP   rh   s              r'   ÚbuildListingInfoListrŽ   ¡   s„  € à˜|¨GÑ4Ó4ô �l ;Ñ/Ó0ˆÜ�l 7Ñ+¨DÑ1Ó2ˆØˆØxÑä�l ;Ñ/Ó0ˆØ˜wÑ'¨Ñ-ˆØ(¨Ñ1Ð2BÑCÀAÑFÀvÑNÐÜÐ0Ó1×7Ñ7¸Ó<Ðô Ð& rÑ*Ó+¨A¨BÐ/€GØ�"ƒ}ØˆàMÜ
ˆg‹,ñ	à
ñ	ô ˆg‹,ñ	ð ñ		ô
 ˆg‹,ñ	ð ð ˜LÓ(Ü�lÐ#4Ñ5Ó6¸Ñ<ˆÜ�L Ñ1Ó2°SÑ8‰ô �l 7Ñ+Ó,¨sÑ2ˆÜ�L Ñ'Ó(¨3Ñ.ˆà‘€JÜ�z 1Ó%€Jð #ØØØØØñð rE   c           
      ór  • S/ 0/nU  H:  n US   S   R                  S[        UR                  5       5      S   S   05        M<     US   $ ! [         aá  n [        R
                  " [        U 5      5        [        R
                  " [        [        UR                  5       5      5      5        [        R
                  " [        U5      5        [        R
                  " S[        U5      -   5         S nAMê  ! [         a2  n[        R
                  " S[        U5      -   5         S nA S nAGM!  S nAff = fS nAff = f)NÚlinksr   r   zgetSplittedBulkList():zEXCEPT HELL: )r   ÚlistÚvaluesr   r   r   r   )Úsplitted_bulkÚbulk_inspect_links_listÚir   s       r'   ÚgetSplittedBulkListr–   Ò   s÷   € Ø '¨˜}˜oÐÛˆð	Ø# AÑ& wÑ/×6Ñ6Øœ˜aŸh™h›jÓ)¨!Ñ,¨VÑ4Ð5öñ ð  # 1Ñ%Ð%øô ó 		ðÜ—’œS Ó/Ô0Ü—’œS¤ a§h¡h£jÓ!1Ó2Ô3Ü—’œSÐ!8Ó9Ô:Ü—’Ð5¼¸A»Ñ>Ô?ÝøÜó Ü—’˜_¬s°1«vÑ5Ô6ß’ûðþð		ús6   Œ6AÁ
D6ÁBC2Ã2
D.Ã<"D)ÄD1Ä)D.Ä.D1Ä1D6c                 óT  • SnSnSnU(       Gd[  [         R                  " S5         UR                  U 5      R                  5       nUS-  nSU;   a  SUS   ;   a  SnUS   $ O$[        R
                  " S[        U 5      -   5        M~  US	:”  aÔ  [        R                  " S
S
SSSS9nUR                  5       nSU;   a  UR                  SUS   45        OUR                  SUS   45         UR                  5       S   n	UR                  5       nUR                  SU	45        UR                  5         US-  nUS	:”  a$  Sn[        R
                  " S[        U 5      -   5        GMX  U(       d  GMZ  g g !    GMj  = f! [         a)    Sn[        R
                  " S[        U 5      -   5         GMŸ  f = f)NFr   r   r   rs   ÚfloatidTz7iteminfo not in r_json with single_inspect_server_url: r3   ÚpostgreszBerufsorientierung1!z23.88.122.57Ú6432)ÚdatabaseÚuserr%   r"   r#   Úitemidz(SELECT floatid FROM history WHERE a = %srC   ú-FLOATID NOT IN AFTER 5 TRIES, GIVING UP WITH z$DELETE FROM items WHERE floatid = %sz-EXECUTED DB QUERY MULTIPLE TIMES, SKIPPING - )ÚtimeÚsleepÚgetRequestNakedr   r   r   r   Úpsycopg2ÚconnectÚcursorrp   ry   Ú	TypeErrorÚcommit)
Úsingle_inspect_server_urlÚ	bulk_at_jÚrequest_managerÚend_floatidÚcounter_floatidÚcounter_execÚr_jsonÚpostgresql_connÚpostgresql_curr˜   s
             r'   ÚfloatidNotInBulkr°   ç   sØ  € Ø€KØ€OØ€LßÜ�
Š
�1Œð	Ø$×4Ñ4Ð5NÓO×TÑTÓVˆFð 	˜1ÑˆØ˜ÓØ˜F :Ñ.Ó.Ø"�Ø˜jÑ)Ð)ð /ô
 �LŠLØIÜÐ/Ó0ñ1ôñ à˜QÓÜ&×.Ò.Ø#ØØ/Ø#ØñˆOð -×3Ñ3Ó5ˆNØ˜9Ó$Ø×&Ñ&Ø>ÀÈ8ÑATÐ@Võð ×&Ñ&Ø>ÀÈ3ÁÐ@QôðØ(×1Ñ1Ó3°AÑ6�ð -×3Ñ3Ó5ˆNØ×"Ñ"Ð#IÈGÈ:ÔVØ×"Ñ"Ô$à˜AÑˆLØ˜aÓØ"�Ü—’ØCÜÐ3Ó4ñ5ôò ÷s ‹køð	ÛûôB ó Ø"�Ü—’ØCÜÐ3Ó4ñ5ôó ðús   ¦E+ Ã'E4 Å+E1Å4.F'Æ&F'c                 ó,  • SnSnU(       Gd]  US-  nUS:”  a#  [         R                  " S[        U 5      -   5        g [        R                  " SU-   5         [
        R                  " S[        U 5      -   5      R                  5       nSU;   aÔ  SUS   ;   a  US   $ S	US   ;   aF  [        R                  " 5        nUR                  S
US   S   US   S	   US   S   45        S S S 5        OwSUS   ;   aL  SS:w  aF  [        R                  " 5        nUR                  S
US   S   US   S   US   S   45        S S S 5        O"[         R                  " S[        U5      -   5        U(       d  GM\  g g !    GMl  = f! , (       d  f       N#= f! , (       d  f       N4= f)NFr   r   é   rž   úhttp://23.88.122.57:1337/?url=rs   r˜   r†   z?UPDATE items SET floatid = %s WHERE ms = %s AND paintindex = %srC   Ú
paintindexÚsÚ0zWTF IS GOING ON??????????)r   r   r   rŸ   r    r   r   r   rn   ro   rp   r`   )Úsingle_inspect_linkÚendr«   r­   rv   s        r'   ÚfloatIdNotInSingler¹   &  s’  € Ø
€CØ€OßØ˜1ÑˆØ˜QÓÜ�LŠLÐHÌ3ÐObÓKcÑcÔdØÜ�
Š
�1�Ñ&Ô'ð	Ü—\’\Ð"BÄSÐI\ÓE]Ñ"]Ó^×cÑcÓeˆFð ˜ÓØ˜F :Ñ.Ó.à˜jÑ)Ð)à˜& Ñ,Ó,Ü#×-Ò-Ô/°3ØŸ™Ø]Ø# JÑ/°Ñ4°f¸ZÑ6HÈÑ6MÈvÐV`ÑOaÐbnÑOoÐqô÷ 0Ð/ð
 ˜F :Ñ.Ó.°3¸#³:Ü#×-Ò-Ô/°3ØŸ™Ø]Ø# JÑ/°Ñ4°f¸ZÑ6HÈÑ6MÈvÐV`ÑOaÐbnÑOoÐqô÷ 0Ð/ô —O’OÐ$?Ä#ÀfÃ+Ñ$MÔN÷9 ‹cøð	Ûú÷ 0Õ/ú÷ 0Õ/ús$   Á0E+ Â7(E4Ä(FÅ+E1Å4
FÆ
Fc                 ó  • SnSnU S    Hý  nUSL a    g US   n[        U5      nUS   n[        U[        U5      :H  5      (       aÂ  [        US   SUS   US   5      nU(       d¢  US-  nUS	:”  a$  [        R                  " S
[        U5      -   5        Sn[
        R                  " S[        U5      -   5      R                  5       n	SU	;   a  [        R                  " S5        M„  SU	;   a  U	S   s  $ [        R                  " S5        M©  Mý  Mÿ     g )Nr   Fr�   Tr   r   r¶   r   r²   z'RETRYING BULK ERROR FAILED 5 TIMES FOR r³   r   rs   )
ÚgetParamsFromInspectLinkr   ÚgenerateInspectLinkr   r   r   r   r   rŸ   r    )
Únew_splitted_bulkÚjÚrepeat_counterr¸   r�   r   Ú
all_paramsr‰   rr   r­   s
             r'   ÚretryBulkErrorrÁ   G  s  € Ø€NØ
€CØ" 7Ô+ˆØ�$Š;Øð0 ð/ �V‰}ˆÜ-¨dÓ3ˆ
Ø˜Q‘-ˆÜˆq”C˜“LÑ ×!Ñ!Ü.¨z¸!©}¸cÀ:ÈaÁ=ÐR\Ð]^ÑR_Ó`ˆLæØ !Ñ#�Ø! AÓ%Ü—L’LÐ!JÌSÐQ]ÓM^Ñ!^Ô_Ø�CÜ!ŸšÐ&FÌÈ\ÓIZÑ&ZÓ[×`Ñ`Ób�Ø˜fÓ$Ü—J’J˜q”MÙà! VÓ+Ø% jÑ1Ò1äŸ
š
 1œÙ ñ ñ  ñ3 ,ð4 rE   c                 óT  •  [        U 5      R                  S5      S   nUR                  S5      S   SS  nUR                  S5      S   R                  S5      S   nUR                  S5      S   nX#U/$ ! [         a)  n[        R                  " [        U5      5         S nAg S nAff = f)Nz%20r   r„   r   r…   )r   r   r   r   r   )rr   Úparams_onlyrˆ   r‰   rŠ   r   s         r'   r»   r»   f  s©   € ðÜ˜,Ó'×-Ñ-¨eÓ4°QÑ7ˆØ×#Ñ# CÓ(¨Ñ+¨A¨BÐ/ˆØ×#Ñ# CÓ(¨Ñ+×1Ñ1°#Ó6°qÑ9ˆØ×#Ñ# CÓ(¨Ñ+ˆð ˜gÐ&Ð&øô ó Ü�Š”S˜“VÔÜûðús   ‚A.A4 Á4
B'Á>B"Â"B'c                 óŒ   • [         R                  R                  U 5      nUR                  SS5      nUR                  SS5      nU$ )Nú%28Ú(ú%29Ú)©r\   r]   ÚquoteÚreplace©rq   Úitem_encodeds     r'   ÚencodeItemNamerÎ   q  s@   € Ü—<‘<×%Ñ% nÓ5€LØ×'Ñ'¨¨sÓ3€LØ×'Ñ'¨¨sÓ3€LØÐrE   c                 óÔ   • [         R                  R                  U 5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nU$ )	NrÅ   rÆ   rÇ   rÈ   Ú|z%7CÚ&z%26rÉ   rÌ   s     r'   ÚencodeAidsItemNamerÒ   x  sd   € Ü—<‘<×%Ñ% nÓ5€LØ×'Ñ'¨¨sÓ3€LØ×'Ñ'¨¨sÓ3€LØ×'Ñ'¨¨UÓ3€LØ×'Ñ'¨¨UÓ3€LàÐrE   c                 ó^   • S[        U 5      -   S-   [        U5      -   S-   [        U5      -   nU$ )Nz/https://steamcommunity.com/market/listings/730/z#buylisting|z|730|2|©r   )rÍ   r†   rC   Úmarket_links       r'   ÚgenerateMarketLinkrÖ   ‚  sM   € à9Ü
ˆlÓ
ñ	à
ñ	ô ˆa‹&ñ	ð ñ		ô
 ˆa‹&ñ	ð ð ÐrE   c                 óØ   • [        U 5      S:w  a.  S[        U 5      -   S-   [        U5      -   S-   [        U5      -   nU$ S[        U5      -   S-   [        U5      -   S-   [        U5      -   nU$ )Nr¶   rƒ   r„   r…   rÔ   )r†   rµ   rC   r‡   rr   s        r'   r¼   r¼   Ž  s¤   € ä
ˆ1ƒv�ƒ}àQÜ�!‹fñàñô �!‹fñð ñ	ô
 �!‹fñð 	ð$ Ðð RÜ�!‹fñàñô �!‹fñð ñ	ô
 �!‹fñð 	ð ÐrE   c                 ó¶   • [        [        U5      SSS9nUR                  [        R                  " U SS95        UR                  S5        UR                  5         g )NrC   úutf-8©Úencodingr3   ©ÚindentÚ,)r+   r   r_   r   ÚdumpsÚclose)Úsingle_dictÚfilenameÚf_cheaps      r'   ÚwriteSingleAnyToFilerä   ¦  s@   € Ü”3�x“= #°Ñ8€GØ‡M�M”$—*’*˜[°Ñ3Ô4Ø‡M�M�#ÔØ‡M�M…OrE   c                 ó¤   • [        SSSS9nUR                  [        R                  " U SS95        UR                  S5        UR	                  5         g )Nzcheap_top_skins.txtrC   rÙ   rÚ   r3   rÜ   rÞ   )r+   r_   r   rß   rà   )Úsingle_cheap_top_dictrã   s     r'   ÚwriteSingleCheapSkinToFilerç   ­  s>   € ÜÐ(¨#¸Ñ@€GØ‡M�M”$—*’*Ð2¸1Ñ=Ô>Ø‡M�M�#ÔØ‡M�M…OrE   c                 ó¬   • [         R                  " SU 5      nUb:  UR                  5       n[         R                  " SS[	        U5      5      nUc  g SU-   $ g )Nz"Market_LoadOrderSpread\( [0-9]+ \)z\Dr2   zhhttps://steamcommunity.com/market/itemordershistogram?country=FR&language=german&currency=3&item_nameid=)r9   r:   r;   Úsubr   )Úlisting_source_pageÚxÚaidsÚnameids       r'   ÚmanuallyGetNameIDURLrî   ´  sZ   € Ü
�	Š	Ð7Ð9LÓM€AØ�}Ø�w‰w‹yˆÜ—’˜˜r¤3 t£9Ó-ˆØ‰>Øð {Øñðð
 rE   ) rW   r9   r   rŸ   Úurllib.parser\   r¢   r   ÚLoggingFormatterr   rn   r‘   r(   r/   r   rD   rl   rw   r{   rŽ   r–   r°   r¹   rÁ   r»   rÎ   rÒ   rÖ   r¼   rä   rç   rî   © rE   r'   Ú<module>rò      s¤   ðÛ Û 	Û Û Û Û Û Û Ý #Û ð˜Dô ò@ð cô ð<< ð <°ô <ò|
òò.òb&ò*=ò~OòBò>	'òòò	òò0òórE   