
    YgU,                         S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKJr  S SK	r	S\
4S jrS rS rS rS rS	 rS
 rS rS rS rS rS rS rS rS rg)    N)loggerreturnc                  L    [         R                  " SSS0SS9R                  5       S   n [         R                  " S[        U 5      -   S-   S	S
9R                  n/ nUR                  5       nU H{  nUR                  S5      S   nUR                  S5      S   nUR                  S5      S   nUR                  S5      S   n	SU-   S-   U	-   S-   U-   S-   U-   n
UR                  U
5        M}     U$ ! [
         a+  n[        R                  " [        U5      5        / s S nA$ S nAff = f)Nz.https://proxy.webshare.io/api/v2/proxy/config/Authorization(wsvwkkpdl6e2zirlyimm1nbkxbrcxf0zr07myh61
   )headerstimeoutproxy_list_download_tokenz5https://proxy.webshare.io/api/v2/proxy/list/download/z/-/any/username/direct/   )r
   :r            zhttp://@)requestsgetjsonstrtext	Exceptionr   error
splitlinessplitappend)dl_token
proxy_listeall_proxiesproxiesproxyhostportusernamepasswordfinal_proxys              -/var/www/highfloat_scraper/HelperFunctions.pygetWebshareProxiesr(      s<   <<<$&PQ
 $&,	. \\C(m'( 	

 $ 	 K##%G{{3"{{3";;s#A&;;s#A&(*S08;cADH3NQUU;'    SV	s   AC. .
D#8 DD#D#c                     [        SS5       nXR                  5       ;   a
   S S S 5        g S S S 5        g! , (       d  f       g = f)Nzexcluded_skins.txtrTF)openread)skin_name_with_conditionfs     r'   checkIfSkinExcludedr/   ,   s7    	"C	(A#vvx/ 
)	( 	 
)	(	(s   55
Ac                    SU S   ;  a$  [        U S   5      n[        U S   S   5      nSnSnO$[        U S   5      nU S   S   nU S   S   S   S   n[        U5      R                  S	5      n[        US
   5      SS  nS[        U5      -   S-   [        U5      -   S-   [        U5      -   nSU ;   a#  [        U S   5      S-  n[        U S   5      S-  n	O"[        U S   5      S-  n[        U S   5      S-  n	X-   n
[        U
S5      n
UUUUU
US.$ )Nmarket_actionsasset	listingidid zZsteam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D0r   link%r   Csteam://rungame/730/76561202255233023/+csgo_econ_action_preview%20MADconverted_priced   converted_feepricefeer   )r6   madr?   high_low)r   r   floatround)listing_inforD   m_parama_paramd_paramfull_inspect_linkspliced_inspect_linkfinal_inspect_linkr?   r@   
full_prices              r'   buildListingInfoListrO   4   sv   |G44 l;/0l7+D12xl;/0w'-(12BCAFvN0177<&r*+AB/GM
g,	
	 g,	 		
 g,	  L(l#456<L12S8 l7+,s2L'(3.Jz1%J #     c           
      r   S/ 0/nU  H:  n US   S   R                  S[        UR                  5       5      S   S   05        M<     US   $ ! [         a  n [        R
                  " [        U 5      5        [        R
                  " [        [        UR                  5       5      5      5        [        R
                  " [        U5      5        [        R
                  " S[        U5      -   5         S nAM  ! [         a2  n[        R
                  " S[        U5      -   5         S nA S nAGM!  S nAff = fS nAff = f)Nlinksr   r6   zgetSplittedBulkList():zEXCEPT HELL: )r   listvaluesr   r   r   r   )splitted_bulkbulk_inspect_links_listir   s       r'   getSplittedBulkListrX   a   s     '}o	#A&w/66ahhj)!,V45   #1%%  		S/0Sahhj!123S!89:5A>? _s1v56		s6   6A
D6BC22
D.<"D)D1)D..D11D6c                 T   SnSnSnU(       Gd[  [         R                  " S5         UR                  U 5      R                  5       nUS-  nSU;   a  SUS   ;   a  SnUS   $ O$[        R
                  " S[        U 5      -   5        M~  US	:  a  [        R                  " S
S
SSSS9nUR                  5       nSU;   a  UR                  SUS   45        OUR                  SUS   45         UR                  5       S   n	UR                  5       nUR                  SU	45        UR                  5         US-  nUS	:  a$  Sn[        R
                  " S[        U 5      -   5        GMX  U(       d  GMZ  g g !    GMj  = f! [         a)    Sn[        R
                  " S[        U 5      -   5         GM  f = f)NFr   r   r   iteminfofloatidTz7iteminfo not in r_json with single_inspect_server_url:    postgreszBerufsorientierung1!z23.88.122.576432)databaseuserr%   r"   r#   itemidz(SELECT floatid FROM history WHERE a = %srB   -FLOATID NOT IN AFTER 5 TRIES, GIVING UP WITH z$DELETE FROM items WHERE floatid = %sz-EXECUTED DB QUERY MULTIPLE TIMES, SKIPPING - )timesleepgetRequestNakedr   r   r   r   psycopg2connectcursorexecutefetchone	TypeErrorcommit)
single_inspect_server_url	bulk_at_jrequest_managerend_floatidcounter_floatidcounter_execr_jsonpostgresql_connpostgresql_curr[   s
             r'   floatidNotInBulkrv   v   s   KOL

1	$445NOTTVF 	1F:.."j)) /
 LLI/01 Q&..#/#O -335N9$&&>8AT@V &&>3@Q(113A6 -335N""#IG:V""$ALa"C345 s k	B  "C345 s   E+ 'E4 +E14.F'&F'c                 ,   SnSnU(       Gd]  US-  nUS:  a#  [         R                  " S[        U 5      -   5        g [        R                  " SU-   5         [
        R                  " S[        U 5      -   5      R                  5       nSU;   a  SUS   ;   a  US   $ S	US   ;   aF  [        R                  " 5        nUR                  S
US   S   US   S	   US   S   45        S S S 5        OwSUS   ;   aL  SS:w  aF  [        R                  " 5        nUR                  S
US   S   US   S   US   S   45        S S S 5        O"[         R                  " S[        U5      -   5        U(       d  GM\  g g !    GMl  = f! , (       d  f       N#= f! , (       d  f       N4= f)NFr   r      rb   http://23.88.122.57:1337/?url=rZ   r[   rA   z?UPDATE items SET floatid = %s WHERE ms = %s AND paintindex = %srB   
paintindexs0zWTF IS GOING ON??????????)r   r   r   rc   rd   r   r   r   
topfloatdb	db_cursorri   critical)single_inspect_linkendrq   rs   curs        r'   floatIdNotInSingler      s   
CO1QLLH3ObKccd

1&'	\\"BSI\E]"]^cceF F:..j))&,,#--/3]#J/4fZ6H6MvV`OabnOoq 0/
 F:..3#:#--/3]#J/4fZ6H6MvV`OabnOoq 0/ OO$?#f+$MN9 c	 0/ 0/s$   0E+ 7(E4(F+E14
F
Fc                    SnSnU S    H  nUSL a    g US   n[        U5      nUS   n[        U[        U5      :H  5      (       a  [        US   SUS   US   5      nU(       d  US-  nUS	:  a$  [        R                  " S
[        U5      -   5        Sn[
        R                  " S[        U5      -   5      R                  5       n	SU	;   a  [        R                  " S5        M  SU	;   a  U	S   s  $ [        R                  " S5        M  M  M     g )Nr   FrR   Tr6   r   r|   r   rx   z'RETRYING BULK ERROR FAILED 5 TIMES FOR ry   r   rZ   )
getParamsFromInspectLinkr   generateInspectLinkr   r   r   r   r   rc   rd   )
new_splitted_bulkjrepeat_counterr   rR   r6   
all_paramsrI   inspect_linkrs   s
             r'   retryBulkErrorr      s   N
C"7+$;0 / V}-d3
Q-qCL !!.z!}c:a=R\]^R_`L!#!A%LL!JSQ]M^!^_C!&F\IZ&Z[``bf$JJqM!V+%j11

1    3 ,4 rP   c                 T    [        U 5      R                  S5      S   nUR                  S5      S   SS  nUR                  S5      S   R                  S5      S   nUR                  S5      S   nX#U/$ ! [         a)  n[        R                  " [        U5      5         S nAg S nAff = f)Nz%20r   r:   r   r;   )r   r   r   r   r   )r   params_onlyrH   rI   rJ   r   s         r'   r   r      s    ,'--e4Q7##C(+AB/##C(+11#6q9##C(+ g&&  SVs   A.A4 4
B'>B""B'c                     [         R                  R                  U 5      nUR                  SS5      nUR                  SS5      nU$ )N%28(%29)urllibparsequotereplacefull_item_nameitem_encodeds     r'   encodeItemNamer      s@    <<%%n5L''s3L''s3LrP   c                     [         R                  R                  U 5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nU$ )	Nr   r   r   r   |z%7C&z%26r   r   s     r'   encodeAidsItemNamer     sd    <<%%n5L''s3L''s3L''U3L''U3LrP   c                 ^    S[        U 5      -   S-   [        U5      -   S-   [        U5      -   nU$ )Nz/https://steamcommunity.com/market/listings/730/z#buylisting|z|730|2|r   )r   rA   rB   market_links       r'   generateMarketLinkr     sM    9
l
	
	 a&	 		
 a&	  rP   c                     [        U 5      S:w  a.  S[        U 5      -   S-   [        U5      -   S-   [        U5      -   nU$ S[        U5      -   S-   [        U5      -   S-   [        U5      -   nU$ )Nr|   r9   r:   r;   r   )rA   r{   rB   rC   r   s        r'   r   r     s    
1v}Q!f !f 	
 !f 	$  R!f !f 	
 !f 	 rP   c                     [        [        U5      SSS9nUR                  [        R                  " U SS95        UR                  S5        UR                  5         g )NrB   utf-8encodingr\   indent,)r+   r   writer   dumpsclose)single_dictfilenamef_cheaps      r'   writeSingleAnyToFiler   5  s@    3x=#8GMM$**[34MM#MMOrP   c                     [        SSSS9nUR                  [        R                  " U SS95        UR                  S5        UR	                  5         g )Nzcheap_top_skins.txtrB   r   r   r\   r   r   )r+   r   r   r   r   )single_cheap_top_dictr   s     r'   writeSingleCheapSkinToFiler   <  s>    (#@GMM$**21=>MM#MMOrP   c                     [         R                  " SU 5      nUb:  UR                  5       n[         R                  " SS[	        U5      5      nUc  g SU-   $ g )Nz"Market_LoadOrderSpread\( [0-9]+ \)z\Dr5   zhhttps://steamcommunity.com/market/itemordershistogram?country=FR&language=german&currency=3&item_nameid=)researchgroupsubr   )listing_source_pagexaidsnameids       r'   manuallyGetNameIDURLr   C  sZ    
		79LMA}wwyr3t9-> {
 rP   )r   r   rc   urllib.parser   rf   r   LoggingFormatterr   r}   rS   r(   r/   rO   rX   rv   r   r   r   r   r   r   r   r   r   r    rP   r'   <module>r      sw    	       # D @*Z&*=~OB>	'	0rP   