RhddgZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z m Z ddlmZdZd Zd Zd Zej(Zd Zd ZGddeZGddZdZGddeZ d#dZdZGddeZ Gdde!Z"Gdde!Z#e#Z$Gdde#Z%Gdde!Z&Gd d!e&Z'Gd"de"Z(y)$Pool ThreadPoolN)util) get_context TimeoutError)waitINITRUNCLOSE TERMINATEc$tt|SN)listmapargss +/usr/lib/python3.12/multiprocessing/pool.pymapstarr/s T cLttj|d|dS)Nrr)r itertoolsstarmaprs r starmapstarr2s"  !!$q'473 44rceZdZdZdZy)RemoteTracebackc||_yrtb)selfrs r__init__zRemoteTraceback.__init__:s rc|jSrrr s r__str__zRemoteTraceback.__str__<s wwrN)__name__ __module__ __qualname__r!r$rrrr9s rrceZdZdZdZy)ExceptionWithTracebackctjt|||}dj|}||_d|z|_y)Nz """ %s""") tracebackformat_exceptiontypejoinexcr)r r1rs rr!zExceptionWithTraceback.__init__@s:  ' 'S 3 ; WWR[ 2%rc>t|j|jffSr) rebuild_excr1rr#s r __reduce__z!ExceptionWithTraceback.__reduce__EsTXXtww///rN)r%r&r'r!r4r(rrr*r*?s & 0rr*c&t||_|Sr)r __cause__)r1rs rr3r3Hs#B'CM Jrc.eZdZdZfdZdZdZxZS)MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be safely sent through the socket.ct||_t||_tt||j|jyr)reprr1valuesuperr8r!)r r1r; __class__s rr!zMaybeEncodingError.__init__Ts39%[   $04::Frc<d|jd|jdS)NzError sending result: 'z '. Reason: '')r;r1r#s rr$zMaybeEncodingError.__str__Ys=AZZ=AXXG Grc<d|jjd|dS)N)r=r%r#s r__repr__zMaybeEncodingError.__repr__]s!^^44d;;r)r%r&r'__doc__r!r$rC __classcell__r=s@rr8r8Ps'G G}t%||d}tjd |z|| | d|ffYd}~d}~wwxYw) NrzMaxtasks {!r} is not valid_writerrz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFz0Possible encoding error while sending result: %szworker exiting after %d tasks) isinstanceintAssertionErrorformatputgethasattrrHclose_readerEOFErrorOSErrorrdebug Exception_helper_reraises_exceptionr* __traceback__r8)inqueueoutqueue initializerinitargsmaxtaskswrap_exceptionrMrN completedtaskjobifuncrkwdsresultewrappeds rworkerrgasz(C'@+3q=9@@JKK ,,C ++Cw " XI  xI,@ 5D < JJ7 8 #' QdD D$/$/0F  , a !4877s7V7d7TDQ 7  xI,@8 JJ.:;3'"  JJB C   $.H"H*1aoo>QZF  ,(F1I6G JJI  a%)* + +  ,sBD D: E4$D76D7: E1$E,,E14 F;=4F66F;c|)z@Pickle-able helper function for use by _guarded_task_generation.r()exs rrVrVs Hrc2eZdZdZddfd ZfdZxZS) _PoolCachez Class that implements a cache for the Pool class that will notify the pool management threads every time the cache is emptied. The notification is done by the use of a queue that is provided when instantiating the cache. Nnotifierc2||_t||i|yr)rmr<r!)r rmrrcr=s rr!z_PoolCache.__init__s   $'$'rc`t|||s|jjdyyr)r< __delitem__rmrM)r itemr=s rrpz_PoolCache.__delitem__s, D! MM  d #r)r%r&r'rDr!rprErFs@rrkrks +/( $ $rrkceZdZdZdZedZ d(dZeje fdZ dZ d Z ed Zed Zd Zed ZedZdZdZdifdZd)dZd)dZ d*dZdZd+dZd+dZdiddfdZ d*dZ d*dZed)dZe dZ!edZ"edZ#edZ$d Z%d!Z&d"Z'd#Z(ed$Z)e d%Z*d&Z+d'Z,y),rzS Class which supports an async version of applying functions to arguments. Tc&|j|i|SrProcess)ctxrrcs rruz Pool.Processss{{D)D))rNr(cg|_t|_|xs t|_|j t j|_|jj|_ t|j|_ ||_ ||_ ||_|tj xsd}|dkr t#d| t%|t&r|dkr t#d|t)|s t+d||_ |j/|j9}t;j<t>j@|j|j|j|jB|j,|j|jD|jF|j|j|j|jH||jf|_%d|jJ_&tN|jJ_|jJjQt;j<t>jR|j|jT|jF|j|jf|_+d|jV_&tN|jV_|jVjQt;j<t>jX|jF|jZ|jf|_.d|j\_&tN|j\_|j\jQt_j`||jb|j|jD|jF|j|j|jJ|jV|j\|jf d |_2tN|_y#t0$rQ|jD]}|j2|j5!|jD]}|j7wxYw) Nrlrz&Number of processes must be at least 1rz/maxtasksperchild must be a positive int or Nonezinitializer must be a callabletargetrT)r exitpriority)3_poolr _stater_ctx _setup_queuesqueue SimpleQueue _taskqueue_change_notifierrk_cache_maxtasksperchild _initializer _initargsos cpu_count ValueErrorrIrJcallable TypeError _processes_repopulate_poolrUexitcode terminater0_get_sentinels threadingThreadr_handle_workersru_inqueue _outqueue_wrap_exception_worker_handlerdaemonr start _handle_tasks _quick_put _task_handler_handle_results _quick_get_result_handlerrFinalize_terminate_pool _terminate)r processesrZr[maxtasksperchildcontextp sentinelss rr!z Pool.__init__sV  ,{}  ++-!% 5 5 7 $*?*?@ !1'!   +!I q=EF F  '.48HA8M !RSS  "8K+@<= =#   ! ! #'') (//''++t 4<<//4::t}}dnn##T^^T5K5K&& 43H3HJ '+#&)# ""$'--%%//4??DNN**dkk+ %)!$'!   "(//''..$//4;;? '+#&)# ""$-- $&&//4==$..$**'')=)=t?Q?Q&& 5  ] ZZ "::%KKM "ZZ     s O''$Q 5Qc|j|k(r<|d|t|t|dd|jj dyyy)Nz&unclosed running multiprocessing pool )sourcer)r}ResourceWarninggetattrrrM)r _warnr s r__del__z Pool.__del__ sT ;;#  :4(C!$ 0t/6B%%))$/C rc |j}d|jd|jd|jdt |j d S)NrA.z state=z pool_size=rB)r=r&r'r}lenr|)r clss rrCz Pool.__repr__sQnnCNN#1S%5%5$67 & _-Q0 1rcj|jjg}|jjg}g||Sr)rrQr)r task_queue_sentinelsself_notifier_sentinelss rrzPool._get_sentinelss< $ 6 67#'#8#8#@#@"A@%@(?@@rcX|Dcgc]}t|dr |jc}Scc}w)Nsentinel)rOr)workersrgs r_get_worker_sentinelszPool._get_worker_sentinelss18F"6:68 88s'cd}ttt|D]A}||}|jt j d|z|j d}||=C|S)zCleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. Fcleaning up worker %dT)reversedrangerrrrTr0)poolcleanedrargs r_join_exited_workerszPool._join_exited_workers!sg %D *+ A!WF* 2Q67 G rc |j|j|j|j|j|j |j |j|j|j|j Sr) _repopulate_pool_staticr~rurr|rrrrrrr#s rrzPool._repopulate_pool1s_++DIIt||,0OO,0JJ ,0NNDbI I Js)A $A  AAA AA c|j|dk(rOt|}|jj|j |j |||j f|S|dkrtdj|tj|||}t|}|jj|j |j t||j fd|DS)zP Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. rzChunksize must be 1+, not {0:n}c3.K|] }|D]}|ywrr(.0chunkrqs r zPool.imap..?U?D?D?) r IMapIteratorrrMr_job _set_lengthrrLr _get_tasksrr rbrrrd task_batchess rimapz Pool.imaps  >!$'F OO  11&++tXN&&  M1} 5<<!#$$ ??49EL!$'F OO  11&++292>@&&  @f? ?rc|j|dk(rOt|}|jj|j |j |||j f|S|dkrtdj|tj|||}t|}|jj|j |j t||j fd|DS)zL Like `imap()` method but ordering of results is arbitrary. rzChunksize must be 1+, not {0!r}c3.K|] }|D]}|ywrr(rs rrz&Pool.imap_unordered..rr) rIMapUnorderedIteratorrrMrrrrrLrrrrs rimap_unorderedzPool.imap_unordereds  >*40F OO  11&++tXN&&  M1} 5<@&&  @f? ?rc|jt|||}|jj|jd|||fgdf|S)z; Asynchronous version of `apply()` method. rN)r ApplyResultrrMr)r rbrrcrrrds rrzPool.apply_asyncsL T8^< v{{AtT4@A4HI rc6|j||t|||S)z9 Asynchronous version of `map()` method. )rrrs r map_asynczPool.map_asyncs" tXw 8  rc|jt|ds t|}|5tt |t |j dz\}}|r|dz }t |dk(rd}t j|||}t||t |||} |jj|j| j||df| S)zY Helper function to implement map, starmap and their async counterparts. __len__Nrrr) rrOrdivmodrr|rr MapResultrrMrr) r rbrmapperrrrextrarrds rrzPool._map_asyncs x+H~H  %c(mS_q5HI IuQ x=A ItXyA 4CM8*8: --fkk.4.:<    rct|||js"|j|js!yy)N)timeout)r emptyrN)rchange_notifierrs r_wait_for_updateszPool._wait_for_updatess2 Y(!'')    !"'')rc tj}|jtk(s|r~|jtk7rk|j ||||||| | | | g|j || }|j|||jtk(rU|r|jtk7rk|jdtjdy)Nzworker handler exiting) rcurrent_threadr}r r rrrrMrrT)rcache taskqueuervrurrrXrYrZr[rr]rrthreadcurrent_sentinelss rrzPool._handle_workerss ))+mms"u)1K   sGYg'h/ A!O#";";D"A NI N   ! !"3_ Emms"u)1K  d +,rc0tj}t|jdD]}\}}d} |D]5}|jt k7rt jdn> ||7|r)t jd|r|dnd} || dz dx}x}} v dx}x}} nt jd t jd|jdt jd |D] } |d  t jd y#t$r;} |dd\} } || j| d| fn#t$rYnwxYwYd} ~ +d} ~ wwxYw#dx}x}} wxYw#t$rt jd YwxYw) Nz'task handler found thread._state != RUNFzdoing set_length()rrztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting) rr iterrNr}r rrTrU_setKeyErrorrMrS) r rMrYrr r taskseq set_lengthr_rer`idxrs rrzPool._handle_taskss))+#' t#< 4 GZD ,#D}}+ #LM!D " #78)-d1g2"37+(,++w'+++w1 44 JJ2 3 J JJH I LL  JJA B D   )*=%!#'8S!!#JOOC%<'! ! !(,++w J JJH I Jsl/E)$D",-E)A E4" E&+E!4E  E! E E!E E!E)!E&&E))E14FFc$tj} |}|j tk7r0|j tk(sJdt j dn;|t j dn#|\}}} ||j||dx}x}}|rk|j tk7rX |}|t j d5|\}}} ||j||dx}x}}|r|j tk7rXt|drIt j d tdD]%}|jjsn |'t j d t||j y#ttf$rt j dYywxYw#t$rY*wxYw#ttf$rt j dYywxYw#t$rY wxYw#ttf$rYwxYw) Nz.result handler got EOFError/OSError -- exitingzThread not in TERMINATEz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelrQz"ensuring that outqueue is not full z7result handler exiting: len(cache)=%s, thread._state=%s)rr rSrRrrTr}r r rrrOrrQpollr)rYrNr r r_r`raobjs rrzPool._handle_results=s ))+ u }}#}} 1L3LL1 IJ| 89KCC c 3' $ #D #3+. 2 u | CDKCC c 3' $ #D #3 2" 8Y ' JJ; < rA#++002E L%j&-- )eX&  KL    X&  KL   X&  sYF F32GG-+3G= $F0/F03 G?G$G*)G*- G:9G:=HHc#tKt|} ttj||}|sy||f)wr)rtuplerislice)rbitsizers rrzPool._get_tasksys= "Xi&&r401A)O s68ctd)Nz:pool objects cannot be passed between processes or pickled)NotImplementedErrorr#s rr4zPool.__reduce__s!J rctjd|jtk(r" ;;# DK*/D '  ! ! % %d + rcdtjdt|_|j y)Nzterminating pool)rrTr r}rr#s rrzPool.terminates  %&  rctjd|jtk(r t d|jt t fvr t d|jj|jj|jj|jD]}|jy)Nz joining poolzPool is still runningzIn unknown state) rrTr}r rr r rr0rrr|)r rs rr0z Pool.joins >" ;;# 45 5 [[ 2 2/0 0 !!# ! !!# A FFH rcptjd|jj|j rw|j j r\|j jtjd|j r|j j rZyyyy)Nz7removing tasks from inqueue until task handler finishedr) rrT_rlockacquireis_aliverQrrtimesleep)rX task_handlerrs r_help_stuff_finishzPool._help_stuff_finishs} LM ##%'//*>*>*@ OO " JJqM##%'//*>*>*@%*@%rc dtjdt|_|j dt|_tjd|j ||t ||jst | dk7r tdt|_|j d|j dtjdtj|ur|j|rHt|ddr9tjd|D]} | j| j!tjdtj|ur|jtjd tj|ur|j|rpt|ddr`tjd |D]E} | jstjd | jz| jGyyy) Nzfinalizing poolz&helping task handler/workers to finishrz/Cannot have cache with result_handler not alivezjoining worker handlerrzterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr)rrTr r}rMr+rr'rKrr r0rOrrpid) rr rXrYrrworker_handlerr*result_handlerr rs rrzPool._terminate_pools $% !*D!'  ;< w c$i@'')E a AC C!*D! T +,  # # %^ ;    ! GDG[1 JJ, - "::%KKM " )*  # # %\ 9     +,  # # %^ ;    ! GDG[1 JJ- . ::<JJ6>?FFH  24rc&|j|Sr)rr#s r __enter__zPool.__enter__s  rc$|jyr)r)r exc_typeexc_valexc_tbs r__exit__z Pool.__exit__s  r)NNr(NNr)NNN)r)-r%r&r'rDr staticmethodrur!warningswarnr rrCrrrrrrrrrrrrrrrrrrr classmethodrrrrr4rPrr0r+rr1r6r(rrrrsO**CE04Of%MMs01 A 88  B''$ 9 96 1 "8IMFJ9 J@:@8&(b4BFKO:"" --(++++Z9)9)v , 33jrcXeZdZdZdZdZddZddZdZe e jZ y) rc||_tj|_t t |_|j|_||_||_ ||j|j <yr) r|rEvent_eventnext job_counterrr _callback_error_callback)r rrrs rr!zApplyResult.__init__sO oo' % kk !-!% DIIrc6|jjSr)r>is_setr#s rreadyzApplyResult.readys{{!!##rcn|jstdj||jS)Nz{0!r} not ready)rErrL_successr#s r successfulzApplyResult.successfuls,zz|.55d;< <}}rNc:|jj|yr)r>r r rs rr zApplyResult.waits !rc|j||jst|jr |jS|jr)r rErrG_valuerJs rrNzApplyResult.gets8 'zz|  ==;; ++ rc^|\|_|_|jr'|jr|j|j|jr'|js|j|j|jj |j |j=d|_yr) rGrLrArBr>setrrr|r rars rrzApplyResult._setss%(" t{ >>dmm NN4;; '      -  KK " rr) r%r&r'r!rErHr rNrr:types GenericAlias__class_getitem__r(rrrrs3&$ "$E$6$67rrceZdZdZdZy)rctj||||d|_dg|z|_||_|dkr9d|_|j j|j|j=y||zt||zz|_y)NrTr) rr!rGrL _chunksize _number_leftr>rNrrbool)r rrlengthrrs rr!zMapResult.__init__sT4,:  < fvo # > !D  KKOO  DII& & 1D)9K4L LD rc|xjdzc_|\}}|r|jr||j||jz|dz|jz|jdk(r`|jr|j |j|j |j =|jjd|_ yy|s|jrd|_||_|jdk(r`|jr|j|j|j |j =|jjd|_ yy)NrrF) rVrGrLrUrArrr>rNr|rB)r rasuccess_resultsuccessrds rrzMapResult._set)s Q( t}}CIDKK$//)1Q3*? @  A%>>NN4;;/KK * !! &t}} % $   A%''((5KK * !! &rN)r%r&r'r!rr(rrrrs  M"rrc0eZdZdZdZddZeZdZdZy)rcB||_tjtj|_t t |_|j|_tj|_ d|_ d|_ i|_||j|j<y)Nr)r|r ConditionLock_condr?r@rr collectionsdeque_items_index_length _unsorted)r rs rr!zIMapIterator.__init__Gsp (()9: % kk !'')   !% DIIrc|Srr(r#s r__iter__zIMapIterator.__iter__Rs rNc|j5 |jj}ddd\}}|r|S|#t$r|j|j k(rd|_td|jj| |jj}n;#t$r/|j|j k(rd|_tdtdwxYwYwxYw#1swYxYwr) r`rcpopleft IndexErrorrdrer| StopIterationr r)r rrqr[r;s rr?zIMapIterator.nextUs ZZ 1 1{{**, 1  L ! 1;;$,,.!%DJ'T1 (1;;..0D!1{{dll2%) +5&D0 1 1 1 1s?C$<A C!B#"C!#8CC!C$ C!!C$$C-c|j5|j|k(r|jj||xjdz c_|j|jvrn|jj |j}|jj||xjdz c_|j|jvrn|jj n||j|<|j|jk(r|j|j=d|_ dddy#1swYyxYwNr) r`rdrcrrfpopnotifyrerrr|rOs rrzIMapIterator._setms ZZ "{{a ""3' q kkT^^3..,,T[[9CKK&&s+KK1$KkkT^^3 !!#$'q!{{dll*KK *!  " " "sCD=A!D==Ec|j5||_|j|jk(r8|jj|j|j =d|_dddy#1swYyxYwr)r`rerdrprrr|)r rXs rrzIMapIterator._set_length~s\ ZZ "!DL{{dll* !!#KK *!  " " "s AA//A8r) r%r&r'r!rhr?__next__rrr(rrrrEs" &,H"""rrceZdZdZy)rcH|j5|jj||xjdz c_|jj |j|j k(r|j |j=d|_dddy#1swYyxYwrn) r`rcrrdrprerrr|rOs rrzIMapUnorderedIterator._setsu ZZ " KK  s # KK1 K JJ   {{dll*KK *!  " " "s BBB!N)r%r&r'rr(rrrrs"rrcZeZdZdZedZd dZdZdZedZ edZ d Z y) rFcddlm}||i|S)Nrrt)dummyru)rvrrcrus rruzThreadPool.Processs"%%%rNc4tj||||yr)rr!)r rrZr[s rr!zThreadPool.__init__s dI{H=rctj|_tj|_|jj|_|jj |_yr)rrrrrMrrNrr#s rrzThreadPool._setup_queuessD))+ **,--++..,,rc0|jjgSr)rrQr#s rrzThreadPool._get_sentinelss%%--..rcgSrr()rs rrz ThreadPool._get_worker_sentinelss rc |jd#tj$rYnwxYwt|D]}|j dy)NF)block)rNrEmptyrrM)rXr*rras rr+zThreadPool._help_stuff_finishsR  % ({{   t A KK  s ,,c.tj|yr)r(r))r rrrs rrzThreadPool._wait_for_updatess 7r)NNr() r%r&r'rr7rur!rrrr+rr(rrrrsUO&&>- /r)Nr(NF))__all__rarrrrr(r-rPr8r,rrr connectionr r r r r countr@rrrUrr*r3r8rgrVdictrkobjectrr AsyncResultrrrrr(rrrs <     '    ioo 5i 00<<"GK+