File: //usr/local/lib/python3.9/site-packages/billiard/__pycache__/pool.cpython-39.pyc
a
X>h� � @ s d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl Z d dl
m Z d dl m
Z
ddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZmZmZ dd l m Z dd
l m Z dd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' d d lm(Z( d d
l)m*Z*m+Z+ ddlm,Z,m-Z-m.Z. dZ/ej0d dkZ1e�2� dk�rRddl3m4Z5 eZ6nd dlm7Z5 ej6Z6z
ej8Z8W n e9�y� dZ8Y n0 ej0dk�r�ej:Z;nej;Z;d Z<dZ=dZ>d Z?dZ@dZAdZBdZCd ZDdZEdZFeGedd�ZHdZIeGedd �ZDd ZJd ZKe�L� ZMejNZNd d � ZOd d!� ZPd"d#� ZQd$d%� ZRdHd&d'�ZSG d(d)� d)e;�ZTG d*d+� d+eU�ZVG d,d-� d-eU�ZWd.d/� ZXG d0d1� d1�ZYG d2d3� d3e �ZZG d4d5� d5eZ�Z[G d6d7� d7eZ�Z\G d8d9� d9eZ�Z]G d:d;� d;eZ�Z^G d<d=� d=�Z_G d>d?� d?�Z`G d@dA� dAe`�ZaG dBdC� dC�ZbG dDdE� dEeb�ZcG dFdG� dGe_�ZddS )I� N)�deque)�partial� )� cpu_count� get_context)�util)� TERM_SIGNAL� human_status� pickle_loads�
reset_signals�
restart_state)� get_errno�mem_rss� send_offset)�
ExceptionInfo)� DummyProcess)�CoroStop�RestartFreqExceeded�SoftTimeLimitExceeded�
Terminated�TimeLimitExceeded� TimeoutError�WorkerLostError�� monotonic��Queue�Empty)�Finalize�debug�warningzEchild process exiting after exceeding memory limit ({0}KiB / {1}KiB)
� �Windows)�kill_processtree)�killg _�B)r! r! � � � �SIGUSR1g $@�EX_OKi, 皙�����?c C s< z
| j }W n ty d }Y n0 |d u r8tt | �� �S |S �N)r �AttributeErrorr �fileno)�
connectionZnative� r/ �7/usr/local/lib/python3.9/site-packages/billiard/pool.py�_get_send_offsetx s
r1 c C s t t| � �S r+ )�list�map��argsr/ r/ r0 �mapstar� s r6 c C s t t�| d | d ��S )Nr r )r2 � itertools�starmapr4 r/ r/ r0 � starmapstar� s r9 c O s t �� j| g|�R i |�� d S r+ )r Z
get_logger�error)�msgr5 �kwargsr/ r/ r0 r: � s r: c C s | t �� ur| �|� d S r+ )� threading�current_thread�stop)�thread�timeoutr/ r/ r0 �stop_if_not_current� s rB c @ sb e Zd ZdZdd� Zer<ddd�Zdd � Zd
d � Zd d
� Z n"ddd�Zdd � Zdd � Zdd
� Z dS )�LaxBoundedSemaphorez^Semaphore that checks that # release is <= # acquires,
but ignores if # releases >= value.c C s | j d8 _ | �� d S �Nr )�_initial_value�acquire��selfr/ r/ r0 �shrink� s zLaxBoundedSemaphore.shrinkr Nc C s t �| |� || _d S r+ ��
_Semaphore�__init__rE �rH �value�verboser/ r/ r0 rL � s z LaxBoundedSemaphore.__init__c C sP | j �6 | jd7 _| jd7 _| j �� W d � n1 sB0 Y d S rD )�_condrE �_value�notifyrG r/ r/ r0 �grow� s zLaxBoundedSemaphore.growc C sP | j }|�2 | j| jk r.| jd7 _|�� W d � n1 sB0 Y d S rD )rP rQ rE �
notify_all�rH Zcondr/ r/ r0 �release� s
zLaxBoundedSemaphore.releasec C s | j | jk rt�| � q d S r+ )rQ rE rK rV rG r/ r/ r0 �clear� s zLaxBoundedSemaphore.clearc C s t �| ||� || _d S r+ rJ rM r/ r/ r0 rL � s c C sR | j }|�4 | jd7 _| jd7 _|�� W d � n1 sD0 Y d S rD )�_Semaphore__condrE �_Semaphore__valuerR rU r/ r/ r0 rS � s
c C sP | j }|�2 | j| jk r.| jd7 _|�� W d � n1 sB0 Y d S rD )rX rY rE � notifyAllrU r/ r/ r0 rV � s
c C s | j | jk rt�| � q d S r+ )rY rE rK rV rG r/ r/ r0 rW � s )r N)r N)
�__name__�
__module__� __qualname__�__doc__rI �PY3rL rS rV rW r/ r/ r/ r0 rC � s
rC c s0 e Zd ZdZ� fdd�Zdd� Zdd� Z� ZS )�MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
safely sent through the socket.c s* t |�| _t |�| _t� �| j| j� d S r+ )�repr�excrN �superrL )rH rb rN �� __class__r/ r0 rL � s
zMaybeEncodingError.__init__c C s d| j jt| �f S )Nz<%s: %s>)re r[ �strrG r/ r/ r0 �__repr__� s zMaybeEncodingError.__repr__c C s d| j | jf S )Nz)Error sending result: '%r'. Reason: '%r'.)rN rb rG r/ r/ r0 �__str__� s �zMaybeEncodingError.__str__)r[ r\ r] r^ rL rg rh �
__classcell__r/ r/ rd r0 r` � s r` c @ s e Zd ZdZdS )�
WorkersJoinedz All workers have terminated.N)r[ r\ r] r^ r/ r/ r/ r0 rj � s rj c C s
t � �d S r+ )r )�signum�framer/ r/ r0 �soft_timeout_sighandler� s rm c
@ s� e Zd Zd dd�Zdd� Zdd � Zd
d � Zd d d
�Zdd� Zdd� Z e
e dfdd�Z dd� Z
dd� Zdd� Zefdd�Zd d � ZdS ) �WorkerNr/ Tc
C sz |d u s t |�tkr |dks J �|| _|| _|| _| | _|| _|| _| | _||| | _ | _
| _ |
| _ | | _
| �| � d S �Nr )�type�int� initializer�initargs�maxtasks�max_memory_per_child� _shutdown�on_exit�
sigprotection�inq�outq�synq�wrap_exception�on_ready_counter�contribute_to_object)
rH ry rz r{ rr rs rt �sentinelrw rx r| ru r} r/ r/ r0 rL � s zWorker.__init__c C s� | j | j| j |_ |_|_| j j�� |_| jj�� |_| jrj| jj�� |_| jj�� |_ t
| jj�|_ nd |_ |_ |_ | j jj
|_| jjj|_t
| j j�|_|S r+ )ry rz r{ �_writerr- �inqW_fd�_readerZoutqR_fdZsynqR_fd�synqW_fdr1 Zsend_syn_offsetZ_send_syn_offset�send�
_quick_put�recv�
_quick_getZsend_job_offset)rH �objr/ r/ r0 r~ � s zWorker.contribute_to_objectc
C s: | j | j| j| j| j| j| j| j| j| j | j
| j | j f fS r+ )
re ry rz r{ rr rs rt rv rw rx r| ru r} rG r/ r/ r0 �
__reduce__
s �zWorker.__reduce__c s� t j� d g�d� �fdd� }|t _t�� }| �� | �� | j|d� ztzt �| j|d�� W nF ty� } z.t d| |dd� | �
|�d |� W Y d }~n
d }~0 0 W | �
|�d d � n| �
|�d d � 0 d S ) Nc s | �d< � | �S ro r/ )�status��_exitZ _exitcoder/ r0 �exit s z Worker.__call__.<locals>.exit��pidzPool process %r error: %rr ��exc_infor )N) �sysr� �os�getpid�_make_child_methods�
after_fork�
on_loop_start�workloop� Exceptionr: �_do_exit)rH r� r� rb r/ r� r0 �__call__ s *zWorker.__call__c C s~ |d u r|rt nt}| jd ur*| �||� tjdkrpz,| j�t||ff� t� d� W t
� |� qzt
� |� 0 n
t
� |� d S )N�win32r ) �
EX_FAILUREr) rw r� �platformrz �put�DEATH�time�sleepr� r� )rH r� �exitcoderb r/ r/ r0 r� + s
zWorker._do_exitc C s d S r+ r/ �rH r� r/ r/ r0 r� ; s zWorker.on_loop_startc C s |S r+ r/ )rH �resultr/ r/ r0 �prepare_result> s zWorker.prepare_resultc
s< |p
t �� }�jj}�j}�j}�j}�jp.d}�j} �j }
�j
� � �fdd�} d} �z�|d u sp|�r�| |k �r�|
� }
|
rX|
\}}|t ks�J �|\}}}}}|t |||� ||ff� � r�| |�}|s�qXzd| ||i |���f}W n t
y� dt� f}Y n0 z|t||||ff� W nt t
�y� } zZt�� \}}}z:t||d �}tt||f�}|t||d|f|ff� W ~n~0 W Y d }~n
d }~0 0 | d7 } |dkrXt� } | dk�r�td� | dkrX| |krXtt�| |�� tW �j| d� S qX|d | � |�r| |k�rtntW �j| d� S tW �j| d� S �j| d� 0 d S )
Nr c s` d}|dkr$t d| �jj�� dd� � � }|rR|\}}|tkrBdS |tksNJ �dS |d7 }qd S )Nr �< z(!!!WAIT FOR ACK TIMEOUT: job:%r fd:%r!!!r r� FT)r: r{ r� r- �NACK�ACK)Zjid�i�req�type_r5 �Z
_wait_for_synrH r/ r0 � wait_for_synM s � z%Worker.workloop.<locals>.wait_for_synTFr z'worker unable to determine memory usage)� completedz worker exiting after %d tasks) r� r� rz r� r� r� rt ru r� � wait_for_jobr� �TASKr� r� r �READYr� r� r` r r: r �MAXMEM_USED_FMT�format�
EX_RECYCLE�_ensure_messages_consumedr� r) ) rH r �nowr� r� r� r� rt ru r� r� r� r� r� r� �args_�jobr� �funr5 r< Zconfirmr� rb �_�tb�wrapped�einfoZused_kbr/ r� r0 r� A sv
�
� �
� �zWorker.workloopc C sJ | j s
dS tt�D ]*}| j j|kr2td|� dS t�t� qtd� dS )zr Returns true if all messages sent out have been received and
consumed within a reasonable amount of time Fz*ensured messages consumed after %d retriesTz<could not ensure all messages were consumed prior to exiting) r} �range�)GUARANTEE_MESSAGE_CONSUMPTION_RETRY_LIMITrN r r� r� �,GUARANTEE_MESSAGE_CONSUMPTION_RETRY_INTERVALr )rH r� �retryr/ r/ r0 r� � s
z Worker._ensure_messages_consumedc C s� t | jd�r| jj�� t | jd�r0| jj�� | jd urF| j| j� t| j d� t
d urft � t
t � zt � t j
t j� W n ty� Y n0 d S )Nr� r� )�full)�hasattrry r� �closerz r� rr rs r rx �SIG_SOFT_TIMEOUT�signalrm �SIGINT�SIG_IGNr, rG r/ r/ r0 r� � s
zWorker.after_forkc s` |j �t|d�rP|jj� t|d�r@|jr@|j�tf�fdd� }q\� �fdd�}n �fdd�}|S )Nr� � get_payloadc s d|� � �fS �NTr/ )rA �loads)r� r/ r0 �_recv� s z'Worker._make_recv_method.<locals>._recvc s � | �rd�� fS dS �NT�FNr/ �rA )�_poll�getr/ r0 r� � s
c s, zd� | d�fW S t jy& Y dS 0 d S �NTr� r� r r� )r� r/ r0 r� � s )r� r� r� �pollr� r
)rH �connr� r/ )r� r� r� r0 �_make_recv_method� s
zWorker._make_recv_methodc C s* | � | j�| _| jr | � | j�nd | _d S r+ )�_make_protected_receivery r� r{ r� )rH r� r/ r/ r0 r� � s
��zWorker._make_child_methodsc s2 | � |�� | jr| jjnd �tf� �fdd� }|S )Nc
s� �r�� r| d� t t��z� d�\}}|s2W d S W n\ ttfy� } z@t|�tjkrdW Y d }~d S | dt|�j� t t ��W Y d }~n
d }~0 0 |d u r�| d� t t ��|S )Nz worker got sentinel -- exiting� �?zworker got %s -- exiting)
�
SystemExitr) �EOFError�IOErrorr
�errnoZEINTRrp r[ r� )r �readyr� rb �Z_receiveZshould_shutdownr/ r0 �receive� s
z/Worker._make_protected_receive.<locals>.receive)r� rv �is_setr )rH r� r� r/ r� r0 r� � s
z Worker._make_protected_receive)
NNr/ NNNTTNN)N)r[ r\ r] rL r~ r� r� r� r� r� r r r� r� r� r� r
r� r� r/ r/ r/ r0 rn � s" �
M rn c sN e Zd Zdd� Zdd� Z� fdd�Zdd� Zdd
d �Zd d
� Zdd� Z � Z
S )�
PoolThreadc O s t �| � t| _d| _d| _d S �NFT)r rL �RUN�_state� _was_started�daemon�rH r5 r< r/ r/ r0 rL � s
zPoolThread.__init__c
C s� z
| � � W S tyZ } z8tdt| �j|dd� tt�� t� t �
� W Y d }~nLd }~0 t y� } z,tdt| �j|dd� t� d� W Y d }~n
d }~0 0 d S )NzThread %r crashed: %rr r� )
�bodyr r: rp r[ �_killr� r� r r� r� r� r� �rH rb r/ r/ r0 �run� s
� �zPoolThread.runc s d| _ tt| �j|i |�� d S r� )r� rc r� �startr� rd r/ r0 r� s zPoolThread.startc C s d S r+ r/ rG r/ r/ r0 �on_stop_not_started s z PoolThread.on_stop_not_startedNc C s | j r| �|� d S | �� d S r+ )r� �joinr� �rH rA r/ r/ r0 r?
s
zPoolThread.stopc C s
t | _d S r+ )� TERMINATEr� rG r/ r/ r0 � terminate s zPoolThread.terminatec C s
t | _d S r+ )�CLOSEr� rG r/ r/ r0 r� s zPoolThread.close)N) r[ r\ r] rL r� r� r� r? r� r� ri r/ r/ rd r0 r� � s
r� c s$ e Zd Z� fdd�Zdd� Z� ZS )�
Supervisorc s || _ t� �� d S r+ )�poolrc rL )rH r� rd r/ r0 rL s zSupervisor.__init__c C s� t d� t�d� | j}z~|j}td|j d�|_td�D ]*}| jtkr:|jtkr:|� � t�d� q:||_| jtkr�|jtkr�|� � t�d� qlW n$ t
y� |� � |� � � Y n0 t d� d S )Nzworker handler startingg�������?�
r r* zworker handler exiting)
r r� r� r� r �
_processesr� r� r� �_maintain_poolr r� r� )rH r� Z
prev_stater� r/ r/ r0 r� s&
zSupervisor.body)r[ r\ r] rL r� ri r/ r/ rd r0 r� s r� c s4 e Zd Z� fdd�Zdd� Zdd� Zdd� Z� ZS ) � TaskHandlerc s, || _ || _|| _|| _|| _t� �� d S r+ )� taskqueuer� �outqueuer� �cacherc rL )rH r� r� r� r� r� rd r/ r0 rL > s zTaskHandler.__init__c
C s~ | j }| j}| j}t|jd �D �]H\}}d }d}z�t|�D ]�\}}| jrTtd� q�z ||� W q: ty� td� Y q�Y q: t y� |d d� \}} z|| �
| dt � f� W n t y� Y n0 Y q:0 q:|r�td� ||d � W q W �qrW q t �yf |�r|d d� nd\}} ||v �rF|| �
| d dt � f� |�rbt
�d� ||d � Y q 0 q td � | �� d S )
N���z'task handler found thread._state != RUNzcould not put task on queuer% Fzdoing set_length()r )r r ztask handler got sentinel)r� r� r� �iterr� � enumerater� r r� r� �_setr �KeyErrorr � tell_others)
rH r� r� r� ZtaskseqZ
set_length�taskr� r� �indr/ r/ r0 r� F sH
zTaskHandler.bodyc C sj | j }| j}| j}z0td� |�d � td� |D ] }|d � q2W n ty\ td� Y n0 td� d S )Nz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got IOError when sending sentinelsztask handler exiting)r� r� r� r r� )rH r� r� r� �pr/ r/ r0 r p s
zTaskHandler.tell_othersc C s | � � d S r+ )r rG r/ r/ r0 r� � s z TaskHandler.on_stop_not_started)r[ r\ r] rL r� r r� ri r/ r/ rd r0 r� < s *r� c sT e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d
� Zd d � Zd
d� Z dd� Z
� Z S )�TimeoutHandlerc s, || _ || _|| _|| _d | _t� �� d S r+ )� processesr� �t_soft�t_hard�_itrc rL )rH r r� r r rd r/ r0 rL � s zTimeoutHandler.__init__c s t � fdd�t| j�D �d�S )Nc 3 s$ | ] \}}|j � kr||fV qd S r+ r� ��.0r� �procr� r/ r0 � <genexpr>� s
�z1TimeoutHandler._process_by_pid.<locals>.<genexpr>�NN)�nextr� r r� r/ r� r0 �_process_by_pid� s
��z TimeoutHandler._process_by_pidc
C sx t d|� | �|j�\}}|s"d S |jdd� zt|jt� W n4 tyr } z t|�tj kr^� W Y d }~n
d }~0 0 d S )Nz soft time limit exceeded for %rT��soft)
r r � _worker_pid�handle_timeoutr� r� �OSErrorr
r� �ESRCH)rH r� �process�_indexrb r/ r/ r0 �on_soft_timeout� s
z TimeoutHandler.on_soft_timeoutc C sz |� � r d S td|� zt|j��W n& tyJ |�|jdt� f� Y n0 | �|j�\}}|j dd� |rv| �
|� d S )Nz hard time limit exceeded for %rFr ) r� r r �_timeoutr� �_jobr r r r � _trywaitkill)rH r� r r r/ r/ r0 �on_hard_timeout� s
z TimeoutHandler.on_hard_timeoutc C s� t d|j� zBt�|j�|jkrDt d|j� t�t�|j�tj� n|�� W n t y` Y n0 |j
j dd�rtd S t d|j� zFt�|j�|jkr�t d|j� t�t�|j�tj � n t
|jt � W n t y� Y n0 d S )Nztimeout: sending TERM to %szIworker %s is a group leader. It is safe to kill (SIGTERM) the whole groupr* r� z/timeout: TERM timed-out, now sending KILL to %szIworker %s is a group leader. It is safe to kill (SIGKILL) the whole group)r �_namer� �getpgidr� �killpgr� �SIGTERMr� r �_popen�wait�SIGKILLr� �rH �workerr/ r/ r0 r � s$ zTimeoutHandler._trywaitkillc # s� | j | j }}t� }| j}| j}dd� }| jtkr�t�| j�� |rXt� fdd�|D ��}� � � D ]j\}}|j
} |j }
|
d u r�|}
|j } | d u r�|} || | �r�||� q`||vr`|| |
�r`||� |�
|� q`d V q(d S )Nc S s" | r|s dS t � | | kr dS d S r� r )r� rA r/ r/ r0 �
_timed_out� s z2TimeoutHandler.handle_timeouts.<locals>._timed_outc 3 s | ]}|� v r|V qd S r+ r/ )r �k�r� r/ r0 r
� � z1TimeoutHandler.handle_timeouts.<locals>.<genexpr>)r r �setr r r� r� �copyr� �items�_time_accepted�
_soft_timeoutr �add) rH r r Zdirtyr r r' r� r� Zack_time� soft_timeoutZ hard_timeoutr/ r) r0 �handle_timeouts� s.
z TimeoutHandler.handle_timeoutsc C sL | j tkr@z | �� D ]}t�d� qW q ty< Y q@Y q 0 q td� d S )Nr� ztimeout handler exiting)r� r� r2 r� r� r r �rH r� r/ r/ r0 r� � s
zTimeoutHandler.bodyc G s@ | j d u r| �� | _ zt| j � W n ty: d | _ Y n0 d S r+ )r r2 r �
StopIteration�rH r5 r/ r/ r0 � handle_event s
zTimeoutHandler.handle_event) r[ r\ r] rL r r r r r2 r� r6 ri r/ r/ rd r0 r � s & r c sT e Zd Zd� fdd� Zdd� Zdd� Zdd d
�Zdd d �Zd
d� Zddd�Z � Z
S )�
ResultHandlerNc s^ || _ || _|| _|| _|| _|| _|| _d | _d| _|| _ | | _
|
| _ | � � t
� �� d S )NF)r� r� r� r� �join_exited_workers�putlockr r �_shutdown_complete�check_timeouts� on_job_ready�on_ready_counters�
_make_methodsrc rL ) rH r� r� r� r� r8 r9 r r; r< r= rd r/ r0 rL s zResultHandler.__init__c C s | j dd� d S )NT)r2 )�finish_at_shutdownrG r/ r/ r0 r� s z!ResultHandler.on_stop_not_startedc sl �j � �j��j��j�� �fdd�}� ���fdd�}dd� }t|t|t|i ��_�fdd�}|�_d S ) Nc s: d�_ z� | �||||� W n ttfy4 Y n0 d S ro )�R�_ackr r, )r� r� �
time_acceptedr� r� )r� r r/ r0 �on_ack( s
z+ResultHandler._make_methods.<locals>.on_ackc s� �d ur�| |||� z � | }W n t y6 Y d S 0 �jr�tt|�� �d �}|r�|�jv r��j| }|�� � | jd7 _W d � n1 s�0 Y |�� s��d ur���� z|� ||� W n t y� Y n0 d S rD )
r r= r r� � worker_pidsZget_lockrN r� rV r� )r� r� r� r� �itemZ
worker_pidr} )r� r< r9 rH r/ r0 �on_ready0 s&
, z-ResultHandler._make_methods.<locals>.on_readyc
S sJ zt �| t� W n4 tyD } z t|�tjkr0� W Y d }~n
d }~0 0 d S r+ )r� r$ r r r
r� r )r� r� rb r/ r/ r0 �on_deathG s
z-ResultHandler._make_methods.<locals>.on_deathc s< | \}}z� | |� W n t y6 td||� Y n0 d S )Nz Unknown job state: %s (args=%s))r r )r �stater5 )�state_handlersr/ r0 �on_state_changeR s
z4ResultHandler._make_methods.<locals>.on_state_change) r� r9 r r< r� r� r� rI rJ )rH rC rF rG rJ r/ )r� r< r9 r rH rI r0 r> " s �
zResultHandler._make_methodsr� c
c s� | j }| j}z||�\}}W n8 ttfyT } z td|� t� �W Y d }~n
d }~0 0 | jrx| jtksjJ �td� t� �|r�|d u r�td� t� �||� |dkr�q�nq�d V q d S )N� result handler got %r -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelr )r� rJ r� r� r r r� r� )rH rA r� rJ r� r rb r/ r/ r0 �_process_resultZ s(
z ResultHandler._process_resultc C sP | j tkrL| jd u r | �d�| _zt| j� W n ttfyJ d | _Y n0 d S ro )r� r� r rL r r4 r )rH r- �eventsr/ r/ r0 r6 u s
zResultHandler.handle_eventc C s\ t d� zD| jtkrBz| �d�D ]}q W q
ty> Y qBY q
0 q
W | �� n
| �� 0 d S )Nzresult handler startingr� )r r� r� rL r r? r3 r/ r/ r0 r� ~ s
zResultHandler.bodyFc
C s� d| _ | j}| j}| j}| j}| j}| j}| j}d } |�r&| jt k�r&|d urT|� z|d�\}
} W n4 t
t fy� } zt d| � W Y d } ~ d S d } ~ 0 0 |
r�| d u r�t d� q4|| � z|dd� W q4 t
�y" t� }
| s�|
} n8|
| dk�rt d� Y �q&t dtt|
| d d ��� Y q40 q4t|d
��r�t d � z,td �D ] }|j�� �sZ �qd|� �qDW n t
t f�y~ Y n0 t d
t|�| j� d S )NTr� rK z&result handler ignoring extra sentinel)�shutdowng @z!result handler exiting: timed outz6result handler: all workers terminated, timeout in %ssr r� z"ensuring that outqueue is not fullr� z7result handler exiting: len(cache)=%s, thread._state=%s)r: r� r� r� r� r8 r; rJ r� r� r� r� r rj r �abs�minr� r� r� �len)rH r2 r� r� r� r� r8 r; rJ Ztime_terminater� r rb r� r� r/ r/ r0 r? � s\
�
�z ResultHandler.finish_at_shutdown)N)r� )NN)F) r[ r\ r] rL r� r> rL r6 r� r? ri r/ r/ rd r0 r7
s �8
r7 c @ s< e Zd ZdZdZeZeZeZeZe Z e
Z
dwdd�Z d d
� Z d d � Z
d
d� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zdxd!d"�Zd#d$� Zd%d&� Zd'd(� Zd)d*� Z d+d,� Z d-d.� Z d/d0� Z d1d2� Z d3d4� Z!dyd5d6�Z"dzd7d8�Z#d9d:� Z$d;d<� Z%d=d>� Z&d?d@� Z'dAdB� Z(dCdD� Z)dEdF� Z*dGdH� Z+dIdJ� Z,di fdKdL�Z-d{dMdN�Z.d|dOdP�Z/d}dQdR�Z0d~dSdT�Z1ddUdV�Z2di ddddddddddf dWdX�Z3dYdZ� Z4d�d[d\�Z5d�d]d^�Z6d�d_d`�Z7e8dadb� �Z9dcdd� Z:dedf� Z;dgdh� Z<e8didj� �Z=dkdl� Z>dmdn� Z?e8dodp� �Z@eAdqdr� �ZBeAdsdt� �ZCeDdudv� �ZEdS )��PoolzS
Class which supports an async version of applying functions to arguments.
TNr/ r Fc K s� |pt � | _|| _| �� t� | _i | _t| _|| _ || _
|| _ || _ || _
|| _|| _|p^t| _|
| _| | _| | _|
| _|| _i | _|| _t|p�| j d up�| j
d u�| _|r�td u r�t � t d�� d }|d u r�| � � n|| _ |p�t!| j d �| _"t#|| p�d�| _#|d u�r t$|��s t%d��|d u�r:t$|��s:t%d��| jj&| _'g | _(i | _)i | _*|| _+|�pjt,| j �| _-t.| j �D ]}| �/|� �qx| �0| �| _1|�r�| j1�2� | �3| j| j4| j5| j(| j�| _6|�r�| j6�2� d | _7| j�r$| �8| j(| j| j
| j �| _9t:� | _;d| _<| �=� |�s6| j9j>| _7nd | _9d| _<d | _;| �?� | _@| j@j>| _A|�rZ| j@�2� tB| | jC| j| jD| j5| j(| j1| j6| j@| j| j9| �E� f
dd�| _Fd S ) N�WSoft timeouts are not supported: on this platform: It does not have the SIGUSR1 signal.�d r z initializer must be a callablez on_process_exit must be callableF� )r5 Z exitpriority)Gr �_ctx�synack�
_setup_queuesr �
_taskqueue�_cacher� r� rA r1 �_maxtasksperchild�_max_memory_per_child� _initializer� _initargs�_on_process_exit�LOST_WORKER_TIMEOUT�lost_worker_timeout�
on_process_up�on_process_down�on_timeout_set�on_timeout_cancel�threadsZreaders�
allow_restart�bool�enable_timeoutsr� �warnings�warn� UserWarningr r� �round� max_restartsr �callable� TypeError�Process�_Process�_pool� _poolctrl�_on_ready_counters�putlocksrC �_putlockr� �_create_worker_processr� �_worker_handlerr� r� r� � _outqueue�
_task_handlerr; r �_timeout_handler�Lock�_timeout_handler_mutex�_timeout_handler_started�_start_timeout_handlerr6 �create_result_handler�_result_handler�handle_result_eventr �_terminate_pool�_inqueue�_help_stuff_finish_args�
_terminate)rH r rr rs ZmaxtasksperchildrA r1 ra rn Zmax_restart_freqrb rc rd re rf Z semaphorerv rg rW Zon_process_exit�contextru ri r< r� r/ r/ r0 rL � s�
�� �
�
�
��z
Pool.__init__c O s | j |i |��S r+ )rr )rH r5 �kwdsr/ r/ r0 rq I s z Pool.Processc C s |� | j|d��S )N)�target)r~ rq r% r/ r/ r0 �
WorkerProcessL s zPool.WorkerProcessc
K s: | j | j| j| j| j| j| j| j| j| j f d| j
i|��S )Nr= ) r7 rz r� rZ � _poll_result�_join_exited_workersrw r r; r< ru )rH � extra_kwargsr/ r/ r0 r� O s ���zPool.create_result_handlerc C s d S r+ r/ )rH r� r� r� r� r/ r/ r0 r< X s zPool.on_job_readyc C s | j | j| jfS r+ )r� r{ rs rG r/ r/ r0 r� [ s z Pool._help_stuff_finish_argsc C s" zt � W S ty Y dS 0 d S rD )r �NotImplementedErrorrG r/ r/ r0 r ^ s zPool.cpu_countc G s | j j|� S r+ )r� r6 r5 r/ r/ r0 r� d s zPool.handle_result_eventc C s d S r+ r/ )rH r& �queuesr/ r/ r0 �_process_register_queuesg s z Pool._process_register_queuesc s t � fdd�t| j�D �d�S )Nc 3 s$ | ] \}}|j � kr||fV qd S r+ r� r
r� r/ r0 r
k s
�z'Pool._process_by_pid.<locals>.<genexpr>r )r r� rs r� r/ r� r0 r j s
��zPool._process_by_pidc C s | j | jd fS r+ )r� rz rG r/ r/ r0 �get_process_queuesp s zPool.get_process_queuesc C s� | j r| j�� nd }| �� \}}}| j�d�}| �| j|||| j| j| j || j
| j | j | j
|d� �}| j�|� | �||||f� |j�dd�|_d|_||_|�� || j|j< || j|j< | jr�| �|� |S )Nr� )rx r| ru r} rq Z
PoolWorkerT)rg rV �Eventr� �Valuer� rn r] r^ r[ r_ rf �_wrap_exceptionr\ rs �appendr� �name�replacer� �indexr� rt r� ru rb )rH r� r ry rz r{ r} �wr/ r/ r0 rx s s,
�
zPool._create_worker_processc C s d S r+ r/ r% r/ r/ r0 �process_flush_queues� s zPool.process_flush_queuesc s| d}dd� t | j�� �D �D ]2}|p(t� }|j\}}|| |jkr | �||� q |rdt| j�sdt � �i i �}t
t t| j���D ]�}| j| }|j } |j
}
|
du s�| dur�td|� |
dur�|�� td|� |�|j< | ||j< | ttfv�rt|dd��std|j|jt| �d d
� | �|� | j|= | j|j= | j|j= q���rxd d� | jD �� t | j�� �D ]�}t� �fd d
�|�� D �d�} | �r�| � || � |� � �s6|� | ��p�d } �� | �} | �r�t| dd��r�|� | � n| � || | � nJ|j!}
|j"}|
�r|
�#� �s| � ||
j� n |�rd|�#� �sd| � ||j� �qd��� D ](}| j$�rB|�s^| �%|� | �$|� �qBt |�� �S g S )z�Cleanup after any worker processes which have exited due to
reaching their specified lifetime. Returns True if any workers were
cleaned up.
Nc S s g | ]}|� � s|jr|�qS r/ )r� � _worker_lost)r r� r/ r/ r0 �
<listcomp>� s �z-Pool._join_exited_workers.<locals>.<listcomp>z!Supervisor: cleaning up worker %dz Supervisor: worked %d joined�_controlled_terminationFz Process %r pid:%r exited with %rr r� c S s g | ]
}|j �qS r/ r� �r r� r/ r/ r0 r� � r* c 3 s" | ]}|�v s|� vr|V qd S r+ r/ �r r� �Zall_pids�cleanedr/ r0 r
� s �z,Pool._join_exited_workers.<locals>.<genexpr>�_job_terminated)&r2 rZ �valuesr r� �_lost_worker_timeout�mark_as_worker_lostrQ rs rj �reversedr� r� r" r r� r� r) r� �getattrr: r� r r� rt ru r rD �on_job_process_downr� r� �_set_terminated�on_job_process_lost� _write_to�_scheduled_for� _is_aliverc �_process_cleanup_queues)rH rN r� r� Z lost_timeZlost_ret� exitcodesr� r& r� �popenZ
acked_by_goner Zwrite_toZ sched_forr/ r� r0 r� � s|
��
�
�
zPool._join_exited_workersc C s d S r+ r/ )rH r� r& r/ r/ r0 �on_partial_read� s zPool.on_partial_readc C s d S r+ r/ r% r/ r/ r0 r� � s z Pool._process_cleanup_queuesc C s d S r+ r/ )rH r� Zpid_goner/ r/ r0 r� � s zPool.on_job_process_downc C s t � |f|_d S r+ )r r� )rH r� r� r� r/ r/ r0 r� � s zPool.on_job_process_lostc C sD zt d�t|�|j���W n$ t y> |�d dt� f� Y n0 d S )Nz(Worker exited prematurely: {0} Job: {1}.F)r r� r r r� r )rH r� r� r/ r/ r0 r� � s
�� zPool.mark_as_worker_lostc C s | S r+ r/ rG r/ r/ r0 � __enter__� s zPool.__enter__c G s | � � S r+ )r� )rH r� r/ r/ r0 �__exit__� s z
Pool.__exit__c C s d S r+ r/ �rH �nr/ r/ r0 �on_grow s z Pool.on_growc C s d S r+ r/ r� r/ r/ r0 � on_shrink s zPool.on_shrinkc C sb t | �� �D ]H\}}| jd8 _| jr2| j�� |�� | �d� ||d kr q^q td��d S )Nr z&Can't shrink pool. All processes busy!)r� �
_iterinactiver� rw rI Zterminate_controlledr� �
ValueError)rH r� r� r& r/ r/ r0 rI s
z Pool.shrinkc C s: t |�D ]"}| jd7 _| jr| j�� q| �|� d S rD )r� r� rw rS r� )rH r� r� r/ r/ r0 rS s
z Pool.growc c s | j D ]}| �|�s|V qd S r+ )rs �_worker_activer% r/ r/ r0 r� s
zPool._iterinactivec C s( | j �� D ]}|j|�� v r
dS q
dS )NTF)rZ r� r� rD )rH r& r� r/ r/ r0 r� s zPool._worker_activec C s� t | jt| j� �D ]j}| jtkr( dS z"|rH|| ttfvrH| j� � W n t
yf | j� � Y n0 | � | � � � t
d� qdS )z�Bring the number of pool processes up to the specified number,
for use after reaping workers which have exited.
Nz added worker)r� r� rQ rs r� r� r) r� r �step�
IndexErrorrx � _avail_indexr )rH r� r� r/ r/ r0 �_repopulate_pool$ s
zPool._repopulate_poolc sD t | j�| jk sJ �tdd� | jD ��� t� fdd�t| j�D ��S )Nc s s | ] }|j V qd S r+ )r� )r r r/ r/ r0 r
5 r* z$Pool._avail_index.<locals>.<genexpr>c 3 s | ]}|� vr|V qd S r+ r/ )r r� ��indicesr/ r0 r
6 r* )rQ rs r� r+ r r� rG r/ r� r0 r� 3 s zPool._avail_indexc C s
| � � S r+ )r� rG r/ r/ r0 � did_start_ok8 s zPool.did_start_okc C s< | � � }| �|� tt|��D ]}| jdur | j�� q dS )zF"Clean up any exited workers and start replacements for them.
N)r� r� r� rQ rw rV )rH Zjoinedr� r/ r/ r0 r� ; s
zPool._maintain_poolc
C s� | j jtkr�| jtkr�z | �� W n\ tyF | �� | �� � Y n: ty~ } z"t|�t j
krht |�� W Y d }~n
d }~0 0 d S r+ ) ry r� r� r� r r� r� r r
r� �ENOMEM� MemoryErrorr� r/ r/ r0 �
maintain_poolD s zPool.maintain_poolc sF � j �� � _� j �� � _� jjj� _� jjj� _ � fdd�}|� _
d S )Nc s � j j�| �rd� �� fS dS r� )rz r� r� r� r� rG r/ r0 r� W s z(Pool._setup_queues.<locals>._poll_result) rV � SimpleQueuer� rz r� r� r� r� r� r� r� �rH r� r/ rG r0 rX Q s zPool._setup_queuesc C sP | j rL| jd urL| j�& | js.d| _| j�� W d � n1 sB0 Y d S r� )rf r| r~ r r� rG r/ r/ r0 r� ] s
zPool._start_timeout_handlerc C s | j tkr | �|||��� S dS )z8
Equivalent of `func(*args, **kwargs)`.
N)r� r� � apply_asyncr� )rH �funcr5 r� r/ r/ r0 �applyf s
z
Pool.applyc C s" | j tkr | �||t|��� S dS )z�
Like `map()` method but the elements of the `iterable` are expected to
be iterables as well and will be unpacked as arguments. Hence
`func` and (a, b) becomes func(a, b).
N)r� r� �
_map_asyncr9 r� �rH r� �iterable� chunksizer/ r/ r0 r8 m s
�z Pool.starmapc C s" | j tkr | �||t|||�S dS )z=
Asynchronous version of `starmap()` method.
N)r� r� r� r9 �rH r� r� r� �callback�error_callbackr/ r/ r0 �
starmap_asyncw s
�zPool.starmap_asyncc C s | j tkr | �|||��� S dS )zx
Apply `func` to each element in `iterable`, collecting the results
in a list that is returned.
N)r� r� � map_asyncr� r� r/ r/ r0 r3 � s
zPool.mapc s� | j tkrdS |p| j}|dkrXt| j|d��| j�� �fdd�t|�D ��jf� �S |dksdJ �t �
� ||�}t| j|d��| j��fdd�t|�D ��jf� dd� �D �S dS )zP
Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
Nr �ra c 3 s( | ] \}}t �j|� |fi ffV qd S r+ �r� r �r r� �x�r� r� r/ r0 r
� s �z Pool.imap.<locals>.<genexpr>c 3 s( | ] \}}t � j|t|fi ffV qd S r+ �r� r r6 r� �r� r/ r0 r
� s �c s s | ]}|D ]
}|V q
qd S r+ r/ �r �chunkrE r/ r/ r0 r
� r* ) r� r� ra � IMapIteratorrZ rY r� r� � _set_lengthrR �
_get_tasks�rH r� r� r� ra � task_batchesr/ r� r0 �imap� s4
� �� �
��z Pool.imapc s� | j tkrdS |p| j}|dkrXt| j|d��| j�� �fdd�t|�D ��jf� �S |dksdJ �t �
� ||�}t| j|d��| j��fdd�t|�D ��jf� dd� �D �S dS )zL
Like `imap()` method but ordering of results is arbitrary.
Nr r� c 3 s( | ] \}}t �j|� |fi ffV qd S r+ r� r� r� r/ r0 r
� s �z&Pool.imap_unordered.<locals>.<genexpr>c 3 s( | ] \}}t � j|t|fi ffV qd S r+ r� r� r� r/ r0 r
� s �c s s | ]}|D ]
}|V q
qd S r+ r/ r� r/ r/ r0 r
� r* ) r� r� ra �IMapUnorderedIteratorrZ rY r� r� r� rR r� r� r/ r� r0 �imap_unordered� s4
� �� �
��zPool.imap_unorderedc C s | j tkrdS | p| j} |
p | j}
| p*| j} | rJtdu rJt�td�� d} | j tk�r |du rd| j n|}|r�| j
dur�| j
� � t | j
||||| |
| | j| j| | jr�| jnd|
d�
}|
s�| r�| �� | jr�| j�t|jd|||ffgdf� n| �t|jd|||ff� |S dS )a
Asynchronous equivalent of `apply()` method.
Callback is called when the functions return value is ready.
The accept callback is called when the job is accepted to be executed.
Simplified the flow is like this:
>>> def apply_async(func, args, kwds, callback, accept_callback):
... if accept_callback:
... accept_callback()
... retval = func(*args, **kwds)
... if callback:
... callback(retval)
NrS )rd re �callbacks_propagate�send_ack�correlation_id)r� r� r1 rA ra r� rj rk rl rv rw rF � ApplyResultrZ rd re rW r� r� rf rY r� r� r r� )rH r� r5 r� r� r� �accept_callback�timeout_callbackZ waitforslotr1 rA ra r� r� r� r/ r/ r0 r� � sB
�
� ��zPool.apply_asyncc C s d S r+ r/ )rH �responser� r� �fdr/ r/ r0 r� � s z
Pool.send_ackc
C sn | � |�\}}|d urjzt||p"t� W n4 ty\ } z t|�tjkrH� W Y d }~nd }~0 0 d|_d|_d S r� ) r r� r r r
r� r r� r� )rH r� �sigr r� rb r/ r/ r0 �
terminate_job� s zPool.terminate_jobc C s | � ||t|||�S )z<
Asynchronous equivalent of `map()` method.
)r� r6 r� r/ r/ r0 r�
s �zPool.map_asyncc s� | j tkrdS t|d�s t|�}|du rPtt|�t| j�d �\}}|rP|d7 }t|�dkr`d}t�|||�}t | j
|t|�||d��| j � � �fdd�t
|�D �df� �S ) zY
Helper function to implement map, starmap and their async counterparts.
N�__len__r& r r �r� c 3 s( | ] \}}t �j|� |fi ffV qd S r+ r� r� ��mapperr� r/ r0 r
' s �z"Pool._map_async.<locals>.<genexpr>)r� r� r� r2 �divmodrQ rs rR r� � MapResultrZ rY r� r� ) rH r� r� r� r� r� r� �extrar� r/ r� r0 r� s(
���zPool._map_asyncc c s0 t |�}tt�||��}|s d S | |fV qd S r+ )r� �tupler7 �islice)r� �it�sizer� r/ r/ r0 r� + s
zPool._get_tasksc C s t d��d S )Nz:pool objects cannot be passed between processes or pickled)r� rG r/ r/ r0 r� 4 s �zPool.__reduce__c C sL t d� | jtkrHt| _| jr(| j�� | j�� | j� d � t
| j� d S )Nz closing pool) r r� r� r� rw rW ry r� rY r� rB rG r/ r/ r0 r� 9 s
z
Pool.closec C s$ t d� t| _| j�� | �� d S )Nzterminating pool)r r� r� ry r� r� rG r/ r/ r0 r� C s
zPool.terminatec C s t | � d S r+ )rB )� task_handlerr/ r/ r0 �_stop_task_handlerI s zPool._stop_task_handlerc C s� | j ttfv sJ �td� t| j� td� | �| j� td� t| j� td� t | j
�D ]2\}}td|d t | j
�|� |j d ur\|�
� q\td� d S )Nzjoining worker handler�joining task handler�joining result handlerzresult handler joinedzjoining worker %s/%s (%r)r zpool join complete)r� r� r� r rB ry r� r{ r� r� rs rQ r" r� )rH r� r r/ r/ r0 r� M s
z Pool.joinc C s | j �� D ] }|�� q
d S r+ )rt r� r+ )rH �er/ r/ r0 �restart\ s z Pool.restartc C s> t d� | j�� |�� r:| j�� r:| j�� t�d� qd S )Nz7removing tasks from inqueue until task handler finishedr ) r Z_rlockrF �is_aliver� r� r� r� r� )�inqueuer� rs r/ r/ r0 �_help_stuff_finish` s
zPool._help_stuff_finishc C s |� d � d S r+ )r� )�clsr� r� r/ r/ r0 �_set_result_sentineli s zPool._set_result_sentinelc C s> t d� |�� |�� |�d � t d� | j|
� |�� | �||� | d urX| �� |r�t|d d�r�t d� |D ]} | �� rv| �� qvt d� | �|� t d� |�� | d ur�t d� | �t � |�r t|d d��r t d � |D ]*} | �
� r�t d
| j � | j d ur�| �
� q�t d � |�r,|�� |�r:|�� d S ) Nzfinalizing poolz&helping task handler/workers to finishr r� zterminating workersr� r� zjoining timeout handlerzjoining pool workerszcleaning up worker %dzpool workers joined)r r� r� r r r� r� r� r? � TIMEOUT_MAXr r� r" r� r� ) r r� r r� r� Zworker_handlerr� Zresult_handlerr� Ztimeout_handlerZhelp_stuff_finish_argsr r/ r/ r0 r� m sD
zPool._terminate_poolc C s dd� | j D �S )Nc S s g | ] }|j j�qS r/ )r" r r� r/ r/ r0 r� � r* z*Pool.process_sentinels.<locals>.<listcomp>)rs rG r/ r/ r0 �process_sentinels� s zPool.process_sentinels)NNr/ NNNNNr NNNNTNFFFNNNF)F)r )r )N)NNN)N)r N)r N)N)NNN)NNN)Fr[ r\ r] r^ r� rn r� r� r r7 r rL rq r� r� r< r� r r� r� r r� rx r� r� r� r� r� r� r� r� r� r� r� rI rS r� r� r� r� r� r� r� rX r� r� r8 r� r3 r� r� r� r� r� r� r� � staticmethodr� r� r� r� r� r� r� r � classmethodr r� �propertyr r/ r/ r/ r0 rR � s� �
z
S
�
�
�
8
�
�
6rR c @ s� e Zd ZdZdZdZdddddedddddf dd�Zdd� Zdd� Z d d
� Z
d d � Z d
d� Z dd� Z
dd� Zd$dd�Zdd� Zd%dd�Zd&dd�Zdd � Zd'd d �Zd d!� Zd"d#� ZdS )(r� Nr/ c C s� |
| _ t� | _t�� | _tt�| _|| _ || _
|| _ || _ || _
|| _|| _|| _| | _|
| _| pdd| _| | _d| _d| _d | _d | _d | _| || j< d S )Nr/ F)r� r} �_mutexr= r� �_eventr � job_counterr rZ � _callback�_accept_callback�_error_callback�_timeout_callbackr r/ r� �_on_timeout_set�_on_timeout_cancel�_callbacks_propagate� _send_ack� _accepted�
_cancelledr r. � _terminated)rH r� r� r� r� r� r1 rA ra rd re r� r� r� r/ r/ r0 rL � s,
zApplyResult.__init__c C s dj | jj| j| j| �� d�S )Nz&<{name}: {id} ack:{ack} ready:{ready}>)r� �idZackr� )r� re r[ r r r� rG r/ r/ r0 rg � s �zApplyResult.__repr__c C s
| j �� S r+ )r r� rG r/ r/ r0 r� � s zApplyResult.readyc C s | j S r+ )r rG r/ r/ r0 �accepted� s zApplyResult.acceptedc C s | � � s J �| jS r+ )r� �_successrG r/ r/ r0 �
successful� s zApplyResult.successfulc C s
d| _ dS )z Only works if synack is used.TN)r rG r/ r/ r0 �_cancel� s zApplyResult._cancelc C s | j �| jd � d S r+ )rZ �popr rG r/ r/ r0 �discard� s zApplyResult.discardc C s
|| _ d S r+ )r �rH rk r/ r/ r0 r� � s zApplyResult.terminatec C s< zt |p
d ��W n$ t y6 | �d dt� f� Y n0 d S �Nr F)r r� r r r/ r/ r0 r� � s zApplyResult._set_terminatedc C s | j r| j gS g S r+ �r rG r/ r/ r0 rD � s zApplyResult.worker_pidsc C s | j �|� d S r+ )r r# r� r/ r/ r0 r# � s zApplyResult.waitc C s. | � |� | �� st�| jr"| jS | jj�d S r+ )r# r� r r rQ � exceptionr� r/ r/ r0 r� � s
zApplyResult.getc
O sb |r^z||i |�� W nF | j y, � Y n2 ty\ } ztd|dd� W Y d }~n
d }~0 0 d S )Nz"Pool callback raised exception: %rr r� )r r� r: )rH r� r5 r< rb r/ r/ r0 �safe_apply_callback� s �z ApplyResult.safe_apply_callbackFc C s, | j d ur(| j| j ||r | jn| jd� d S )N)r rA )r r# r/ r )rH r r/ r/ r0 r s
�zApplyResult.handle_timeoutc C s� | j �� | jr| �| � |\| _| _| j�� | jrD| j�| j d � | j
r`| jr`| � | j
| j� | jd ur�| j r�| js�| � | j | j� W d � n1 s�0 Y d S r+ )
r
r r rQ r r+ r rZ r r r
r# r �rH r� r� r/ r/ r0 r� s&
�
���zApplyResult._setc
C s� | j ��t | jrP| jrPd| _|r@| �t|| j|�W d � S W d � d S d| _|| _|| _| �� rz| j �
| jd � | j r�| � | | j | j
� t}| j�rNzrz| �||� W n. | jy� t}� Y n ty� t}Y n0 W | j�rN|�rN| �||| j|�W d � S n<| j�rL|�rL| �||| j|� Y W d � S 0 | j�rn|�rn| �||| j|� W d � n1 �s�0 Y d S r� )r
r r r r� r r. r r� rZ r r r/ r r� r Z_propagate_errorsr� )rH r� rB r� r� r� r/ r/ r0 rA % s@
��
� zApplyResult._ack)N)N)N)F)r[ r\ r] r� r� r� r` rL rg r� r r r r r� r� rD r# r� r# r r� rA r/ r/ r/ r0 r� � s2 �
r� c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d
� Zd S ) r� c C s� t j| |||d� d| _|| _d g| | _dg| | _d g| | _d g| | _|| _|dkrvd| _ | j
� � || j = n|| t
|| � | _ d S )Nr� TFr )r� rL r �_lengthrQ r r r. �
_chunksize� _number_leftr r+ r rh )rH r� r� �lengthr� r� r/ r/ r0 rL M s �
zMapResult.__init__c C s� |\}}|rv|| j || j |d | j �< | jd8 _| jdkr�| jrT| �| j � | jrj| j�| jd � | j� � n>d| _
|| _ | j r�| � | j � | jr�| j�| jd � | j� � d S )Nr r F) rQ r& r' r
r rZ r r r r+ r r )rH r� Zsuccess_result�successr� r/ r/ r0 r� _ s"
zMapResult._setc G sj || j }t|d | j | j�}t||�D ]"}d| j|< || j|< || j|< q*| �� rf| j� | j
d � d S �Nr T) r& rP r% r� r r r. r� rZ r r )rH r� rB r� r5 r� r? �jr/ r/ r0 rA s s
zMapResult._ackc C s
t | j�S r+ )�allr rG r/ r/ r0 r } s zMapResult.acceptedc C s dd� | j D �S )Nc S s g | ] }|r|�qS r/ r/ r� r/ r/ r0 r� � r* z)MapResult.worker_pids.<locals>.<listcomp>r! rG r/ r/ r0 rD � s zMapResult.worker_pidsN)r[ r\ r] rL r� rA r rD r/ r/ r/ r0 r� K s
r� c @ sZ e Zd ZdZefdd�Zdd� Zddd�ZeZdd � Z d
d � Z
d d
� Z dd� Z dd� Z
dS )r� Nc C sZ t �t �� �| _tt�| _|| _t� | _ d| _
d | _ d| _ i | _
g | _|| _| || j< d S r )r= � Conditionr} rP r r r rZ r �_itemsr r% �_ready� _unsorted� _worker_pidsr� )rH r� ra r/ r/ r0 rL � s
zIMapIterator.__init__c C s | S r+ r/ rG r/ r/ r0 �__iter__� s zIMapIterator.__iter__c C s� | j �� z| j�� }W np ty� | j| jkr:d| _t�| j �|� z| j�� }W n, ty� | j| jkrxd| _t�t �Y n0 Y n0 W d � n1 s�0 Y |\}}|r�|S t
|��d S r� ) rP r. �popleftr� r r% r/ r4 r# r r� )rH rA rE r) rN r/ r/ r0 r � s& .zIMapIterator.nextc C s� | j �� | j|krn| j�|� | jd7 _| j| jv rb| j�| j�}| j�|� | jd7 _q,| j �� n
|| j|< | j| jkr�d| _| j | j
= W d � n1 s�0 Y d S r* ) rP r r. r� r0 r rR r% r/ rZ r r$ r/ r/ r0 r� � s
zIMapIterator._setc C sV | j �< || _| j| jkr4d| _| j �� | j| j= W d � n1 sH0 Y d S r� )rP r% r r/ rR rZ r )rH r( r/ r/ r0 r� � s
zIMapIterator._set_lengthc G s | j �|� d S r+ )r1 r� )rH r� rB r� r5 r/ r/ r0 rA � s zIMapIterator._ackc C s | j S r+ )r/ rG r/ r/ r0 r� � s zIMapIterator.readyc C s | j S r+ )r1 rG r/ r/ r0 rD � s zIMapIterator.worker_pids)N)r[ r\ r] r� r` rL r2 r �__next__r� r� rA r� rD r/ r/ r/ r0 r� � s
r� c @ s e Zd Zdd� ZdS )r� c C sj | j �P | j�|� | jd7 _| j �� | j| jkrHd| _| j| j= W d � n1 s\0 Y d S r* ) rP r. r� r rR r% r/ rZ r r$ r/ r/ r0 r� � s
zIMapUnorderedIterator._setN)r[ r\ r] r� r/ r/ r/ r0 r� � s r� c @ s: e Zd ZddlmZ eZd dd�Zdd� Zed d
� �Z dS ) �
ThreadPoolr )rq Nr/ c C s t �| |||� d S r+ )rR rL )rH r rr rs r/ r/ r0 rL � s zThreadPool.__init__c s: t � � _t � � _� jj� _� jj� _� fdd�}|� _d S )Nc s, zd� j | d�fW S ty& Y dS 0 d S r� )r� r r� rG r/ r0 r� � s z.ThreadPool._setup_queues.<locals>._poll_result)r r� rz r� r� r� r� r� r� r/ rG r0 rX � s
zThreadPool._setup_queuesc C sT | j �: | j�� | j�d gt|� � | j �� W d � n1 sF0 Y d S r+ )� not_empty�queuerW �extendrQ rT )r r� r� r/ r/ r0 r � s
z ThreadPool._help_stuff_finish)NNr/ )
r[ r\ r] �dummyrq r rL rX r r r/ r/ r/ r0 r5 � s
r5 )N)er, r� r7 r� r� r� r� r= r� rj � collectionsr � functoolsr � r r r �commonr r r
r r �compatr
r r r� r r9 r �
exceptionsr r r r r r r r r7 r r r r r r� � version_infor_ �systemZ_winr# r� r$ r$ r r, � SemaphorerK r� r� r� r� r� r� r� r� r) r� r� r� r� r` r� r� �countr r} r1 r6 r9 r: rB rC r� r` rj rm rn r� r� r� r r7 rR r� r� r� r� r5 r/ r/ r/ r0 �<module>
s� $
; )%K : o =R