
    sh                         S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKJ	r	J
r
  S SKJr  S SKJrJrJr  S SKJr  S SKJrJrJr  S SKJr  S SKJr  S SKJrJr  S	S
KJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&  S	SK'J(r(J)r)  / SQr*\RV                  " \,5      r-S@S jr.\r/  " S S\	5      r0 " S S5      r1\" SSSS9 " S S5      5       r2\" SSSS9 " S S5      5       r3 " S S5      r4 " S S5      r5S\5S\2S S4S! jr6 " S" S#\	5      r7 " S$ S%\75      r8 " S& S'\5      r9 " S( S)5      r: " S* S+\	5      r; " S, S-\;5      r<S.\:S \=4S/ jr> " S0 S15      r? " S2 S35      r@ " S4 S55      rA " S6 S75      rB " S8 S9\!5      rCS:\#S;\DS \\   4S< jrES=\S>\0S:\#S \C4S? jrFg)A    N)ABCabstractmethod)	dataclass)datetime	timedeltatimezone)Enum)AnyCallableOptional)Store)construct_and_record_rdzv_event	NodeState   )	RendezvousClosedErrorRendezvousErrorRendezvousGracefulExitErrorRendezvousHandlerRendezvousInfoRendezvousParametersRendezvousStateErrorRendezvousStoreInfoRendezvousTimeoutError)_delay_PeriodicTimer)RendezvousBackendRendezvousTimeoutRendezvousSettingsDynamicRendezvousHandlercreate_handlerc                     [        [        R                  " 5       5      U :  a"  [        R                  " 5       U    R                  $ g)Nno_method_name)leninspectstackfunction)depths    ڋ/Users/tiagomarins/Projetos/claudeai/copy_bank/venv/lib/python3.13/site-packages/torch/distributed/elastic/rendezvous/dynamic_rendezvous.pyget_method_namer)   3   s/    
7==?e#}}u%...    c                       \ rS rSrSr\\S\4S j5       5       r\S\	\
\\4      4S j5       r\ SS\S\	\   S\	\
\\\4      4S	 jj5       rS
rg)r   =   z4Represent a backend that holds the rendezvous state.returnc                     g)zGet the name of the backend.N selfs    r(   nameRendezvousBackend.name@       r*   c                     g)a8  Get the rendezvous state.

Returns:
    A tuple of the encoded rendezvous state and its fencing token or
    ``None`` if no state is found in the backend.

Raises:
    RendezvousConnectionError:
        The connection to the backend has failed.
    RendezvousStateError:
        The rendezvous state is corrupt.
Nr/   r0   s    r(   	get_stateRendezvousBackend.get_stateE   r4   r*   Nstatetokenc                     g)a  Set the rendezvous state.

The new rendezvous state is set conditionally:

  - If the specified ``token`` matches the fencing token stored in the
    backend, the state will be updated. The new state will be returned
    to the caller along with its fencing token.
  - If the specified ``token`` does not match the fencing token stored
    in the backend, the state won't be updated; instead the existing
    state along with its fencing token will be returned to the caller.
  - If the specified ``token`` is ``None``, the new state will be set
    only if there is no existing state in the backend. Either the new
    state or the existing state along with its fencing token will be
    returned to the caller.

Args:
    state:
        The encoded rendezvous state.
    token:
        An optional fencing token that was retrieved by a previous call
        to :py:meth:`get_state` or ``set_state()``.

Returns:
    A tuple of the serialized rendezvous state, its fencing token, and
    a boolean value indicating whether our set attempt succeeded.

Raises:
    RendezvousConnectionError:
        The connection to the backend has failed.
    RendezvousStateError:
        The rendezvous state is corrupt.
Nr/   )r1   r8   r9   s      r(   	set_stateRendezvousBackend.set_stateT   r4   r*   r/   N)__name__
__module____qualname____firstlineno____doc__propertyr   strr2   r   tuplebytesTokenr6   boolr;   __static_attributes__r/   r*   r(   r   r   =   s    >+c +  + 8E%,$78   59""#+E?"	%ud*+	," "r*   r   c                   6   \ rS rSr% Sr\" S5      r\" SS9\" SS9\" SS9\" SS9S.r\\S	'   \\S
'   \\S'   \\S'       SS\	\   S\	\   S\	\   S\	\   SS4
S jjr
\S\4S j5       r\S\4S j5       r\S\4S j5       r\S\4S j5       rS\	\   4S jrSrg)r   z   aJ  Hold the timeout configuration of a rendezvous.

Args:
    join:
        The time within which the rendezvous is expected to complete.
    last_call:
        An additional wait amount before completing the rendezvous once the
        rendezvous has the minimum number of required participants.
    close:
        The time within which the rendezvous is expected to close after a
        call to :py:meth:`RendezvousHandler.set_closed` or
        :py:meth:`RendezvousHandler.shutdown`.
    heartbeat:
        The time within which a keep-alive heartbeat is expected to
        complete.
r   iX  seconds      join	last_callclose	heartbeat_join
_last_call_close
_heartbeatNrQ   rR   rS   rT   r-   c                 $    U R                  XX4S9  g )NrP   )_set_timeouts)r1   rQ   rR   rS   rT   s        r(   __init__RendezvousTimeout.__init__   s     	% 	 	
r*   c                     U R                   $ )zGet the join timeout.)rU   r0   s    r(   rQ   RendezvousTimeout.join   s     zzr*   c                     U R                   $ )zGet the last call timeout.)rV   r0   s    r(   rR   RendezvousTimeout.last_call        r*   c                     U R                   $ )zGet the close timeout.)rW   r0   s    r(   rS   RendezvousTimeout.close        {{r*   c                     U R                   $ )z%Get the keep-alive heartbeat timeout.)rX   r0   s    r(   rT   RendezvousTimeout.heartbeat   ra   r*   timeoutsc                     UR                  5        HH  u  p#Uc  U R                  U   nX0R                  ::  a  [        SU SU S35      e[	        U SU-   U5        MJ     g )NzThe z
 timeout (z) must be positive._)items_DEFAULT_TIMEOUTS_ZERO
ValueErrorsetattr)r1   rg   r2   timeouts       r(   rZ   RendezvousTimeout._set_timeouts   s_    %^^-MD006**$ 4vZy@S!TUUD#*g. .r*   r/   )NNNN)r>   r?   r@   rA   rB   r   rl   rk   __annotations__r   r[   rC   rQ   rR   rS   rT   rZ   rI   r/   r*   r(   r   r   z   s*   " aLE #&r*2&q)	  %))-%))-	
y!	
 I&	
 	"		

 I&	
 
	
 i   9   y   9  /(; /r*   r   FT)repreqfrozenc                   V    \ rS rSr% Sr\\S'   \\S'   \\S'   \\S'   \	\S'   \\S'   S	r
g
)r      aO  Hold the settings of the rendezvous.

Attributes:
    run_id:
        The run id of the rendezvous.
    min_nodes:
        The minimum number of nodes to admit to the rendezvous.
    max_nodes:
        The maximum number of nodes to admit to the rendezvous.
    timeout:
        The timeout configuration of the rendezvous.
    keep_alive_interval:
        The amount of time a node waits before sending a heartbeat to keep
        it alive in the rendezvous.
    keep_alive_max_attempt:
        The maximum number of failed heartbeat attempts after which a node
        is considered dead.
run_id	min_nodes	max_nodesro   keep_alive_intervalkeep_alive_max_attemptr/   N)r>   r?   r@   rA   rB   rD   rq   intr   r   rI   r/   r*   r(   r   r      s*    & KNN""r*   r   )rs   orderrt   c                   F    \ rS rSr% Sr\\S'   \\S'   \\S'   S\4S jrSr	g	)
	_NodeDesc   zDescribe a node in the rendezvous.

Attributes:
    addr:
        The FQDN of the node or user specified local node address.
    pid:
        The id of the process in which the rendezvous handler runs.
    local_id:
        A process-wide unique id.
addrpidlocal_idr-   c                 R    U R                    SU R                   SU R                   3$ )Nri   )r   r   r   r0   s    r(   __repr___NodeDesc.__repr__   s%    ))AdhhZq88r*   r/   N)
r>   r?   r@   rA   rB   rD   rq   r|   r   rI   r/   r*   r(   r   r      s$    	 I	HM9# 9r*   r   c                   h    \ rS rSr% Sr\R                  \S'   \\S'   SS jr	SS\
\   S\4S	 jjrS
rg)_NodeDescGenerator   zGenerate node descriptors.

A node descriptor is a combination of an FQDN, a process id, and an auto-
incremented integer that uniquely identifies a node in the rendezvous.
_lock	_local_idr-   Nc                 F    [         R                  " 5       U l        SU l        g )Nr   )	threadingLockr   r   r0   s    r(   r[   _NodeDescGenerator.__init__   s    ^^%
 r*   
local_addrc                    U R                      U R                  nU =R                  S-  sl        S S S 5        [        U=(       d    [        R                  " 5       [
        R                  " 5       W5      $ ! , (       d  f       NJ= f)Nr   )r   r   r   socketgetfqdnosgetpid)r1   r   r   s      r(   generate_NodeDescGenerator.generate  sR     ZZ~~HNNaN 
 7v~~'7hOO Zs   "A33
B)r   r   r-   Nr=   )r>   r?   r@   rA   rB   r   r   rq   r|   r[   r   rD   r   r   rI   r/   r*   r(   r   r      s=     >>NP8C= PI P Pr*   r   c                       \ rS rSr% Sr\\S'   \\S'   \\	   \S'   \\S'   \
\\4   \S'   \\   \S'   \\   \S	'   \
\\	4   \S
'   SS jrSrg)_RendezvousStatei  a  Hold the state of a rendezvous.

Attributes:
    round:
        The current round of the rendezvous.
    complete:
        A boolean value indicating whether the current round of the
        rendezvous is complete.
    deadline:
        The time at which the current round of the rendezvous will be
        considered complete if it is still waiting for nodes to join.
    closed:
        A boolean value indicating whether the rendezvous is closed.
    participants:
        A dictionary of the participants and their corresponding ranks.
    wait_list:
        A set of nodes that are waiting to participate in the next round of
        the rendezvous.
    redundancy_list:
        A set of nodes that are redundant in the current round and can join
        the next rendezvous without triggering re-rendezvous.
    last_heartbeats:
        A dictionary containing each node's last heartbeat time.
roundcompletedeadlineclosedparticipants	wait_listredundancy_listlast_heartbeatsNc                     SU l         SU l        S U l        SU l        0 U l        [        5       U l        [        5       U l        0 U l        g )Nr   F)	r   r   r   r   r   setr   r   r   r0   s    r(   r[   _RendezvousState.__init__2  sA    
"u!r*   )r   r   r   r   r   r   r   r   r   )r>   r?   r@   rA   rB   r|   rq   rH   r   r   dictr   r   r[   rI   r/   r*   r(   r   r     s[    2 JNx  Ly#~&&9~^#)X-.."r*   r   r8   settingsr-   c                 ~   U R                   (       aF  U R                  (       d4  Sn[        R                  U5        SU l         U =R                  S-  sl        g g [        U R                  5      UR                  :  aC  S[        U R                  5       SUR                   S3n[        R                  U5        S U l        g g )NzHNo participants left in the rendezvous, marking rendezvous as incompleteFr   zNumber of participants z) less thanmin_nodes z, clearning deadline in state)r   r   loggerdebugr   r#   rx   r   )r8   r   msgs      r(   _remove_participant_epiloguer   =  s     ~~!!\CLL"ENKK1K " u!!"X%7%77)#e.@.@*A)B C%//00MO  LL!EN 8r*   c                   l    \ rS rSrSr\\S\4S j5       5       r\S\	\
   4S j5       r\S	S j5       rSrg)
_RendezvousStateHolderiR  z9Hold the shared rendezvous state synced with other nodes.r-   c                     g)zGet the local state.Nr/   r0   s    r(   r8   _RendezvousStateHolder.stateU  r4   r*   c                     g)zRead or writes the latest state.

Returns:
    A boolean value indicating whether the local state, in case marked
    as dirty, was successfully synced with other nodes.
Nr/   r0   s    r(   sync_RendezvousStateHolder.syncZ  r4   r*   Nc                     g)zMark the local state as dirty.Nr/   r0   s    r(   
mark_dirty!_RendezvousStateHolder.mark_dirtyc  r4   r*   r/   r   )r>   r?   r@   rA   rB   rC   r   r   r8   r   rH   r   r   rI   r/   r*   r(   r   r   R  sZ    C#' #  # htn   - -r*   r   c            	           \ rS rSr% Sr\\S'   \\S'   \\S'   \	\S'   \
\S'   \\S'   \\S	'   \\   \S
'    SS\S\S\	SS4S jjr\R"                  4S\S\4S jjr\S\4S j5       rS\\   4S jrSS jrSS jrSrg)_BackendRendezvousStateHolderih  a?  Hold the rendezvous state synced with other nodes via a backend.

Args:
    backend:
        The rendezvous backend to use.
    settings:
        The rendezvous settings.
    cache_duration:
        The amount of time, in seconds, to cache the last rendezvous state
        before requesting it from the backend again.
_backend_state	_settings_cache_duration_token_dirty_last_sync_time_dead_nodesbackendr   cache_durationr-   Nc                 ~    Xl         [        5       U l        X l        X0l        S U l        SU l        SU l        / U l        g )NF)	r   r   r   r   r   r   r   r   r   )r1   r   r   r   s       r(   r[   &_BackendRendezvousStateHolder.__init__~  s<      &(!-!r*   message
node_statec                     [        U R                  R                   S[        5        3U R                  R
                  UUS9  g )N.)r2   rw   r   r   )r   	__class__r>   r)   r   rw   r1   r   r   s      r(   _record%_BackendRendezvousStateHolder._record  s:    'NN++,Ao.?-@A>>((!		
r*   c                     U R                   $ See base class.)r   r0   s    r(   r8   #_BackendRendezvousStateHolder.state  rd   r*   c                    SnSnU R                   (       aP  Sn[        R                  " U R                  5      nU R                  R                  XR                  5      nUb  Uu  pnOnSnU R                  S:  a;  U R                  [        [        R                  " 5       U R                  -
  S5      :  a  gU R                  R                  5       nUb  Uu  pUb   [        R                  " U5      U l        O[!        5       U l        U(       a  U R"                  (       a  [$        R'                  [(        R*                  5      (       ac  SR-                  S U R"                   5       5      nSU SU R.                  R0                   S	3nU R3                  US
9  [$        R5                  U5        X l        SU l         [        R                  " 5       U l        U R7                  5         U$ ! [        R                   a  n[        S5      UeSnAff = f)r   NFr   zAThe rendezvous state is corrupt. See inner exception for details.z, c              3   .   #    U  H  nS U S 3v   M     g7f)'Nr/   ).0	dead_nodes     r(   	<genexpr>5_BackendRendezvousStateHolder.sync.<locals>.<genexpr>  s     !UDTyAi["2DTs   z*As part of the sync operation the node(s) z( have been removed from the rendezvous 'z' since they had no heartbeat.r   )r   pickledumpsr   r   r;   r   r   r   maxtime	monotonicr6   loadsPickleErrorr   r   r   r   isEnabledForloggingDEBUGrQ   r   rw   r   r   	_sanitize)	r1   
state_bitsr9   has_setset_responseget_responseexc	node_listr   s	            r(   r   "_BackendRendezvousStateHolder.sync  s   &*
 ;;Gdkk2J==22:{{KL'-9*
7G##a' ''3NN$t';';;Q,   ==224L'$0!
!$ll:6 +,DKt''F,?,?,N,N		!UDDTDT!UUI =YK H#~~4455SU  LLL%LL#~~/3 %% *Ws   G H3G??Hc                    U R                   n[        R                  " [        R                  5      U R
                  R                  U R
                  R                  -  -
  nUR                  R                  5        VVs/ s H  u  p4XB:  d  M  UPM     snnU l
        SnU R                   Hs  nSU S3n[        R                  U5        UR                  U	  UR                  U	 Sn UR                  R!                  U5         UR"                  R!                  U5        Mu     U(       a  [%        XR
                  5        g g s  snnf ! [         a     Nlf = f! [         a     N`f = f! [         a     M  f = f)NFzDetected dead node 'z"', removing it from the rendezvousT)r   r   nowr   utcr   rz   r{   r   rj   r   r   r   r   KeyErrorr   remover   r   )r1   r8   expire_timenodelast_heartbeatparticipant_removedr   r   s           r(   r   '_BackendRendezvousStateHolder._sanitize  sY   ll8<<0NN..1V1VV
 ).(=(=(C(C(E
(E$+ (E
 $))I(3UVCLL%%i0&&y1&*#&&y1%%,,Y7% *,  )? =
"  
  
  sB   <D=D=E$E E#
EE
E E #
E10E1c                     SU l         g)zSee base class.

If the local rendezvous state is dirty, the next sync call will try to
write the changes back to the backend. However this attempt might fail
if another node, which had the same state, also made changes and wrote
them before us.
TN)r   r0   s    r(   r   (_BackendRendezvousStateHolder.mark_dirty  s     r*   )r   r   r   r   r   r   r   r   )r   r   )r>   r?   r@   rA   rB   r   rq   r   r   r|   rG   rH   floatlistr   r[   r   RUNNINGrD   r   rC   r8   r   r   r   r   rI   r/   r*   r(   r   r   h  s    
  !!MLi   	" % 	
 
 =F<M<M 
s 
	 
 '  ;htn ;z)@Vr*   r   c                   L    \ rS rSrSrSrSrSrSrSr	Sr
S	rS
rSrSrSrSrSrSrg)_Actioni  zDSpecifies the possible actions based on the state of the rendezvous.r            rO            	   
            r/   N)r>   r?   r@   rA   rB   
KEEP_ALIVEADD_TO_PARTICIPANTSADD_TO_WAIT_LISTADD_TO_REDUNDANCY_LISTREMOVE_FROM_PARTICIPANTSREMOVE_FROM_WAIT_LISTREMOVE_FROM_REDUNDANCY_LISTMARK_RENDEZVOUS_COMPLETEMARK_RENDEZVOUS_CLOSEDSYNCERROR_CLOSEDERROR_TIMEOUTFINISHrI   r/   r*   r(   r   r     sN    NJ "# DLMFr*   r   c                   R    \ rS rSr% Sr\\S'   \\S'   \\S'   S\S\S\SS4S jr	S	r
g)
_RendezvousContexti  a   Holds the context of the rendezvous.

Attributes:
    node:
        The node descriptor associated with the current rendezvous handler
        instance.
    state:
        The current state of the rendezvous.
    settings:
        The rendezvous settings.
r   r8   r   r-   Nc                 (    Xl         X l        X0l        g r=   )r   r8   r   )r1   r   r8   r   s       r(   r[   _RendezvousContext.__init__0  s     	
 r*   )r   r   r8   )r>   r?   r@   rA   rB   r   rq   r   r   r[   rI   r/   r*   r(   r  r    s?    
 O  !!&6!BT!	!r*   r  c                   b    \ rS rSrSr\ S
S\\\/\	4   S\S\
\\/\4      SS4S jj5       rS	rg)_RendezvousOpExecutori8  zExecute rendezvous operations.Nstate_handlerr   update_deadliner-   c                     g)a+  Execute a rendezvous operation.

An operation is run inside a state machine and is expected to transition
the rendezvous from one state to another.

Args:
    state_handler:
        A callable that is expected to return the next state transition
        action based on the current state of the rendezvous.
    deadline:
        The time, in seconds, at which the operation will be considered
        timed-out.
    update_deadline:
        Function to generate a new operation deadline if the current
        node may participate in the next rendezvous.
Nr/   )r1   r  r   r  s       r(   run_RendezvousOpExecutor.run;  r4   r*   r/   r=   )r>   r?   r@   rA   rB   r   r   r  r   r   r   r   r  rI   r/   r*   r(   r  r  8  se    (
 CG	!3U ;W DE  "(I;+=">?	
 
 r*   r  c            
       &   \ rS rSr% Sr\\S'   \\S'   \\S'   \	\S'   S\S\S	\	S
S4S jr
\R                  4S\S\S
S4S jjr SS\\\/\4   S\S\\\/\4      S
S4S jjrSS jrSS jrSS jrSS jrSS jrSS jrSS jrSS jrSS jrSrg)  _DistributedRendezvousOpExecutoriT  aG  Execute rendezvous operations using a shared state.

Args:
    node:
        The node descriptor associated with the current rendezvous handler
        instance.
    state_holder:
        The ``RendezvousStateHolder`` to use to sync the rendezvous state
        with other nodes.
    settings:
        The rendezvous settings.
_noder   _state_holderr   r   state_holderr   r-   Nc                 (    Xl         X l        X0l        g r=   )r"  r#  r   )r1   r   r$  r   s       r(   r[   )_DistributedRendezvousOpExecutor.__init__g  s     
)!r*   r   r   c           
         [        U R                  R                   S[        5        3U R                  R
                  UUU R                  R                  U R                  R                  U R                  R                  S9  g )Nr   )r2   rw   r   r   hostnamer   r   )
r   r   r>   r)   r   rw   r"  r   r   r   r   s      r(   r   (_DistributedRendezvousOpExecutor._recordq  s]    'NN++,Ao.?-@A>>((!ZZ__

ZZ((	
r*   r  r   r  c                 ,   SnU[         R                  :w  Ga  U R                  R                  5       nUbz  U(       a(  SU R                   SU R
                  R                   S3nO'SU R                   SU R
                  R                   S3nU R                  US9  [        R                  U5        U R                  R                  U l        [        U R                  U R                  U R
                  5      nU" Xr5      nU[         R                  :X  a  GM  U[         R                  :X  a  [        eU[         R                  :X  a  [         eU[         R"                  :X  a  [%        SS9  GOU[         R&                  :X  a  U R)                  5         GOVU[         R*                  :X  a  U R-                  5         GO0U[         R.                  :X  a  U R1                  5         GO
U[         R2                  :X  a  U R5                  5         OU[         R6                  :X  a  U R9                  5         OU[         R:                  :X  a  U R=                  5         OU[         R>                  :X  a>  U RA                  5         U(       a&  U" U R
                  RB                  RD                  5      nOIU[         RF                  :X  a  U RI                  5         O$U[         RJ                  :X  a  U RM                  5         U R                  RO                  5         U[         R                  :w  a  GM  gg)	r   N
The node 'zP' has successfully synced its local changes with other nodes in the rendezvous ''.z]' has a stale state and failed to sync its local changes with other nodes in the rendezvous 'r   r   rL   )(r   r  r#  r   r"  r   rw   r   r   r   r8   r   r  r  r   r  r   r  r   r  _keep_aliver	  _add_to_participantsr
  _add_to_wait_listr  _add_to_redundancy_listr  _remove_from_participantsr  _remove_from_wait_listr  _remove_from_redundancy_listro   rQ   r  _mark_rendezvous_completer  _mark_rendezvous_closedr   )r1   r  r   r  actionr   r   ctxs           r(   r  $_DistributedRendezvousOpExecutor.run|  sc    & ((--/G"$TZZL 1::>..:O:O9PPRT  %TZZL 1GGK~~G\G\F]]_a 
 S)S!,,22DK$TZZdnnMC #31F'---++...,,% q!W///$$&w:::--/w777**,w===002w???224w<<<//1wBBB557&#24>>3I3I3N3N#Ow???224w===002 ""--/C &&r*   c                 $   SU R                    SU R                  R                   S3nU R                  US9  [        R                  U5        [        R                  " [        R                  5      U R                  R                  U R                   '   g )Nr+  z<' updated its keep-alive heartbeat time for the rendezvous ''. Pending sync.r   )r"  r   rw   r   r   r   r   r   r   r   r   r   r1   r   s     r(   r-  ,_DistributedRendezvousOpExecutor._keep_alive  sp     %%%&&68 	 	S!S2:,,x||2L##DJJ/r*   c                    SU R                    SU R                  R                   SU R                  R                   S3nU R                  US9  [        R                  U5        U R                  n UR                  R                  U R                   5        SUR                  U R                   '   U R                  5         [        UR                  5      U R                  R                  :X  aJ  [        R                   " ["        R$                  5      U R                  R&                  R(                  -   Ul        [        UR                  5      U R                  R,                  :X  a  U R/                  5         g g ! [         a     Nf = f)Nr+  z,' added itself to the participants of round  of the rendezvous 'r:  r   r   )r"  r   r   r   rw   r   r   r   r   r   r   r   r-  r#   rx   r   r   r   r   ro   rR   r   ry   r4  r1   r   r8   s      r(   r.  5_DistributedRendezvousOpExecutor._add_to_participants  s/   $P{{  !!5dnn6K6K5LL\^ 	 	S!S	OO""4::. *+4::&u!!"dnn&>&>>X\\*T^^-C-C-M-MM N u!!"dnn&>&>>**, ?  		s   0%E4 4
F Fc                    SU R                    SU R                  R                  S-    SU R                  R                   S3nU R                  US9  [        R                  U5        U R                   U R                  R                  ;   a/  U R                  R                  R                  U R                   5        U R                  R                  R                  U R                   5        U R                  5         g )Nr+  z)' added itself to the wait list of round r   r>  r:  r   )r"  r   r   r   rw   r   r   r   r   r   r   addr-  r;  s     r(   r/  2_DistributedRendezvousOpExecutor._add_to_wait_list  s    $M{{  1$%%9$..:O:O9PP`b 	 	S!S::444KK''..tzz:!!$**-r*   c                 L   SU R                    SU R                  R                  S-    SU R                  R                   S3nU R                  US9  [        R                  U5        U R                  R                  R                  U R                   5        U R                  5         g )Nr+  z/' added itself to the redundancy list of round r   r>  r:  r   )r"  r   r   r   rw   r   r   r   r   rB  r-  r;  s     r(   r0  8_DistributedRendezvousOpExecutor._add_to_redundancy_list  s    $S{{  1$%%9$..:O:O9PP`b 	 	S!S##''

3r*   c                 f   SU R                    SU R                  R                   SU R                  R                   S3nU R                  US9  [        R                  U5        U R                  nUR                  U R                   	 UR                  U R                   	 [        X R                  5        g )Nr+  z0' removed itself from the participants of round r>  r:  r   )r"  r   r   r   rw   r   r   r   r   r   r   r?  s      r(   r1  :_DistributedRendezvousOpExecutor._remove_from_participants  s    $T{{  !!5dnn6K6K5LL\^ 	 	S!Stzz*!!$**- 	%UNN;r*   c                 n   SU R                    SU R                  R                  S-    SU R                  R                   S3nU R                  US9  [        R                  U5        U R                  R                  R                  U R                   5        U R                  R                  U R                   	 g )Nr+  z-' removed itself from the wait list of round r   r>  r:  r   )r"  r   r   r   rw   r   r   r   r   r   r   r;  s     r(   r2  7_DistributedRendezvousOpExecutor._remove_from_wait_list  s    $Q{{  1$%%9$..:O:O9PP`b 	 	S!S$$TZZ0KK''

3r*   c                 n   SU R                    SU R                  R                  S-    SU R                  R                   S3nU R                  US9  [        R                  U5        U R                  R                  R                  U R                   5        U R                  R                  U R                   	 g )Nr+  z1' removed itself from the redunant list of round r   r>  r:  r   )r"  r   r   r   rw   r   r   r   r   r   r   r;  s     r(   r3  =_DistributedRendezvousOpExecutor._remove_from_redundancy_list&  s    $U{{  1$%%9$..:O:O9PP`b 	 	S!S##**4::6KK''

3r*   c                    SU R                    SU R                  R                   SU R                  R                   S3nU R                  U[        R                  S9  [        R                  U5        U R                  nSUl
        S Ul        [        [        UR                  5      5       H  u  p4X2R                  U'   M     g )Nr+  z' marked round r>  z' as complete. Pending sync.r   r   T)r"  r   r   r   rw   r   r   	SUCCEEDEDr   r   r   r   	enumeratesortedr   )r1   r   r8   rankr   s        r(   r4  :_DistributedRendezvousOpExecutor._mark_rendezvous_complete2  s    ODKK4E4E3F G%%&&BD 	 	SY-@-@AS $F5+=+=$>?JD'+t$ @r*   c                     SU R                    SU R                  R                   S3nU R                  U[        R
                  S9  [        R                  U5        SU R                  l	        g )Nr+  z' marked the rendezvous 'z' as closed. Pending sync.rM  T)
r"  r   rw   r   r   rN  r   r   r   r   r;  s     r(   r5  8_DistributedRendezvousOpExecutor._mark_rendezvous_closedC  s_    $=dnn>S>S=T U  	 	SY-@-@AS!r*   )r"  r   r   r#  r=   r   ) r>   r?   r@   rA   rB   r   rq   r   r   r   r[   r   r   rD   r   r   r  r   r   r   r   r  r-  r.  r/  r0  r1  r2  r3  r4  r5  rI   r/   r*   r(   r!  r!  T  s     ))!!"" -" %	"
 
" =F<M<M 	
s 	
	 	
RV 	
 CG	I0!3U ;W DEI0 I0 "(I;+=">?	I0
 
I0VM-:
<$
4
4,""r*   r!  r7  c                      U R                   R                  U R                     nU[        R
                  " [        R                  5      U R                  R                  -
  :*  $ ! [         a     gf = f)z8Determine whether a keep-alive heartbeat should be sent.F)
r8   r   r   r   r   r   r   r   r   rz   )r7  r   s     r(   _should_keep_aliverV  N  s^    22388<
 	(,,x||4s||7W7WWW  s   #A# #
A0/A0c                   .    \ rS rSrSrS\S\S\4S jrSr	g)	_RendezvousExitOpiZ  z&Represent a rendezvous exit operation.r7  r   r-   c                     UR                   UR                  R                  ;   a9  [        R                  " 5       U:  a  [
        R                  $ [
        R                  $ [
        R                  $ r=   )	r   r8   r   r   r   r   r  r  r  r1   r7  r   s      r(   __call___RendezvousExitOp.__call__]  sI    88syy---~~(*,,,333~~r*   r/   N
r>   r?   r@   rA   rB   r  r   r   r[  rI   r/   r*   r(   rX  rX  Z  s     0. % G r*   rX  c                   .    \ rS rSrSrS\S\S\4S jrSr	g)	_RendezvousJoinOpie  z&Represent a rendezvous join operation.r7  r   r-   c                 f   UR                   nUR                  (       aO  UR                  UR                  ;   a%  SUR                  R
                   S3n[        U5      e[        R                  $ UR                  UR                  ;   a  SUR                   S3n[        R                  U5        [        UR                  5      UR                  R                  :X  a0  [        U5      (       a  [        R                  $ [        R                   $ SUR                   S3n[        R                  U5        [        R"                  $ UR                  UR                  ;   nUR$                  (       a  U(       a  [        R&                  $ [(        R*                  " 5       nXb:  a[  SnXbU-   ::  aA  U(       a  [        R,                  $ UR                  UR.                  ;   a  [        R0                  $ [        R2                  $ UR$                  (       a  [        UR                  5      UR                  R                  :  a,  UR                  UR.                  ;  a  [        R4                  $ GO[        UR                  5      UR                  R                  :  aD  UR                  UR                  ;  a*  UR                  UR.                  ;  a  [        R6                  $ GO2U(       Ga  [        UR                  5      UR                  R8                  :  a  [        UR                  5      UR                  R                  ::  a  UR:                  b  UR:                  [<        R>                  " [@        RB                  5      :  a5  SUR                   S3n[        R                  U5        [        RD                  $ SUR                   S	3n[        R                  U5        O6SUR                   S
3n[        R                  U5        O[        RF                  $ [        U5      (       a  [        R                  $ [        R                   $ )NzThe rendezvous 'z,' is closed, terminating pending rendezvous.z	The node z is in redunancy listz is removed from redunancy listrO   r+  zE' marking the rendezvous complete, quorum established within deadlinez-' can't complete rendezvous: deadline reachedz4' can't complete rendezvous: not enough participants)$r8   r   r   r   r   rw   r   r   r  r   r   r#   r   ry   rV  r  r  r  r   r  r   r   r  r   r  r  r
  r  rx   r   r   r   r   r   r  r	  )r1   r7  r   r8   r   is_participantr   rollback_periods           r(   r[  _RendezvousJoinOp.__call__h  s   		 <<xx5000()<)<(==ij1#66'''88u,,,chhZ'<=CLL5%%&#,,*@*@@%c**"---"<<' "#((+JKS!:::U%7%77 >>n>>!nn>O 00 "";;; 88u."888(((>> 5%%&)?)??885??2"333 3U''(CLL,B,BBHHE$9$997"999
 E&&'3<<+A+AA**+s||/E/EENN.>>HLL$>>$SXXJ /= >  LL%";;;&sxxj0]^CLL%"388*,`aS! ...c""%%% ||r*   r/   Nr]  r/   r*   r(   r_  r_  e  s$    0b. b% bG br*   r_  c                   .    \ rS rSrSrS\S\S\4S jrSr	g)	_RendezvousCloseOpi  z'Represent a rendezvous close operation.r7  r   r-   c                     UR                   R                  (       a  [        R                  $ [        R
                  " 5       U:  a  [        R                  $ [        R                  $ r=   )r8   r   r   r  r   r   r  r  rZ  s      r(   r[  _RendezvousCloseOp.__call__  s@    99>>!>>h&(((---r*   r/   Nr]  r/   r*   r(   re  re    s     1.. .% .G .r*   re  c                   .    \ rS rSrSrS\S\S\4S jrSr	g)	_RendezvousKeepAliveOpi  z3Represent a rendezvous keep-alive update operation.r7  r   r-   c                     [        U5      (       a9  [        R                  " 5       U:  a  [        R                  $ [        R
                  $ [        R                  $ r=   )rV  r   r   r   r  r  r  rZ  s      r(   r[  _RendezvousKeepAliveOp.__call__  s>    c""~~(*,,,%%%~~r*   r/   Nr]  r/   r*   r(   ri  ri    s     =. % G r*   ri  c                   v   \ rS rSr% Sr\" 5       r\\S'   \	\S'   \
\S'   \\S'   \\S'   \\S'   \R                  \S	'   \\   \S
'   \    S4S\
S\S\S\S\S\\
   S\\   S\S\4S jj5       rS\S\	S\
S\S\SS4S jr\R2                  S4S\
S\S\\   SS4S jjrS\R8                  4S  jr\S\	4S! j5       rS\
4S" jr \S\!4S# j5       r"S\#4S$ jr$S\!4S% jr%S5S& jr&S\4S' jr'S\
4S( jr(S\!4S) jr)S5S* jr*\+S5S+ j5       r,S5S, jr-S5S- jr.S5S. jr/S\0\\4   4S/ jr1S\S\4S0 jr2S\4S1 jr3S\4S\54S2 jr6S3r7g)6r   i  zCRepresent a handler that sets up a rendezvous among a set of nodes.
_this_noder   _backend_name_storer#  _op_executor_heartbeat_lock_keep_alive_timerNrw   storer   rx   ry   r   ro   rz   r{   c
           
          U R                   R                  U5      n
[        UUUU=(       d
    [        5       [	        US9U	S9n[        X;5      nU " XUR                  X,5      $ )a  Create a new :py:class:`DynamicRendezvousHandler`.

Args:
    run_id:
        The run id of the rendezvous.
    store:
        The C10d store to return as part of the rendezvous.
    backend:
        The backend to use to hold the rendezvous state.
    min_nodes:
        The minimum number of nodes to admit to the rendezvous.
    max_nodes:
        The maximum number of nodes to admit to the rendezvous.
    local_addr:
        The local node address.
    timeout:
        The timeout configuration of the rendezvous.
    keep_alive_interval:
        The amount of time a node waits before sending a heartbeat to keep
        it alive in the rendezvous.
    keep_alive_max_attempt:
        The maximum number of failed heartbeat attempts after which a node
        is considered dead.
rL   rz   r{   )_node_desc_generatorr   r   r   r   r   r2   )clsrw   rs  r   rx   ry   r   ro   rz   r{   r   r   r$  s                r(   from_backend%DynamicRendezvousHandler.from_backend  se    L ''00<%*(* )2E F#9
 5WG47<<EEr*   r   r   backend_namer$  r-   c                    UR                   (       d  [        S5      eUR                  S:  a  [        SUR                   S35      eUR                  UR                  :  a&  [        SUR                   SUR                   S35      eXl        X l        X0l        X@l        XPl        [        U R                  U R                  U R
                  5      U l
        [        R                  " 5       U l        S U l        S U l        S U l        g )Nz&The run id must be a non-empty string.r   zThe minimum number of nodes (z) must be greater than zero.zThe maximum number of nodes (z@) must be greater than or equal to the minimum number of nodes (z).)rw   rm   rx   ry   rm  r   rn  ro  r#  r!  rp  r   r   rq  rr  _shared_tcp_store_server_bootstrap_store_info)r1   r   r   rz  rs  r$  s         r(   r[   !DynamicRendezvousHandler.__init__'  s     EFF!/0B0B/CC_`   2 22/0B0B/C D33;3E3E2FbJ 
 !))<OOT//
  )~~/!% ?C%DH"r*   r   r   rQ  c                    [        U R                  R                   S[        5        3U R                  R
                  UUU R                  R                  U R                  R                  U R                  R                  US9  g )Nr   )r2   rw   r   r   r(  r   r   rQ  )
r   r   r>   r)   r   rw   rm  r   r   r   )r1   r   r   rQ  s       r(   r    DynamicRendezvousHandler._recordT  sf     	(NN++,Ao.?-@A>>((!__))##__--		
r*   c                 0    [         R                  " UUSSS9$ )NT)	host_nameport	is_mastermulti_tenant)distTCPStore)r1   master_addrmaster_ports      r(   _create_tcp_store_server1DynamicRendezvousHandler._create_tcp_store_servere  s    }}!	
 	
r*   c                     U R                   $ )z#Get the settings of the rendezvous.)r   r0   s    r(   r   !DynamicRendezvousHandler.settingsm  s     ~~r*   c                     U R                   $ r   )rn  r0   s    r(   get_backend$DynamicRendezvousHandler.get_backendr  s    !!!r*   c                 6    [         R                  " SS5      S:g  $ )r   "TORCH_DISABLE_SHARE_RDZV_TCP_STORE01)r   getenvr0   s    r(   use_agent_store(DynamicRendezvousHandler.use_agent_storev  s     yy=sCsJJr*   c                 0   SU R                    SU R                  R                   S3nU R                  US9  [        R                  U5         U R                  5         U R                  R                  R                  S:X  a	  [        SS9  [        5       n[        5       nU R                  U R                  R                  R                  5      nU R                   R#                  X$5        U R                   R#                  X4U R                  5        U R%                  5         U R'                  5       u  pVU R)                  5       nSU R                    SU R                  R                  R                   SU R                  R                   SU SU S3nU R                  XS9  [        R                  U5        [6        R8                  " SS5      S:X  a7  [:        R<                  " XWU R                   R>                  S9n	[A        UUUU	5      $ U RB                  cy  Sn
US:X  aA  U RE                  U R                   R>                  U
5      U l#        U RF                  RH                  n
[:        R<                  " UUU R                   R>                  U
S9U l!        U RB                  c   eUS:X  a  U RF                  c   e[A        UUUU RB                  5      $ ! [*         aE  nU R                  [-        U5      R.                   S[1        U5       3[2        R4                  S	9  e S
nAff = f)r   r+  z5' attempts to join the next round of the rendezvous 'r,  r   r   )r   g333333?rL   : rM  Nz' has joined round r>  z
' as rank z in a world of size r   )r   rQ  r  r  r  )r   )r   server_port)%rm  r   rw   r   r   info_stop_heartbeatsr#  r8   r   r   rX  r_  _get_deadlinero   rQ   rp  r  _start_heartbeats
_get_world
_get_store	Exceptiontyper>   rD   r   FAILEDr   r  r   buildr   r   r}  r  r|  r  )r1   r   exit_opjoin_opr   rQ  
world_sizers  ebootstrap_store_infor  s              r(   next_rendezvous(DynamicRendezvousHandler.next_rendezvous{  s    ) *%%&b* 	 	S!C	!!#
 !!''--2x(')G')G))$..*@*@*E*EFH!!'4!!'T5G5GH""$#0DOO%E ))<T=O=O=U=U=[=[<\ ]#~~445ZvEYl! 	
 	S,C 9993?3F#6#<#<(<(<$  "$	  %%- Kqy040M0MOO((+1- #;;@@)<)B)B??//'	*D& ))5551900<<<&&	
 	
a  	LL7++,Bs1vh7$++   	s   C3K 
LA LLc                 j    U R                      U R                  R                  5         U R                  R                  R                  sSSS5        $ ! , (       d  f       g= f! [
         aE  nU R                  [        U5      R                   S[        U5       3[        R                  S9  e SnAff = fr   Nr  rM  )rq  r#  r   r8   r   r  r   r  r>   rD   r   r  r1   r  s     r(   	is_closed"DynamicRendezvousHandler.is_closed  s    	%%""'')))//66 &%%
  	LL7++,Bs1vh7$++   	s5   A# :A	A# 
A A#  A# #
B2-A B--B2c                     U R                      U R                  5         SSS5        g! , (       d  f       g= f! [         aE  nU R                  [	        U5      R
                   S[        U5       3[        R                  S9  e SnAff = fr  )	rq  rW   r  r   r  r>   rD   r   r  r  s     r(   
set_closed#DynamicRendezvousHandler.set_closed  sp    	%% &%% 	LL7++,Bs1vh7$++   	s*   9 (9 
69 9 
BA BBc                 |    U R                      U R                  R                  5         [        U R                  R                  R
                  5      sSSS5        $ ! , (       d  f       g= f! [         aE  nU R                  [        U5      R                   S[        U5       3[        R                  S9  e SnAff = fr  )rq  r#  r   r#   r8   r   r  r   r  r>   rD   r   r  r  s     r(   num_nodes_waiting*DynamicRendezvousHandler.num_nodes_waiting  s    	%%""'')4--33==> &%%
  	LL7++,Bs1vh7$++   	s6   A, AA	A, 
A)%A, )A, ,
B;6A B66B;c                 .    U R                   R                  $ r   )r   rw   r0   s    r(   
get_run_id#DynamicRendezvousHandler.get_run_id  s    ~~$$$r*   c           	         U R                  5          U R                  5         g! [         az  nSU R                   SU R                  R
                   S[        U5      R                   S3nU R                  U[        R                  S9  [        R                  U5         SnAgSnAf[         aE  nU R                  [        U5      R                   S	[        U5       3[        R                  S9  e SnAff = f)
r   Tr+  z)' has failed to shutdown the rendezvous '' due to an error of type r   rM  NFr  )r  rW   r   rm  r   rw   r  r>   r   r   r  r   warningr  rD   )r1   exr   r  s       r(   shutdown!DynamicRendezvousHandler.shutdown  s    	KKM 	T__- .NN))**DT"XEVEVDWWXZ  LL1A1ALBNN3 	LL7++,Bs1vh7$++   	s"   # 
C4A0B""C4/A C//C4c                 `   [        5       nU R                  U R                  R                  R                  5      nU R
                  R                  X5        SU R                   SU R                  R                   S3nU R                  U[        R                  S9  [        R                  U5        g )Nr+  z' has closed the rendezvous 'r,  rM  )re  r  r   ro   rS   rp  r  rm  rw   r   r   rN  r   r  )r1   opr   r   s       r(   rW   DynamicRendezvousHandler._close  s    !%%dnn&<&<&B&BCb+4??++HI^I^H__abSY-@-@ACr*   c                 :    U " 5       nUb  UR                  5         g g r=   )r-  )	weak_selfr1   s     r(   _keep_alive_weak)DynamicRendezvousHandler._keep_alive_weak!  s     { r*   c           	         U R                   R                  5         [        5       nU R                  U R                  R
                  R                  5      n U R                  R                  X5        SU R                   SU R                  R                   S3nU R                  US9  [        R                  U5        U R                   R)                  5         g ! [         az  nSU R                   SU R                  R                   S[        U5      R                    S3nU R                  U["        R$                  S9  [        R'                  U5         S nANS nAff = f! U R                   R)                  5         f = f)	Nr+  z5' has sent a keep-alive heartbeat to the rendezvous 'r,  r   z?' has failed to send a keep-alive heartbeat to the rendezvous 'r  r   rM  )rq  acquireri  r  r   ro   rT   rp  r  rm  rw   r   r   r   r   r  r>   r   r  r  release)r1   r  r   r   r  s        r(   r-  $DynamicRendezvousHandler._keep_alive'  sF   $$&#%%%dnn&<&<&F&FG	+!!"/ T__- .NN))*".  LLL%LL   ((*  	 T__- .#~~4455OPTUWPXPaPaObbce  LL1A1ALBNN3	    ((*s,   A&C 
E A0EE EE E9c                 $   [        U R                  R                  U R                  [        R
                  " U 5      5      U l        U R                  R                  SU R                  R                   35        U R                  R                  5         g )NRendezvousKeepAliveTimer_)r   r   rz   r  weakrefrefrr  set_namerm  r   startr0   s    r(   r  *DynamicRendezvousHandler._start_heartbeatsA  so    !/NN..0E0Ew{{SWGX"
 	'''(@(@'AB	
 	$$&r*   c                 T    U R                   c  g U R                   R                  5         g r=   )rr  cancelr0   s    r(   r  )DynamicRendezvousHandler._stop_heartbeatsL  s#    !!)%%'r*   c                     U R                   R                  nUR                  U R                     [	        UR                  5      4$ r=   )r#  r8   r   rm  r#   )r1   r8   s     r(   r  #DynamicRendezvousHandler._get_worldR  s8    ""((!!$//2C8J8J4KKKr*   c                     SU R                   R                   SU R                  R                  R                   3n[
        R                  " X!5      $ )Nztorch.rendezvous.r   )r   rw   r#  r8   r   r  PrefixStore)r1   rs  
key_prefixs      r(   _wrap_store$DynamicRendezvousHandler._wrap_storeW  sH     5 56a8J8J8P8P8V8V7WX 	 
22r*   c                 8    U R                  U R                  5      $ r=   )r  ro  r0   s    r(   r  #DynamicRendezvousHandler._get_store^  s    ,,r*   c                 N    [         R                  " 5       UR                  5       -   $ r=   )r   r   total_seconds)r1   ro   s     r(   r  &DynamicRendezvousHandler._get_deadlinea  s    ~~'"7"7"999r*   )
rn  r}  rq  rr  rp  r   r|  r#  ro  rm  )NNrO   r   r   )8r>   r?   r@   rA   rB   r   rv  r   rq   r   rD   r   r   r  r   r   r   r   classmethodr   r|   r   rx  r[   r   r   r   r  r  r  rC   r   r  rH   r  r   r  r  r  r  r  r  rW   staticmethodr  r-  r  r  rE   r  r  r  r   r   r  rI   r/   r*   r(   r   r     sd   M ./!!M))''^^#// %)/3#$&'2F2F 2F #	2F
 2F 2F SM2F +,2F !2F !$2F 2Fh+I+I %+I 	+I
 +I -+I 
+I` !* 1 1"	

 
 sm	

 

"
DMM 
 ,  "S " K K KS
 S
j4 
3 %C %$ 0	  
+4	'(LE#s(O L
3 35 3-E -:Y :5 :r*   r   paramskeyc                 D    U R                  US-   5      nUc  g [        US9$ )N_timeoutrL   )
get_as_intr   )r  r  ro   s      r(   _get_timeoutr  e  s*    j 01GW%%r*   rs  r   c                 $    [        [        US5      [        US5      [        US5      [        US5      5      nUR                  SS5      nUc  [        S5      eUR                  S	S
5      nUc  [        S5      e[        R                  UR                  U UUR                  UR                  UR                  UUUS9	$ ! [         aJ  n[        [        U5      R                   S[        U5       3UR                  [        R                   S9  e SnAff = f)a  Create a new :py:class:`DynamicRendezvousHandler` from the specified parameters.

Args:
    store:
        The C10d store to return as part of the rendezvous.
    backend:
        The backend to use to hold the rendezvous state.

+-------------------+------------------------------------------------------+
| Parameter         | Description                                          |
+===================+======================================================+
| join_timeout      | The total time, in seconds, within which the         |
|                   | rendezvous is expected to complete. Defaults to 600  |
|                   | seconds.                                             |
+-------------------+------------------------------------------------------+
| last_call_timeout | An additional wait amount, in seconds, before        |
|                   | completing the rendezvous once the minimum number of |
|                   | nodes has been reached. Defaults to 30 seconds.      |
+-------------------+------------------------------------------------------+
| close_timeout     | The time, in seconds, within which the rendezvous is |
|                   | expected to close after a call to                    |
|                   | :py:meth:`RendezvousHandler.set_closed` or           |
|                   | :py:meth:`RendezvousHandler.shutdown`. Defaults to   |
|                   | 30 seconds.                                          |
+-------------------+------------------------------------------------------+
| heartbeat         | The time, in seconds, within which a keep-alive      |
|                   | heartbeat is expected to complete                    |
+-------------------+------------------------------------------------------+
rQ   rR   rS   rT   rz   rO   NzJYou passed 'keep_alive_interval=None' as a rendezvous configuration optionr{   r   zMYou passed 'keep_alive_max_attempt=None' as a rendezvous configuration optionru  r  )r   rw   r   )r   r  r  	TypeErrorr   rx  rw   rx   ry   r   r  r   r  r>   rD   r   r  )rs  r   r  ro   rz   r{   r  s          r(   r    r    l  s)   @##(-)-	
 %//0EqI&\  "(!2!23KQ!O!)_  (44MM 3#9 5 

 
	
  'Aw''(3q6(3== ''	

 	s   B8B; ;
DAD

D)r   )Gr$   r   r   r   r   r   r   r  abcr   r   dataclassesr   r   r   r   enumr	   typingr
   r   r   torch.distributeddistributedr  r    torch.distributed.elastic.eventsr   r   apir   r   r   r   r   r   r   r   r   utilsr   r   __all__	getLoggerr>   r   r)   rG   r   r   r   r   r   r   r   r   r   r   r  r  r!  rH   rV  rX  r_  re  ri  r   rD   r  r    r/   r*   r(   <module>r     s     	      # ! 2 2  * *   # W
 
 
 * 
		8	$ 	 G: :zE/ E/P %-    . 8 d$t,9 9 -9(P P6+" +"\""'9"	"*-S -,b$: bJd $! !2C 8w"'< w"t	. 	4 	 e eP. . :0 :D&- &C &HY<O &CC,C6JCCr*   