
    sh                        % S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKrS SKrS SKrS SKrS SKJr  S SKJr  S SKJr  S SKJr  S SKJrJrJr  S SKJr  S SKJrJrJ r J!r!J"r"  S S	K#J$r$  S S
K%J&r&  S SK'r'S SK(r'S SK)r'S SK*J+r,  S SK-J.r.  S SK/J0r0  S SK1J2r2  S SK3J4r4J5r5J6r6J7r7J8r8J9r9J:r:J;r;J<r<J=r=J>r>J?r?  S SK@JArAJBrBJCrC  S SKDrD\R                  " \R                  S9  \R                  " \H5      rI " S S\5      rJ0 S\J" SS5      _S\J" SS5      _S\J" SS5      _S\J" SS5      _S\J" SS 5      _S!\J" S"S#5      _S$\J" S%S&5      _S'\J" S(S)5      _S*\J" S+S,5      _S-\J" S.S/5      _S0\J" S1S25      _S3\J" S4S55      _S6\J" S7S85      _S9\J" S:S;5      _S<\J" S=S>5      _S?\J" S@SA5      _SB\J" SCSD5      _SE\J" SFSG5      0ErK\ " SH SI5      5       rLSJ rMSK rNSL rOSM rPSN rQSO rRSP rSSQ rTSR rUSS rVST rWSU rXSV rYSW rZSX r[SY r\SZ r]S[ r^S\ r_S]\'R                  S^\aS_\aS`\b4Sa jrc\7SbScSd\" SeSf9SdSgSd4Sh j5       rd\;(       a  SireO\a" \R                  " SjSk5      5      reSlSm0rg\:(       a  Sn\gSo'   SSp jrhS`\a4Sq jri\Sr 5       rjSSs\aSt\aSu\a4Sv jjrkSt\aSw\l4Sx jrmSqn\\	R                     \pSy'   SSz\\l   S`S4S{ jjrqSS| jrrS}rs " S~ S\<5      rt " S S\t5      ruS\v\l\w\!   4   S\"S\!4S jrxSqyS`\b4S jrzS r{S\e\s4S jr| " S S\<5      r} " S S\2R                  5      r " S S\2R                  5      r\SS j5       r " S S\'GR                  GR                  Rx                  5      r " S S\t5      r " S S\<5      rg)    N)contextmanager)	dataclass)	timedelta)Enum)partialreducewraps)StringIO)
NamedTupleOptionalUnionAnyCallable)patch)	trace_log)
DeviceType)_SymmetricMemory)FILE_SCHEMAfind_free_portIS_SANDCASTLEretry_on_connect_failuresskip_but_pass_in_sandcastleskip_but_pass_in_sandcastle_ifTEST_WITH_ROCMTEST_WITH_TSANTestCase	run_testsTEST_HPUTEST_XPU)_install_threaded_pg_uninstall_threaded_pgProcessLocalGroup)levelc                   *    \ rS rSr% \\S'   \\S'   Srg)TestSkip<   	exit_codemessage N)__name__
__module____qualname____firstlineno__int__annotations__str__static_attributes__r)       ~/Users/tiagomarins/Projetos/claudeai/copy_bank/venv/lib/python3.13/site-packages/torch/testing/_internal/common_distributed.pyr%   r%   <   s    NLr2   r%   backend_unavailableH   z5Skipped because distributed backend is not available.small_worldsizeI   z Skipped due to small world size.odd_worldsizeW   zSkipped due to odd world size.no_cudaJ   zCUDA is not available.zmulti-gpu-1K   zNeed at least 1 CUDA devicezmulti-gpu-2M   zNeed at least 2 CUDA deviceszmulti-gpu-3P   zNeed at least 3 CUDA deviceszmulti-gpu-4Q   zNeed at least 4 CUDA deviceszmulti-gpu-5R   zNeed at least 5 CUDA deviceszmulti-gpu-6S   zNeed at least 6 CUDA deviceszmulti-gpu-7T   zNeed at least 7 CUDA deviceszmulti-gpu-8U   zNeed at least 8 CUDA devicesncclL   z#c10d not compiled with NCCL support
skipIfRocmN   zTest skipped for ROCmno_peer_accessO   z'Test skipped because no GPU peer accessgenericV   zHTest skipped at subprocess level, look at subprocess log for skip reasonimporterrorX   z"Test skipped due to missing importno_acceleratorY   zaccelerator is not available.c                       \ rS rSr0 r1 Sk\S'   \" 5       \S'   SS1\S'   SS1\S'   0 r1 S	k\S
'   1 S	k\S'   1 S	k\S'   1 S	k\S'   \" 5       \S'   \(       a  S1\S'   \(       a	  S1\S'   Sr	gSr	g)DistTestCases[   >   mpiuccrD   allgather_coalescedr   rD   rT   zsendrecv anysourcezcpu barrier>   rT   gloorD   gpucudaddpsubgrouppluginhcclhpuxcclxpur)   N)
r*   r+   r,   r-   skip_collectivesetbackend_featurer   r   r1   r)   r2   r3   rQ   rQ   [   s     O-CO)* #OH-3UOO()&,e_OM" O4OE5OF4OE"9OJ #OH"("( r2   rQ   c                 0   ^  [        T 5      U 4S j5       nU$ )zSkips if the world size exceeds the number of GPUs, ensuring that if the
test is run, each rank has its own GPU via ``torch.cuda.device(rank)``.c                    > [         R                  R                  5       (       d'  [        R                  " [
        S   R                  5        [        [        R                  S   5      n[         R                  R                  5       U:  a*  [        R                  " [
        SU 3   R                  5        [        (       aH  [         R                  R                  U:  a*  [        R                  " [
        SU 3   R                  5        [        (       aH  [         R                  R                  U:  a*  [        R                  " [
        SU 3   R                  5        T" U 0 UD6$ )Nr:   
WORLD_SIZE
multi-gpu-z
multi-xpu-)torchrX   is_availablesysexit
TEST_SKIPSr'   r.   osenvirondevice_countr   r]   r   r_   )argskwargs
world_sizefuncs      r3   wrapperskip_if_no_gpu.<locals>.wrapperu   s    zz&&((HHZ	*445L12
::""$z1HHZ*ZL 9:DDE8		..;HHZ*ZL 9:DDE8		..;HHZ*ZL 9:DDET$V$$r2   r	   rr   rs   s   ` r3   skip_if_no_gpurw   q   s"     4[% % Nr2   c                 0   ^  [        T 5      U 4S j5       nU$ )Nc                     > [         R                  S   S:w  aG  [        [         R                  S   5      S:  a'  [        R                  " [
        S   R                  5        T" U 0 UD6$ )NBACKENDrS   re      r6   rl   rm   r.   ri   rj   rk   r'   ro   rp   rr   s     r3   rs   (skip_if_small_worldsize.<locals>.wrapper   sR    JJy!U*BJJ|4L0MPQ0QHHZ 12<<=T$V$$r2   ru   rv   s   ` r3   skip_if_small_worldsizer           
4[% % Nr2   c                 0   ^  [        T 5      U 4S j5       nU$ )Nc                     > [         R                  S   S:w  aJ  [        [         R                  S   5      S-  S:X  a'  [        R                  " [
        S   R                  5        T" U 0 UD6$ )Nrz   rS   re         r8   r|   r}   s     r3   rs   &skip_if_odd_worldsize.<locals>.wrapper   sW    JJy!U*BJJ|4L0MPQ0QUV0VHHZ0::;T$V$$r2   ru   rv   s   ` r3   skip_if_odd_worldsizer      r   r2   c                    ^ ^ UU 4S jnU$ )Nc                 6   >^  [        T 5      UU U4S j5       nU$ )Nc                     > TS:X  aM  [         R                  R                  5       T:  a+  [        R                  " [
        ST 3   R                  5        g T" U 0 UD6$ NrD   rf   )rg   rX   rn   ri   rj   rk   r'   )ro   rp   backendrr   ns     r3   rs   Crequire_n_gpus_for_nccl_backend.<locals>.decorator.<locals>.wrapper   sM    & UZZ%<%<%>%Bj$45??@T,V,,r2   ru   )rr   rs   r   r   s   ` r3   	decorator2require_n_gpus_for_nccl_backend.<locals>.decorator   s     	t	- 
	- r2   r)   )r   r   r   s   `` r3   require_n_gpus_for_nccl_backendr      s     r2   c                      S n U $ )Nc                 0   ^  [        T 5      U 4S j5       nU$ )Nc                     >  SSK JnJn  T" U 0 UD6$ ! [         a*    [        R
                  " [        S   R                  5         g f = f)Nr   )AutoModelForMaskedLM
BertConfigrL   )transformersr   r   ImportErrorri   rj   rk   r'   )ro   rp   r   r   rr   s       r3   rs   ?import_transformers_or_skip.<locals>.decorator.<locals>.wrapper   sE    >
 T,V,, >M2<<=>s    1AAru   rv   s   ` r3   r   .import_transformers_or_skip.<locals>.decorator   s     	t		> 
		> r2   r)   )r   s    r3   import_transformers_or_skipr      s     r2   c                     [         R                  R                  5       =(       a!    [         R                  R                  5       U :  $ N)rg   rX   rh   rn   )xs    r3   at_least_x_gpur      s,    ::""$G)@)@)Ba)GGr2   c                    ^  U 4S jnU$ )Nc                 4   >^  [        T 5      U U4S j5       nU$ )Nc                    > [         R                  R                  5       (       a*  [         R                  R                  5       T:  a  T" U 0 UD6$ [        (       a*  [         R
                  R                  5       T:  a  T" U 0 UD6$ [        (       a*  [         R                  R                  5       T:  a  T" U 0 UD6$ [        R                  " [        ST 3   R                  5        g )Nrf   )rg   rX   rh   rn   r   r]   r   r_   ri   rj   rk   r'   )ro   rp   rr   r   s     r3   rs   4skip_if_lt_x_gpu.<locals>.decorator.<locals>.wrapper   s    zz&&((UZZ-D-D-F!-KT,V,,xEII2249T,V,,xEII2249T,V,,HHZ*QC 01;;<r2   ru   )rr   rs   r   s   ` r3   r   #skip_if_lt_x_gpu.<locals>.decorator   s     	t	= 
	= r2   r)   )r   r   s   ` r3   skip_if_lt_x_gpur      s     r2   c                    ^ ^ U U4S jnU$ )Nc                 6   >^  [        T 5      UU U4S j5       nU$ )Nc                    > TS:w  a  T" U 0 UD6$ [         R                  R                  5       (       a*  [         R                  R                  5       T:  a  T" U 0 UD6$ [        R
                  " [        ST 3   R                  5        g r   )rg   rX   rh   rn   ri   rj   rk   r'   )ro   rp   r   rr   r   s     r3   rs   9nccl_skip_if_lt_x_gpu.<locals>.decorator.<locals>.wrapper   sp    & T,V,,zz&&((UZZ-D-D-F!-KT,V,,HHZ*QC 01;;<r2   ru   )rr   rs   r   r   s   ` r3   r   (nccl_skip_if_lt_x_gpu.<locals>.decorator   s     	t	= 
	= r2   r)   )r   r   r   s   `` r3   nccl_skip_if_lt_x_gpur      s    	 r2   c                     U R                  5       nSU;   d   eSU;   d   eSU;   d   eUS   nUR                  S5      S:X  a  UOUR                  S5      S   nXC;   d   SU SU 35       eg )	N	iteration	has_errorerrorz
Exception raised from r   zDid not find expected z in ddp logging data error: )_get_ddp_logging_datafindsplit)	model_DDP
err_substrddp_logging_datalogging_erractuals        r3   verify_ddp_error_loggedr      s     668********&&&&"7+K ??56"< 	89!<  	R	x'CK=QRr2   c                 0   ^  [        T 5      U 4S j5       nU$ )a6  
Convenience decorator to set/unset TORCH_NCCL_BLOCKING_WAIT flag. Note that use of
this decorator will override the setting of TORCH_NCCL_ASYNC_ERROR_HANDLING for
the particular test. After the test, both TORCH_NCCL_BLOCKING_WAIT and
TORCH_NCCL_ASYNC_ERROR_HANDLING will be restored to their original values.
c                    >  [         R                  S   n[         R                  S	  [         R                  S   nS[         R                  S'    T" U 0 UD6nUUb  U[         R                  S'   Ub  U[         R                  S'   $ $ ! [         a    S n Nmf = f! [         a    S n Nkf = f! S[         R                  S'   f = f! Ub  U[         R                  S'   Ub  U[         R                  S'   f f = f)NTORCH_NCCL_ASYNC_ERROR_HANDLINGTORCH_NCCL_BLOCKING_WAIT1)rl   rm   KeyError)ro   rp    cached_nccl_async_error_handlingcached_nccl_blocking_waitretrr   s        r3   rs   (with_nccl_blocking_wait.<locals>.wrapper  s   	4AC1B, 

<=	9:<***;% 69BJJ12	S''C 0; 5 

5 )49R

56 51  	4/3,	4  	-(,%	- 69BJJ12 0; 5 

5 )49R

56 5s@   $B B 	C BBB'$B* &B''B* *B?/C1ru   rv   s   ` r3   with_nccl_blocking_waitr      s%     4[ S  SD Nr2   c                    ^  U 4S jnU$ )zC
Runs a test for each distributed debug level specified in levels.
c                 4   >^  [        T 5      U U4S j5       nU$ )Nc                    > [         R                  R                  SS 5      nT H`  nU[         R                  S'   [        R                  " 5         T" U 0 UD6n[        R
                  " 5         Uc  MM  U[         R                  S'   Mb     W$ )NTORCH_DISTRIBUTED_DEBUG)rl   rm   getc10dset_debug_level_from_envbarrier)ro   rp   	old_levelr#   r   rr   levelss        r3   rs   :with_dist_debug_levels.<locals>.decorator.<locals>.wrapper.  sq    

'@$GI8=

45--/D+F+(<EBJJ89   Jr2   ru   )rr   rs   r   s   ` r3   r   )with_dist_debug_levels.<locals>.decorator-  s     	t	 
	 r2   r)   )r   r   s   ` r3   with_dist_debug_levelsr   (  s    
$ r2   c                  J    [        [        R                  " 5       (       + S5      $ )Nz+c10d was not compiled with the Gloo backend)r   r   is_gloo_availabler)   r2   r3   requires_gloor   B  !    )""$$5 r2   c           	         [         R                  " 5       (       d  [        S5      $ [        [        R
                  R                  R                  5       U :  SU  S[        R
                  R                  R                  5        SU 35      $ )N+c10d was not compiled with the NCCL backendz0Requires NCCL version greater than or equal to: z	, found: z
, reason: )r   is_nccl_availabler   r   rg   rX   rD   version)r   msgs     r3   requires_nccl_versionr   I  sy    !!##*9
 	
 .JJOO##%/>wiyQVQ[Q[Q`Q`QhQhQjPkkuvyuz{
 	
r2   c                  J    [        [        R                  " 5       (       + S5      $ )Nr   )r   r   r   r)   r2   r3   requires_ncclr   U  r   r2   c                  J    [        [        R                  " 5       (       + S5      $ )Nz*c10d was not compiled with the UCC backend)r   r   is_ucc_availabler)   r2   r3   requires_uccr   [  !    )!!##4 r2   c                  J    [        [        R                  " 5       (       + S5      $ )Nz*c10d was not compiled with the MPI backend)r   r   is_mpi_availabler)   r2   r3   requires_mpir   a  r   r2   c                      [         R                  R                  5       =(       a%    [        R                  " [
        R                  S5      n [        U (       + S5      $ )Nr   z"multicast support is not available)rg   rX   rh   r   has_multicast_supportr   CUDAr   )r   s    r3   requires_multicast_supportr   h  sI    

! 	G22:??AF  *!!, r2   c                 >   ^  ST l         [        T 5      U 4S j5       nU$ )zSkips a test for ROCmTc                  z   > [         (       d  T" U 0 UD6$ [        R                  " [        S   R                  5        g )NrF   )r   ri   rj   rk   r'   r}   s     r3   rs   *skip_if_rocm_multiprocess.<locals>.wrapperw  s/    ~(((L)334r2   )skip_if_rocm_multiprocessr	   rv   s   ` r3   r   r   s  s(    %)D"
4[5 5
 Nr2   c                  <    [        [        R                  S:H  S5      $ )Nwin32z8This unit test case is not supported on Windows platform)r   ri   platformr)   r2   r3   skip_if_win32r     s    )B r2   devicemajorminorreturnc                     U R                   S:w  a  [        S5      e[        R                  R                  b  g[        R
                  R                  U 5      X4:  $ )z
Returns True if the device's compute capability is (major, minor) or higher.
Error out if the device is not a CUDA device.
Returns False if device is a RoCM device.
rX   z3sm_is_or_later() is only supported for CUDA devicesF)type
ValueErrorrg   r   hiprX   get_device_capability)r   r   r   s      r3   sm_is_or_higher_thanr     sK     {{fNOO}}$::++F3~EEr2   	localhostr   T   )minutesFc           	          [        5       nU(       a@  [        U[        SS9-  5      n[        R                  R
                  R                  XXU5      $ [        R                  " XXXFS9$ )zD
Creates a TCP store. Retries if the chosen port is already in use.
r   )milliseconds)wait_for_workers	use_libuv)r   r.   r   rg   classes	dist_c10dTCPStorer   )	addrrq   	is_mastertimeoutr   	jit_classr   porttimeout_milliseconds	            r3   create_tcp_storer
    sd     D!'I1,E"EF}}&&//
/B
 	
 }}
@P
 	
r2   i  !DISTRIBUTED_TESTS_DEFAULT_TIMEOUT300test_ddp_uneven_inputsi     test_join_kwargsc                     [         R                  S:X  d  U c  [        R                  R	                  SS9$ [        R                  R	                  U S9$ )Nr   z	127.0.0.1)hostname	interface)ri   r   r   ProcessGroupGloocreate_devicer  s    r3   r  r    sG    
||w)"3$$22K2HH$$22Y2GGr2   c                 Z    [         R                  U R                  S5      S   [        5      $ N.r   )TIMEOUT_OVERRIDEr   r   TIMEOUT_DEFAULT)test_ids    r3   get_timeoutr    s#    c 22 6HHr2   c               #   D  #    [        5       [        5       p[        R                  [        R                  p2 Xs[        l        [        l        [        R                  [        R                  4v   X#s[        l        [        l        g ! X#s[        l        [        l        f = f7fr   )r
   ri   stdoutstderr)new_outnew_errold_outold_errs       r3   captured_outputr$    sa     z8:Wzz3::W2!(
CJjj#**$$!(
CJ
CJs   3B 8B .B BB rankrq   
num_inputsc                    SS[         S[         S[         S[         4S jjnS[         4S jn[        USS9[        US	S9[        US
S9[        USS9[        US	S9[        US
S94 VVs/ s HQ  n[        U5       Vs/ s H  nU" X -  U-   X!-  5      PM     sn[        U5       Vs/ s H  od" XRU-  5      PM     sn4PMS     snn$ s  snf s  snf s  snnf )z
Generate a number of basic test cases for sparse reduction.
These cover tensors with a varying number of sparse dimensions and a varying
number of dense dimensions. The only reduction operation we support is sum.
r   r%  rq   sparse_dims
dense_dimsc           	         [         R                  " [         R                  " U S-   5      SU S-   45      nU/[        U5       Vs/ s H  nSPM     sn-   n[        US-
  5       HD  n[         R                  " U[         R
                  " SU S-   5      45      nUR                  U5        MF     [         R                  " U S-   /[        U5       Vs/ s H  nSPM     sn-   5      n[         R                  " XGU5      $ s  snf s  snf )Nr   r   )	rg   reshapearangerangecatzerosappendonessparse_coo_tensor)r%  rq   r(  r)  indices_shapevaluess           r3   generate,simple_sparse_reduce_tests.<locals>.generate  s     --TAX 6D1HF5+<=+<a+<=={Q'Aii%++a*B CDGLL$ ( TAXJU:5F)G5F!5F)GGH&&w>>  > *Hs   C8C=
c           
      ~    [        [        R                  [        U5       Vs/ s H
  o " X!5      PM     sn5      $ s  snf r   )r   operatoraddr-  )fnrq   r%  s      r3   compute_sum/simple_sparse_reduce_tests.<locals>.compute_sum  s6    LLE*<MN<MD2d/<MN
 	
Ns   :
)r(  r      )r)  )r   r   )r.   r   r-  )r%  rq   r&  r7  r=  r<  is          r3   simple_sparse_reduce_testsrA    s    
?s 
? 
?# 
?s 
?
C 
 H!,H!,H!,H+H+H+

B	 z**A :$q(**AB* @EZ?PQ?P![*45?PQ	

  Rs$   &C9CC"C6C
Cr   c           
      |   [         R                  R                  5       n[        (       a  [         R                  R                  5       n[
        (       a  [         R                  R                  5       n[        U5      nSnX:  a  X -  n[        U 5       Vs0 s H  nU[        X5U-  US-   U-   5      _M     nnU$ s  snf )zMultigpu tests are designed to simulate the multi nodes with multi
GPUs on each node. Nccl backend requires equal #GPUs in each process.
On a single node, all visible GPUs are evenly
divided to subsets, each process only uses a subset.
r   )	rg   rX   rn   r   r]   r   r_   r-  list)rq   r   nGPUsvisible_devicesnGPUs_per_processr@  rank_to_GPUs          r3   init_multigpu_helperrH    s     JJ##%Ex		&&(x		&&(ElO !/ z""A 	
4$5 5QBS8STUU"   	s   !B9tmp_dirinit_methodc                    [         R                  " 5       q[        R                  [        R
                  S'   [        R                  " [        R                  R                  [        R                  S5      5        [        R                  " [        R                  R                  [        R                  S5      5        [        R                  R                  [        R                  S5      n[        R                  " U5        U b  U [        R
                  S'   g [        [        R                  R                  US5      -   [        R
                  S'   g )NTEMP_DIRr   test_dirinit_dirINIT_METHODshared_init_file)
tempfileTemporaryDirectoryrI  namerl   rm   mkdirpathjoinr   )rJ  init_dir_paths     r3   initialize_temp_directoriesrX    s    ))+G$\\BJJzHHRWW\\',,	23HHRWW\\',,
34GGLLz:MHH]$/

=!$/"'',,-3
 %


=!r2   c                  <    [         b  [         R                  5         g g r   )rI  cleanupr)   r2   r3   cleanup_temp_dirr[  ,  s     r2      c            	       n  ^  \ rS rSrSrSrS\4S jr\S\4S j5       r	\S\
4S j5       rS rS S	\S
\SS4U 4S jjjrS!U 4S jjrS!U 4S jjrS\4S jrS!S jrS!S jr " S S\5      r\S\
4S j5       r\S\
S\S\SS4S j5       rS\SS4S jrS!S jrS!S jrS!S jrS!S jr\S\4S j5       rSr U =r!$ )"MultiProcessTestCasei@  r   
   r   c                     g)NFr)   selfs    r3   _should_stop_test_suite,MultiProcessTestCase._should_stop_test_suiteI  s    r2   c                     g)NTr)   ra  s    r3   destroy_pg_upon_exit)MultiProcessTestCase.destroy_pg_upon_exitQ  s    r2   c                     [         $ r   DEFAULT_WORLD_SIZEra  s    r3   rq   MultiProcessTestCase.world_sizeU      !!r2   c                 X   ^ [        T5      U4S j5       n[        R                  " X 5      $ )Nc                 l   > U R                   U R                  :X  a  U R                  T5        g T" 5         g r   )r%  MAIN_PROCESS_RANK_join_processesrb  r<  s    r3   rs   1MultiProcessTestCase.join_or_run.<locals>.wrapperZ  s(    yyD222$$R(r2   r	   types
MethodTyperb  r<  rs   s    ` r3   join_or_run MultiProcessTestCase.join_or_runY  ,    	r	 
	 ..r2   method_name
methodNameNc                    > US:w  a  Un[         TU ]  U5         [        X5      n[        XU R	                  U5      5        g ! [
         a,  nUS:w  a  [        SU R                   SU 35      Ue S nAg S nAff = fNrunTestzno such test method in z: super__init__getattrsetattrrw  AttributeErrorr   	__class__rb  rz  r{  r<  er  s        r3   r  MultiProcessTestCase.__init__g       "$K%	b+BDt'7'7';< 	bY& !#:4>>:J"ZL!YZ`aa '	b   &A 
A7"A22A7c                    > [         TU ]  5         / U l        / U l        U R                  U l        [        R                  " SS9R                  U l	        0 U l
        g )NF)delete)r  setUpskip_return_code_checks	processesro  r%  rQ  NamedTemporaryFilerS  	file_namepid_to_piperb  r  s    r3   r  MultiProcessTestCase.setUpv  sH    ')$**	!44EBGGr2   c                 v   > [         TU ]  5         U R                   H  nUR                  5         M     / U l        g r   )r  tearDownr  	terminate)rb  pr  s     r3   r  MultiProcessTestCase.tearDown  s.    AKKM   r2   c                 F    U R                  5       R                  S5      S   $ r  idr   ra  s    r3   _current_test_name'MultiProcessTestCase._current_test_name  s    wwys#B''r2   c                    / U l         [        [        U R                  5      5       H  n[        R
                  R                  5       u  p4U" U R                  R                  S[        U5      -   X R                  5       U R                  U4S[        U SS5      0S9nUR                  5         [        R                  SX%R                   5        X0R"                  UR                   '   U R                   R%                  U5        M     g )Nzprocess fake_pgF)targetrS  ro   rp   zStarted process %s with pid %s)r  r-  r.   rq   rg   multiprocessingPiper  _runr0   r  r  r  startloggerinfopidr  r0  )rb  procr%  parent_conn
child_connprocesss         r3   _start_processes%MultiProcessTestCase._start_processes  s    #doo./D&+&;&;&@&@&B#K~~**#d)+335t~~zRwtY>	G MMOKK8$L,7W[[)NN!!'* 0r2   c                 x    [         R                  R                  S5      R                  nU R	                  U5        g )Nspawn)rg   r  get_contextProcessr  )rb  r  s     r3   _spawn_processes%MultiProcessTestCase._spawn_processes  s,    $$009AAd#r2   c                       \ rS rSrSrSrg)MultiProcessTestCase.Eventi  r   r)   N)r*   r+   r,   r-   GET_TRACEBACKr1   r)   r2   r3   Eventr    s    r2   r  r%  c                    [         R                  SU5         [        R                  R	                  X/5      nX;   a  U R
                  (       a  [         R                  SU5        g U R                  5       n[         R                  SXB5        U[        R                  R                  :X  a  [        R                  " SS9 n[        R                  " U5        UR                  5         UR                  S5        U R!                  UR#                  5       5        [         R                  SU5        S S S 5        X;   a  g GM#  ! , (       d  f       N= f)Nz*Starting event listener thread for rank %sz:Pipe closed for process %s, stopping event listener threadzReceived event %s on process %szr+)moder   zProcess %s sent traceback)r  r  r  
connectionwaitclosedrecvr^  r  r  rQ  r  faulthandlerdump_tracebackflushseeksendread)parent_pipesignal_piper%  ready_pipeseventtmp_files         r3   _event_listener$MultiProcessTestCase._event_listener  s    @$G)4499;:TUK)%%KKTVZ #((*=uK066DDD!44$?8$33H= ( a(#((9$?F @ )5   @?s   =A-D;;
E		test_namer  c                 N    U " U5      nXl         X6l        UR                  X$5        g r   )r%  r  run_testclsr%  r  r  r  rp   rb  s          r3   r  MultiProcessTestCase._run  s!    9~	"i-r2   c           	         [         R                  R                  SS9u  p4[        R                  " [
        R                  X#U R                  4SS9nUR                  5         [        R                  S:w  a3  [        R                  S:w  a  [         R                  R                  S5        S[        R                  S'    [        X5      " 5         Ub  UR;                  S 5        Uc   eUR=                  5         UR?                  5         U R@                  (       a   [B        RD                  " 5         g g ! [         R"                   a\  n[$        R'                  S	U R                  U[)        U5      5        [        R*                  " [,        S
   R.                  5         S nANS nAf[0         a    [$        R3                  S[4        R6                  " 5       U R                  [
        R8                  5        UR;                  [4        R6                  " 5       5        [        R*                  " [
        R8                  5         GNhf = f! Ub  UR;                  S 5        Uc   eUR=                  5         UR?                  5         f = f! [F        [H        4 a     g f = f)NF)duplexT)r  ro   daemonr   darwinr   TORCH_SHOW_CPP_STACKTRACESz4Process %s skipping test %s for following reason: %srJ   z;Caught exception: 
%s exiting process %s with exit code: %s)%rg   r  r  	threadingThreadr^  r  r%  r  ri   r   _C'_set_print_stack_traces_on_fatal_signalrl   rm   r  unittestSkipTestr  r  r0   rj   rk   r'   	Exceptionr   	traceback
format_excTEST_ERROR_EXIT_CODEr  rV  closerf  r   destroy_process_groupAssertionErrorr   )rb  r  r  signal_recv_pipesignal_send_pipeevent_listener_threadses          r3   r  MultiProcessTestCase.run_test  s   -2-B-B-G-Gu-G-U* ) 0 0'77;!

 	##%<<7"s||x'? HH<<TB36

/0	 D$&   + %%d+(444!&&($$ **,	 %/    	6KKF		S\^abd^e HHZ	*4455 	@LL0$$&		3G3\3\ Y1134HH)>>?	@  + %%d+(444!&&( #J/ sJ   ;D- I5 -H4AFH7 BH40H7 3H44H7 7;I25JJc                    / n[        U R                  5       Hi  u  p#UR                  b  M  U R                  UR                     n UR                  [        R                  R                  5        UR                  X$45        Mk     U H  u  pd UR                  S5      (       aQ  UR                  (       a  [        R                  SU5        ME  UR!                  5       n[        R                  SXg5        Mm  [        R                  SU5        M     g ! [         a"  n[        R                  SX%5         S nAGM  S nAff = f! [         a!  n[        R                  SXe5         S nAM  S nAff = f)NzBEncountered error while trying to get traceback for process %s: %sr   z5Pipe closed for process %s, cannot retrieve tracebackz)Process %s timed out with traceback: 

%sz6Could not retrieve traceback for timed out process: %s)	enumerater  exitcoder  r  r  r^  r  r  r0  ConnectionErrorr  r   pollr  r  r  )rb  pipesr@  r  piper  r%  r  s           r3   _get_timedout_process_traceback4MultiProcessTestCase._get_timedout_process_traceback  s'   #DNN3JA'''4II288FFGLL!+ 4  JD99Q<<{{SUY ! $		ILLEt LLPRV   ' LL\^_ 0 # XZ^ s<   ;D=D>&D>4D>
D;D66D;>
E)E$$E)c                 6   [        U R                  5       5      n[        R                  " 5       nSn  [        U R                  5       Hz  u  pVUR
                  [        R                  :X  d  M%  [        SU SUR
                   S35        [        R                  R                  5       nU H  nUR                  5         M     Sn  O   U(       a  O[        S U R                   5       5      (       a  Oy[        R                  " 5       U-
  n	X:  aC  U R                  5         [        SU S35        U R                   H  nUR                  5         M     O[        R                  " S	5        GM6  [        R                  " 5       U-
  n
XR                   ;   a  U R#                  U
5        OU R%                  U
5        U R&                  R)                  5        H  nUR+                  5         M     g ! U R&                  R)                  5        H  nUR+                  5         M     f = f)
NFTProcess z terminated with exit code z", terminating remaining processes.c              3   <   #    U  H  oR                   S Lv   M     g 7fr   )r  ).0r  s     r3   	<genexpr>7MultiProcessTestCase._join_processes.<locals>.<genexpr>9  s     F~!zz-~s   zTiming out after z" seconds and killing subprocesses.g?)r  r  timer  r  r  r^  r  printrg   r  active_childrenr  allr  sleepr  _check_no_test_errors_check_return_codesr  r6  r  )rb  r<  r  
start_timesubprocess_errorr@  r  r  acelapsedelapsed_timer  s               r3   rp  $MultiProcessTestCase._join_processes#  s   dggi(YY[
 *	'7FQ zz%9%N%NN&qc)DQZZLPrs +0*?*?*O*O*Q"1BLLN #2+/( 8 $Ft~~FFF))+
2$88:+G94VW "^^ ,

3= @  99;3L111**<8((6 ((//1

 2((//1

 2s   7G% -EG% %3Hc                     [        U R                  5       HJ  u  p#UR                  c  [        SU SU S35      eU R	                  U R
                  UR                  5        ML     g)zF
Checks that we didn't have any errors thrown in the child processes.
Nr  z timed out after  seconds)r  r  r  RuntimeErrorassertNotEqualr  )rb  r  r@  r  s       r3   r  *MultiProcessTestCase._check_no_test_errorsS  s_     dnn-DAzz!"qc!2<.I   9 91::F .r2   c                 `   U R                   (       d  [        R                  S5        gU R                   S   n[        U R                   5       VVs/ s H(  u  p4UR                  [
        R                  :X  d  M%  X44PM*     nnnU(       a\  SnU HI  u  p7U R                  UR                     R                  5       nUSU S[
        R                   SU S3-  nMK     [        U5      e[        U R                   5       Hf  u  p4UR                  c  [        SU S	U S
35      eU R                  UR                  UR                  SU SUR                   SUR                   3S9  Mh     [        R                  5        Hy  n	UR                  U	R                  :X  d  M  [        (       a1  [        R!                  SU R#                  5       U	R$                  5          g[&        R(                  " U	R$                  5      e   U R                  UR                  SSUR                   SUR                   3S9  gs  snnf )z
Checks that the return codes of all spawned processes match, and skips
tests if they returned a return code indicating a skipping condition.
z<Note: no subprocesses were spawned, test was likely skipped.Nr    r  z exited with error code z and exception:

 terminated or timed out after r  zExpect process z+ exit code to match Process 0 exit code of z
, but got )r   6Skipping %s on sandcastle for the following reason: %sz Expected zero exit code but got z
 for pid: )r  r  warningr  r  r^  r  r  r  r  r  assertEqualrk   r6  r'   r   r  r  r(   r  r  )
rb  r  first_processr@  r  errored_processesr   r  error_messageskips
             r3   r  (MultiProcessTestCase._check_return_codes^  s.    ~~NNYZq) "$..1
1zz1FFF QF1 	 

 E/
 $ 0 0 = B B Dqc!9:N:c:c9d e''4oR9 0 u%% dnn-DAzz!"qc!@hW  

&&%aS(STaTjTjSkkuvw  wA  wA  vB  C   . %%'D%%7 =
 KKPRVRYRYR[]a]i]i "++DLL99 ( 	""2=3I3I2J*UbUfUfTgh 	 	
S
s   $H*7H*c                      U R                   S:H  $ )Nr   r%  ra  s    r3   r  MultiProcessTestCase.is_master  s    yyA~r2   )r  r  r  r%  r  r~  r~  r   N)"r*   r+   r,   r-   ro  r  boolrc  propertyrf  r.   rq   rw  r0   r  r  r  r  r  r  r   r  staticmethodr  classmethodr  r  r  rp  r  r  r  r1   __classcell__r  s   @r3   r^  r^  @  s<   
   d   "C " "/bC b bUY b b(C (+"$    < . . . .W[ . .1# 1t 1f#J.`	G>
@ 4  r2   r^  c                   H   ^  \ rS rSrU 4S jrS rS\4S jrS rS r	Sr
U =r$ )	DistributedTestBasei  c                 B   > [         TU ]  5         U R                  5         g r   r  r  r  r  s    r3   r  DistributedTestBase.setUp      r2   c                 f     [         R                  " U R                  5        g ! [         a     g f = fr   )rl   remover  OSErrorra  s    r3   r  DistributedTestBase.tearDown  s)    	IIdnn% 		s    # 
00r   c                 .    SU;   a  gSU;   a  gSU;   a  gg)NrX   rD   r]   r\   r_   r^   rV   r)   )rb  r   s     r3   r   DistributedTestBase.backend  s$    Vf_f_r2   c                    [         R                  " U5      R                  5       n[         R                  R	                  U R
                  U5      n[         R                  R                  U R                  U5      U R                  U R                  US9  SU R                  U5      ;   d  SU R                  U5      ;   a)  [         R                  R                  U R                  5        [         R                  R                  R                  5       $ )Nr   rq   r%  storerD   r^   )rg   get_device_modulern   distributed	FileStorer  init_process_groupr   rq   r%  acceleratorset_device_indexdistributed_c10d_get_default_group)rb  r   num_visible_devicesr/  s       r3   	create_pgDistributedTestBase.create_pg  s    #55f=JJL!!++DNN<OP,,LL(	 	- 	
 T\\&))Vt||F7K-K..tyy9  11DDFFr2   c                     [         R                  " U5      R                  5       n[        U R                  5       Vs0 s H  o3X2-  /_M
     sn$ s  snf r   )rg   r0  rn   r-  rq   )rb  r   r8  r@  s       r3   rank_to_device"DistributedTestBase.rank_to_device  sH    #55f=JJL6;DOO6LM6LA+,,6LMMMs   Ar)   )r*   r+   r,   r-   r  r  r0   r   r9  r<  r1   r  r   s   @r3   r"  r"    s+      GN Nr2   r"  subtest_configtest_fntest_kwargsc                    [        UR                  5       5      nU Vs/ s H  ofS   PM	     nnU Vs/ s H  ofS   PM	     nn[        R                  " U6  H  n	[	        [        Xy5      5      n
U R                  " S0 U
D6   [        R                  R                  5         U" U0 UDU
D6  [        R                  R                  5         SSS5        [        R                  " 5         M     gs  snf s  snf ! , (       d  f       N2= f)a0  
Runs a test function given by ``test_fn`` as a subtest according to the
configurations specified by ``subtest_config``. This amortizes the
costly setup overhead (including process spawn and initializing the
process group) over the subtests.

Args:
    subtest_config (Dict[str, List[Any]]): A mapping from subtest
        keyword argument name to a list of its possible values.
    test_fn (Callable): A callable that runs the actual test.
    test_args: Positional arguments to pass to ``test_fn``.
    test_kwargs: Keyword arguments to pass to ``test_fn``.
r   r   Nr)   )rC  items	itertoolsproductdictzipsubTestrg   _dynamoresetr   r   )cls_instr>  r?  	test_argsr@  subtest_config_itemsitemsubtest_config_keyssubtest_config_valuesr6  subtest_kwargss              r3   run_subtestsrQ    s    * 9=^=Q=Q=S8T:N%O:N$1g:N%OBV-WBV$1gBV-W##%:;c"5>?//MM!Y@+@@MM! 0 	 < &P-W 0/s   C'C,>AC11
C?	c                      [         b  [         $  [        R                  " / SQSS9R                  S:H  q [         $ ! [         a
    Sq  [         $ f = f)z
If shell command `fi_info -p efa -t FI_EP_RDM` returns exit code 0 then we assume that the machine has
Libfabric EFA interfaces and EFA software components installed,
see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html.
)fi_infoz-pefaz-t	FI_EP_RDMF)checkr   )EFA_PROBE_RESULT
subprocessrun
returncodeFileNotFoundErrorr)   r2   r3   has_efar\    sZ     #!NNFeT__cdd 	
   ! !s   $9 AAc                  ,    [        5       (       a  SS/$ S$ )aw  
If the machine has Libfabric EFA interfaces and EFA software components installed it may cause
'RuntimeError: In operator() at tensorpipe/common/ibv.h:172 "": Operation not supported' if tensorpipe
uses InfiniBand transport, so we exclude it from tensorpipe transports,
see https://github.com/pytorch/pytorch/issues/73885 and https://github.com/pytorch/pytorch/issues/65022
shmuvN)r\  r)   r2   r3   tp_transportsr`    s     $IIE4=/4/r2   c                 b   ^ ^^ T c  [        [        UTS9$ S m[        T 5      UU U4S j5       nU$ )z#
Wrapper to use with a test method
)r  rq   c                    ^ ^^^ [        5       m[        R                  " 5       nU4S jmUUU 4S jn/ n[        T 5       H;  n[        R
                  " X5TU4S9nUR                  5         UR                  U5        M=     U$ )Nc                  >   > T [         R                  R                  :H  $ r   r   r6  _worldworlds   r3   world_is_validaspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads.<locals>.world_is_valid!      D118888r2   c                   > [         R                  " SU TUS9   T" 5         T" 5       (       a  [         R                  " 5         g g ! [         aT  n[        R                  R                  U [        R                  " 5       45        [        R                  " U5         S nAN|S nAff = f! T" 5       (       a  [         R                  " 5         f f = f)Nthreadedr   r%  rq   r/  )r   r3  BaseExceptionMultiThreadedTestCaseexception_queueputri   exc_infor"   exception_handler  )r%  world_pgr/  excallbackrh  rq   s       r3   workerYspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads.<locals>.worker$  s    ##"*E1
 "##..0 $ ! 7%55994:PQ!222667
 "##..0 $s*   A 
B"A
BB% B""B% %$C	r  ro   )r    r   	HashStorer-  r  r  r  r0  )	rq   rv  global_storerw  threadsr%  trg  rh  s	   ``     @@r3   #_run_test_method_with_multi_threadsIspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads  sj    $&~~'	9	1 *%D  E<5PQAGGINN1 &
 r2   c                 Z  >^ ^^ [         R                  R                  R                  S5         T" TUUUU 4S j5      n[        R                  UT5        [         R                  R                  R                  S5        g ! [         R                  R                  R                  S5        f = f)NTc                     > T" T/T Q70 TD6$ r   r)   )ro   rr   rp   rb  s   r3   <lambda>?spawn_threads_and_init_comms.<locals>.wrapper.<locals>.<lambda>@  s    dSWNiZ^NibhNir2   F)rg   r  _distributed_c10d_set_thread_isolation_modero  _join_threads)rb  ro   rp   r|  r~  rr   rq   s   ``` r3   rs   -spawn_threads_and_init_comms.<locals>.wrapper;  sq     	""==dC	I9*FijG!//>HH&&AA%HEHH&&AA%Hs   &A? ?+B*)r   spawn_threads_and_init_commsr	   )rr   r  rq   rs   r~  s   ` ` @r3   r  r    sD     |('j
 	

< 4[I I Nr2   c                   $  ^  \ rS rSrSr\R                  " 5       rSrS r	SS\
S\
SS4U 4S	 jjjrS
 rS rSU 4S jjrU 4S jrS r\S 5       rS r\S 5       r\S 5       r\S\4S j5       r\S\
4S j5       rSSS.S jjrSSS.S jjrSrU =r$ )ro  iI  a  
Test runner that runs all tests with the in-proc process group using
multiple threads with the threaded process group.

Each test spawns world_size threads and run the test method in each thread.

Difference from regular MultiProcess test runner:
Must explicitly defines SetUp and call self._spawn_threads() to run the tests.
Cannot use setUp / tearDown (must use perThreadSetup / perThreadShutdown)
    to set up / tear down each thread when running each test.
No global state possible
    How bad of a limitation is this?
r   c                 X   ^ [        T5      U4S j5       n[        R                  " X 5      $ )Nc                    > U R                   U R                  :X  a  U R                  U R                  T5        g T" 5         g r   )r%  MAIN_THREAD_RANKr  r|  rq  s    r3   rs   2MultiThreadedTestCase.join_or_run.<locals>.wrapper\  s.    yyD111""4<<4r2   rs  rv  s    ` r3   rw  !MultiThreadedTestCase.join_or_run[  ry  r2   rz  r{  r   Nc                    > US:w  a  Un[         TU ]  U5         [        X5      n[        XU R	                  U5      5        g ! [
         a,  nUS:w  a  [        SU R                   SU 35      Ue S nAg S nAff = fr}  r  r  s        r3   r  MultiThreadedTestCase.__init__e  r  r  c                     g r   r)   ra  s    r3   perThreadSetUp$MultiThreadedTestCase.perThreadSetUpt  s    r2   c                     g r   r)   ra  s    r3   perThreadTearDown'MultiThreadedTestCase.perThreadTearDownx  s    r2   c                 x   > [         TU ]  5         U R                  U l        / U l        S[
        R                  S'   g)zy
setUp only set up things in the main thread, if you want to configure things
in the spawned threads, use perThreadSetUp
r   r  N)r  r  r  r%  r|  rl   rm   r  s    r3   r  MultiThreadedTestCase.setUp{  s1    
 	))	36

/0r2   c                 0   > [         TU ]  5         / U l        g)z
tearDown only set up things in the main thread, if you want to configure things
in the spawned threads, use perThreadTearDown
N)r  r  r|  r  s    r3   r  MultiThreadedTestCase.tearDown  s    
 	r2   c                   ^ [         R                  R                  R                  S5        U R                  n[        5       m[        R                  " 5       U R                  l	        U4S jnU" 5       (       d  [        S5      e[        U R                  5       Hc  n[        R                  " U R                  R                  XU R                  4S9nUR!                  5         U R"                  R%                  U5        Me     g)z[
class method to spawn threads and run test, use this method in the SetUp of your TestCase
Tc                  >   > T [         R                  R                  :H  $ r   rd  rf  s   r3   rh  <MultiThreadedTestCase._spawn_threads.<locals>.world_is_valid  rj  r2   zInvalid worldry  N)rg   r  r  r  r  r    r   rz  r  r{  r  r-  rq   r  r  r  r  r|  r0  )rb  r  rh  r%  r}  rg  s        @r3   _spawn_threads$MultiThreadedTestCase._spawn_threads  s     	""==dC++	$&&*nn&6#	9 //$//*D  (;(;9TXTcTcBdeAGGILL" +r2   c                    U " U5      nX%l         [        US5      (       aX  [        R                  " 5       Ul        [
        R                  UR                  l        [
        R                  UR                  l	        UR                  XU5        g )N_tls)r%  hasattrr  localr  r   
_precision	precision_rel_tolrel_tolrun_test_with_threaded_pg)r  r  r%  rq   rp   rb  s         r3   r  MultiThreadedTestCase._run  sa    9~	 4  !)DI"*"5"5DII ( 1 1DII&&y
Cr2   c                    [         R                  " SX#U R                  R                  S9  U R	                  5          [        X5      " 5         [         R                  " 5         U R                  5         g! [         aP  nU R                  R                  U[        R                  " 5       45        [        R                  " U5         SnAN{SnAff = f! [         R                  " 5         U R                  5         f = f)zT
Run the current test associated with `test_name` using the threaded process group.
rl  rm  N)r   r3  r  r{  r  r  rn  rp  rq  ri   rr  r"   rs  r  r  )rb  r  r%  rq   ru  s        r3   r  /MultiThreadedTestCase.run_test_with_threaded_pg  s     	THcHc	
 		%D$&
 &&(""$  	3  $$dCLLN%;<..r22	3 &&(""$s*   A2 2
C<ACC CC 'C6c           
         [         n [        U5       Hl  u  pEUR                  [        SU5      5        UR	                  5       (       d  M7  [
        R                  R                  U[        [        SU S35      S 445        Mn     [        R                  " 5         / nU R                  R                  5       (       dL  U R                  R                  5       nUR                  U5        U R                  R                  5       (       d  ML  [        5         [        R                   R"                  R%                  S5        U R'                  XcU5        g ! [        5         [        R                   R"                  R%                  S5        f = f)Nr   zRank failed to join in under r  F)r  r  rV  maxis_alivero  rp  rq  TimeoutErrorr"   rI  emptyr   r0  r!   rg   r  r  r  r  )r  r|  r<  r  idxthreadfailed_ranksfailures           r3   r  #MultiThreadedTestCase._join_threads  s-   !	I(1C7O,??$$)99== , ,&CG9H$U!" !%	  2 ##%L))//11--113##G, ))//11 #$HH&&AA%Hr: #$HH&&AA%Hs   >E 
B9E 5F c           	         SnSnU GHN  u  pgUS   n[        U[        R                  5      (       a=  [        R	                  SXc[        U5      5        US:  a  [        S   R                  nMe  Mg  [        U[        5      (       a)  SU SU S	3n	[        R                  U	5        [        U	5      e[        U[        5      (       aG  SR                  [        R                  " U6 5      n	[        R                  S
X5        USU SU	 S3-  nGM  [        U[        5      (       d  GM  [!        UR"                  5      [$        :X  d  GM9  US:  d  GMB  UR"                  nGMQ     ['        U5      S:  a  [        U5      eUS:  ax  [        R)                  5        H_  n
XZR                  :X  d  M  [*        (       a"  [        R	                  SX:R,                  5          g [        R                  " U
R,                  5      e   g g )Nr  r   r   z3Thread %s skipping test %s for following reason: %sr   rJ   zThread r  z	 seconds
z'Caught exception: 
%s exiting thread %sz exited with exception:
r  r  )
isinstancer  r  r  r  r0   rk   r'   r  r   r  r  rV  r  format_exception
SystemExitr   coder.   lenr6  r   r(   )r  r  r  r<  	error_msg	skip_coder%  rr  excr   r  s              r3   r  )MultiThreadedTestCase._check_return_codes  s    		*ND1+C#x0011I4UXY\U] q= *9 5 ? ?I !C..v%DWIZXS!"3''C++ggi88(CD> dV#<SED	 C,,>S(Y] #I- +2 y>Ay))q="))+.$}TVXZfZf &//== , r2   c                     [         $ r   ri  ra  s    r3   rq    MultiThreadedTestCase.world_size  rl  r2   c                 F    U R                  5       R                  S5      S   $ r  r  ra  s    r3   r  (MultiThreadedTestCase._current_test_name  s     wwys#B''r2   r   r  c                J    U R                   U:X  a  U R                  XU5        gg)z
The reason why we have this util function instead of
self.assertEqual is all threads are sharing one CPU RNG
so the assertion result is only reliable on rank 0
N)r%  r  rb  r   yr   r%  s        r3   assertEqualOnRank'MultiThreadedTestCase.assertEqualOnRank  s%     99Q3' r2   c                H    U R                   U:X  a  U R                  X5        g g r   )r%  r  r  s        r3   assertNotEqualOnRank*MultiThreadedTestCase.assertNotEqualOnRank'  s!    99% r2   )r  r%  r|  r  r  r   )r*   r+   r,   r-   __doc__queueQueuerp  r  rw  r0   r  r  r  r  r  r  r  r  r  r  r  r  r.   rq   r  r  r  r1   r  r   s   @r3   ro  ro  I  s     kkmO/bC b bUY b b	7#* D D%& ; ;: .> .>` "C " " (C ( (( (&1 & &r2   ro  c                      ^  \ rS rSrS\\R                  \R                  4   S\	SS4U 4S jjr
S\R                  S\R                  4S jrS	rU =r$ )
SaveForwardInputsModulei,  forward_inputscast_forward_inputsr   Nc                 r   > [         TU ]  5         [        R                  " SS5      U l        Xl        X l        g )Nd   )r  r  nnLinearlr  r  rb  r  r  r  s      r3   r   SaveForwardInputsModule.__init__-  s.    
 	3$,#6 r2   r   c                     XR                   U '   U R                  U R                  (       a3  UR                  U R                  R                  R
                  5      5      $ U5      $ r   )r  r  r  toweightdtyperb  r   s     r3   forwardSaveForwardInputsModule.forward7  sG    $%D!vv43K3Kadd466==../SSQRSSr2   )r  r  r  r*   r+   r,   r-   rE  r  Modulerg   Tensorr  r  r  r1   r  r   s   @r3   r  r  ,  sZ    7RYY457 "7 
	7T T%,, T Tr2   r  c                      ^  \ rS rSrS\\R                  \R                  4   S\	SS4U 4S jjr
S\R                  S\R                  4S jrS	rU =r$ )
SaveForwardInputsModeli<  r  r  r   Nc                 n   > [         TU ]  5         [        X5      U l        [        X5      U l        Xl        g r   )r  r  r  c1c2r  r  s      r3   r  SaveForwardInputsModel.__init__=  s.    
 	).N).N,r2   r   c                 ^    XR                   U '   U R                  U R                  U5      5      $ r   )r  r  r  r  s     r3   r  SaveForwardInputsModel.forwardG  s'    $%D!wwtwwqz""r2   )r  r  r  r  r   s   @r3   r  r  <  sV    -RYY45- "- 
	-# #%,, # #r2   r  c              #     #    U(       d  [         R                  R                  U 5        S[        R                  S'   S[        R                  S'   U(       ap  U(       aT  [         R
                  R                  R                  R                  R                  5       n[        R                  " SUU US9  O[        R                  " SXS9  [         R                  R                  5         [         R                  R                  R                  R!                  5          S v   [         R                  R                  5         [         R                  R                  R                  R!                  5         U(       a  [        R"                  " 5         g g ! [         R                  R                  5         [         R                  R                  R                  R!                  5         U(       a  [        R"                  " 5         f f = f7f)	Nr   MASTER_ADDR6789MASTER_PORTfaker.  rD   r%  rq   )rg   r4  r5  rl   rm   testing	_internalr1  r  	FakeStorer   r3  rH  rI  utilscountersclearr  )r%  rq   init_pgr  r/  s        r3   _dynamo_dist_per_rank_initr  K  sG     **40 +BJJ} &BJJ}MM++77??IIKE##%	 ##FM	MM	MM  &&()$$**,&&(  	$$**,&&( s    DG;F	 A.G;	A/G88G;c                   L   ^  \ rS rSrSr\U 4S j5       r\U 4S j5       rSrU =r	$ )#DynamoDistributedSingleProcTestCaseii  z
Test harness for single-process dynamo distributed tests,
initializes dist process group.

Prefer this for simple tests, as it's easier to debug.
c                 b  > [         TU ]  5         U R                  R                  [        R
                  " [        R                  SSS.5      5        SU l        SU R                   3U l	        SU R                  ;   a  S OU R                  /U l
        [        R                  " SU R                  SS	9  g )
Nr   12355)r  r  r   zcuda:rX   rD   r   r  )r  
setUpClass_exit_stackenter_contextr   rE  rl   rm   r%  r   
device_idsr   r3  r  r  s    r3   r  .DynamoDistributedSingleProcTestCase.setUpClassq  s    %%JJ

#.#*	
 SXXJ'
!'3::!5CHH:SXX!Dr2   c                 L   > [         R                  " 5         [        TU ]  5         g r   )r   r  r  tearDownClassr  s    r3   r  1DynamoDistributedSingleProcTestCase.tearDownClass  s    ""$r2   r)   )
r*   r+   r,   r-   r  r  r  r  r1   r  r   s   @r3   r  r  i  s2     E E"    r2   r  c            	       t   ^  \ rS rSrSrU 4S jrU 4S jr\S\4S j5       r	\
S\S\S	\SS
4S j5       rSrU =r$ )"DynamoDistributedMultiProcTestCasei  a  
Use this for tests that actually run on multiple GPUs.

Decorate tests with @skip_if_lt_x_gpu(ngpu)

Note: MultiProcTestCase spawns processes per test and is slow.
Prefer MultiThreadedTestCase for most tests. Perhaps use this one
sparingly for integration tests.
c                 B   > [         TU ]  5         U R                  5         g r   r$  r  s    r3   r  (DynamoDistributedMultiProcTestCase.setUp  r&  r2   c                    > [         TU ]  5          [        R                  " U R                  5        g ! [
         a     g f = fr   )r  r  rl   r(  r  r)  r  s    r3   r  +DynamoDistributedMultiProcTestCase.tearDown  s5    	IIdnn% 		s    2 
??r   c                 >    [         R                  R                  5       $ r   )rg   rX   rn   ra  s    r3   rq   -DynamoDistributedMultiProcTestCase.world_size  s    zz&&((r2   r%  r  r  Nc                     [         R                  " [        R                  " 5       5        U " U5      nXl        X6l        UR                  X$5        g r   )r   
addHandlerloggingNullHandlerr%  r  r  r  s          r3   r  'DynamoDistributedMultiProcTestCase._run  s:    W0023 9~	"i-r2   )r  r%  )r*   r+   r,   r-   r  r  r  r  r.   rq   r  r0   r  r1   r  r   s   @r3   r  r    s^      )C ) ) . . . .W[ . .r2   r  c            	         ^  \ rS rSr% Sr\\S'   Sr\\S'   Sr\	\
   \S'   \" SS	9r\\S
'   \\R                  S\
4S j5       5       r\SS j5       r\U 4S j5       r\U 4S j5       r\ SS\S\S\	\
   4S jj5       rSrU =r$ )MultiProcContinousTesti  r   rq   r   r%  N	rdvz_filex   )secondsr  r   c                     [        S5      e)zc
ProcessGroup backend str.
To be customized by sub test classes, e.g. "nccl".
Here we raise error.
z/Please implement backend_str in your test class)NotImplementedError)r  s    r3   backend_str"MultiProcContinousTest.backend_str  s     ""STTr2   c                     g)zs
ProcessGroup init options.
To be customized by sub test classes, e.g. ProcessGroupNCCLOpTest
Here we return None.
Nr)   )r  high_priority_streams     r3   optsMultiProcContinousTest.opts  s     r2   c           	      t  > [         TU ]  5         SU R                  s=::  a  U R                  :  d'  O  [	        SU R                   SU R                   35      eU R
                  (       a,  [        R                  " U R
                  U R                  5      nOSnU R                  5       nU R                  5       n[        SU< 35        [        R                  " UU R                  U R                  UUU R                  S9  [        R                  R                  5       U l        [        SU R                   S35        g)	zm
Class-scope test fixture. Run once for entire test class, before any test starts.
Set up the process group.
r   zBRank must be set and in the range of 0 to world_size. World size: z Rank: NzTesting backend=)r   rq   r%  r/  
pg_optionsr  Rank z setup complete)r  r  r%  rq   r  r  r   r2  r  r  r  r3  r  r6  r7  pg)r  r/  r  r   r  s       r3   r  !MultiProcContinousTest.setUpClass  s     	CHH-s~~-"~~.gchhZA  ==NN3==#..AE Exxz//#!
#$~~KK	
 &&99;chhZ/0r2   c                   > [         R                  " 5         [        TU ]  5         U R                  (       a!   [
        R                  " U R                  5        [        SU R                   S35        g! [         a     N&f = f)zp
Class-scope test fixture. Run once for entire test class, after all tests finish.
Tear down the process group.
r#  z teardown completeN)
r   r  r  r  r  rl   r(  r)  r  r%  r  s    r3   r  $MultiProcContinousTest.tearDownClass  sd     	""$==		#--( 	chhZ123  s    A1 1
A>=A>c                 <    Xl         X l        X0l        [        5         g)a$  
This is an entry point for each rank to run the tests in `MultiProcContinousTest`.
In this entry point, we set the class variables for the test class.
Then we run all tests.

Note:
- This helper only works for a subclass of `MultiProcContinousTest`.

Example:
- See `test_c10d_ops_nccl.py`.
N)r%  rq   r  r   )r  r%  rq   r  s       r3   run_rankMultiProcContinousTest.run_rank  s    & #!r2   r)   )Fr   )r*   r+   r,   r-   rq   r.   r/   r%  r  r   r0   r   r  r  abcabstractmethodr  r  r  r  r)  r1   r  r   s   @r3   r  r    s     JD#N#Ix}#"3/GY/ UC U  U   1 1> 4 4 
 $(	  C=	 r2   r  r   )r   r  )TF)r+  r  rC  r  r  rl   r  rX  ri   rQ  r  r  r  rt  r  
contextlibr   dataclassesr   datetimer   enumr   	functoolsr   r   r	   ior
   typingr   r   r   r   r   unittest.mockr   torch._logging._internalr   rg   torch._dynamo.test_casetorch.cuda.nccltorch.distributedr1  r   torch._C._autogradr   torch._C._distributed_c10dr   torch.nnr  $torch.testing._internal.common_utilsr   r   r   r   r   r   r   r   r   r   r   r   5torch.testing._internal.distributed.multi_threaded_pgr    r!   r"   r:  basicConfigINFO	getLoggerr*   r  r%   rk   rQ   rw   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r.   r  r   r
  r  getenvr  r  r  r$  rA  r0   rH  rI  rR  r/   rX  r[  rj  r^  r"  rE  rC  rQ  rW  r\  r`  r  ro  r  r  r  r  rH  	test_caser  r  r  r)   r2   r3   <module>rC     s        	   
       % !   , ,  = =  .      ) 7     
    ',, '			8	$z 
8
C x$FG	
 Xb"BC x45 8B => 8B >? 8B >? 8B >? 8B >? 8B >? 8B >? 8B >? HR>?  (267!" hr#LM#$ x
V%* 8B DE+, hr#BC-
4 * * **6&H$R$+\4	

F Fc F# F$ F  	a 
 
0 O"))$GOPO,c2  +.'(HIC I 2 2(S (c (s (XS 3 2 26(--	. 5
Xc] 
d 
"  `8 `N%N. %NNd3i( 
 F   &0 
3E5p`&H `&FTbii T #RYY # ) ): %--*A*A*J*J  @!.)= !.HfX fr2   