
    sh;              	          S SK r S SKrS SKJr  S SKJr  S SKJr  S SKJ	r	  S SK
r
S SKJr  S SKJs  Js  Jr  S SKJrJrJr  \ R,                  " \5      r " S S5      rS	\
R4                  R6                  S
\\\\\\   4   4   4S jr g)    N)defaultdict)Iterator)contextmanager)Enum)_apply_to_modules_get_module_fsdp_stateclean_tensor_namec                       \ rS rSr%  " S S\\5      r\" \5      r	\
\\4   \S'   \" 5       r\\   \S'   \SS j5       r\\S	\S\S   4S
 j5       5       r\S\SS4S j5       rSrg)SimpleProfiler   c                   ,    \ rS rSrSrSrSrSrSrSr	Sr
g	)
SimpleProfiler.Type   all
all_gatherall_gather_object
reshardingH2DD2H N)__name__
__module____qualname____firstlineno__ALL	ALLGATHERALLGATHER_OBJ
RESHARDINGr   r   __static_attributes__r       w/Users/tiagomarins/Projetos/claudeai/copy_bank/venv/lib/python3.13/site-packages/torch/distributed/fsdp/_debug_utils.pyTyper      s      	+!
r    r"   results	profilingreturnNc                 l    U R                   R                  5         U R                  R                  5         g )N)r#   clearr$   )clss    r!   resetSimpleProfiler.reset"   s"    r    profile_typec              #     #    XR                   ;  d
   U S35       eU R                   R                  U5        [        R                  " 5       n S v   [        R                  " 5       nU R                  U==   X2-
  -  ss'   U R                   R                  U5        g ! [        R                  " 5       nU R                  U==   X2-
  -  ss'   U R                   R                  U5        f = f7f)Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r$   addtime	monotonicr#   remove)r(   r+   beginends       r!   profileSimpleProfiler.profile'   s      ==0 	
n  	
0 	,' 	/.."CKK%4%MM  . .."CKK%4%MM  .s    A
C)B A
C)AC&&C)msgc                     [         R                  " 5       S:X  aQ  [         R                  " 5       [         R                  R                  :X  a   [
        R                  SXR                  5        U R                  5         g )Nr   z%s %s)	distget_rankget_debug_level
DebugLevelINFOloggerinfor#   r)   )r(   r5   s     r!   dump_and_resetSimpleProfiler.dump_and_reset9   sG     ==?aD$8$8$:doo>R>R$RKKkk2		r    r   )r%   N)r   r   r   r   strr   r"   r   floatr#   dict__annotations__setr$   classmethodr)   r   r   r3   r>   r   r   r    r!   r   r      s    sD  !,E 2GT#u*2%Is3x  /3 /8D> /  /     r    r   modelr%   c                     S nS nS/n0 n[        U UUU R                  5        VVs/ s H  u  pVUPM	     snnUU5      $ s  snnf )al  
It is used for composable fully_shard() code path, it returns
  1. sharded module tree info: each line reprents a submodule name that contats the
submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
is like this:
    [CompositeModel] FULLY SHARDED
        l1[Linear]
        u1[UnitModule] FULLY SHARDED
            u1.l1[Linear]
            u1.seq[Sequential]
                u1.seq.0[ReLU]
                u1.seq.1[Linear]
                u1.seq.2[ReLU]
            u1.l2[Linear]
        u2[UnitModule] FULLY SHARDED
            u2.l1[Linear]
            u2.seq[Sequential]
                u2.seq.0[ReLU]
                u2.seq.1[Linear]
                u2.seq.2[ReLU]
            u2.l2[Linear]
        l2[Linear]
  2. a dict mapping from the concated module FQN and class name to a list of its managed
original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
        {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
         'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
         'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
        }
All FQNs are prefixed starting from ``model``.

Args:
    model (torch.nn.Module): Root module (which may or may not be passed to
                             composable `fully_shard()`).
c                 b   US-  n[        U5      S:  a  US   S:X  a  US S OUnUS-   U R                  R                  -   S-   nSU-  U-   n[        U 5      n	U	c  US==   US-   -  ss'   g U	R                  R                  U S 5      n
U
(       a  US==   US	-   S-   -  ss'   OUS==   US-   -  ss'   U
(       ar  U
R                  n[        U[        R                  5      (       d   eUR                   Vs/ s H  n[        X-   5      PM     nnXt;   a  XG   R                  U5        g XU'   g g s  snf )
N   r   .[] 
z FULLY SHARDED)len	__class__r   r   _fully_sharded_module_to_handleget
flat_param
isinstanceflat_param_fileFlatParameter_fqnsr	   extend)moduleprefix
tree_levelsharded_tree_infosharded_module_name_to_fqns
num_spacestrimed_prefixprefixed_module_nameprinted_prefixed_module_namestatehandleparamnameglobal_fqnss                 r!   	module_fnD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnj   s[     !^
K!Or
c0AF3BK 	  -s2V5E5E5N5NNQTT'*Z'7:N'N$&v.=a $@4$GG 66::64Ha ,/??$F  a $@4$GG %%Ee_%B%BCCCC=B[[=HT!&-0[   $B+AHHUDO,@A s   4D,c                     U S   U4$ )Nr   r   )r]   r^   s     r!   	return_fnD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fn   s     #%@@@r     )r   named_parameters)rF   rh   rk   r]   r^   key_s          r!   1_get_sharded_module_tree_with_module_name_to_fqnsrq   B   se    P"PHA
 	$ 9; 1134334#  	5s   ?)!loggingr.   collectionsr   collections.abcr   
contextlibr   enumr   torchtorch.distributeddistributedr7   "torch.distributed.fsdp._flat_paramfsdp_flat_paramrV   $torch.distributed.fsdp._common_utilsr   r   r	   	getLoggerr   r<   r   nnModuletupler@   rB   listrq   r   r    r!   <module>r      s      # $ %     < <  
		8	$) )X[88??[
3S$s)^$$%[r    