
    shw)                   
   % S r SSKJr  SSKrSSKrSSKrSSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKrSSKrSSKrSSKJrJr  SSKJr  SSKJrJrJrJrJr  SSKJr  SSKJrJrJrJrJ r J!r!J"r"  SS	K#J$r$J%r%  SSK&r&SSK'r&SSK(r&SSK)J*s  J+r,  SS
K-J.r.  SSK/J0r0  SSK1J2r2  SSK3J4r4  SSK5J6r6  SSK7J8r8  SSK9J:r:  SSK;J<r<J=r=  SSK>J?r?  SSK@JArA  SSKBJCrD  SSKEJFrFJGrG  SSKHJIrI  SSKJJKrKJLrL  SSKMJNrN  SSKOJCrC  SSKPJQrQ  SSKRJSrS  \	R                  " \U5      rV\&R                  R                  rX\&R                  R                  rY\rZ\"\Z\&R                  R                  4   r\ " S S\ 5      r] " S S \ 5      r^ " S! S"\ 5      r_\!" S#5      r`\"\&R                  R                  R                  \c4   rd " S$ S%5      re\e" 5       rf S       SS& jjrg " S' S(5      rh " S) S*\i5      rj\"\h\j4   rkSS+ jrl " S, S-5      rm " S. S/\5      rn " S0 S1\n5      ro " S2 S3\n5      rp " S4 S5\n5      rq " S6 S7\n5      rr " S8 S9\n5      rs\t\S:4   ru " S; S<\s5      rv " S= S>\v5      rw " S? S@\v5      rx " SA SB\v5      ry " SC SD\s5      rz " SE SF\z5      r{ " SG SH\z5      r| " SI SJ\z5      r} " SK SL\n5      r~ " SM SN\n5      r " SO SP\n5      r " SQ SR5      r " SS ST\ 5      r\GR                   " SU SV5      5       r\GR                   " SW SX\5      5       r\GR                   " SY SZ\5      5       r\GR                   " S[ S\\5      5       rSS] jrSS^ jr S         SS` jjr\SSaSS_4                     SSb jjr\A\c   " 5       r            SSc jr\" \5      GR                  Sd-  Se-  r/ rSf\Sg'   \SSaS_4                     SSh jjr\DGR(                  " S_Si9  S           SSj jj5       r  S           SSk jjr\4S_Sl.         SSm jjjr\4S_Sl.         SSn jjjrSSo jr\GR4                  " Sp5      rSSq jrSSr jrSSs jrSSt jrSSu jrSSv jr " Sw Sx5      rSSy jr    S           SSz jjr\&GRJ                  " 5       S{SS|.         SS} jj5       r\&GRN                  " 5       SS~ j5       rSS jrSS jrSS jrSS jrSS jr\A\c   " 5       r S       SS jjrSS jrSS jrg)a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)
Collection	GeneratorIterableMappingSequence)Path)AnyCallableNoReturnOptionalProtocolTypeVarUnion)SelfTypeIs)enable_python_dispatcher)counters)is_integer_dtype)unset_fake_temporarily)make_fxstatically_known_true)	_get_attr)immutable_dictimmutable_list)GraphTransformObserver)
OrderedSet   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_typec                  *    \ rS rSr% S\S'   SS jrSrg)SearchFnZ   str__name__c                    g N selfargskwargss      s/Users/tiagomarins/Projetos/claudeai/copy_bank/venv/lib/python3.13/site-packages/torch/_inductor/pattern_matcher.py__call__SearchFn.__call__]       #    r5   Nr8   r   r9   r   returnr   )r2   
__module____qualname____firstlineno____annotations__r;   __static_attributes__r5   r>   r:   r/   r/   Z   s
    M=r>   r/   c                      \ rS rSrSS jrSrg)	ReplaceFn`   c                    g r4   r5   r6   s      r:   r;   ReplaceFn.__call__a   r=   r>   r5   Nr?   r2   rA   rB   rC   r;   rE   r5   r>   r:   rG   rG   `   s    =r>   rG   c                  .    \ rS rSr        SS jrSrg)TraceFnd   c                    g r4   r5   )r7   fnr8   r9   s       r:   r;   TraceFn.__call__e   s    "r>   r5   N)rP   zUnion[SearchFn, ReplaceFn]r8   r   r9   r   r@   torch.fx.GraphModulerK   r5   r>   r:   rM   rM   d   s'    #,#58#DG#	#r>   rM   Tc                      \ rS rSrSS jrSrg)Multiplep   c                8    S[        5       ;  d  U [        L d   eg g )NMULTIPLE)globalsrX   r7   s    r:   __init__Multiple.__init__q   s    *dh.>>>.>*r>   r5   Nr@   None)r2   rA   rB   rC   r[   rE   r5   r>   r:   rU   rU   p   s    ?r>   rU   c                   SSK JnJn  [        R                  R
                  (       aw  U R                  S/ 5      R                  5       nUR                  U" XUR                  5      5        U R                  S UR                  R                  5        5       5        XPS'   g U R                  S UR                  R                  5        5       5        g )Nr   )
NodeSourceNodeSourceAction	from_nodec              3     #    U  H4  u  pU[         R                  R                  R                  ;   d  M/  X4v   M6     g 7fr4   torchfxproxy_COPY_META_FIELDS.0kvs      r:   	<genexpr>!_transfer_meta.<locals>.<genexpr>   3      
-EHHNN444 QF-   .>
>c              3     #    U  H4  u  pU[         R                  R                  R                  ;   d  M/  X4v   M6     g 7fr4   rd   ri   s      r:   rm   rn      ro   rp   )torch.fx.tracebackr`   ra   r$   traceenabledgetcopyappendREPLACEupdatemetaitems)new_metaold_node	pass_namer`   ra   new_from_nodes         r:   _transfer_metar   z   s     @
 || ["5::<Z=M=U=UVW 
 ++-
 	

 !. 
 ++-
 	
r>   c                    ^  \ rS rSr% SrS\S'   S\S'   S\S'   S	\S
'   S\S'   S\S'   S\S'     S         SU 4S jjjr\SS j5       rSS jr	S S jr
S!S jrS"S jrS#S jrS$S jr      S%S jr  S&         S'S jjrSrU =r$ )(Match   ao  
Represents a successfully matched pattern.

The `Match` object is returned to represent a successfully matched
pattern. Included in the Match are the pattern that was matched, the graph
nodes matched, and any args that were used during the matching.

The args and kwargs are specific to the type of pattern that was matched and
provide hints about what was matched.
PatternExprpattern	list[Any]r8   dict[str, Any]r9   list[torch.fx.Node]nodesz'dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxzOptional[torch.fx.GraphModule]replacement_graphc                   > [         TU ]  5         X l        [        U=(       d    / 5      U l        U=(       d    0 U l        / U l        0 U l        Xl        S U l	        g r4   )
superr[   r   listr8   r9   r   r   r   r   )r7   r   r   r8   r9   	__class__s        r:   r[   Match.__init__   sK     	$	l
!%r>   c                .    U R                   R                  $ r4   )r   graphrZ   s    r:   r   Match.graph   s    xx~~r>   c                B   U R                   (       az  [        U R                   R                  5       5      [        UR                   R                  5       5      -   H/  nU R                   U   UR                   U   :w  d  M%  [        SU5      e   U R                  R                  UR                  5        U R                  R                  UR                  5        U R                   R                  UR                   5        U R                  R                  UR                  5        g )Nzkwarg mismatch: {})	r9   r!   keysFailedMatchr8   extendr   ry   r   )r7   otherkeys      r:   r   Match.extend   s    ;;!$++"2"2"45
5<<CTCTCV8WW;;s#u||C'88%&:C@@ X 			$

%++&5<<(EMM*r>   c                p    U R                   (       a  [        U R                   5      /U l         U $ / U l         U $ r4   )r8   tuplerZ   s    r:   bundleMatch.bundle   s.    *.))U499%&	 :<	r>   c                <    SU R                    SU R                   S3$ )NzMatch(..., , )r8   r9   rZ   s    r:   __repr__Match.__repr__   s    TYYKr$++a88r>   c                    U R                   n[        U R                  5       H:  nUR                  (       a  M  UR                  (       a  M)  UR                  U5        M<     g r4   )r   reversedr   _erasedusers
erase_node)r7   r   ns      r:   erase_nodesMatch.erase_nodes   s=    

$**%A999QWWW  # &r>   c                    U R                   R                   Vs/ s H!  nUb  U R                   R                  U   OS PM#     sn$ s  snf r4   )r   outputspattern_to_node)r7   ps     r:   output_nodesMatch.output_nodes   sK     XX%%
% -.MTXX%%a(tC%
 	
 
s   (Ac                B    [        S U R                  5        5       5      $ )Nc              3  6   #    U  H  o(       d  M  Uv   M     g 7fr4   r5   )rj   r   s     r:   rm   $Match.output_node.<locals>.<genexpr>   s     82!aAA2s   
	)nextr   rZ   s    r:   output_nodeMatch.output_node   s    8t002888r>   c                X    [         R                  X R                  R                  X5        g r4   )ReplacementPatternEntryreplace_with_graphr   r   )r7   r   r8   s      r:   r   Match.replace_with_graph   s      	 22((.."3	
r>   c                n   SSK JnJn  [        UR                  U5      (       a  UR                  c  UR                  O[
        R                  " 5       nU   Uc  [        R                  " [        US9nU" U[        R                  R                  US 5      5      n[        U R                  5      S:X  a?  UR                  R                   H%  n	[!        U	R"                  U R                  S   SS9  M'     [$        R'                  U U R(                  R                  UU5        SSS5        g! , (       d  f       g= f)	zReplace with a graph generated by tracing the replacement_fn.

Args:
    run_functional_passes (bool). If we should run passes that
        assume functional IR (like DCE, remove_noop_ops), on the
        replacement graph.

r   )NullHandlerVN)run_functional_passesc                     U R                   S   $ Nvalrz   )args    r:   <lambda>*Match.replace_by_example.<locals>.<lambda>  s    388E?r>   r+   replace_by_exampler|   r}   r~   )torch._inductor.virtualizedr   r   
isinstance	fake_mode
contextlibnullcontext	functoolspartialfwd_onlyre   rf   map_arglenr   r   r   rz   r   r   r   )
r7   replacement_fnr8   trace_fnr   r   r   contextreplacementr   s
             r:   r   Match.replace_by_example   s     	? q{{K88Q[[=P KK'') 	 $,,4I # 0 07R SK 4::!#$**00A"!"!%A"6 1 $66	! WWs   C	D&&
D4)r8   r   r9   r   r   r   r   )NN)
r   r   r   r   r8   zOptional[Sequence[Any]]r9   zOptional[dict[str, Any]]r@   r^   )r@   torch.fx.Graph)r   r   r@   r^   )r@   r   r@   r1   r]   )r@   zlist[Optional[torch.fx.Node]])r@   torch.fx.Node)r   r   r8   Sequence[Any]r@   r^   NT)
r   rG   r8   r   r   zOptional[TraceFn]r   boolr@   r^   )r2   rA   rB   rC   __doc__rD   r[   propertyr   r   r   r   r   r   r   r   r   rE   __classcell__r   s   @r:   r   r      s   	 
O44	55 )-+/&& & &	&
 )& 
& &&  +
9$
9
!/
7D
	
 '+&*,!, , $	,
  $, 
, ,r>   r   c                  B    \ rS rSr% SrS\S'   S
S jrSS jrSS jrSr	g	)r   i  zs
Represents a unsuccessful match.

The `FailedMatch` object is returned to represent a failure to match a
pattern.
r1   format_stringc                b    Xl         [        U5      S:  a  [        SU 35      eX l        X0l        g )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r   r   RuntimeErrorr8   r9   )r7   r   r8   r9   s       r:   r[   FailedMatch.__init__   s<    * }#hivhwx  	r>   c                b    U R                   R                  " U R                  0 U R                  D6$ r4   )r   formatr8   r9   rZ   s    r:   __str__FailedMatch.__str__+  s&    !!(($))Ct{{CCr>   c                    gNFr5   rZ   s    r:   __bool__FailedMatch.__bool__.      r>   )r8   r   r9   N)r   r1   r8   r   r9   r   r@   r^   r   r@   r   )
r2   rA   rB   rC   r   rD   r[   r   r   rE   r5   r>   r:   r   r     s     	Dr>   r   c                    [        U 5      $ )zp
TypeIs cannot act on `self`. Thus this function exists to let mypy
recognize FailedMatch.__bool__ as a TypeIs.
)r   )ms    r:   is_matchr   5  s    
 7Nr>   c                  t    \ rS rSr% SrS\S'   S\S'   S\S'   S	\S
'    S       SS jjrSS jrSS jrSr	g)r   i=  z;
Internal state needed while running PatternExpr._match().
list[Optional[PatternExpr]]r   z*dict[PatternExpr, Optional[torch.fx.Node]]r   r   r   zlist[NodeOrConstant]exclusive_node_setNc               T    Xl         Uc  0 O
[        U5      U l        X0l        / U l        g r4   )r   dictr   r   r   )r7   r   r   r   s       r:   r[   MatchContext.__init__G  s*     %4%<r$BW
"$r>   c                    XR                   ;   a)  U R                   U   U:X  a  [        X5      $ [        S5      $ UR                  X 5      nXR                   ;  d   eU(       a  UOSU R                   U'   U$ )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   r   r   _match)r7   r   noder   s       r:   matchMatchContext.matchS  sq    ***##G,4T++"#=>>NN4&2222201tW%r>   c                    U R                   R                  5        VVs0 s H#  u  pUR                  5       (       d  M  Uc  M!  X_M%     snn$ s  snnf r4   )r   r{   has_multiple_users)r7   r   r   s      r:   filter_multi_user_patterns'MatchContext.filter_multi_user_patterns_  sR     "&!5!5!;!;!=
!=))+ 04 GM!=
 	
 
s   AAA)r   r   r   r   r4   )r   r   r   z*Optional[dict[PatternExpr, torch.fx.Node]]r   r   r@   r^   )r   r   r   NodeOrConstantr@   MatchResult)r@   z dict[PatternExpr, torch.fx.Node])
r2   rA   rB   rC   r   rD   r[   r  r  rE   r5   r>   r:   r   r   =  s^     )(??,,
 GK
%,
% D
%
 
% 

%

r>   r   c                  j    \ rS rSrSr\SS j5       rSS jrSS jrSS jr	      SS jr
SS jrS	rg
)r   ig  z#
Base class for types of patterns.
c                    g r4   r5   r7   r   r   s      r:   r   PatternExpr._matchl  s    MPr>   c                ~     [        U /UR                  S9R                  X5      $ ! [         a  nUs S nA$ S nAff = fNr   )r   r   r  r   r7   r   es      r:   r  PatternExpr.matcho  s:    	djj9??KK 	H	s   #& 
<7<<c                    gr   r5   rZ   s    r:   r  PatternExpr.has_multiple_usersu  r   r>   c                4    U R                   R                  S-   $ )Nz())r   r2   rZ   s    r:   r   PatternExpr.__repr__x  s    ~~&&--r>   c              #  N   #    XR                   ;   a  UR                   U    v   g g 7fr4   )r   r7   r   searcheds      r:   find_anchor_nodesPatternExpr.find_anchor_nodes{  s)      &&&%%d++ 's   #%c                ,    [        XR                  5      $ )z
Compare two `PatternExpr`s and return true if they are the
same. Note this is NOT matching a pattern - it is comparing the pattern
structures (for debugging).
)r   r   )r7   r   s     r:   
pattern_eqPatternExpr.pattern_eq  s     %00r>   r5   Nr   r   r   r   r@   r  r   r   r@   r  r   r   r   r   r  OrderedSet[torch.fx.Node]r@   z.Generator[Optional[torch.fx.Node], None, None]r   r   r@   r   )r2   rA   rB   rC   r   r   r   r  r  r   r  r  rE   r5   r>   r:   r   r   g  sH     P P.,,+D,	7,1r>   r   c                  "    \ rS rSrSrSS jrSrg)Argi  zb
Capture an arg which will become an input to the handler.  Args are
passed in depth first order.
c                    [        X U/S9$ )N)r8   r   r  s      r:   r   
Arg._match  s    SdV,,r>   r5   Nr   r  r   r   r@   r  r2   rA   rB   rC   r   r   rE   r5   r>   r:   r%  r%    s    
-r>   r%  c                  6    \ rS rSrSrSS jrS	S jrS
S jrSrg)Ignoredi  z,
Match an arg, but don't pass it to handler
c                    [        X 5      $ r4   r'  r  s      r:   r   Ignored._match  s    Sr>   c                    g)N*r5   rZ   s    r:   r   Ignored.__repr__  s    r>   c                    g)Nz	Ignored()r5   )r7   pps     r:   pretty_printIgnored.pretty_print  s    r>   r5   Nr)  r   r3  PatternPrettyPrinterr@   r1   )	r2   rA   rB   rC   r   r   r   r4  rE   r5   r>   r:   r,  r,    s     r>   r,  c                  T   ^  \ rS rSrSrSU 4S jjrS	S jrS
S jrSU 4S jjrSr	U =r
$ )
KeywordArgi  <
Capture a kwarg which will become an input to the handler.
c                .   > [         TU ]  5         Xl        g r4   r   r[   namer7   r=  r   s     r:   r[   KeywordArg.__init__      	r>   c                $    SU R                   < S3$ )NzKeywordArg(r   r=  rZ   s    r:   r   KeywordArg.__repr__  s    TYYM++r>   c                .    [        X U R                  U0S9$ )Nr9   )r   r=  r  s      r:   r   KeywordArg._match  s    S		4'899r>   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       a    U R
                  UR
                  :H  $ r4   typingcastr   r   r  r=  r7   r   r   s     r:   r  KeywordArg.pattern_eq  5    D%(w!%(DTYY%**-DDr>   rB  r=  r1   r@   r^   r   r)  r#  r2   rA   rB   rC   r   r[   r   r   r  rE   r   r   s   @r:   r9  r9    s#    ,:E Er>   r9  c                  `   ^  \ rS rSr% SrS\S'   S
U 4S jjrSS jrSS jrSU 4S jjr	S	r
U =r$ )ExclusiveKeywordArgi  r:  r1   r=  c                .   > [         TU ]  5         Xl        g r4   r<  r>  s     r:   r[   ExclusiveKeywordArg.__init__  r@  r>   c                $    SU R                   < S3$ )NzExclusiveKeywordArg(r   rB  rZ   s    r:   r   ExclusiveKeywordArg.__repr__  s    %dii]!44r>   c                    XR                   ;   a  [        S5      $ UR                   R                  U5        [        X U R                  U0S9$ )Nzexclusive arg appears twicerE  )r   r   rw   r   r=  r  s      r:   r   ExclusiveKeywordArg._match  sD    )))<==%%d+S		4'899r>   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       a    U R
                  UR
                  :H  $ r4   rH  rK  s     r:   r  ExclusiveKeywordArg.pattern_eq  rM  r>   rB  rN  r   r)  r#  )r2   rA   rB   rC   r   rD   r[   r   r   r  rE   r   r   s   @r:   rQ  rQ    s*     I5:E Er>   rQ  c                     ^  \ rS rSr% SrS\S'   S\S'    S     SU 4S jjjr\\SS j5       5       r	SS	 jr
SS
 jrSS jr      SS jrSS jrSS jrSU 4S jjrSrU =r$ )_TargetExpri  z/
Base class for filtering match by node.target
zlist[FnsType]fnszOrderedSet[FnsType]fns_setc                |  >^ [         TU ]  5         [        U5      (       d  [        U[        5      (       a  U/O
[        U5      nU HW  m[        T[        R                  R                  5      (       d  M.  UR                  U4S jTR                  5        5       5        MY     Xl        [        U5      U l        X l        g )Nc              3  <   >#    U  H  n[        TU5      v   M     g 7fr4   )getattr)rj   overloadrP   s     r:   rm   '_TargetExpr.__init__.<locals>.<genexpr>  s     PX72x00s   )r   r[   callabler   r1   r   re   _opsOpOverloadPacketr   	overloadsr\  r!   r]  r   )r7   r\  r   rP   r   s      @r:   r[   _TargetExpr.__init__  s     	}}
3(<(<se$s)B"ejj99::

PPP  !#
r>   c                    g r4   r5   rZ   s    r:   op_TargetExpr.op  s    r>   c                   U R                   S   n[        U[        5      (       d  UR                  n[	        U R                   5      S:  a  SU S3$ U R                   S   [        [        US 5      L a  SU 3$ U R                   S   [        [        US 5      L a  SU 3$ [        U R                   S   [        R                  R                  5      (       a  [        U R                   S   5      $ U$ )Nr   r+   [z, ...]torch.z	operator.)
r\  r   r1   r2   r   r`  re   operatorrd  
OpOverload)r7   
first_reprs     r:   fns_repr_TargetExpr.fns_repr  s    XXa[
*c**#,,Jtxx=1zl&))XXa[GE:t<<J<((XXa[GHj$??zl++UZZ%:%:;;txx{##r>   c                    U R                   [        L a  SnO#U R                   S:w  a  SU R                    S3nOSnU R                  R                   SU R	                  5        U S3$ )Nz
, MULTIPLEr+   r   r    ()r   rX   r   r2   rq  )r7   comma_userss     r:   r   _TargetExpr.__repr__  s^    ::!&KZZ1_tzzl!,KK..))*!DMMO+<[MKKr>   c                b    [        U R                  [        5      =(       d    U R                  S:  $ )Nr+   )r   r   rU   rZ   s    r:   r  _TargetExpr.has_multiple_users  s     $**h/A4::>Ar>   c                    [         er4   NotImplementedErrorr  s      r:   r  _TargetExpr.find_anchor_nodes  s
     "!r>   c                    [        U[        R                  R                  5      =(       a8    UR                  U R                  :H  =(       a    [        U5      U R                  ;   $ r4   )r   re   rf   Noderi  extract_targetr]  )r7   r   s     r:   
_match_fns_TargetExpr._match_fns  sD    tUXX]]+ 5477"5t$4	
r>   c                    XR                   ;   =(       d;    U R                  [        L =(       d"    [        UR                  5      U R                  :H  $ r4   )r   r   rX   r   r  s      r:   _match_users_TargetExpr._match_users  s;    KK -zzX%-4::$**,	
r>   c                  > [         R                  " [        U5      n[        TU ]  U5      =(       aY    U R
                  UR
                  :H  =(       a9    U R                  UR                  :H  =(       a    U R                  UR                  :H  $ r4   )rI  rJ  r   r   r  ri  r\  r   rK  s     r:   r  _TargetExpr.pattern_eq  sf    D%(Gu% *588#*EII%* 

ekk)		
r>   )r\  r]  r   )r+   )r\  z!Union[FnsType, Sequence[FnsType]]r   zUnion[Multiple, int]r@   r^   r   r   r!  r   r   r@   r   )r   r   r   r   r@   r   r#  )r2   rA   rB   rC   r   rD   r[   r   r   ri  rq  r   r  r  r  r  r  rE   r   r   s   @r:   r[  r[    s     
   UV4=Q	     LB""+D"	7"



 
r>   r[  .c                     ^  \ rS rSrSrSS.         SU 4S jjjr\      SS j5       r\      SS j5       rSS jr	SS	 jr
SS
 jr      SS jrSU 4S jjrSrU =r$ )_TargetArgsExpri&  z=
Base class for filtering match by node.{target,args,kwargs}
r+   )_usersc               r  > [         TU ]  X5        [        U5      U l        [	        U5      U l        [        S [        R                  " X4R                  5       5       5       5      (       a  U R                  U l        OU R                  U l        U R                  U R                  U R
                  5      U l        g )Nc              3  X   #    U  H   n[        U[        [        [        45      v   M"     g 7fr4   )r   r   r   r   rj   xs     r:   rm   +_TargetArgsExpr.__init__.<locals>.<genexpr>5  s(      
; q4u-..;s   (*)r   r[   r   r8   r   r9   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)r7   r\  r  r8   r9   r   s        r:   r[   _TargetArgsExpr.__init__+  s     	%$K	6l 
__T==?;
 
 
  ..DL..DL $TYY Dr>   c                l    / U QUR                  5       Q7n[        U 5      /UR                  5       Q7nX#4$ r4   )r  r   r   )r8   r9   r  specs       r:   r  _TargetArgsExpr.simple_flatten>  s8     +4*&--/*D	*FKKM*|r>   c                   ^^ [         [        [        [        [        [        0mSUU4S jjm[
        R                  " TX4U4S jS9n[
        R                  " U5      u  p4X44$ )Nc                   > [        U 5      nTR                  U5      nUb   [        R                  " TU" U 5      U4S jS9$ U $ )Nc                    > [        U 5      T;   $ r4   typer  type_mappings    r:   r   F_TargetArgsExpr.pytree_flatten.<locals>.convert_type.<locals>.<lambda>W  s    d1g&=r>   is_leaf)r  ru   pytreetree_map)r  cls
convert_fnconvert_typer  s      r:   r  4_TargetArgsExpr.pytree_flatten.<locals>.convert_typeP  sG    q'C%))#.J% qM= 
 Hr>   c                    > [        U 5      T;   $ r4   r  r  s    r:   r   0_TargetArgsExpr.pytree_flatten.<locals>.<lambda>^  s    d1g5r>   r  )r  r   r@   r   )r   r   r   r   r   r  r  tree_flatten)r8   r9   normalized_args_treeflatr  r  r  s        @@r:   r  _TargetArgsExpr.pytree_flattenF  s^    
 E%D*
		 		  &N5 

 (()=>
zr>   c                   U R                  5       /[        [        U R                  5      QU R                  R                  5        VVs/ s H  u  pU SU 3PM     snnQnU R                  [        L a  UR                  S5        O.U R                  S:w  a  UR                  SU R                   35        U R                  R                   SSR                  U5       S3$ s  snnf )N=_users=MULTIPLEr+   _users=ru  r   r   )rq  mapreprr8   r9   r{   r   rX   rw   r   r2   join)r7   rk   rl   r8   s       r:   r   _TargetArgsExpr.__repr__c  s    MMO
tyy!
 &*[[%6%6%89%8TQ1QCj%89

 ::!KK)*ZZ1_KK'$**./..))*!DIIdO+<A>> :s   Cc           
       ^ U R                  5       /U4S jU R                   5       QU R                  R                  5        VVs/ s H  u  p#U STR	                  U5       3PM     snnQnU R
                  [        L a  UR                  S5        O.U R
                  S:w  a  UR                  SU R
                   35        SnU R                  R                   SUR                  U5       S3$ s  snnf )	Nc              3  F   >#    U  H  nTR                  U5      v   M     g 7fr4   )r4  )rj   r  r3  s     r:   rm   /_TargetArgsExpr.pretty_print.<locals>.<genexpr>r  s     4)Qbooa  )s   !r  r  r+   r  r   ru  r   )rq  r8   r9   r{   r4  r   rX   rw   r   r2   r  )r7   r3  rk   rl   r8   
joiner_strs    `    r:   r4  _TargetArgsExpr.pretty_printo  s    MMO
4$))4
 7;kk6G6G6IJ6Ida1R__Q'()6IJ

 ::!KK)*ZZ1_KK'$**./
..))*!JOOD,A+B!DD Ks   "C,c                   U R                  U5      (       a,  [        UR                  5      [        U R                  5      :w  a  [        SX5      $ U R	                  X5      (       d  [        SU 5      $ UR                  nUR
                  n[        U5      [        U R
                  5      :  a  SSKJn  [        UR                  5      (       d   eU" UR                  UR                  UR
                  5      nUc  [        SX5      $ Uu  p4[        U5      [        U R                  5      :X  aI  [        U5      [        U R
                  5      :  a'  U Vs0 s H  owU R
                  ;   d  M  XtU   _M     nnO2[        SX5      $ U Vs0 s H  owU R
                  ;   d  M  XtU   _M     nnU R                  X45      u  pU R                  u  pX:w  a  [        SX5      $ [        U5      [        U
5      :X  d   e[        X 5      n[        [        R                  " 5       X5       H  u  p}n[!        U["        5      (       a8  UR%                  X5      n['        U5      (       d  Us  $ UR)                  U5        MS  [!        U[*        R,                  R.                  5      (       d  X:w  d  M  [        SX5      s  $    UR0                  R3                  U5        UR                  UR4                  U '   U$ s  snf s  snf )Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionzargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})r  r   r8   r   r  r9   torch.fx.operator_schemasr  rc  targetr  r  r   zipr  countr   r   r  r   r   re   rf   r  r   rw   r   )r7   r   r   _args_kwargsr  normalized_args_and_kwargsi
node_items	node_spec
self_items	self_specr   r   
child_nodechild_matchs                   r:   r   _TargetArgsExpr._match}  s`   t$$DII#dii.(HGTT  ++2D99		++w<#dkk**DDKK(((();TYY*& *1"#KTXX!;u:TYY/CLCDT4T6=RgdkkAQ}q!*}gGRG&@$  /6Jgdkk9I}q!*}gGJ $U <
 $ 5 5
!5yLL:#j/111#&))//*;Z&T"A
';//!ii<,,&&%J66*:O"94  'U 	
t++		$7 S Ks   K$'	K$K)	K)c              #    #    XR                   ;   a  UR                   U    v   gU R                  S    H  n[        U[        5      (       d  M  UR	                  X5       Hu  n[        U[
        R                  R                  5      (       d  M.  UR                   H7  nXR;  d  M
  U R                  U5      (       d  M"  Uv   UR                  U5        M9     Mw     M     g7f)z
This is used when we are matching a pattern with multiple outputs.
There is a partial match (stored in ctx) and we want to walk
this pattern to find a connection to an already-matched node.

Yields candidate nodes that `self._match` might like.
Nr   )r   r  r   r   r  re   rf   r  r   r  add)r7   r   r  r   
other_noder   s         r:   r  !_TargetArgsExpr.find_anchor_nodes  s      &&&%%d++,,Q/G';//")";";C"JJ%j%((--@@  * 0 0/#t44&*
 (T 2	 !1 #K 0s   ACAC"C:"Cc                  > [         R                  " [        U5      n[        TU ]  U5      =(       a\    U R
                  S   UR
                  S   :H  =(       a6    [        S [        U R
                  S   UR
                  S   5       5       5      $ )Nr+   c              3  z   #    U  H1  u  p[        U[        5      (       a  UR                  U5      OX:H  v   M3     g 7fr4   r   r   r  rj   abs      r:   rm   -_TargetArgsExpr.pattern_eq.<locals>.<genexpr>  s4      TDA $.a#=#=Q16IT   9;r   )rI  rJ  r   r   r  r  allr  rK  s     r:   r  _TargetArgsExpr.pattern_eq  s    D%(Gu% %%a(E,B,B1,EE  5 5a 8%:P:PQR:ST 	
r>   )r8   r  r  r9   )
r\  z/Union[torch.fx.node.Target, str, Sequence[Any]]r8   r   r  zUnion[int, Multiple]r9   r   r@   r^   )r8   r   r9   zMapping[Any, Any]r@   z9tuple[Sequence[Any], Union[_SimpleSpec, pytree.TreeSpec]]r   r6  r  r!  r#  )r2   rA   rB   rC   r   r[   staticmethodr  r  r   r4  r   r  r  rE   r   r   s   @r:   r  r  &  s     ()	E<E E %	E
 E 
E E& %6	B  %6	B 8
?E1f33+D3	732	
 	
r>   r  c                      \ rS rSrSrSrSrg)CallFunctioni  zJ
Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
call_functionr5   Nr2   rA   rB   rC   r   ri  rE   r5   r>   r:   r  r    s     
Br>   r  c                      \ rS rSrSrSrSrg)
CallMethodi  zO
Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
call_methodr5   Nr  r5   r>   r:   r  r         
Br>   r  c                      \ rS rSrSrSrSrg)
CallModulei  zH
Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
call_moduler5   Nr  r5   r>   r:   r  r    r  r>   r  c                  "    \ rS rSrSrSS jrSrg)_TargetExprVarArgsi  zS
Matches a call_function node with any arguments which are passed into the pattern
c                   U R                  U5      (       d  [        S5      $ U R                  X5      (       d  [        S5      $ [        X 5      nUR                  R                  U5        UR                  UR                  U '   UR                  R                  UR                  5        UR                  R                  UR                  5        U$ )Nfunction_mismatchmultiple_users)r  r   r  r   r   rw   r  r   r8   r   r9   ry   )r7   r   r   r   s       r:   r   _TargetExprVarArgs._match  s    t$$233  ++/00#	t++		$	dii 	$r>   r5   Nr  r*  r5   r>   r:   r  r    s    r>   r  c                      \ rS rSrSrSrg)CallFunctionVarArgsi  r  r5   Nr2   rA   rB   rC   ri  rE   r5   r>   r:   r  r    s    	Br>   r  c                      \ rS rSrSrSrg)CallMethodVarArgsi  r  r5   Nr  r5   r>   r:   r  r        	Br>   r  c                      \ rS rSrSrSrg)CallModuleVarArgsi	  r  r5   Nr  r5   r>   r:   r  r  	  r  r>   r  c                  X   ^  \ rS rSrSrSS	U 4S jjjrS
S jrSS jrSU 4S jjrSr	U =r
$ )ListOfi  z
Matches a repeated pattern
c                h   > [         TU ]  5         [        U[        5      (       d   eXl        X l        g r4   )r   r[   r   r   r   r   )r7   r   r   r   s      r:   r[   ListOf.__init__  s+    ';////r>   c                N    U R                   R                   SU R                   S3$ Nru  r   )r   r2   r   rZ   s    r:   r   ListOf.__repr__  $    ..))*!DLL>;;r>   c                J   [        U[        [        45      (       a  [        U5      S:X  a  [	        S5      $ [        X 5      nUR                  5       nSn[        U5       H  u  pg[        UR                  XGR                  S9nUR                  U R                  U5      n	UR                  5       n[        U	5      (       d!  U R                  (       d  [	        SXi5      s  $ M  SnUR                  U	R!                  5       5        M     U(       d  [	        S5      $ UR!                  5       $ )Nr   non_listFr  zlist[{}]: {}Tzlist: no_match)r   r   r   r   r   r   r  	enumerater   r   r   r  r   r   r   r   r   )
r7   r   r   r   r   matchedr  r  	child_ctxr  s
             r:   r   ListOf._match  s    $u..#d)q.z**# 88:&t_MA$_4D4DI $//$,,
CK'BBDOK((||&~qFFGHH['')* - /00xxzr>   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       aE    U R
                  R	                  UR
                  5      =(       a    U R                  UR                  :H  $ r4   )rI  rJ  r   r   r  r   r   rK  s     r:   r  ListOf.pattern_eq3  sU    D%(Gu% .''6.-	
r>   )r   r   F)r   r   r   r   r@   r^   r   )r   r   r   r   r@   r  r#  rO  r   s   @r:   r  r    s&     <0
 
r>   r  c                     ^  \ rS rSr% S\S'   SU 4S jjr\SS j5       rSS jrSS jr	SS jr
      SS	 jrSS
 jrSU 4S jjrSrU =r$ )MultiOutputPatterni<  r   r   c                   > [         TU ]  5         [        US   [        5      (       d   e[	        S U 5       5      (       d   U5       e[        U5      U l        US   R                  U l        g )Nr   c              3  V   #    U  H  oS L =(       d    [        U[        5      v   M!     g 7fr4   )r   r   r  s     r:   rm   .MultiOutputPattern.__init__.<locals>.<genexpr>B  s!     LGq9:
1k ::Gs   '))r   r[   r   r[  r  r   r   ri  )r7   r   r   s     r:   r[   MultiOutputPattern.__init__?  s[    '!*k2222LGLLLUgULG}!*--r>   c                j    [         R                  " [        U R                  S   5      nUR                  $ Nr   )rI  rJ  r[  r   r\  )r7   outputs     r:   r\  MultiOutputPattern.fnsF  s&     [$,,q/:zzr>   c                N    U R                   R                   SU R                   S3$ r  )r   r2   r   rZ   s    r:   r   MultiOutputPattern.__repr__L  r   r>   c                    U R                    Vs/ s H  o!R                  U5      PM     nnSS 3nU R                  R                   SUR	                  U5       3nU S3nU$ s  snf )Nz,
z  z([z
]))r   r4  r   r2   r  )r7   r3  r  r8   r  str_outs         r:   r4  MultiOutputPattern.pretty_printO  sg    ,0LL9Lq"L94&\
^^,,-R
0E/FGIT"	 :s   A!c                >   [         R                  " [        U R                  S   5      nUR	                  X15      n[        U5      (       d  U$ U R                  SS   H>  nUc  M  U R                  XR5      n[        U5      (       d  Us  $ UR                  U5        M@     U$ )Nr   r+   )rI  rJ  r[  r   r  r   _match_from_anchorsr   )r7   r   r   r  r   r   r  s          r:   r   MultiOutputPattern._matchV  s    [$,,q/:IIf#{{H||AB'G227@KK((""HH[! ( r>   c                    [        UR                  5      n[        S5      nUR                  U[	        5       5       H8  nUR                  X5      n[        U5      (       a  Us  $ [        U5      Ul        M:     U$ )Nzno anchor found)r   r   r   r  r!   r  r   )r7   r   r   priorr   r   s         r:   r  &MultiOutputPattern._match_from_anchorsf  sh     S(()$%67--c:<@D		'(A{{"&u+C A r>   c                     [        U R                  UR                  S9R                  X5      $ ! [         a  nUs S nA$ S nAff = fr  )r   r   r   r  r   r  s      r:   r  MultiOutputPattern.matchs  s<    	DJJ?EEdQQ 	H	s   ,/ 
AA A Ac                *  > [         R                  " [        U5      n[        TU ]  U5      =(       ab    [        U R                  5      [        UR                  5      :H  =(       a0    [        S [        U R                  UR                  5       5       5      $ )Nc              3  z   #    U  H1  u  p[        U[        5      (       a  UR                  U5      OX:H  v   M3     g 7fr4   r  r  s      r:   rm   0MultiOutputPattern.pattern_eq.<locals>.<genexpr>~  s4      <DA $.a#=#=Q16I<r  )	rI  rJ  r   r   r  r   r   r  r  rK  s     r:   r  MultiOutputPattern.pattern_eqy  sm    D%(Gu% DLL!S%77 emm< 	
r>   )ri  r   )r   zSequence[Optional[PatternExpr]]r@   r^   )r@   z-Union[Callable[..., Any], str, Sequence[Any]]r   r6  r  )r   r   r   r   r@   r  r   r#  )r2   rA   rB   rC   rD   r[   r   r\  r   r4  r   r  r  r  rE   r   r   s   @r:   r  r  <  s]    ((   
< ")5		
 	
r>   r  c                  ^   ^  \ rS rSrSrSU 4S jjr\S	S j5       rS
S jrSU 4S jjr	Sr
U =r$ )RepeatedExpri  zh
Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
c                P   > [         TU ]  5         Xl        UR                  U l        g r4   )r   r[   inner_patternri  )r7   r(  r   s     r:   r[   RepeatedExpr.__init__  s!    *""r>   c                .    U R                   R                  $ r4   )r(  r\  rZ   s    r:   r\  RepeatedExpr.fns  s    !!%%%r>   c                   UR                  U R                  U5      n[        U5      (       d  U$ UR                  R	                  U R                  5        U R                  R                  U[        5       5       HW  n[        U /UR                  S9R                  U R                  U5      n[        U5      (       d  Us  $ UR                  U5        MY     U$ r  )
r  r(  r   r   popr  r!   r   r   r   )r7   r   r   r   anchor_nodeanchor_ms         r:   r   RepeatedExpr._match  s    IId(($/{{H	
  --??Z\RK#TF$**=CC""KH H%%HHX S r>   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       a%    U R
                  R	                  UR
                  5      $ r4   )rI  rJ  r   r   r  r(  rK  s     r:   r  RepeatedExpr.pattern_eq  sF    D%(w!%( 
T-?-?-J-J.
 	
r>   )r(  ri  )r(  r[  r@   r^   )r@   zSequence[FnsType]r  r#  )r2   rA   rB   rC   r   r[   r   r\  r   r  rE   r   r   s   @r:   r&  r&    s0    #
 & &"
 
r>   r&  c                  x    \ rS rSrSrS	S jr\\R                  " S5      S
SS jj5       5       r	SS jr
SS jrSrg)r7  i  z
Serializes Patterns to executable python.
XXX: currently only used and tested for fuse attention patterns. May not cover
all patterns.
c                z    [         R                  R                  R                  5       U l        0 U l        0 U l        g r4   )re   rf   r   
_Namespace	namespacememoized_objs_namesmemoized_objs_pprZ   s    r:   r[   PatternPrettyPrinter.__init__  s*    224;= 8:r>   Nc                (   [        5       n[        U S5      (       d   eU R                  US9nUR                   Vs/ s H$  nUR                  U    SUR                  U    3PM&     nnUR                  U SU 35        SR                  U5      $ s  snf )zE
Serializes obj to python code with obj written out to `output_name`
r4  )r3  z = 
)r7  hasattrr4  r7  r8  rw   r  )objoutput_namer3  out_strr   r  s         r:   runPatternPrettyPrinter.run  s     "#sN++++""b") --
- %%c*+3r/B/B3/G.HI- 	 

 	S	23yy  
s   +Bc                    [        U[        5      (       a5  U R                  R                  U5      =n(       a  U$ U R	                  U5      $ [        US5      (       a  UR                  U 5      $ [        U5      $ )Nr4  )r   r  r7  ru   memoizer<  r4  r  )r7   r=  memoized_names      r:   r4  !PatternPrettyPrinter.pretty_print  sh    c?++ $ 8 8 < <S AA}A$$||C((3''##D))Cyr>   c                    UR                  U 5      nUR                  5       nS H  nUR                  US5      nM     U R                  R	                  US 5      nXPR
                  U'   X R                  U'   U$ )N)zaten.rm  zprims.rt  )r4  rq  replacer6  create_namer7  r8  )r7   r=  obj_strobj_nameprefixtmp_names         r:   rC  PatternPrettyPrinter.memoize  sr    ""4(<<>3F''3H 4 >>--h=(0  %%,c"r>   )r7  r8  r6  r]   )r  )r=  r   r>  r1   r@   r1   )r=  r   r@   r1   )r=  r  r@   r1   )r2   rA   rB   rC   r   r[   r  r   	lru_cacher@  r4  rC  rE   r5   r>   r:   r7  r7    s=    ;
 !  !$		r>   r7  c                  &    \ rS rSr    SS jrSrg)_PassDictsTypei  c                    g r4   r5   )r7   rk   s     r:   __getitem___PassDictsType.__getitem__  s     r>   r5   N)rk    tuple[str, torch.fx.node.Target]r@   list[PatternEntry])r2   rA   rB   rC   rR  rE   r5   r>   r:   rP  rP    s    !1!	!r>   rP  c                  T    \ rS rSr% S\S'   S\S'   S
S jr  S       SS jjrS	rg)PatternEntryi  r   r   Callable[[Match], bool]extra_checkc                    [         er4   r{  r7   r  r   r   s       r:   applyPatternEntry.apply  s    !!r>   Nc                >   UcK  [        U R                  S5      (       d   eU R                  R                   H  nU R                  XUS9  M     g [	        U[
        [        45      (       ay  [        U R                  S5      (       d   eU(       a+  XR                  R                  U4   R                  SU 5        g XR                  R                  U4   R                  U 5        g [        R                  " [        [           U5      nU H  nU R                  XRUS9  M     g )Nr\  prependri  r   )r<  r   r\  registerr   r   PatternMatcherPassri  insertrw   rI  rJ  r   rP  )r7   
pass_dictsr  r`  rP   r  s         r:   ra  PatternEntry.register  s     >4<<////ll&&jg> '
T+=$>??4<<....LLOOV45<<QELLOOV45<<TBXn%=zJJa9  r>   r5   r  r   r   r   r   r   r@   r^   r   )rd  /Union[_PassDictsType, Sequence[_PassDictsType]]r  z!Union[torch.fx.node.Target, None]r`  r   r@   r^   )r2   rA   rB   rC   rD   r\  ra  rE   r5   r>   r:   rW  rW    sK    ((" 59	:C: 2: 	:
 
: :r>   rW  c                  *    \ rS rSr% S\S'   SS jrSrg)LoweringPatternEntryi  Callable[..., Any]handlerc                   [         R                  " U R                  5      " [         R                  " U R                  U5      5      nUR	                  U5         UR                  U[        UR                  5      UR                  5      nUR                  R                  UR                  5        UR                  U5        S S S 5        UR                  S   UL d   eUR                  5         g ! , (       d  f       N3= f)N)r   wrapsrk  r   inserting_beforer  r   r8   r9   rz   ry   replace_all_uses_withr   r   )r7   r  r   r   rk  r   s         r:   r\  LoweringPatternEntry.apply
  s    //$,,/	0A0A$,,PU0VW##D)--guUZZ7H%,,WK##DII.&&{3 * {{2$&&& *)s   A'C++
C9r5   Nrf  )r2   rA   rB   rC   rD   r\  rE   r5   r>   r:   ri  ri    s    r>   ri  c                  .    \ rS rSr% SrS\S'   SS jrSrg)	GraphPatternEntryi  z0
A pattern that runs a function on the FX graph
rj  rk  c                    UR                  U5         U R                  " U/UR                  Q70 UR                  D6  S S S 5        g ! , (       d  f       g = fr4   )ro  rk  r8   r9   r[  s       r:   r\  GraphPatternEntry.apply  s:    ##D)LL<<u||< *))s   +A
Ar5   Nrf  )r2   rA   rB   rC   r   rD   r\  rE   r5   r>   r:   rs  rs    s      =r>   rs  c                  R    \ rS rSr% S\S'   \          SS j5       rS	S jrSrg)
r   i!  zCallable[..., list[Any]]normalize_argsc                &  ^^^^^  " U4S jS[         R                  R                  5      nU R                  5       n[	        U5      S:X  a  US   nOUS   (       d   e[        US   R                  R                  5      nU Vs/ s HA  n[        U[         R                  R                  5      (       d  M.  UR                  U5      U4PMC     n	n[        U	[        R                  " S5      S9S   n          S	S jmTR                  U5         [        U[         R                  R                  5      (       d   eU" U5      R                   " T6 n
[        U
[         R                  R                  5      (       a  U
/n
S
S jm      SUUUUU4S jjm[	        U5      [	        U
5      :X  a  [#        XZ5       H  u  pT" X5        M     O[	        U5      S:X  d   eT" US   U
5        S S S 5        U R%                  5         g s  snf ! , (       d  f       N$= f)Nc                  <   >^  \ rS rSrSrSrSrSU U4S jjrSrU =r	$ )<ReplacementPatternEntry.replace_with_graph.<locals>.Replaceri,  Nc                ^  > UR                   S;   a  [        TU ]	  U5      $ UR                   S:X  a  UR                  nU R	                  U5      u  p4[        U5      (       d   eTR                  X#U5      n[        UR                  USS9  SUR                  ;   a  SUR                  ;  av  UR                  S   UR                  S'   [        UR                  S   [        R                  5      (       a.  SUR                  ;   d   eUR                  S   UR                  S'   U$ [        SU 35      e)N)placeholderr  r  Interpreter_Replacerr   r   tensor_metaz
unhandled )ri  r   run_noder  fetch_args_kwargs_from_envrc  r  r   rz   r   re   Tensorr|  )r7   r   r  r8   r9   resultr   r   s         r:   r  EReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_node1  s	   7777 7+D1177o-![[F#'#B#B4#HLD#F++++"00vFF"!'!%"8
 		)e6;;.F-1YYu-=E*%dii&6EE#0DII#==#=9==9QFKK6!M)Jtf*=>>r>   r5   r   r   r@   r   )
r2   rA   rB   rC   r  r  get_attrr  rE   r   )r   r   s   @r:   Replacerrz  ,  s    KKH? ?r>   r  r+   r   )r   c                B   U /n[         [        R                  R                     " 5       nU(       ao  UR	                  5       nXe;  aP  Xc;  aK  [        US5      (       a:  UR                  U5        X&R                  U'   UR                  UR                  5        U(       a  Mn  g g )Nrz   )
r!   re   rf   r  r-  r<  r  rz   r   all_input_nodes)r   tag_name	tag_valueinput_stopsqueuevisitedr   s          r:   percolate_tagsBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tagsT  sx     FE /1Giik&.V,,KK$)2HHX&LL!4!45 %r>   c                    U R                   S:w  a  g U R                  [        R                  :w  a  g [	        U R
                  5      S:X  d   eU R
                  S   $ )Nr  r"   r+   )ri  r  rn  getitemr   r8   r   s    r:   maybe_getitemAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitemn  sJ    77o-;;("2"22499~***yy|#r>   c           	       > U c  Ub   eg [        U [        R                  R                  5      (       d   eUc#  U R	                  S 5        TR                  U 5        g [        U[        R                  R                  5      (       a  SUR                  ;  a%  UR                  R                  U R                  5        S H4  nX R                  ;   d  M  T	" XU R                  U   [        T5      5        M6     U R	                  U5        TR                  U 5        g [        U R                  R                  5       5      nU H$  nT" U5      nUc  [        S5      eT
" XAU   5        M&     TR                  U 5        g )Nr   )	recomputeac_graph_idzcan't handle)r   re   rf   r  rp  r   rz   ry   r!   r   r   r   AssertionError)oldnewr  old_usesuseridxr8   r   r  r  rG  s         r:   rG  ;ReplacementPatternEntry.replace_with_graph.<locals>.replacev  s8    ;;&;!#uxx}}5555;--d3$$S)c588==11CHH,1 %A#xx/* #sxx/A:dCS %A --c2$$S):  		 01$D'-C{,^<<Dc(+	 %
   %r>   )
r   r   r  r1   r  r1   r  r"  r@   r^   r  )r  zUnion[torch.fx.Node, None]r  z3Union[torch.fx.Node, Sequence[torch.fx.Node], None]r@   r^   )re   rf   Interpreterr   r   r   r   r   r   r  indexminrn  
itemgetterro  GraphModuler@  r  r   )r  r   r   r8   r  r   	last_noder   r   indicesr   r  r  r  r  rG  s    ` `         @@@r:   r   *ReplacementPatternEntry.replace_with_graph%  s   	?uxx++ 	?4 ))+|!$QI?"?a..445E &%Aa/ $Q#%  
 G)<)<Q)?@CI	6	6	6 	6 3		6
 	6( ##I./1E1EFFFF"#45994@K+uxx}}55*m$C&/C&HC& C& C&J < C$44 #L >HCC% !? <(A---Q5q /t 	k6 /.s   -G=4G=CH
Hc           
         UR                   c   eU R                  UUUR                   U R                  " UR                  0 UR                  D65        g r4   )r   r   rw  r8   r9   r[  s       r:   r\  ReplacementPatternEntry.apply  sL    &&222##<u||<		
r>   r5   N)
r  r   r   r   r   z+Union[torch.fx.Graph, torch.fx.GraphModule]r8   zSequence[torch.fx.Node]r@   r^   rf  )	r2   rA   rB   rC   rD   r  r   r\  rE   r5   r>   r:   r   r   !  sV    ,,\\\ G\ &	\
 
\ \|
r>   r   c                    gr   r5   )r  s    r:   _return_truer    s    r>   c                F    [         R                  SU R                  U5        g )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginfor2   )	search_fnr  s     r:   log_trace_failurer    s    HHJ	r>   Fc                  ^^ [         R                  U 5      mUR                  T5      nU(       d'  UT   R                  U(       a  [	        U5      OS5        gUc#  U(       a  g[
        R                  " SU4S j5        [	        U5      nU H0  mUT:X  d  M  U(       a    g[
        R                  " SUU4S j5        M2     UR                  U5        g)a.  
Check if a pattern is a duplicate. Because we ignore certain types in searching, but not
in matching, use the graph to distinguish equivalent search patterns.

Returns True if a duplicate is found and `skip_duplicates=True` is passed in. Errors if
`skip_duplicates` is False and a duplicate is found.
NFTc                    > ST  S3$ )NDuplicate pattern: z with no graphr5   )pattern_reprs   r:   r   1check_and_add_duplicate_pattern.<locals>.<lambda>  s    ),~Fr>   c                    > ST ST  S3$ )Nr  z with duplicated match graph  r5   )	graph_strr  s   r:   r   r    s    ),7TU^T__`ar>   )r7  r@  ru   rw   r1   re   _check)r   r   seen_patternsskip_duplicatesequiv_pattern_reprsnew_graph_strr  r  s         @@r:   check_and_add_duplicate_patternr    s     (++G4L'++L9l#**3u:DI}F	

 JM(		)a	
 ) }-r>   r5   c
           
     2  ^ ^^^^^^^^ / [         R                  " T 5      R                  R                  5       QmSUUUUUUU UU4	S jjn
S	U4S jjnT[        L a  [
        R                  " 5       (       a  g[        R                  " SS9   U Vs/ s H0  n[        U[
        R                  5      =(       a    UR                  PM2     snmTc  [        T UTTT5      u  pOTnSn[        U[        5      (       a  UOU/ HR  n[        U[        5      (       d  M  [        UU(       a  UR                   OSUR"                  U	S9(       d  MJ    SSS5        g   [%        UU
US9nUR'                  U5        UR(                  sSSS5        $ s  snf ! , (       d  f       g= f)
a  
Create a replacement rule based on example functions that get traced
to create patterns.  This supports both training and inference when
run on a joint forward+backward graph.

Args:
    search_fn: traced to give original pattern
    replace_fn: traced to give replacement graph
    example_inputs: example inputs for initial trace
    trace_fn: fwd_only or joint_fwd_bwd
    pass_dict: dict of passes to register to
    extra_check: additional check to run on match(using real shapes)
c           
     	  >	^^ [        T5      nU H-  nX R                  ;  d  M  [        SU SU R                   35      e   [        [        R                  R                  U Vs/ s H  o R                  U   PM     snS 5      5      m/ n[        R                  R                  R                  T5         [        T5       GH9  u  pE[        TU   [        R                  5      (       d  M*  U(       a'  [        TU   R                  5      (       a
    SSS5        g[        R                  " TU   R                  5       TU   R!                  5       TU   R                  TU   R"                  US9TU'   [$        R&                  " TU   R(                  TU   R!                  5       5       HQ  m[        T[        R*                  5      (       d  M$  [-        U4S jU 5       5      (       d  M@  UR/                  T5        MS     GM<     TnU(       Gd7  U(       Ga  SUU4S jjn T" XsT-   5      n/ n
[3        [5        [7        U5      [7        T5      -   5      UR8                  R:                  5       H  u  pKU[7        U5      :  a  U
R/                  UR<                  5        M1  UR8                  R?                  U5         UR8                  RA                  X[7        U5      -
     5      nURB                  Ul        URE                  U5        UR8                  RG                  U5        SSS5        M     X-   nO
 T" TT5      n[I        UUTTS	9nU RK                  5       S
   nUc   eURM                  U5      n[O        U5      (       a  T" U5      (       az  T" TT5      U l(        [7        U R:                  5      S:X  aI  U RP                  R8                  R:                   H%  n[S        URT                  U R:                  S
   SS9  M'      SSS5        g SSS5        gs  snf ! [         a  n	[1        TU	5         Sn	A	SSS5        gSn	A	ff = f! , (       d  f       GM  = f! [         a  n	[1        TU	5         Sn	A	SSS5        gSn	A	ff = f! , (       d  f       g= f)z
Often shapes get burned into the pattern, so our initial match ran with
`ignore_types=(int, ...)`.

Recheck the match with the correct shapes.
z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                     U R                   S   $ r   r   )r   s    r:   r   8register_replacement.<locals>.check_fn.<locals>.<lambda>0  s    QVVE]r>   NF)dtypedevicerequires_gradc              3  @   >#    U  H  n[        TU:g  5      v   M     g 7fr4   r   )rj   r  rl   s     r:   rm   9register_replacement.<locals>.check_fn.<locals>.<genexpr>C  s!      ?CKa1!q&998s   c                 >   > T" U [        U 5      [        T5      -
  S  6 $ r4   )r   )args_newr8   r  s    r:   search_fn_new=register_replacement.<locals>.check_fn.<locals>.search_fn_newV  s#    ((3x=3t93L3N*OPPr>   )argnamesexclusive_arg_namesscalar_workaroundr   r+   r   r   T)r  r   r@   r   )+r   r9   r   re   rf   r   _dynamoutilsdetect_fake_moder  r   r  r   r  empty_stridedsizestrider  r  r  shapeSymIntr  rw   r  r  ranger   r   r   r  inserting_afterr|  r=  rp  r   fx_to_patternr   r  r   r   r   rz   )r  r  r=  sym_argsr  gradspecific_patternr  specific_graphr  sym_arg_namesr|  new_noder   specific_pattern_matchr   r8   rl   argnames_staticr  rY  
replace_fnr  r  r  search_fn_patternr   s                   @@r:   check_fn&register_replacement.<locals>.check_fn  s    (D<<'"99A
/RWR^R^Q_a   HH089d#9;R
 (*]]  11$7$]3d1gu||44 0a ? ?$	 87 $11QQ("1gmm#Aw~~&*DG '__T!W]]DGNN<LM%a663 ?CK? < < %OOA.	 N 4*  1#Q Q%)1-D)Q %'M*-c(mc$i78&,,22+ s8},)001C1CD$+11AA+N'5';';'G'G (S]): ;(H /7mmHO'==hG*00;;KH ON+   -7H%)1)T)B
 $1"%(;&7	$  %%'*D###%5%;%;D%A".//K@V4W4W*2:t*D'u{{#q("44::@@&%&VV%*[[^&3 A s 87t u 87 :V ( %))Q7$Q 87L% ON ( %))Q7$E 87@%A 87s   $P	42Q7*%Q7B,Q7Q7$5Q7P%BQ71A'P9Q7*	Q3CQ7?Q7
P6P1$Q71P66Q79
Q		Q7
Q4Q/"Q7/Q44Q77
Rc                   > T Vs/ s H  oR                  U5      PM     nn[        S[        U 5      S-   5       H1  nSU 3U ;  a    O'UR                  U R                  SU 35      5        M3     U (       a   SU < 35       eU$ s  snf )Nr+   	tangents_zleftover kwargs: )r-  r  r   rw   )r9   r=  r8   r  r  s       r:   rw  ,register_replacement.<locals>.normalize_args  s    -<=_T

4 _=q#f+/*A1#f,KK

Yqc?34 + 9.vj99z >s   BFfunctionalize_rng_opsN)r  )r   rY  rw  r  r   r@   r   )r9   r   r@   r   )inspect	signature
parametersr   joint_fwd_bwdre   is_inference_mode_enabledfunctorch_configpatchr   r  r  gen_pattern_and_search_gmr   rb  r  r   r  r   ra  r   )r  r  example_inputsr   rd  rY  r  r  r  r  r  rw  r  r   gmpattern_matcher_passr  r  s   `` ` ````       @@r:   register_replacementr    sw   2 H)))4??DDFGOp pd =  **,, 
		e	<ES%
ESJq%,,';AOO;^%
 $3!#KGR (GB %Z::JL ! .0BCC2 "BHH(66$3	  !5 
=	<" M * )

 	$E 
=	<%
 
=	<s1   
F7FAF,FF)FF
Fc                l   SS jn[         R                  5       (       d  [        S[          35      eUR                  nSSKJn  UR                  " SS9   [        XX45      nS S S 5        [        R                  WU S9n	U[        ;  a  Sn
[        R                  U5        OS	n
U" 5       n[        [         U S
3-  U
5       nU
S:X  a  UR                  U5        OUR                  S5        UR                  U	5        UR                  S5        S S S 5        U$ ! , (       d  f       N= f! , (       d  f       U$ = f)Nc                    [         R                  " S5      n [         R                  " S5      R                  U S9n/ n[        [        R
                  R                  5       Hl  n[        [        R
                  R                  U5      n[        U[        5      (       d  M>  [        U[        [        45      (       d  M[  UR                  U5        Mn     SR                  U5      nSU S3nU U 3$ )Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            a               # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor
            import operator

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr   dirre   	_inductorpattern_matcherr`  r   r  
issubclassr   r[  rw   r  )auto_generated_msgfile_templatepattern_matcher_importsr=  attrformatted_importss         r:   get_file_template-_serialize_pattern.<locals>.get_file_template  s    %__
 !
 &'&
( 	  #%778D5??::DAD$%%*TK;U*V*V'..t4 9
 %MM*ABPQbPccij!2 344r>   z0Could not find serialized patterns directory at r   r#   Fr  )r>  wr  z.pyz

r;  r   )SERIALIZED_PATTERN_PATHis_dirr   r2   torch._functorchr$   r  gen_patternr7  r@  _serialized_patternsr  openwrite)unique_namer  r  r   r  r  pattern_namer  r   serialized_pattern
write_moder  fs                r:   _serialize_patternr    s   !5F #))++>?V>WX
 	
 %%L;			e	<iU 
= .11'{1S//
  .
%'M	%<.(<<j	IQGGM"GGFO	"#	 
J N) 
=	< 
J	I Ns   D=AD$
D!$
D3	fx_passesserialized_patternszvlist[tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsc
                    [        U5      nS[        R                  ;   a  [        XX4U5      n
O]UR                  n[
        R                  " SU 35      nU(       a  [        X5      (       d  [        R                  SU 5        [        X5      n
[        R                  " U5       H0  n[        U[        5      (       d  M  UR                  c  M)  S Ul        M2     [         R#                  XXGU
45        [%        UUUUUUUUU
U	S9
  g )NPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)r  r  )r   osenvironr  r2   	importlibimport_moduler<  r  warningr`  r  	tree_iterr   r(   constantr  rw   r  )r  r  r  r  r   rd  rY  r  r  r  patr  r   r   s                 r:   gen_register_replacementr"  #  s     >*N+ N>O
 !))##<\NK
 //KKV a%/c:&&3<<+C
  CL 0  &&	HE 'r>   r  c                r   / [         R                  " U 5      R                  R                  5       QnUc  0 n/ nSnU H5  nX;   a  UR	                  X8   5        M  UR	                  X   5        US-  nM7     U" X5      n	[        U	[        [        [        [        R                  [        R                  4UUUS9U	4$ )Nr   r+   )ignore_typesr  r  r  )r  r  r  r   rw   r  intfloatr   re   r  r  )
r  r  r   r  r  r  flat_inputs	input_idxargname	search_gms
             r:   r  r  [  s     A""9-88==?@H KI'09:~89NI  0IudELL%++F/ 3	
 		 	r>   c                "    [        XX#U5      S   $ r  )r  )r  r  r   r  r  s        r:   r  r  ~  s!     %8@S	 	r>   r_  c               $   ^ ^^^ SUUU U4S jjnU$ )z
Register an aten to inductor IR replacement pattern.  The decorated
function is saved and then called a lowering time allowing direct
pattern to inductor IR conversion.
c                l   > [        U 5      (       d   e[        TTU S9R                  TTS9  SU l        U $ )Nr   rY  rk  r_  T)rc  ri  ra  _inductor_lowering_functionrk  rY  	pass_dictr   r`  s    r:   	decorator,register_lowering_pattern.<locals>.decorator  s>        g	

(9g(
..2+r>   rk  rj  r@   rj  r5   r   rY  r1  r`  r2  s   ```` r:   register_lowering_patternr6    s      r>   c               $   ^ ^^^ SUUU U4S jjnU$ )z_
Register a pattern that runs a function on the FX graph, allowing
custom transformation code.
c                ^   > [        U 5      (       d   e[        TTU S9R                  TTS9  U $ )Nr.  r_  )rc  rs  ra  r0  s    r:   r2  )register_graph_pattern.<locals>.decorator  s6        g	

(9g(
.r>   r4  r5   r5  s   ```` r:   register_graph_patternr:    s      r>   c                B    U[        [        U R                  5      5      L $ r4   )r   iterr   )r   r   s     r:   is_start_of_fx_graphr=    s    4U[[)***r>   z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)c                    U R                   S:w  a  gU [        R                  R                  R                  R
                  [        R                  R                  R                  R
                  4;   $ )NinductorF)r6  re   opsr?  accumulate_grad_defaultresize_storage_bytes_)ri  s    r:   "fixme_incorrect_inductor_schema_oprD    sW    	||z! 		++33		0088  r>   c                   [        U R                  [        R                  R                  5      (       a:  [        U R                  5      (       d   U R                  R                  R                  $ [        U R                  [        R                  R                  R                  5      (       a  gU R                  S:X  aL  [        U R                  5      (       d   e[        R                  U R                  R                  5      (       a  gOVU R                  S:X  aF  [        U R                  [         5      (       d   e[        R                  U R                  5      (       a  gU R"                  R%                  S5      S L$ )NFr  Tr  out)r   r  re   rd  ro  rD  _schema
is_mutable_higher_order_opsauto_functionalizeAutoFunctionalizedri  rc  _mutation_op_researchr2   r1   r9   ru   r  s    r:   is_mutation_oprN    s   UZZ** 0=={{""---	U,,??RR
 
 ww/!$$$$!!$++"6"677 8	M	!$++s++++!!$++..;;??5!--r>   c                    SU R                   ;   d   eSUR                   ;   d   eU R                   S   UR                   S   :H  $ Nmutation_region_idr   )r  r  s     r:   same_mutation_regionsrR    sE    166)))166)))66&'1662F+GGGr>   c                P   UnSUR                   ;  a>  [        X5      (       d.  UR                  nSUR                   ;  a  [        X5      (       d  M.  UR                   R                  SS5      nX!La5  UR                  n[        U5      (       a  US-  nX2R                   S'   X!La  M5  U$ )NrQ  r   r+   )rz   r=  prevru   r   rN  )r   r   r   rQ  s       r:   get_mutation_region_idrU    s    A
aff
,5I%5S5SFF aff
,5I%5S5S$8!<
-FF!!#'9#$	 -
 r>   c                X    S[        [        U R                  5      5      R                  ;  $ rP  )r   r<  r   rz   r  s    r:   "should_compute_mutation_region_idsrW    s!    tD,='>'C'CCCr>   c                t    SnU R                    H&  n[        U5      (       a  US-  nXR                  S'   M(     g )Nr   r+   rQ  )r   rN  rz   )r   rQ  nds      r:   compute_mutation_region_idsrZ    s9    kk"!#(:$% r>   c                  V   ^  \ rS rSr S   SU 4S jjjrS	S jrS
S jrSS jrSrU =r	$ )rb  i  c                ~   > [         TU ]  5         [        [        5      U l        Xl        [        [        5      U l        g r4   )r   r[   r   r   patternsr~   r  )r7   r~   r   s     r:   r[   PatternMatcherPass.__init__  s8     	  	 # >I=Nr>   c                     U R                   U   $ r4   )r]  )r7   items     r:   rR  PatternMatcherPass.__getitem__  s    }}T""r>   c                   U R                   (       d  g[        U[        R                  R                  5      (       a  UR
                  nOO[        U[        R                  R                  5      (       a  UnUR                  nO[        S[        U5       35      e[        U5      (       a  [        U5        [        R                  " [        U5      nSn/ nSnU R                    H.  u  pxUS:X  a  SnM  UR                  UR!                  XxSS95        M0     U(       a  UR                  UR!                  SSS95        U R"                  b  U R"                  OSn	[        U[        R                  R                  5      (       d   e[%        X5         ['        [(        R*                  R-                  U5      SS	9 GH  n
[/        U
5      nU
R0                  S:X  a  U
R0                  U4U R                   ;  a  M=  [3        U
SS
9(       a  MN  U R                   U
R0                  U4    GHD  nU
R4                  (       a    M  UR6                  R9                  U
5      n[;        U5      (       a-  [=        [?        [A        X<RB                  5      5      5      S:w  a  Mp  [D        RF                  RI                  S5      U
RJ                  :X  a+  [L        RO                  SXRP                  XR6                  5        [;        U5      (       d  M  URS                  U5      (       d  M  US-  nURU                  XU
5        [V        S   S==   S-  ss'   [V        S   S==   [=        URB                  5      -  ss'   GMG     GM     S S S 5        U$ ! , (       d  f       U$ = f)Nr   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Fr  T)ri  r  sort)ri  rc  r  )reverse)allow_cpu_inputsr+   !TORCHINDUCTOR_PATTERN_MATCH_DEBUGz
%s%s %s %sr?  pattern_matcher_countpattern_matcher_nodes),r]  r   re   rf   r  r   Graphowning_moduler   r  rW  rZ  r   r   rU  rw   
find_nodesr~   r    sortedr  r  from_iterabler  ri  r-   r   r   r  r   r   r!   r  r   r  r  ru   r=  r  r  r8   rY  r\  r   )r7   r  r   get_mutation_region_id_partialr  r   has_call_moduleri  r  r~   r   entryr   s                r:   r\  PatternMatcherPass.apply  s   }}b%((..//HHEEHHNN++E$$B\]abd]e\fg  .e44'.)2):):"E*
& --JB]""&U---OP	 (
 LL))])GH&*nn&@DNNFW	"ehh223333#B2y<<UCTR'-77m+(= 
 9PUV!]]DGGV+<=E||++D1A !&s+I77'ST 
 !zz~~&IJdiiWL$		1mmT{{u'8'8';';
Ad3 ,-DEJE ,-DEQWWUE' > S 3B C 32B s   	E'M24M2AM22
Nc                8    U R                   R                  5         g r4   )r]  clearrZ   s    r:   rs  PatternMatcherPass.clearU  s    r>   )r~   r]  r  r4   )r~   Optional[str]r@   r^   )r`  rT  r@   rU  )r  +Union[torch.fx.GraphModule, torch.fx.Graph]r@   r%  r]   )
r2   rA   rB   rC   r[   rR  r\  rs  rE   r   r   s   @r:   rb  rb    s=     $(O O 
O O #>@ r>   rb  c                     [         er4   r{  r   s     r:   _not_implementedrx  Y  s    
r>   c                <  ^^^^	^
^ U=(       d    0 nUR                  5        VVs0 s H  u  pVXe_M	     snnm
[        T
5      [        U5      :X  d   e S     SUU
4S jjjm[        R                  " 5       m	 " UU	UUU4S jS[        R
                  R                  5      n[        U [        R
                  R                  5      (       d   eU" U 5      R                  5       n[        U[        5      (       d  [        [        R                  " U5      5      $ U$ s  snnf )z
Convert an FX graph into a PatternExpr.  This is useful for simple
patterns that can only match single functions and fixed-length lists.
c                   > Ub  UOTn[        U [        [        45      (       a  U T;   a  [        TU    5      $ [	        U 5      U;   a
  [        5       $ [        U [        5      (       a(  [        S U  5       5      (       a  U (       a
  [        5       $ U $ )Nc              3  B   #    U  H  n[        U[        5      v   M     g 7fr4   )r   r,  )rj   ys     r:   rm   5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>x  s     &Iq!z!W'='=qs   )r   r&  r%  r9  r  r,  r   r  )r  ignore_types_overridecurrent_ignore_typesr$  inv_scalar_workarounds      r:   process_arg"fx_to_pattern.<locals>.process_argn  s     &;%F!L 	 a%&&10E+E3A6777**9a3&Iq&I#I#Ia9r>   c                     >^  \ rS rSr\r\r\r        SUUU4S jjr        SUU4S jjr	SU 4S jjr
SrU =r$ )	 fx_to_pattern.<locals>.Converteri~  c                   > [        T5      nU[        T5      :  a  TU   nO<T(       a  UR                  S5      (       d   eUnO[        R                  " SSU5      nUnUT;   a  [        U5      $ [        U5      $ )Ntangentz_\d+$rt  )r   r   
startswithresubrQ  r9  )	r7   r  r8   r9   r   r=  r  argnumr  s	         r:   r|  ,fx_to_pattern.<locals>.Converter.placeholder  sx     VA3x= {((3333"f5***400!$''r>   c                r  > T	nU[         R                  :X  a"  [        S T 5       5      4     SU	4S jjjnUn[        R                  " XBU45      u  p#[
        T;   aA  U Vs/ s H
  od" U5      PM     nnUR                  5        VVs0 s H  u  pvXt" U5      _M     nnn[        U/UQ70 UD6$ s  snf s  snnf )Nc              3  :   #    U  H  o[         Ld  M  Uv   M     g 7fr4   )r%  )rj   ts     r:   rm   Afx_to_pattern.<locals>.Converter.call_function.<locals>.<genexpr>  s      Q#/aC<<s   	c                   > T" X5      $ r4   r5   )r  r~  r  s     r:   process_arg_fn_implKfx_to_pattern.<locals>.Converter.call_function.<locals>.process_arg_fn_impl  s     'q@@r>   r  rS   r~  zOptional[Sequence[type[Any]]]r@   zUnion[T, KeywordArg, Ignored])rn  r  r   r  r  r   r{   r  )
r7   r  r8   r9   process_arg_fnr  r  rk   r$  r  s
           r:   r  .fx_to_pattern.<locals>.Converter.call_function  s     )N))) LQ Q#/Q LAA+HA
 3A A "5!??>&>JLD|#3784aq)48;A<<>J>41!^A..>J8888 9Js   "B.B3c                  > [         TU ]  U5      nUR                  S:X  a  [        U[        5      (       ap  UR
                  S   n[        U[        5      (       d   e[        U5      [        U5      :X  d   e[        X#5       H  u  pE[        UR                  5      Ul	        M!     U$ [        UR                  5      Ul	        U$ )Nr  r   )
r   r  ri  r   r   r8   r   r   r  r   )r7   r   rvr8   rr   r   s         r:   r  )fx_to_pattern.<locals>.Converter.run_node  s    !!$BttxJr5$9$9vvay!$
33332w#d)+++!"mFA!#))nAG , I qww<Ir>   r5   )r  r1   r8   r   r9   Mapping[str, Any]r@   z&Union[ExclusiveKeywordArg, KeywordArg])r  r1   r8   r   r9   r  r@   r   )r   r   r@   r   )r2   rA   rB   rC   rx  r  r  r  r|  r  r  rE   r   )r   r  r  r  r$  r  s   @r:   	Converterr  ~  s|    &&#	(	(  	( &		(
 4	( 	((	9	9  	9 &		9
 	9 	96
	 
	r>   r  r4   r  )r{   r   r  r  re   rf   r  r   r  r@  r   r  r  tree_leaves)r  r$  r  r  r  rk   rl   r  r   r  r  r  s    `` `    @@@r:   r  r  ]  s    */R.?.E.E.GH.GdaQT.GH$%->)???? FJ%B	&  __F> >EHH(( >@ b%((..////m!Gg{++!&"4"4W"=>>No Is   DT)r   get_decomp_fnc               "   [        5          Ub  U" 5       O	[        5       n[        XSS9" U6 nSSS5        SSKJn  U(       a,  U" WR
                  5        UR
                  R                  5         WR                  5         U$ ! , (       d  f       NY= f)z>Build a normalized inference graph, for use with fx_to_patternNreal)tracing_moder+   remove_noop_ops)r   r,   r   fx_passes.post_gradr  r   eliminate_dead_code	recompile)rP   r8   r   r  decompositionsr  r  s          r:   r   r     sr     
"	#,8MO>Q>S 	 Rf=tD	 
$ 5!
$$&LLNI 
$	#s   "B  
Bc           
       ^ Sm        SU4S jjn[         R                  R                  S5         [        U S U[	        5       SSS9" U6   SSS5        T(       d   eSSKJn  U" TR                  5        SS	KJ	n  [        5       n[        [         R                  R                  R                  R                  [!        S
5      [!        S5      5      n[#        Xd[$        S9R'                  UR(                  5        UR+                  TR                  5        [         R,                  R                  R/                  5       TR                  l        TR                  R3                  5         TR5                  5         T$ ! , (       d  f       GN0= f)z=Build a normalized training graph, for use with fx_to_patternNc                D   > T(       a   e[        U 5      m[        X40 UD6$ r4   )clone_graphr'   )joint_graphinputsr9   r  s      r:   record_joint_graph)joint_fwd_bwd.<locals>.record_joint_graph  s&     v% ???r>   c                    [        U 5      $ r4   )r&   )gr  s     r:   r   joint_fwd_bwd.<locals>.<lambda>  s	    +r>   TF)partition_fnr  keep_inference_input_mutations
enable_logr+   r  )pointless_viewr   r  )r   rk  rY  )r  rR   r  r   r9   r   r@   z1tuple[torch.fx.GraphModule, torch.fx.GraphModule])re   _guardstracingr%   r,   r  r  r   fx_passes.joint_graphr  rb  r  r@  atenviewrB  r9  rs  r  ra  r]  r\  rf   CodeGen_codegenr  r  )rP   r8   r  r  r  matcher_passr   r  s          @r:   r  r    s=    *.B@)@3@@LO@	:@ 
		t	$++.0+/	
 	 
% I24BHH5%'L		##Z%6
68JG \h|$$%rxx  ..0BHHHH  "LLNI? 
%	$s   E77
Fc                    / n[         R                  R                  U R                  U R                  4UR
                  5        U$ r4   )re   rf   r   r8   r9   rw   )r   r8   s     r:   r  r    s1    )+D	HHaffahh'5Kr>   c                   [        [        U R                  5      5      n[        [        R
                  R                     " 5       n[        [         5      nS nU(       a  UR                  5       n[        U5       Vs/ s H  ofU;  d  M
  UPM     nnU(       a  X7S      R                  U5        OdUR                  U5        U(       a   UR                  ULa  UR                  U5        UnUR                  [        UR                  US5      5      5        U(       a  M  U(       d"  [        U5      [        U R                  5      :X  d   eg s  snf )Nrm  r5   )r   r   r   r!   re   rf   r  r   r-  r  rw   r  r   r   r   )r   pendingreadywaitingcursorr   r  waiting_fors           r:   stable_topological_sortr    s     8EKK()G uxx}}%'E $G F
{{}"'+@+Q%q+@ O$++D1IIdO&++T1d#F NN8GKKb$9:; '  3u:U[[)99999 As   :	EEc                |   ^  [         R                  " S5      [         R                  " T 5      SU 4S jj5       5       nU$ )z0Wrapper around lazy init functions in fx_passes/Nc                 X  > [         S   R                  5       n [        R                  R	                  S 5         [        5          [        5          T" 5       nS S S 5        S S S 5        S S S 5        U [         S'   W$ ! , (       d  f       N)= f! , (       d  f       N2= f! , (       d  f       N;= f)Nr?  )r   rv   re   r  r  r   r)   )counters_refr  rP   s     r:   	lazy_init%init_once_fakemode.<locals>.lazy_init;  ss      
+002]]""4(*@*BNDTTF EU*B(  , EUDT*B*B((s;   BB
A9B
B9
BB


B	B
B))r@   r   )r   rN  rn  )rP   r  s   ` r:   init_once_fakemoder  8  s;     __R	  	 r>   c                   ^  SU 4S jjnU$ )z2Function for extra_check to put pass behind a flagc                $   > [        [        T5      $ r4   )r`  r$   )r  r=  s    r:   
flag_checkconfig_flag.<locals>.flag_checkN  s    vt$$r>   )r  r   r@   r   r5   )r=  r  s   ` r:   config_flagr  K  s    % r>   c                L     " S S[         5      nU" U 5      R                  5       $ )Nc                  ,   ^  \ rS rSrSU 4S jjrSrU =r$ )clone_graph.<locals>.CopyGraphiU  c                T  > [         TU ]  U5      n[        U[        R                  R
                  5      (       an  UR                  R                  R                  UR                  5        U R                  R                  R                  UR                  S 5      UR                  l        U$ r4   )r   r  r   re   rf   Proxyr   rz   ry   	new_graph_graph_namespacerH  r=  )r7   r}   r  r   s      r:   r  'clone_graph.<locals>.CopyGraph.run_nodeV  st    w'1H(EHHNN33""))(--8%)^^%D%D%P%PMM4&" Or>   r5   )r}   r   r@   r   )r2   rA   rB   rC   r  rE   r   r   s   @r:   	CopyGraphr  U  s    	 	r>   r  )r*   	transform)input_graphr  s     r:   r  r  T  s$    K  [!++--r>   c                    [        U R                  5      U:  a  U R                  U   $ Uc  g U R                  R                  U5      $ r4   )r   r8   r9   ru   )r   
arg_number
kwarg_names      r:   get_arg_valuer  f  s?     499~
"yy$$		{{z**r>   c           	     (   U/n[        U[        R                  R                  5      (       a9  UR	                  UR                  5        Vs/ s H  n[        X5      PM     sn5        U  Vs/ s H  oDR                  U;   d  M  UPM     sn$ s  snf s  snf r4   )r   re   rd  re  r   rf  r`  r  )r   rP   r\  ra  r   s        r:   filter_nodesr  q  sm    $C"ejj1122

",,.I.hGB).IJ"9UTkkS&8DU99 J9s   
B
*BBc                    U R                   S:X  aU  [        U R                  [        5      (       d   e[	        U R
                  R                  U R                  5      R                  $ U R                  $ )zFor call_function and call_method, we directly use the target function;
For call_module, the target is string, and we treat the module class
 as a function.
r  )ri  r   r  r1   r   r   rj  r   r  s    r:   r  r  y  sR    
 ww-$++s++++114;;?III;;r>   )rt  )r|   r   r}   r   r~   r1   r@   r^   )r   r  r@   zTypeIs[Match]r  )r  rj  r  r   r@   r^   r	  )
r   r   r   zOptional[torch.fx.Graph]r  zdict[str, list[Optional[str]]]r  r   r@   r   )r  r/   r  rG   r  Iterable[Any]r   rM   rd  rg  rY  rX  r  )Union[dict[str, Union[float, int]], None]r  Sequence[str]r  zUnion[PatternExpr, None]r  r   r@   r   )r  r1   r  r/   r  r   r   rM   r  r  r@   r   )r  r1   r  r/   r  rG   r  r  r   rM   rd  rg  rY  rX  r  r  r  r  r  r   r@   r^   )Nr5   )r  r/   r  r   r   rM   r  r  r  r  r@   z(tuple[PatternExpr, torch.fx.GraphModule])r  r/   r  r   r   rM   r  r  r  r  r@   r   )
r   r   rY  rX  r1  rP  r`  r   r@   z2Callable[[Callable[..., Any]], Callable[..., Any]])r   r   r   r   r@   r   )ri  ztorch._ops.OpOverloadr@   r   r  )r  r   r  r   r@   r   )r   r   r   r   r@   r%  )r   r   r@   r   )r   r   r@   r^   )r8   r   r9   r   r@   r   )r5   r5   Nr5   )r  rv  r$  zSequence[type[Any]]r  r  r  r  r  r  r@   r   )
rP   rj  r8   r   r   r   r  zOptional[Callable[..., Any]]r@   rR   )rP   rj  r8   r   r@   rR   )r   r   r@   zlist[torch.fx.node.Argument])rP   rj  r@   zCallable[[], Any])r=  r1   r@   zCallable[[Match], Any])r  rR   r@   rR   r4   )r   r   r  r%  r  ru  r@   r   )r   zIterable[torch.fx.Node]rP   r   r@   r   )r   r   r@   ztorch.fx.node.Target)r   
__future__r   r   dataclassesr   r  r  r  loggingrn  r  r  r  rI  abcr   r   collectionsr   collections.abcr   r   r	   r
   r   pathlibr   r   r   r   r   r   r   r   typing_extensionsr   r   re   torch._guardstorch.fxtorch.utils._pytreer  _pytreer  torch._dispatch.pythonr   torch._dynamo.utilsr   torch._prims_commonr   torch._subclasses.fake_tensorr   "torch.fx.experimental.proxy_tensorr   %torch.fx.experimental.symbolic_shapesr   torch.fx.graph_moduler   torch.fx.immutable_collectionsr   r   (torch.fx.passes.graph_transform_observerr    torch.utils._ordered_setr!   
_functorchr$   r  _functorch.aot_autogradr%   r&   _functorch.partitionersr'   _subclassesr(   r)   rf   r*   rt  decompositionr,   loweringr-   	getLoggerr2   r  r@  r  primsConstantr  r  r/   rG   rM   rS   r   Targetr1   FnsTyperU   rX   r   r   r   r   r  r   r   r   r%  r,  r9  rQ  r[  r   _SimpleSpecr  r  r  r  r  r  r  r  r  r  r&  r7  rP  	dataclassrW  ri  rs  r   r  r  r  r  r  r  __file__parentr  r  rD   r"  r  r  r  r6  r:  r=  compilerL  rD  rN  rR  rU  rW  rZ  rb  rx  r  no_gradr   enable_gradr  r  r  r  r  r  _seen_patternsr  r  r  r5   r>   r:   <module>r     s  !F #         	 	   # # N N  N N N *    $ $ ; ( 0 @ 6 G + I K / 3 C 7 4   . ; !yy~~		x./>x >> >#h # CL $$c)
*? ? : IK

(5
BE
	
4 D, 8 E;&''
 '
T 1#  1F-+ -k E E(E+ E4O
+ O
d CHol
k l
^?    (, * * ,
[ ,
^F
 F
R#
; #
L4 4n!X ! : : :: 
< 
 
 	= 	= 	= i
l i
 i
X "	''#' 2' 	'
 
'` ,8CG)+26! " 	
 @ ) A ' 0  
D "#( GGG "G 	G
 AG GT x.//+=@UU     $ ,8CG)+!555 5 "	5
 5 @5 )5 A5 '5 5 
5p e4
 DH)+!  A	
 ' . 5L DH)+				!		 		 A			
 '		 		 ,8
 ( 	
  84 ,8
 ( 	
  8,+ **VW	.(H
D;U Up )+ CG)+e3e%e e A	e
 'e eP 
 #'26
  	
 0  2 + +\!:H&. C" GK+
+%(+6C++:r>   