shh %SSKJr SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSK r SSK r SSK r SSKJ r J r SSKJr SSK JrJrJrJrJrJrJrJrJr SSKJr SSKrSSKrSSKrSSKJr SSKJ r! SS K"J#r# SS K$J%r% SS K&J'r( SS K)J*r*J+r+J,r, SS K-J.r.J/r/ SSK0J1r1J2r2 SSK3J4r4 SSK5J6r6J7r7 SSK8J9r9J:r:J;r;Jr>J?r?J@r@JArAJBrBJCrC SSKDJErEJFrFJGrGJHrHJIrIJJrJ \(aWSSKKJLrLJMrMJNrN SSKOJPrPJQrQJRrRJSrS SSKTJUrU SSKVJWrWJXrXJYrY SSKZJ[r[ \"S5r\\\\X/\W4r]\^\[r_\\`\R4rb\`rc\RR\fS5rg\R"\f5riSSjrj"SS\R5rk\="SS 9"S!S"55rl\R"S#S$55rn\R"S%S&55ro\R"S'S(55rp\R"S)S*55rq\R"S+S,55rr\\l\n\o\q\p4rs0rtS-\uS.'"S/S05rv0rwS1\uS2'SSS3jjrx"S4S5\ 5rySS6jrzSS7jr{SS8jr|SSS9jjr}\R"S5SS:j5rSS;jrGSS<jrGSS=jr\GR\GR\GR \GR0\GR \GR\GR\GR\GR\GR\GR\GR\GR\GR\GR 4 Vs0sHnX_M snErS>\uS?'GSS@jr"SASB5r"SCSD\(5r'"SESF5r\ GR*"SG\ GR,SH9rGSSIjr"SJSK\6\\F\5r\R"SLSM55r\"GS0SN\"\GR8SOSPSQ9_SR\"\GR8SSSTSUSV9_SW\"\GR8SXSYSZSV9_S[\"\GR8S\S]S^SV9_S_\"\GR8S`SaSbSV9_Sc\"\GR8SdSeScSf9_Sg\"\GR8ShSiSjSV9_Sk\"\GR8SlSmSnSkSo9_Sp\"\GR8SqSpSQ9_Sr\"\GR8SsSrSQ9_St\"\GR8SuSvSQ9_Sw\"\GR8SxSySQ9_Sz\"\GR8S{S|S}SzS~9_S\"\GR8SSSSf9_S\"\GR8SSSSV9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSSV9_S\"\GR8SSSSV9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_S\"\GR8SSSQ9_6rS\uS'GSSjr"SS\:5r"SS\<5r"SS\5r\R"SS55r"SS5r\"5r"SS5r"SS5r\"S\`S9r\"S\\S9r\(a\\GRT\H\\\\S444r"SS\\\45r"SS5r"SS\\\5r\R"SS55r\R"S5GSSj5r"SS5r"SS\75rgs snf() annotationsN)autoEnum)chain) AnyCallablecastClassVarGeneric NamedTupleOptional TYPE_CHECKINGUnion)TypeVar)ELEMENTWISE_TYPE_PROMOTION_KIND)_pytree) OrderedSet)int_oo) PythonPrinter)free_symbol_is_typesymbol_is_typeSymT) bound_sympy ValueRanges)configmetrics)DtypePropagationOpsHandler)BasicMathOpsMixinDefaultHandler) boolean_opsDeferredLineBasegenerate_assertIndentedBuffer ir_dataclass ScopedDict sympy_dotsympy_index_symbol sympy_subs triton_typeunique)ops OpsHandlerOpsValue ReductionType StoreModeV)IteratorMutableMappingSequence)Buffer ChoiceCaller FixedLayoutIRNodeLoopBody)BaseScheduling Scheduler SchedulerNodePythonWrapperCodegen_Tschedulec[R[R5(a[R SU5 gg)NzData type propagation: %s) schedule_log isEnabledForloggingDEBUGdebug)msgs r/Users/tiagomarins/Projetos/claudeai/copy_bank/venv/lib/python3.13/site-packages/torch/_inductor/codegen/common.pydata_type_loggerrKPs-  //6<0cH\rSrSrSrSrSr\S Sj5r\S Sj5r Sr g) WorkspaceZeroModeUrr>rcX:XdU[R:XaU$U[R:XaU$[SU<SU<S35e)NzWorkspaceZeroMode.combine(, ))rN UNINITIALIZEDNotImplementedErrorabs rJcombineWorkspaceZeroMode.combineZsI 6Q+999H !// /H!$>qe2aU!"LMMrLcPU(a[R$[R$N)rN ZERO_ON_CALLrS) zero_fills rJ from_boolWorkspaceZeroMode.from_boolbs $11 1 ...rLN)rVrNrWrNreturnrN)r]boolrarN) __name__ __module__ __qualname____firstlineno__rSr\ZERO_PER_GRAPH staticmethodrXr^__static_attributes__r`rLrJrNrNUs9MLNNN//rLrNT)frozenc0\rSrSr%SrS\S'S\S'S\S'S \S 'S rS \S '\Rr S \S'\ SSSjj5r \ SSj5r \ S Sj5r \ S Sj5rS!Sjr\rS"SjrS#Sjr\S#Sj5r\r\r\rS$SjrS$SjrS%SjrS&SjrSrg)' WorkspaceArgiaA temporary buffer used for a single kernel, then discarded. Not registered as a traditional buffer since there are no users, so it would be dead code eliminated. Args: nbytes: The size of the buffer in bytes. zero_fill: Whether the buffer should be initialized to zero. sympy.ExprcountrN zero_mode torch.devicedevicestr outer_namews_ptr inner_name torch.dtypedtypecPU[[RR53$r[)nextr1graph workspace_id)prefixs rJ unique_nameWorkspaceArg.unique_name}s!$qww334566rLcURUR:H=(a9 URUR:H=(a URUR:H$r[)rvrxrrrUs rJcan_joinWorkspaceArg.can_joins@ LLALL ( XQWW-? XAHHPQPXPXDX rLc [URUR-[RURUR5UR UR URURS9$N)rorprxrrrvrt) rlrorNrXrprxrrrvrtrUs rJjoinWorkspaceArg.joinsS''AGG#'// Q[[I''88||||   rLc URUR:Xa4URUR:XaURUR:Xde[[R "UR UR 5[RURUR5URURURURS9$r) rxrrrvrlsympyMaxrorNrXrprtrUs rJmaximumWorkspaceArg.maximums GGqww 188qxx#7ALLALL)rrrxsizestride)irr7rrrxro)rr7s rJ get_layoutWorkspaceArg.get_layouts.$;;****3   rLc"UR5$r[)rrs rJlayoutWorkspaceArg.layouts  rLcUR/$r[)rors rJget_sizeWorkspaceArg.get_sizes |rLc8[RR/$r[)rSOners rJ get_strideWorkspaceArg.get_strides }rLcUR$r[)rtrs rJget_nameWorkspaceArg.get_names rLc/$r[r`rs rJget_inputs_that_alias_output)WorkspaceArg.get_inputs_that_alias_outputs rLr`N) workspace_)r}rsrars)rVrlrWrlrarb)rVrlrWrlrarl)rarq)rarw)rar7)ralist[sympy.Expr]rars)ra list[str])rcrdrerf__doc____annotations__rvtorchuint8rxrhr~rrrrget_device_or_errorrrpropertyrget_output_specmaybe_get_output_specmaybe_get_layoutrrrrrir`rLrJrlrlis    OJE;$77        % !!!O&!rLrlcx\rSrSr%S\S'S\S'S\S'\R RrS\S'S r S \S 'S r g ) TensorArgrsnamebufferrwrxrnoffsetN Optional[str]alias_ofr`) rcrdrerfrrrZerorrrir`rLrJrrs. I K FJ%"Hm"rLrc>\rSrSr%S\S'S\S'\S Sj5rSrg) SizeArgrsrrnexprcgr[r`rs rJrSizeArg.alias_ofsrLr`Nrar)rcrdrerfrrrrir`rLrJrrs I  rLrc \rSrSr%S\S'Srg) ConstexprArgrsrr`Nrcrdrerfrrir`rLrJrr IrLrc \rSrSr%S\S'Srg)TMADescriptorArgrsrr`Nrr`rLrJrrrrLrc8\rSrSr%S\S'S\S'SrS\S'S rg) DeviceCodegenSchedulingConstructor schedulingWrapperConstructorwrapper_codegenNOptional[WrapperConstructor]cpp_wrapper_codegenr`)rcrdrerfrrrir`rLrJrrs%%''8<5) CppScheduling) CppWrapperCpu)CppWrapperCpuArrayRef) CppWrapperGpu)CUDACombinedScheduling)HalideScheduling)MetalScheduling)TritonSchedulingr?cpu)cpphalidetritonc6>T[R"U5$r[)r cpu_backend)r cpu_backendss rJ+init_backend_registration..s|F,>,>? KrLcuda)rIrHc6>T[R"U5$r[)r cuda_backend)r cuda_backendss rJrMrNs}V-@-@A*MrLxpumps privateuseoner)_get_custom_mod_func Schedulingr@CppWrapperCodegen)rGr>cpp_wrapper_cpur?cpp_wrapper_cpu_array_refr@cpp_wrapper_gpurAcuda_combined_schedulingrBrHrCrTrDrIrEwrapperr@r1r r aot_inductorallow_stack_allocationr_C_get_privateuse1_backend_name torch.utils.backend_registrationrV RuntimeError)r>r?r@rArBrCrDrEr@private_backendrVrrrrLrRs @@rJr.r.sm".@.@($(- '/ && $  K ""99 "  !(0-&  $  M   !'/#     !'/#     hh<<>O?* %o 6 >I  4\ B 23IJO"67J"K  _9L+#%#' :M_  ? +    s ;E EEcJSSKJn /UQ[XRU55P$)Nr)FlexibleLayout)rrfr'contiguous_strides)index index_varssizesrfs rJindex_prevent_reorderingrks* $ UU TIj*K*KE*RS TTrLcU[U'gr[)r)rrdevice_op_overridess rJregister_device_op_overridesrns(;V$rLc[U[5(de[(dSSKJnJn SSKJn SSKJn [U$)Nr>)cpu_device_op_overridesmps_device_op_overrides)rm) r/rsrrprqrOrmrS)rrrprqrmxpu_op_overridess rJget_device_op_overridesrts/ fc " "" " # #F-@ #F ++rLzdict[torch.dtype, torch.dtype]DTYPE_TO_COMPUTATION_DTYPEcU[5;a[R$US;aSU;aUS$US$US;a[R$US;a[R$US:XaSU;aUS$US$US:XaSU;aUS$US$US ;a$USn[ R RU5$US :XaSU;aUS$US $g ) zC Given op name and a list of input dtypes, deduce the output dtype )to_dtype index_exprrx)randrandn) get_index randint64 load_seed reductionr>constant)loadstorestore_reductionto_dtype_bitcastN)r!rrbfloatint64r1r{r)op_nameargskwargsbuf_names rJdeduce_output_dtype_by_namers +-zz  #*V"3vgAbA  {{   {{ K ")V"3vg@a@ J ")V"3vgAbA   7ww  ** & &")V"3vgAbA rLcx\rSrSrS SjrS SjrSSjrS SjrSSjrSSjr \ SSj5r \ SS j5r S r g )DataTypePropagationi,cXlSURR0UlURR 5Hup#URURU'M gNroot)body root_blockr{graphs subblocksitems)rrkvs rJ__init__DataTypePropagation.__init__-sL DOO))B  NN((*DAWWDKKN+rLcURnUVs/sHBn[U[RR5(dM.UR S:wdM@UPMD nn[ U5S:Xag[SU55nU(dg[R"[RUVs/sH)o3R[RRPM+ sn5$s snfs snf)N placeholderrc3# UHRn[RUR;=(a) UR[RRSLv MT g7fr[)OptimizationContextkeymetarx).0ns rJ BDataTypePropagation.deduce_node_dtype_by_inputs..=sR) ! # #qvv - B*../55TA B sAA)all_input_nodesr/rfxNodeoplenall functoolsreduce promote_typesrrrrx)rnodeinputsr input_nodesall_input_nodes_propagateds rJdeduce_node_dtype_by_inputs/DataTypePropagation.deduce_node_dtype_by_inputs5s%% !Auxx}}!=A!$$-BWAv   { q %() !) & " *   rmasked_subblock)rrrroperatorgetitemdeduce_node_dtyper/rs startswithrrrr)rr output_dtypes rJr%DataTypePropagation.deduce_node_dtypePs 77m # ;;( "s499~': ;;(** *))$))A,7 7$++s++++ ;; ! !"3 4 455d; ;8 ++ L    //55rLcUR(deSnURHn[RUR;aUR[RnO [5nUR U5UlXCR[R'UR S:XdMUR nM U$)Nr)nodesrrrrrxr)rr{ graph_dtyperopt_ctxs rJr#DataTypePropagation.propagate_graphks{{{-1 KKD"&&$))3))$7$;$;<-/ 2248GM18II)-- .{{h&%mm  rLc>URURS5$r)rrrs rJ propagateDataTypePropagation.propagate}s##DKK$788rLc.U"U5R5$r[)r)clsrs rJpropagate_loopbody&DataTypePropagation.propagate_loopbodys4y""$$rLcSSKJn SSKJn [ X5(de[ UR U5(de[ RUR 5$)Nrr9)r=) loop_bodyr: schedulerr=r/_bodyrr)rrr:r=s rJpropagate_scheduler_node,DataTypePropagation.propagate_scheduler_nodesE(-$....$**h////"55djjAArL)rrN)rr:raNone)r torch.fx.NoderaOptional[torch.dtype])rrrarw)r{ztorch.fx.Graphrar)rar)rr:rar)rr=rar)rcrdrerfrrrrrr classmethodrrrir`rLrJrr,sJ% * 66$9%%BBrLrcD^\rSrSrSSS.SU4SjjjrSrU=r$)riT)simplifypc>U(ag[U[R5(aH[[R S5(a)[R R RU5n[TU]%U5$)Nsizevars) r/rExprhasattrr1r{rrsuperdoprint)rrrr __class__s rJrPythonPrinter.doprintsQ  444*9U9U77##,,T2Dwt$$rLr`)rrnrrbrrbrars)rcrdrerfrri __classcell__rs@rJrrs248D%%-1%=A% %%rLrc\rSrSrSr\SSj5r\SSj5r\SSj5r\SSj5r \SSj5r \SSj5r \SS j5r \SS j5r \SS j5r\SS j5r\SS j5r\SSj5r\SSj5r\SSj5r\SSj5r\SSj5r\SSj5r\SSj5rSrg)OpDecompositionsiz Decomposes inductor ops cU$r[r`)values rJidentityOpDecompositions.identitys  rLcv[R"[R"S[R5U5$Nr>)r,truedivrrint32xs rJ reciprocalOpDecompositions.reciprocals"{{3<<5;;7;;rLc.[R"X5$r[)r,mulrs rJsquareOpDecompositions.squareswwq}rLc[R"[R"S[R5[R "U55$r)r,subrrfloat32erfrs rJerfcOpDecompositions.erfcs*wws||Au}}5swwqzBBrLc[R"[R"[R"U55[R"U55$r[)r,rexprrrs rJerfcxOpDecompositions.erfcxs,wwswwszz!}-sxx{;;rLc[R"[R"U5[R"S[R 55$r)r,rrrrrrs rJexpm1OpDecompositions.expm1s*wwswwqz3<<5==#ABBrLc [R"[R"U5[R"S[R"S5- [ R 55$)Nr> r,rlogrmathrrrs rJlog10OpDecompositions.log10s7wwswwqz3<<DHHRL0@%--#PQQrLc [R"[R"U5[R"S[R"S5- [ R 55$)Nr>rrrs rJlog2OpDecompositions.log2s6wwswwqz3<<DHHQK#OPPrLc [R"[R"U[R"[R "S5[ R555$)Nr)r,rrrr rrrrs rJexp2OpDecompositions.exp2s3wwswwq#,,txx{EMM"JKLLrLc [R"[R"U[R"S[R 555$r)r,raddrrrrs rJlog1pOpDecompositions.log1ps+wwswwq#,,q%++">?@@rLc [R"S[R5n[R"U[R "U[R "[R"U5555$r)r,rrrrrrnegrones rJsigmoidOpDecompositions.sigmoidsCll1ekk*{{3SWWSWWQZ-@ ABBrLcv[R"U[R"S[R55$Nr)r,rrrrrs rJreluOpDecompositions.relus"{{1cll1ekk:;;rLcX[R"[R"X5U5$r[)r,rrryzs rJfmaOpDecompositions.fmaswwswwq}a((rLcX[R"[R"U5U5$r[)r,rwfloorrVrxs rJ floor_to_intOpDecompositions.floor_to_int||CIIaL%00rLcX[R"[R"U5U5$r[)r,rwceilr(s rJ ceil_to_intOpDecompositions.ceil_to_ints||CHHQK//rLcX[R"[R"U5U5$r[)r,rwtruncr(s rJ trunc_to_intOpDecompositions.trunc_to_intr+rLc [R"X5n[R"[R"U[R"S[ R 55[R"[R"U5[R"U555n[R"U[R"X!5U5$r) r,modand_nerrrsignbitwherer)rVrWrconds rJ remainderOpDecompositions.remaindersu GGAMxx FF1cll1ekk2 3 FF3;;q>3;;q> 2 yyswwq}a00rLcX[R"[R"U5U5$r[)r,rwroundr(s rJ round_to_intOpDecompositions.round_to_intr+rLr`N)rOpVarTrarBrrBrarB)rrBr"rBr#rBrarB)rVrBrxrwrarBrVrBrWrBrarB)rcrdrerfrrhrrrrrrr r rrrrr$r)r.r2r<r@rir`rLrJrrs}<<CC<<CCRRQQMMAACC<<))1100111111rLrz[a-z0-9_.]+|\([^)]*\)|)flagscUSS:wd[U5S:agSn[USS5H8up#US:XaUS- nO US:XaUS-nUS:XdM$U[U5S- :wdM8 g US:Xdeg)Nr(rFr>rRT)r enumerate)stringroichars rJ_all_in_parensrLs ayC3v;? EVABZ( 3; QJE S[ QJE A:!s6{Q. ) A:: rLc\rSrSr\S)Sj5r\S*Sj5r\S+Sj5r\S+Sj5r\S+Sj5r \S+Sj5r \S+Sj5r \S+S j5r \S+S j5r \S+S j5r\S,S j5r\S-S j5r\S-Sj5r\S-Sj5r\S-Sj5r\S-Sj5r\S.Sj5r\S/Sj5rS0S1SjjrS2SjrS3SjrS4S5SjjrS6SjrS7SjrS8SjrS9SjrS:S;SjjrSS#jr$S?S$jr%\S@S%j5r&\'SAS&j5r(\'SBS'j5r)S(r*g)C OpOverridesic[U[5(d*[RU5(d[ U5(aU$SUS3$NrGrR)r/ CSEVariable_RE_PAREN_NOT_NEEDED fullmatchrL)rIs rJparenOpOverrides.parensB v{ + +#--f55f%%M6(!}rLc[U5$r[)repr)rrxs rJrOpOverrides.constant s E{rLc [R"S[R5n[R"U[R "U[R "[R"U5555$r)r,rrrrr libdevice_exprrs rJlibdevice_sigmoidOpOverrides.libdevice_sigmoidsEll1ekk*{{3S->->swwqz-J KLLrLc.[R"U5$r[)r,absrs rJ libdevice_absOpOverrides.libdevice_abswwqzrLc.[R"U5$r[)r,sqrtrs rJlibdevice_sqrtOpOverrides.libdevice_sqrtsxx{rLc.[R"U5$r[)r,cosrs rJ libdevice_cosOpOverrides.libdevice_cosrarLc.[R"U5$r[)r,sinrs rJ libdevice_sinOpOverrides.libdevice_sin!rarLc.[R"U5$r[)r,rrs rJ libdevice_logOpOverrides.libdevice_log%rarLc.[R"U5$r[)r,rrs rJrZOpOverrides.libdevice_exp)rarLc2S[RU53$)N~rNrTrs rJ bitwise_notOpOverrides.bitwise_not-s;$$Q'())rLc2[RU5S3$)Nz == 0ru)rVs rJ logical_notOpOverrides.logical_not1s##A&'u--rLc\[RU5S[RU53$)Nz & rurr"s rJ bitwise_andOpOverrides.bitwise_and5+##A&'s;+<+> rur|s rJbitwise_right_shiftOpOverrides.bitwise_right_shiftErrLc.[R"X5$r[)r,rrUs rJ int_truedivOpOverrides.int_truedivIs {{1  rLcX[R"U[R"U55$r[)r,rrInteger)rrs rJr~OpOverrides.load_seedPsxxemmF344rLTc*[[U55$r[)r(rs)rvarrcheckwrap_negs rJindirect_indexingOpOverrides.indirect_indexingTs"#c(++rLcD[[U5RS35e)Nz,: check_bounds should be handled by CSEProxyrTr0rcrrrloweruppers rJ check_boundsOpOverrides.check_bounds]s'"Dz""##O P  rLcD[[U5RS35e)Nz$: load should be handled by CSEProxyrrrrhs rJrOpOverrides.loadds%!Dz""##G H  rLNcD[[U5RS35e)Nz%: store should be handled by CSEProxyrrrrhrmodes rJrOpOverrides.storeis'"Dz""##H I  rLcD[[U5RS35e)Nz/: store_reduction should be handled by CSEProxyrrrrhrs rJrOpOverrides.store_reductionps%!Dz""##R S  rLcD[[U5RS35e)Nz): reduction should be handled by CSEProxyrrrx src_dtypereduction_typers rJrOpOverrides.reductionus'"Dz""##L M  rLcD[[U5RS35e)Nz$: scan should be handled by CSEProxyrrdtypes combine_fnvaluess rJscanOpOverrides.scans'"Dz""##G H  rLcD[[U5RS35e)Nz$: sort should be handled by CSEProxyrrrrstable descendings rJsortOpOverrides.sorts'"Dz""##G H  rLcD[[U5RS35e)Nz): bucketize should be handled by CSEProxyrrr boundariesboundary_indicesindexing_dtyperightsortersorter_indicess rJ bucketizeOpOverrides.bucketizes'"Dz""##L M  rLcD[[U5RS35e)Nz2: halide_clamp only implemented for Halide backendr)rrrrs rJ halide_clampOpOverrides.halide_clamps%!Dz""##U V  rLr>) constraintsrxis_purepackcD[[U5RS35e)Nz<: inline_asm_elementwise only implemented for Triton backendr)rasmrrxrrrs rJinline_asm_elementwise"OpOverrides.inline_asm_elementwises'"Dz""##_ `  rLcD[[U5RS35e)Nz.: ops.output should not appear at codegen timeAssertionErrorr0rc)rrs rJrOpOverrides.outputs%Dz""##Q R  rLcD[[U5RS35e)Nz3: ops.placeholder should not appear at codegen timerrrhs rJrOpOverrides.placeholders%Dz""##V W  rLc4^SU4SjjnTUlSUlU$)NcJ>[[U5RST35e)Nz does not implement ops.r)rrrrs rJ unimplemented1OpOverrides._unimplemented..unimplementeds*%:&&''?vF rLT)rrNrrrrrarB)rcis_unimplemented)rrs` rJ_unimplementedOpOverrides._unimplementeds   "& )- &rLc[XS5n[[US5nU(+=(d X#:H=(d [USS5$)NrF)getattrr-)rrfn default_fns rJ_is_unimplementedOpOverrides._is_unimplementeds; S %Zt4 vS)SWR9KU-SSrLc`US;dU5e[R5Hup#[X15nUc5URU5(a[ XUR U55 MFMHX R ;dSUSUR35eX$l[ X[U55 M g)N)rIrGcppvecrHrTzmultiple definitions of z on ) pointwise_overrides_datarrrsetattrr__dict__rcrh)rrfuncnamedataimpls rJ_initialize_pointwise_overrides+OpOverrides._initialize_pointwise_overridessEEMvME6<<>NH4(D|((22C3+=+=h+GH3 ||3.xjS\\NK3!) |D'9:?rLr`)rIrBrarB)rzUnion[bool, float, int]rxrwrarBrC)rVrBrarB)rrBr"rBrarBrD)rrsrrBrarBTT) rrBrUnion[sympy.Expr, int]rrbrrbra sympy.Symbol rrnrrnrrbrrbrar)rrsrhrnrarBr[) rrsrhrnrrBrr0rar)rrsrhrnrrBrar) rxrwrrwrr/r!Union[OpVarT, tuple[OpVarT, ...]]rar)rtuple[torch.dtype, ...]rzFCallable[[tuple[OpVarT, ...], tuple[OpVarT, ...]], tuple[OpVarT, ...]]rtuple[OpVarT, ...]rar) rrrrrrbrrbrarNN)rrBr.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr]rrBrrwrrbr Optional[tuple[str, sympy.Expr]]rzOptional[OpVarT]rarB)rrBrrnrrbrarB)rrBrrsrrrxrwrrbrrrarB)rrBrar)rhrrarB)rrsrazCallable[..., OpVarT]rrsrarb)rrsrar)+rcrdrerfrhrTrr[r_rdrhrlrorZrvryr}rrrrrr~rrrrrrrrrrrrrrrrrrrrir`rLrJrNrNsMM**..BBBBBBCCCC!! 55 , ,%, ,  ,  ,  &0 9= FJ   NR  * 39 AJ        &  1  +    '     #       '  #        $48+/    C  !  $    1  )      &*"]]      #             TT  ; ;rLrNc\rSrSr%S\S'S\S'SrS\S'SrS\S '\Rr S \S 'Sr S\S 'Sr S\S 'Sr g) OverridesDatairsrzCallable[..., str]rGNzOptional[Callable[..., str]]rIrrtype_promotion_kindrHrTr`) rcrdrerfrrIrrDEFAULTrrHrTrir`rLrJrrsQ I +/F (/+/F (/'//8,0F (/(,C %,rLrairy_aicSUS3$)Nzairy_ai_forward(rRr`rs rJrMrMs(1-rLspecial_airy_ai)rrGr bessel_j0cSUS3$)Nzbessel_j0_forward(rRr`rs rJrMrM*1#Q/rLcSUS3$)Nz libdevice.j0(rRr`rs rJrMrM =1-rLspecial_bessel_j0)rrGrIr bessel_j1cSUS3$)Nzbessel_j1_forward(rRr`rs rJrMrMrrLcSUS3$)Nz libdevice.j1(rRr`rs rJrMrMrrLspecial_bessel_j1 bessel_y0cSUS3$)Nzbessel_y0_forward(rRr`rs rJrMrM rrLcSUS3$)Nz libdevice.y0(rRr`rs rJrMrM rrLspecial_bessel_y0 bessel_y1cSUS3$)Nzbessel_y1_forward(rRr`rs rJrMrMrrLcSUS3$)Nz libdevice.y1(rRr`rs rJrMrMrrLspecial_bessel_y1digammacSUS3$)Nz calc_digamma(rRr`rs rJrMrMs aS*rLc US3$)Nz .digamma()r`rs rJrMrMs A3j)rL)rrGrrrcSUS3$)Nz calc_erfcx(rRr`rs rJrMrM A3a(rLcSUS3$)Nzlibdevice.erfcx(rRr`rs rJrMrMs+A3a0rL special_erfcxr$cSUSUSUS3$)Nz std::fma(rQrRr`r!s rJrMrM#sis"QCr!A6rLcSUSUSUS3$)Nzfmadd(rQrRr`r!s rJrMrM$ss"QCr!A6rLcSUSUSUS3$)Nzlibdevice.fma(rQrRr`r!s rJrMrM%ss"QCr!A>rL)rrGrrIrigammacSUSUS3$Nz calc_igamma(rQrRr`r|s rJrMrM+<s"QCq1rLigammaccSUSUS3$Nz calc_igammac(rQrRr`r|s rJrMrM0=2aS2rLgammainccSUSUS3$rr`r|s rJrMrM5rrLspecial_gammainc gammaincccSUSUS3$r"r`r|s rJrMrM:r#rLspecial_gammaincci0cSUS3$)Nzcalc_i0(rRr`rs rJrMrM? 1orLcSUS3$Nzlibdevice.cyl_bessel_i0(rRr`rs rJrMrM@3A3a8rLc US3$)Nz.i0()r`rs rJrMrMAs A3erL)rrGrIrri0ecSUS3$)Nz calc_i0e(rRr`rs rJrMrMF !A&rLc US3$)Nz.i0e()r`rs rJrMrMGs A3frL special_i0ei1cSUS3$)Nzcalc_i1(rRr`rs rJrMrMLr,rLcSUS3$Nzlibdevice.cyl_bessel_i1(rRr`rs rJrMrMMr/rL special_i1i1ecSUS3$)Nz calc_i1e(rRr`rs rJrMrMRr3rL special_i1elog_ndtrcSUS3$)Nzcalc_log_ndtr(rRr`rs rJrMrMWs qc+rLspecial_log_ndtrmodified_bessel_i0cSUS3$)Nzmodified_bessel_i0_forward(rRr`rs rJrMrM]3A3a8rLcSUS3$r.r`rs rJrMrM^r/rLspecial_modified_bessel_i0modified_bessel_i1cSUS3$)Nzmodified_bessel_i1_forward(rRr`rs rJrMrMcrCrLcSUS3$r9r`rs rJrMrMdr/rLspecial_modified_bessel_i1modified_bessel_k0cSUS3$)Nzmodified_bessel_k0_forward(rRr`rs rJrMrMirCrLspecial_modified_bessel_k0modified_bessel_k1cSUS3$)Nzmodified_bessel_k1_forward(rRr`rs rJrMrMnrCrLspecial_modified_bessel_k1ndtrcSUS3$)Nz calc_ndtr(rRr`rs rJrMrMts 1#Q'rL special_ndtrndtricSUS3$)Nz calc_ndtri(rRr`rs rJrMrMyrrL special_ndtri polygammacUSUSUSUS3$)Nz == 0 ? calc_digamma(z) : calc_polygamma(rQrRr`r|s rJrMrM~s A33A36I!BqcQRSrLscaled_modified_bessel_k0cSUS3$)Nz"scaled_modified_bessel_k0_forward(rRr`rs rJrMrM:1#Q?rL!special_scaled_modified_bessel_k0scaled_modified_bessel_k1cSUS3$)Nz"scaled_modified_bessel_k1_forward(rRr`rs rJrMrMrZrL!special_scaled_modified_bessel_k1spherical_bessel_j0cSUS3$)Nzspherical_bessel_j0_forward(rRr`rs rJrMrMs4QCq9rLspecial_spherical_bessel_j0zetacSUSUS3$)Nzzeta(rQrRr`r|s rJrMrMs52aS*rL special_zetachebyshev_polynomial_tcSUSUS3$)Nzchebyshev_polynomial_t_forward(rQrRr`r|s rJrMrM:1#Rs!DrLspecial_chebyshev_polynomial_tchebyshev_polynomial_ucSUSUS3$)Nzchebyshev_polynomial_u_forward(rQrRr`r|s rJrMrMrgrLspecial_chebyshev_polynomial_uchebyshev_polynomial_vcSUSUS3$)Nzchebyshev_polynomial_v_forward(rQrRr`r|s rJrMrMrgrLspecial_chebyshev_polynomial_vchebyshev_polynomial_wcSUSUS3$)Nzchebyshev_polynomial_w_forward(rQrRr`r|s rJrMrMrgrLspecial_chebyshev_polynomial_wlegendre_polynomial_pcSUSUS3$)Nzlegendre_polynomial_p_forward(rQrRr`r|s rJrMrM9!BqcCrLspecial_legendre_polynomial_pshifted_chebyshev_polynomial_tcSUSUS3$)Nz'shifted_chebyshev_polynomial_t_forward(rQrRr`r|s rJrMrMB1#Rs!LrL&special_shifted_chebyshev_polynomial_tshifted_chebyshev_polynomial_ucSUSUS3$)Nz'shifted_chebyshev_polynomial_u_forward(rQrRr`r|s rJrMrMrxrL&special_shifted_chebyshev_polynomial_ushifted_chebyshev_polynomial_vcSUSUS3$)Nz'shifted_chebyshev_polynomial_v_forward(rQrRr`r|s rJrMrMrxrL&special_shifted_chebyshev_polynomial_vshifted_chebyshev_polynomial_wcSUSUS3$)Nz'shifted_chebyshev_polynomial_w_forward(rQrRr`r|s rJrMrMrxrL&special_shifted_chebyshev_polynomial_whermite_polynomial_hcSUSUS3$)Nzhermite_polynomial_h_forward(rQrRr`r|s rJrMrMs82aSBrLspecial_hermite_polynomial_hhermite_polynomial_hecSUSUS3$)Nzhermite_polynomial_he_forward(rQrRr`r|s rJrMrMrtrLspecial_hermite_polynomial_helaguerre_polynomial_lcSUSUS3$)Nzlaguerre_polynomial_l_forward(rQrRr`r|s rJrMrMrtrLspecial_laguerre_polynomial_lzdict[str, OverridesData]rc^[U4Sj[RR[RR[RR [RR 455$)Nc3.># UH nTU;v M g7fr[r`)rrrs rJr$is_buffer_removed..s  A   s)anyr1r{removed_bufferskernelinplaced_to_removers`rJis_buffer_removedrsU  GG # # HH $ $ GG & & HH ' '   rLcD^\rSrSrSrSU4SjjrSSjrS SjrSrU=r $) DeferredLineizHA line that can be 'unwritten' by adding name to V.graph.removed_buffersc^>[TU]U5 Xl[U[5(aegr[)rrrr/r")rrliners rJrDeferredLine.__init__s,  d$455555rLcP[UR5(d UR$gr[)rrrrs rJ__call__DeferredLine.__call__s ++99 rLc.[URU5$r[)rr)rrs rJ _new_lineDeferredLine._new_linesDIIt,,rLr)rrsrrsr)rrsrar) rcrdrerfrrrrrirrs@rJrrsR6  --rLrc"\rSrSrSSSjjrSrg) BracesBuffericL^^[RSUU4Sjj5nU"5$)Nc3># [T5H)nTRS5 T=RS- slM+ [T*5H)nT=RS-slTRS5 M+ Sv [T*5H)nTRS5 T=RS- slM+ [T5H)nT=RS-slTRS5 M+ g7f)N{r>})range writeline_indent)_rrs rJctx BracesBuffer.indent..ctxs6]s# ! #F7^ ! s#$ F7^s# ! $6] ! s##sC(C+)raIterator[None]) contextlibcontextmanager)rrrs`` rJindentBracesBuffer.indents$  " " $ # $u rLr`N)r>)rrraz'contextlib.AbstractContextManager[None])rcrdrerfrrir`rLrJrrs rLrc*\rSrSr%S\S'S\S'Srg)InplacedBufferi rsrvr other_namesr`Nrr`rLrJrr s OrLrc8\rSrSr%S\S'SrS\S'S SjrSrg ) ArgNameirsrFrb is_constexprcLURUR(aS3$S3$)Nz : tl.constexprrr)rrrs rJ full_nameArgName.full_names*))$2C2C.LMMLMMrLr`Nr)rcrdrerfrrrrir`rLrJrrs IL$NrLrc\rSrSrSSjrSrg) RemovedArgicg)NREMOVEDr`rs rJ__str__RemovedArg.__str__srLr`Nr)rcrdrerfrrir`rLrJrrsrLrc\rSrSr\SSj5rSSjrSSjr\SSj5rSSjr SSjr SSjr SS jr SS jr S S jrS!S jrS"S jrS#SjrS$SjrS%SjrS&SjrS'SjrS(SjrS)SjrS*SjrSrg)+ KernelArgsi cURU[5n[U[5(aU[ U53=X'nU$U$r[)getrr/rr)r}odictrresult new_results rJ_lookupKernelArgs._lookup!sE */4)A fj ) )*0#e*'> >EK*  rLcJ0Ul0Ul0Ul0Ul/Ulgr[) input_buffersoutput_buffersinplace_buffersrworkspace_argsrs rJrKernelArgs.__init__-s)-/ACMO/1 24rLc SRSR[[URUR UR UR/555$)NzKernelArgs({})rQ)formatrmaprWrrrrrs rJ__repr__KernelArgs.__repr__4sS&& II**++,,    rLc"[U[5$r[)r/rrs rJ_buffer_is_marked_removed$KernelArgs._buffer_is_marked_removedCs$ ++rLcT[RR(a3[RRRR X5nU[RR ;dU5eXR ;a[[UR U5$XR;a'[[URU5R$URS5(aURSURU5$URSURU5$)Nseedin_ptr)r1r{rmutation_real_namerrrr rsrrrvrrrrs rJinputKernelArgs.inputHs 77  77$$77;;DGD1772228D82 && &T0067 7 '' '(<(R!R  /01)C03  ,03  -! s E*EE+EcF[U[RU5[RR 5[R 5S9n[UR5HupE[RXS5(a>URn[RXS5URU'URU4s $URUR:waURUR:waMe URRU5 URS4$)a Allocate or extend a workspace buffer of nbytes bytes. This function manages the allocation of a workspace buffer. It either creates a new WorkspaceArg or extends an existing one. Note: - Calling this function will in-place mutate the args by adding or updating a WorkspaceArg. - The codegen for generating the Python argdefs and call_defs will check this field and allocate the buffer accordingly. - A new argument "ws_ptr" will be present in the generated code. Args: nbytes (sympy.Expr): The number of bytes to allocate. zero_fill (bool): Whether to initialize the buffer to zero. Returns: Tuple[str, int]: A tuple containing: - "ws_ptr": A string identifier for the workspace pointer. - offset: An integer representing the byte offset in the workspace. )rorprrrtr)rlrNr^r1r{get_current_device_or_throwr~rHrrrorrvrtr)rnbytesr]argrJ existing_argrs rJ workspaceKernelArgs.workspacevs.'11)<77668#//1   ))<)<=OA$$\77%++)5):):<)M##A&#..66''3>>9 ++s~~= > > ""3'~~q  rLc [RR5n[U[R [ RSSURSUR3US9nURH&nURUR:XdMX4:XaM&e URRU5 UR$)a Lazily allocate a graph-wide semaphores buffer with at least min_size. This is a single buffer shared by all kernels and zero initialized once at graph start. Each kernel must leave the buffer zeroed on exit. Warning: multiple calls to this function will return the same buffer. Args: min_size: the number of int32 semaphores required Returns: name of the semaphores buffer sem_ptr semaphores_r)rorprxrvrtrr) r1r{rrlrNrgruint32r0rhrrvr)rmin_sizecurrent_devicerrs rJ semaphoresKernelArgs.semaphoress<<>'66,, $^%8%8$9>;O;O:PQ!  !//L&&#..8***0 ""3'~~rLcz^[U[5(d[U5U45e[R"U5nX R ;aUR U$TUR R 5;a1T[U4SjUR R 5553mTUR U'T$)Nc3V># UHoRT5(dMSv M g7f)r>N)r)rrrs rJr)KernelArgs.seed_offset..sU(>1,,tBTQQ(>s) ))r/rr0rrrrsum)rrrs ` rJ seed_offsetKernelArgs.seed_offsets%%%;U U';;% e$ MM !==' ' 4=='') )&U (<(<(>UUVW  $ e rLc[U[R5(d[U5U45eURS:XaSUR U'gUR SUR U5$)Nrks)r/rSymbolr0rrrrs rJrKernelArgs.sizes[$ --AT D/AA- 99 "(DMM$ ||D$--66rLc[URR5URR5URR55$r[)rrkeysrrrs rJ call_namesKernelArgs.call_namessA    # # %t':':'?'?'A4==CUCUCW  rLc&URRUS5nUb![U[5(d UR$UR RUS5nUb[U[5(dU$UR RUS5$)z+ Returns inner name of a given outer name. N)rrr/rrvrr)rrinplacedrs rJarg_nameKernelArgs.arg_names''++D$7   8Z(H(H&& &))--dD9  ":k:+N+N !!%%dD11rLcU$r[r`)rrrxs rJ wrap_ptr_argKernelArgs.wrap_ptr_args rLc[U5$r[)rs)rrs rJ wrap_size_argKernelArgs.wrap_size_args 4yrLc SSKJnJn /n/n/n[URR 55Hn[ U[5(aMURSnURn[RRU5n Xn URU SU35 URURXy55 URU S35 M URR!5HupxXpR;aM[RRU5n Xn URSU SU35 URURXy55 URSU S35 M UR"R!5Hup{XpR;d[ U [5(aM+[RRU5n Xn URU SU 35 URURXy55 URU S35 M UR$R!5HupxURSUSU35 URUR'U55 URSU35 [RR((dMq[RR(R+U5 M UR,(aS5eXCU4$) Nr>) DTYPE_TO_CPP INDEX_TYPEryz* *zconst  zWorkspace not supported on CPU ) cpp_utilsrrr+rrr/rrrvr1r{rrr rrrrr wrapper_codeensure_size_computedr) rrr call_argsarg_defs arg_typesr outerinnerrx cpp_dtype maybe_inners rJ cpp_argdefsKernelArgs.cpp_argdefss7  t33::<=H(J//((,E''EGG%%e,E$+I OOykE73 4   T..u< =    {!_ ->!..446LE,,,GG%%e,E$+I OOfYKr%9 :   T..u< =   vi[2 37#'"5"5";";"= E,,, ; 0S0SGG%%e,E$+I OOykK=9 :   T..u< =    {!_ -#>!MM//1LE OOfZL%9 :   T//6 7   vj\2 3ww###$$99%@ 2 &&I(II&I--rLc /n/n/n/n[URR55Hn[U[5(aMUR [ UR55 UR URS5 UR [RRURS55 UR [URURS[RRURS5S95 M [URR5UR R55HupgX`R;d[U[5(aM+UR [ U55 UR U5 UR [RRU55 UR [UU[RRU5S95 M UR"R5HupgUR [ U55 UR U5 UR [%U55 UR ['Xv55 [RR((dM[RR(R+U5 M UR,HnnUR [ UR55 UR UR.5 UR U5 UR UR05 Mp XXC4$)Nry)rrrx)r+rrr/rrrrvrr1r{rrrrrrrr0rrrrrtrx) rrrrprecompile_argsr rrrs rJpython_argdefsKernelArgs.python_argdefss#%! ') /1t33::<=H(J// OOGH$7$78 9   X11"5 6   QWW..x/C/CB/GH I  " "!,,#//3''++H,@,@,DE  >"    $ $ &(;(;(A(A(C LE,,, 5*0M0M OOGEN +   U #   QWW..u5 6  " " ''++E2  !MM//1LE OOGEN +   U #   T%[ )  " "75#8 9ww###$$99%@ 2&&C OOGCNN3 4   S^^ ,  " "3 '   SYY ' ' O>>rLc## [URR55Hn[U[5(aMUR HnU[ RR;dU[ RR;aMAX R;aURUUR4v X R;dM~[[URU5UR4v M M g7fr[)r+rrr/rrr1r{rrrrvrr rs)rr others rJaliasesKernelArgs.aliases@st33::<=H(J//!--QWW777 ; ;;...,,U3X5H5HHH///sD$7$7$>?ATATTT.>s CC> 4C>c[URRU[5[5=(a. [UR RU[5[5$r[)r/rrrrrrs rJ is_removedKernelArgs.is_removedOsK    # #D' 2J N--11$@*M NrLc[5n[URR55H8n[ U[ 5(aMUR URS5 M: URR5H<up4X0R;d[ U[ 5(aM+UR U5 M> U$)Nry) rr+rrr/rrrrr)r live_outsr rrs rJlive_output_buffersKernelArgs.live_output_buffersWsL t33::<=H(J// MM(..r2 3>!//557LE,,, 5*0M0M MM% 8rL)rrrrrN)r}rsrz6Union[dict[_T, Union[str, RemovedArg]], dict[_T, str]]rrArarsrarr)rrrarbr)rrsrrsrar)rrnr]rbraztuple[str, int])rrnrars)rrsrrrars)rrrars)raz Iterator[str])rrsrar)rrsrxrwrars)r SymbolLikerars)raz&tuple[list[str], list[str], list[str]])raz?tuple[list[ArgName], list[str], list[KernelArgType], list[Any]])razIterator[tuple[str, str]]r)razOrderedSet[str])rcrdrerfrhrrrrrrrrrrrrr r rr!r%r)r,r0rir`rLrJrr s  E    5  ,, @B44'!R8 7 2'.R/? H/?b UN rLrcr^\rSrSrSrS S U4SjjjrS SjrS SjrSSjrSSjr S Sjr S r U=r $)rQida4A CSEVariable is just a name for an expression but it is useful to be able to annotate them on a backend dependent basis. To do so, the backends can simply overload `Kernel.create_cse_var` The "CSEVariable.update_on_args" method gives you a hook for annotations See example of TritonCSEVariable in triton.py c>[TU]5 [U[5(deXlX lSUlX0lgr)rrr/rrbounds use_countrx)rrr6rxrs rJrCSEVariable.__init__ks9 &+....   rLcUR$r[rrs rJrCSEVariable.__str__xs yyrLc,[UR5$r[)hashrrs rJ__hash__CSEVariable.__hash__{sDIIrLcb[U[5=(a URUR:H$r[)r/rQr)rr(s rJ__eq__CSEVariable.__eq__~s!%-I%** 2IIrLcgr[r`)rrrrs rJupdate_on_argsCSEVariable.update_on_argss rLcPURRSUR<S3$rP)rrcrrs rJrCSEVariable.__repr__s$..))*!DII=::rL)r6rxrr7r[)rrsr6ValueRanges[Any]rxrr)rar)r(objectrarb)rrsrrrrrar) rcrdrerfrrrr=r@rCrrirrs@rJrQrQdsR(,  ! %  J ;;rLrQ AugmentedKeyT)defaultCSEVariableType)boundrJ.cf\rSrSrSrSSSjjrSSjrSSjrSSjrSSjr SS jr SS jr SS jr SS jr \R"5S S SS.SSjjr\R"5S4SSjjr\R"5S4SSjjrSrg)CSEiz Common subexpression eliminationNcXlX l0UlX0lU=(d 0UlU=(d 0UlU=(d [ R"5Ul[5Ul U=(d 0Ul gr[) r}suffix_cache name_prefix store_cachereduction_cache itertoolsroiter_buffer_idsrinvalidated_stores varname_map)rr}rPrR iter_buffersrSrTrXs rJr CSE.__init__sg  FH &ALARPR  !r 6B5VY__EV3=<7B7HbrLcL/URR5QH4up#X1;dM URU URRU5 M6 U(a<URR5VVs0sHupEXQ;dM XE_M snnUlg0Ulgs snnfr[)rSrrWrrQ)r keep_varsrtmprrs rJ invalidateCSE.invalidates44++1134ID#$$T*''++D15 ,0KK,=,=,?R,?DA1>414,?RDKDKSs 9 B B c [U5"URURURURUR UR URS9$)N)r}rPrRrYrSrXrT)r0r}rPrRrVrSrXrTrs rJclone CSE.clonesNDz;;;;((--(((( 00  rLcUR5n[UR5Ul[UR5Ul[UR5UlU$)zNReturn a copy of using ScopedDict so changes to *_cache aren't visible in self)rar&rQrTrS)rnew_cses rJ scoped_copyCSE.scoped_copysH**,#DKK0",T-A-A"B()9)9:rLc"[[U5$)z@Override this method to augment cache key with backend specifics)r rIr cache_keys rJ augment_keyCSE.augment_keysM9--rLc>X RURU5'gr[rQrj)rrirs rJputCSE.puts36 D$$Y/0rLc>URU5UR;$r[)rjrQrhs rJcontains CSE.containss *dkk99rLcXURRURU5S5$r[)rQrrjrhs rJtry_get CSE.try_gets"{{t// :DAArLc>URURU5$r[rmrhs rJrCSE.gets{{4++I677rLT)r6write assignmentrxc B[U[5(a URnU(d U(de[U[5(aEURR U5UlU=R S- sl[[U5$[U[5(aUR5nO;[U[5(a URnO[U[5(deUnURU5nU(GdURX65nUR!Xx5 U(Ga["R$R&(a(["R$R&R)USS9 [U[5(aUU(a UR+UR,US35 UR/U5 UR+UR05 U$[U[5(aRU(deUR+UR3UR,USURUR0355 U$U(a UR,USUUR03n OUUR03n UR+U 5 U(aQ[4R6R8(a2Ub/SUR,US[;U5S3n UR+U 5 U$URR U5UlU=R S- slU$) Nr>T) only_oncez =z = tl.static_assert( .dtype == rR)r/r.rrQr6tightenr7r rKr$getvaluer"rrsrtnewvarrnr1r current_nodecodegen_originating_inforr}splicerPrr test_configsruntime_triton_dtype_assertr*) rrrr6rxryrxrirr assert_lines rJgenerate CSE.generatesz dH % %::D "" dK ( (++--f5DK NNa N. . n - - I . / / IdC(( ((Ill9%++f,C HHY $88((HH))BB$CdN33!((DKK=R)@AMM$'$$T[[12 1 &677%%:$$$++se3tyyk$++'WX, %""&++se3tfT[[MJ"& }5$$T*#"//KK!-(9$++se:VabgVhUiij&k ((5  ++F3CJ MMQ M rLcUR[UR53n[RR X1U5nX@R U'U$r[)rRrzrVr1rcreate_cse_varrX)rr6rxvar_namers rJr CSE.newvarsN &&'T-A-A(B'CDhh%%h>%(" rLc^[R"TUR;U4Sj5 [RR TX#5nX@RT'U$)Nc>ST3$)Nzduplicate name: r`rsrJrMCSE.namedvar..0s 4DTF2KrL)r _check_valuerXr1rr)rrr6rxrs ` rJnamedvar CSE.namedvar)sO  (( (*K hh%%dF:!$ rL) rQrWrVrRr}rTrSrPrX)rrrrr]NNNN)r}rsrPrsrRrsrYzOptional[itertools.count[int]]rSz.Optional[MutableMapping[str, CSEVariableType]]rTz[TU]5 [R"5Ulgr[)rrr ExitStack exit_stackrrs rJrCodeGen.__init__8s $..0rLc:URR5 U$r[)r __enter__rs rJrCodeGen.__enter__<s !!# rLc<URRXU5 gr[)r__exit__)rexc_typeexc_valexc_tbs rJrCodeGen.__exit__@s   F;rL)rr2rrrrrrrrar) rcrdrerfrrrrirrs@rJrr7s1<[TU]5 U(a[=RS- slU=(d [ 5Ul[ 5Ul[ 5Ul[ 5Ul SUl SUl [URUR5Ul[ [""5Ul[ [""5UlSUlSUlSUlSUl[ [""5Ul[ [""5Ul0UlSUlSUlg)Nr>r)rrrgenerated_kernel_countrrr$loadscomputestoresnum_load num_reductionrNrrPcserrsmust_keep_buffersstore_buffer_names _load_mask _load_otherrnode_to_boundsrrinplace_update_buffersmin_elem_per_thread kernel_name)rrincrease_kernel_countrs rJrKernel.__init__Is   * *a / *(JL #% %' $&  .1$2D2Ddkk.R!+C!2",S/"3)-4859OS)#0",S/"3 79##$ *.rLc## URnXlURR5R5UlSv X lg!X lf=f7fr[)rrr6 get_boundsr)rrpriors rJset_current_nodeKernel.set_current_nodeksH!! "jj//1<<> &  %  sAAAAAAc## UcUnUSL=n(a [5nURnURnURnURnXlX lX0lUR 5UlSv XPlX`lXplXlU(aU(aS5egg!XPlX`lXplXlU(aU(aS5eff=f7f)Nz$unexpected store inside swap_buffers)r$rrrrre) rlbcbsbdisallow_storesrrrrs rJ swap_buffersKernel.swap_buffersus :B Dj (? (!B ,,hh   ??$ F J"L KHEEEv2 J"L KHEEEv2sA2C5B(9/C(0CCc[er[rrs rJr Kernel.loadrrLcURnURUlURX5X0l$!X0lf=f)z+A load the depends on an index we have read)rrr)rrrhrs rJ indirect_loadKernel.indirect_loads2  DJ99T)JJs!6>c[er[rrs rJrKernel.store_reductionrrLc[er[rrs rJr Kernel.store "!rLc[er[rrs rJrKernel.reduction "!rLc[er[rrs rJr Kernel.scans "!rLc[er[rrs rJr Kernel.sortrrLc[er[rrs rJ var_rangesKernel.var_rangesrrLc[e)z# See [Note: Inductor bucketize op] rrs rJrKernel.bucketizes "!rLc[er[rrs rJassert_functionKernel.assert_functionrrLc [U[5(a [U5n[U[5(deUb[U[5(deUb[U[5(deU(a!U(aSUSUSUSUS3 nUSUSU3nO#U(a USU3nUnOU(deUSU3nUnU(a SUSUS3nURSUSUS3$) NrGz <= z) & (z < rRz) | ~(z, "index out of bounds: z"))r/rQrsr)rrrrmaskr; cond_prints rJindirect_assertKernel.indirect_asserts c; ' 'c(C#s####} 5# 6 666} 5# 6 666 UugT#eC5E7![TU]5 UR(deURR [ R "[XR5555 URR [ R"U55 U$r[) rrrr enter_contextr1set_ops_handlerCSEProxyset_kernel_handlerrs rJrKernel.__enter__sj ~~~ %%  ht^^-=> ?  %%a&:&:4&@A rLcF>UR5 [TU] XU5 gr[)remove_kernel_local_buffersrr)rrrrrs rJrKernel.__exit__s ((* F3rLc^^[RRmT(dg[U4SjUR55n[[ "5mURHXnX R ;dMX RR;dM/TRX!5(dMGTRU5 MZ THnX RR;aURRUn[U[5(aML[U4SjUR55nU(aUR!U5 UR"RU5 MUR%U5 M g)z Any buffers that are both created and have a last use in the same kernel can be removed. Note that V.graph.scheduler can be None when codegening triton template kernels. Nc3># UH4nUTR;dMTRUR5v M6 g7fr[) name_to_bufdefining_op_name)rrrs rJr5Kernel.remove_kernel_local_buffers..s?& .i+++ :I ! !# & 7 7 9 9.s?$?c3,># UH oT;v M g7fr[r`)rrnames_to_removes rJrr,sK?a/1?s)r1r{rrrrsrrr$can_buffer_be_removed_through_fusionrrr/rrrremove_inplace_bufferr remove_buffer)rfused_node_namesrrremoverrs @@rJr"Kernel.remove_kernel_local_buffers sGG%%  %& ..&   %S/+++D222 7 77BB ##D),$Dyy000ii//5c:..K3??KK..t4''++D1""4($rLc[RSU5 [URRU'UR R U5 g)Nzremove_buffer(%r))rrHrrrrrrs rJrKernel.remove_buffer3s; %t,)0   &   &rLc[RSU5 [URRU'UR R U5 g)Nzremoving_inplace_buffer(%r))rrHrrrrrrs rJrKernel.remove_inplace_buffer;s9 /6*1 !!$'   &rLc [U[[45(a!UVs/sHo RU5PM sn$[R R RU5n[URSS9nUVs0sH^n[U[R[R[R45(dMCX RR!U5_M` nn[#X5$s snfs snf)NcUR$r[r)ss rJrM(Kernel.rename_indexing..Hs!&&rL)r)r/listtuplerename_indexingr1r{rrsorted free_symbolsrr UNBACKED_INTSIZEPRECOMPUTED_SIZErrr))rrhrsorted_symbols replacementss rJr Kernel.rename_indexing@s edE] + +5:;U((+U; ;  ))%0 2 28HI$  #%%II)) !Ayy~~a #   %..<  sC1?AC6C6c[U0UD6$r[)rQ)rrrs rJrKernel.create_cse_varWsD+F++rLc\UcgURRUR55$)z3 Returns arg name of a given input or output node. N)rr r)rrs rJr Kernel.arg_nameZs' <yy!!$--/22rL)rrrrrrrrrrrrrrrrrr)NT)rzOptional[KernelArgs]rrbrar)rr=rarr)rr$rOptional[IndentedBuffer]rrrarrrsrhrnrarQrrsrhrnrrQrarr[ rrsrhrnrrQrr0rar rxrwrrwrr/r+Union[CSEVariable, tuple[CSEVariable, ...]]rarrrrzUCallable[[tuple[CSEVariable, ...], tuple[CSEVariable, ...]], tuple[CSEVariable, ...]]rtuple[CSEVariable, ...]rar rrrrrrbrrbrar)razdict[sympy.Symbol, sympy.Expr]rrQrrrrQrrwrrbrrrzOptional[CSEVariable]rarQr) rzUnion[CSEVariable, str]rrrrrz!Optional[Union[CSEVariable, str]]rarsr)rhrnrarsrrr2rrsrar)rhz;Union[list[sympy.Expr], tuple[sympy.Expr, ...], sympy.Expr]rarn)rrrrrarQ)rr8rar)%rcrdrerfrrrPrrrrrrrrrrrrrrrrrrrrrrrrrr rr rirrs@rJrrDsM3FC9=I6=PT /( /HL /  / /D&&(,'+ F F %F % F  FF8""SW"" *"3>"FO" " """& " ; " 5 ""'" " ( " !""'"(" "  " ! ""4804 " "C "& " $ "  "1 ". "  """37 V $VV V 0 V V<""&0"9="FJ" " "4%)N'' /P/ /.,33rLrc@\rSrSr%SrS\S'SrS\S'SrS \S 'S rg) ricrz ClassVar[str]rNrrxrrrsops_namer`) rcrdrerfrrrxr$rir`rLrJrrcs!"C"#'E 'HcrLrc^SSKnURURS9$![a gf=f)Nr) undefined)jinja2 EnvironmentStrictUndefined ImportError)r's rJ jinja2_envr+ks?!!,,"   s  ,,c\rSrSrSr\S S Sjj5r\S Sj5r\SSj5rSSjr SSjr SSjr S r g )KernelTemplateiwz[ Base class for defining kernel templates. Children classes: TritonTemplate, CUDATemplate cURS5n[U5S:a"USSVs/sHnSU-U-U-PM snUSS&SRU5$s snf)NTr>rrr) splitlinesrr)source num_indentsindents_spacinglinesrs rJindent_except_first"KernelTemplate.indent_except_first~sh!!$' u:>INqrIR&4<E!"Iwwu~sAc[5nUcg[RURS'SSKJn UR U5$!Uan"SSU5nU"U5UeSnAff=f)Nr4r)TemplateSyntaxErrorc6^\rSrSrSU4SjjrSSjrSrU=r$)IKernelTemplate._template_from_string..DetailedTemplateSyntaxErroric>[TU]URURURUR 5 Xlgr[)rrmessagelinenorfilenameoriginal_error)rr>rs rJrRKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__init__s<G$&..&--&++&//  +9'rLcXSURS3nUSURS3- n[URS5(aURRR S5nUS- n[ SURS- 5n[[U5URS-5n[X45HwnXPRS- :XaSXS-S X%S3- n[URS 5(a'US S URRS- --S -- nMfMhXS-SX%S3- nMy U$)NzError in template at line  zError message: r0z Context: rrr>z: --> columnz rz^ z: ) r<r;rr>r0splitmaxminrrrB)r error_infor3startendrJs rJrQKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__str__s:#=dkk]"!MJODLL>"DDJt22H== $ 3 3 : : @ @ F"l2 #At{{Q 7!#e*dkkAo>!&u!2A KK!O3 *QvehZr.J J #*4+>+>#I#I$.(/*-1D1D1K1Ka1O*P)Q*/)0%&J$J!+QvehZr.J J "3&%rL)r>)r>r7rarr)rcrdrerfrrrirrs@rJDetailedTemplateSyntaxErrorr9s9&&rLrJ)r+r-r4filtersr'r7 from_string)r0envr7erJs rJ_template_from_string$KernelTemplate._template_from_stringsjl ;-;-O-O )*.# 8??6* *"! 8 &.A &>.a0a 7C! 8sAA# AA#c0^^[RRm[U[[ 45(a0UVs0sH!oR 5UR5_M# snmO UR 5UR50mSUU4SjjnU$s snf)Nc@>TRU5nUbU$T"U5$r[)r)rr_get_dtype_reallookups rJr1KernelTemplate._fake_get_dtype..get_dtypes'ZZ%F! "4( (rL)rrsrarw)r1r{rr/r r r) fake_outsrrrSrTs @@rJ_fake_get_dtypeKernelTemplate._fake_get_dtypes|''++ i$ / /AJK#llncmmo5KF((*I,?,?,ABF ) ) Ls(BcXlgr[rrs rJrKernelTemplate.__init__s rLc URUR"S0UD65 g![aNn[R SU[ U5[R 5[R:S9 UsSnA$SnAff=f)z Maybe generates a new ChoiceCaller and appends it into existing choices. Returns None if success, otherwise returns the error. choices: A list of ChoiceCallers. kwargs: Additional kwargs to be passed to self.generate() to generate a new ChoiceCaller. Nz3Cannot Append Choice: %s. KernelTemplate type is %s) stack_infor`) rrrTrinfor0getEffectiveLevelrFINFO)rchoicesrrNs rJmaybe_append_choice"KernelTemplate.maybe_append_choicesl  NN4==262 3"  HHET 002W\\A   H s!$ A<AA71A<7A<c [e)z= Generates a ChoiceCaller instance from the given arguments. r)rrs rJrKernelTemplate.generates "!rLrN))r0rsr1rr2rrars)r0rsrar)rVzUnion[list[Buffer], Buffer]razCallable[[str], torch.dtype]r")r`z list[Any]rrrazOptional[NotImplementedError])rrrar6) rcrdrerfrrhr4rOrWrrarrir`rLrJr-r-ws >?"%8; *8*8X. %" ,/ &."rLr-cL^\rSrSrSrSU4SjjrSSjrSSjrSSSjjrSSjr SSjr SSjr SSS jjr SS jr SS jrSS jrSS jrSSSjjrSrU=r$) ric^>[TU]5 SSKJn U"5UlXlX lg)NrValueRangeAnalysis)rrr6ri vr_analysisrparent_handler)rrrkrirs rJrCSEProxy.__init__s' /-/ ,rLc^^^^^^^UR"T/TQ70TD6m[URT5"T0TD6m[5mSmSUUUUUUU4Sjjn[R "UT5$)Nrc>[RRb][RR5RnUS:Xa[ R S:HOUS:wa[ RS:HOSnOSnU(a&T S:Xa T RnO[TT 5"T0T D6nOSn[RRR[RRUTUS9n[ RR(acU(a\SSKJn [%U[&[(45(aUT n[RRR+S US U"U5S 35 T S - m UR-T TT 5 U$) NrFrIrTFmasked)r6rxr)r*r|r}rRr>)r1r{rrr0rrKrQrxrrrrrrrtorch._inductor.codegen.tritonr*r/r r rrC) r device_strtriton_backendrcsevarr*rr6 dtype_handlerrr output_idxrs rJdo_cse!CSEProxy._default..do_cseshww%%1WW@@BGG "U*&&(2"U* ,,8 "'8##(;;L#*%$$' &$'L $ XX\\**  " +F"">>>FlT5M::#/ #;L  **'xz+l:S9TTUV !OJ  ! !$f 5MrL)rrsrarQ)_bound_variablerrkrpytreetree_map) rrrrrvr6rtrurs ``` @@@@rJ_defaultCSEProxy._defaultsd%%d# UHoTR;v M g7fr[)r)rrfx_nodes rJr+CSEProxy._bound_variable..D sV0U1&0Us) set_indirectrrc[U[5(a UR$[U[R5(a [ U5$U$r[)r/rQr6rrrrs rJ arg_to_bound.CSEProxy._bound_variable..arg_to_boundM s8a--88O5::..&q>)HrL)rrrar)r6riselect_algorithmr~r/r1rrr interpreterrrrdictrrcompute_all_boundsrrr rrrj) rrrrrir~r arg_boundsrs @rJrxCSEProxy._bound_variable2 s 0; ahh 4 5 5&&( (--,, >>T !dkk&@&@&Ldkk88$?? ??;;--11';;N;N;PQ Q  & &73E+L+LV0UVVV"**,,  : c,56J4++T2J? ?""$$rLcV[U[5(a[R"U5n[U[R5(dU5eUR R S:GaU(a[R"U[R"U[R55nUR RS:a.[R"US5n[R"XeU5nOUn[R "5nUR [R "5:wa[U[R"5(aUR [[$*S5-n[UR U-URU-5nUR RS:a!UR [S[$5-n Xy-nUR&R(R+UR&R,XWS9nUR.R1XU5n [3U5(avUR R S:+n [U[R"5(+=(d UR RU:+n UR&R5XX5 U $)Nrry)r6)r/rrrrr6rr,rrxrlongrltr9rrNumberrrrrrrkrr#r) rrrrrstmr new_bounds neg_boundspos sympy_var assert_lower assert_uppers rJrCSEProxy.indirect_indexingY s dC ==&D$ ++1T1+ ::  a ggc3>>$ #CD::##q(QB))BS1C%,,.Jzz[0022z$ 7U7U!ZZ+vgr*BB ($$t+Z-=-=-D ::##q(**{1f'==C!+!1J++//**4;;+>+>*WC''99#UK 5 ! ! # 0 0A6L)$ ==   4BL KK $ $Yl QrLc:URRXX45$r[)rrrs rJrCSEProxy.check_bounds s{{''EAArLcXRRR;a)[RRR U5 [ U[R5(aURRX5$URRRnX;aX1$URRX5nURS:XaUR=RS- sl U$r)rrrWr1rrrrTMPrrSrr7r)rrrhrSouts rJr CSEProxy.load s ;;??55 5 HH & & * *4 0 udhh / /;;,,T9 9kkoo11  $ $kkt+ ==A  KK A %  rLczX RRRU'URR(a~U[R R ;a_URRRU5nUR5H%nX RRRU'M' gggr[) rrrSrr1r{name_to_buffer get_output get_mutations)rrrr other_names rJ_update_store_cacheCSEProxy._update_store_cache s,1 ##D) ;; # #0F0F(F++**55d;C!//1 :? ++J72)G #rLcURRRU5 UcURX5 U[R R ;aURRXX4S9$g)N)r)rrrrr1r{rrrs rJrCSEProxy.store s^ &&**40 <  $ $T 1 qww.. .;;$$T%$C CrLcURRRU5 URX5 U[R R ;aURRXU5$gr[)rrrrr1r{rrrs rJrCSEProxy.store_reduction sV &&**40   - qww.. .;;..tEB B /rLcxUR=RS- slURRXX45$r)rrrrs rJrCSEProxy.reduction s0 !!Q&!{{$$U~MMrLc:URRXU5$r[)rrrs rJr CSEProxy.scan s{{F;;rLc:URRXX45$r[)rrrs rJr CSEProxy.sort s{{CCrLc DURRUUUUUUU5$)a [Note: Inductor bucketize op] Inputs: ------- values: the values to be bucketized. boundaries: a tuple containing (a) the name of the boundaries tensor (which must be sorted, unless the sorting tensor is present), (b) the length of the tensor in the last dimension (i.e. the length of one set of boundaries), (c) the number of elements in the underlying storage (i.e. the length of the flattened tensor, ignoring striding), and (d) the stride of the tensor in the last dimension. boundary_indices: indices into a flattened version of the boundaries tensor, of the same size and shape as "values". Each index points to the first element in the set of boundaries to be used for the corresponding value. indexing_dtype: the dtype to use when indexing into the boundaries tensor. This must be int64 or int32. This additionally specifies the dtype of the return value. right: see "Details" below. sorter: an optional tuple containing (a) the name of an optional sorting tensor, used to access unsorted boundaries without reordering the boundaries tensor, and (b) the stride of the tensor in the last dimension. The values in the sorting tensor are used as indices into the *last* dimension of the boundaries tensor, with all other indices matching. The size of the sorting and boundaries tensors must be equivalent. sorter_indices: must be present if the sorting array is present; see "boundary_indices" for the equivalent definition for the boundaries tensor. Output: ------- The buckets each value belongs in, within a given set of boundaries. 0 indicates a position before the first boundary, and len(boundaries_set) represents a position after the last boundary. Details: -------- Given a value and a set of boundaries, calculate the bucket that each value belongs to. This works differently in 1-D and N-D cases. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [0, 4, 4, 8], right=True return = [[ 0, 1, 1, 1], [1, 3, 3, 4]]. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [[0, 4], [4, 8]], right=True return = [[ 0, 1, 1, 1], [0, 1, 1, 2]] Note that in the N-D boundaries case, the shape of "values" and "boundaries" must match in every dimension _except_ the last. When right == False, bucket i refers to range (boundaries[i], boundaries[i+1]]. When right == True, bucket i refers to range [boundaries[i], boundaries[i+1]). Boundaries must be non-decreasing, or a sorter must be provided which would re-index offsets in a non-decreasing order (e.g. the second output of torch.sort(offsets)). Otherwise, the result is undefined. )rrrs rJrCSEProxy.bucketize s1L{{$$         rL)rrkrj)rz Kernel[Any]rkzOpsHandler[Any])rrsrztuple[Any, ...]rzdict[str, Any]rar)rrsrrrrrarGr) rrQrrrrbrrbrarrr)rrsrrQrarr[rrrrr rr!)rcrdrerfrrr{rxrrrrrrrrrrrirrs@rJrrs D-;.z%%V / /%/ /  /  /bBB&0B9=BFJB B "@SW *3>FO CNNN& N ; N 5 N <' <  <( < ! <D'D(D D  D ! D 4804N N CN & N $ N  N 1N .N  N N rLr)rIrsrarr[) rrrsrrrrrrrar)rrUnion[torch.device, str, None]razOrderedSet[BackendFeature])rrrr6r"rarb)rrrsrazOptional[SchedulingConstructor])F)rrrsr:rbrarr2)rhSequence[sympy.Expr]rirrjrrar)rrrsrmrrar)rrrsrar)rrsrrrrrar)rIrsrarbr`r)rar) __future__rr dataclassesenumrrUrFr rretypingrrrrrr r r r r rrtyping_extensionsrrrtorch.fxtorch._prims_commonr torch.utilsrrytorch.utils._ordered_setrtorch.utils._sympy.numbersrtorch.utils._sympy.printersr_PythonPrintertorch.utils._sympy.symbolrrrtorch.utils._sympy.value_rangesrrrrrrdtype_propagationr ops_handlerrr utilsr!r"r#r$r%r&r'r(r)r*r+ virtualizedr,r-r.r/r0r1collections.abcr2r3r4rr5r6r7r8rr:rr;r<r=r]r@rArr0rrsrr3rB_logginggetArtifactLoggerrcrD getLoggerrrKrNrl dataclassrrrrr KernelArgTyperrrrr r"r2r7r1r< lru_cacher.rkrnrtbfloat16rfloat16rbrfloat64int8int16rrruint16ruint64rurrrcompile IGNORECASErRrLrNrr INT_TO_FLOATrrrrrrrrrrQrIrKr rxReductionCacheKeyrNrrrr+r-rrs0rJrs7 "      & ?)/-GOOD:;    QPBB>>$DD- B$hy&9%:N%JK23sELL()JF~~//*E  != / /(T[[[| ###    === lIw8H,VW ,.).3"3"l:<6;8@D  ,/!=    &T &3 *33$3 *35C3 3U &+   " !  TJJZU U$U U U; ;&7; ; , NNEKK MM5;;> JJ MM MM JJ KK KK KK KK LL LL LL    E    >:,' ' '' 'T_B_BD%N%S1S1lzz";2==Q a;#%5z#a;H  - - - 6:_6 ;HH -  _6 ;HH /-  _6;HH /- _6$;HH /- %_60;HH /- 1_6< ;HH *)   =_6L ;HH (0   M_6X ;HH 66>   Y_6h ;HH 1  i_6r ;HH 2  s_6|;HH 1 }_6F;HH 2 G_6P;HH %8$  Q_6^ ;HH &%   __6j;HH %8  k_6v ;HH &  w_6@;HH + A_6L%;HH 88 ) M_6X%;HH 88 ) Y_6d%;HH 8 )e_6n%;HH 8 )o_6z ;HH '  {_6D ;HH (  E_6N;HH S O_6\,;HH ? 0]_6f,;HH ? 0g_6r&;HH 9 *s_6| ;HH *  }_6F);HH D -G_6P);HH D -Q_6Z);HH D -[_6d);HH D -e_6n(;HH C ,o_6x$1;HH L 5$y_6B$1;HH L 5$C_6L$1;HH L 5$M_6V$1;HH L 5$W_6`';HH B +a_6j(;HH C ,k_6t(;HH C ,u_62_D -#-">*Z  NNN ,AAH !;!;H5 +; T  k5c!1223 5a'/=0 1aH < <\3Wgo.\3~  Tp"p"fz ~z w7s" a+