
    sh                        S SK Jr  S SKrS SKrS SKrS	S jrS
S jrSSS jjr  S         SS jjrSS jr	SSS jjr
g)    )annotationsNc                |   U R                  S5      (       dG  U R                  S5      (       d1   SS KnUR                  U S5       nUR	                  5       sS S S 5        $ SS KnUR                  U 5      nUR                  5         UR                  $ ! [         a  n[        S5      UeS nAff = f! , (       d  f       N]= f)Nzhttp://zhttps://r   Oblobfile is not installed. Please install it by running `pip install blobfile`.rb)	
startswithblobfileImportErrorBlobFilereadrequestsgetraise_for_statuscontent)blobpathr   efr   resps         a/Users/tiagomarins/Projetos/claudeai/copy_bank/venv/lib/python3.13/site-packages/tiktoken/load.py	read_filer      s    y))(2E2Ej2Q2Q	
 x.!668 /. <<!D<<  	a	 /.s#   B B-
B*B%%B*-
B;c                R    [         R                  " U 5      R                  5       nX!:H  $ N)hashlibsha256	hexdigest)dataexpected_hashactual_hashs      r   
check_hashr      s"    ..&002K''    c                   SnS[         R                  ;   a  [         R                  S   nO\S[         R                  ;   a  [         R                  S   nO4SS Kn[         R                  R	                  UR                  5       S5      nSnUS:X  a  [        U 5      $ [        R                  " U R                  5       5      R                  5       n[         R                  R	                  X55      n[         R                  R                  U5      (       aR  [        US5       nUR                  5       nS S S 5        Ub  [        WU5      (       a  W$  [         R                  " U5        [        U 5      n	U(       a"  [        X5      (       d  [#        S	U  S
U S35      eSS Kn
 [         R&                  " USS9  US-   [)        U
R+                  5       5      -   S-   n[        US5       nUR-                  U	5        S S S 5        [         R.                  " X5        U	$ ! , (       d  f       N= f! [          a     Nf = f! , (       d  f       NG= f! [          a    U(       a  e  U	$ f = f)NTTIKTOKEN_CACHE_DIRDATA_GYM_CACHE_DIRr   zdata-gym-cacheF r   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)exist_ok.z.tmpwb)osenvirontempfilepathjoin
gettempdirr   r   sha1encoder   existsopenr   r   removeOSError
ValueErroruuidmakedirsstruuid4writerename)r   r   user_specified_cache	cache_dirr)   	cache_key
cache_pathr   r   contentsr4   tmp_filenames               r   read_file_cachedr@       s   rzz)JJ34			+JJ34	GGLL!4!4!68HI	$B""X__./99;Ii3J	ww~~j!!*d#q668D $ Jt]$C$CK	IIj! "HZ@@5hZ{=/ ZH I
 	

 	
I-!C'#djjl*;;fD,%GGH &
		,+ O? $#  		 &%     OsI   H	H AH? H.-H? 
H
H+*H+.
H<8H? ?IIc                
  ^ [        S5       Vs/ s H6  n[        U5      R                  5       (       d  M#  [        U5      S:w  d  M4  UPM8     nnU Vs0 s H  n[        U5      U_M     snmSn[        S5       H1  nXE;  d  M
  UR                  U5        UT[        SU-   5      '   US-  nM3     [	        U5      S:X  d   e[        X5      R                  5       nUR                  S5      SS  Vs/ s H  n[        UR                  5       5      PM     n	nS
U4S jjn
[        U5       VVs0 s H  u  p[        U/5      U_M     nnn[	        U5      nU	 H  u  pXlU
" U5      U
" U5      -   '   US-  nM     SS KnUR                  [        X5      5      nUR                  5        VVs0 s H  u  nnU
" U5      U_M     nnnUR                  SS 5        UR                  S	S 5        UU:X  d   eU$ s  snf s  snf s  snf s  snnf s  snnf )N    r      
c                .   > [        U4S jU  5       5      $ )Nc              3  .   >#    U  H
  nTU   v   M     g 7fr    ).0bdata_gym_byte_to_bytes     r   	<genexpr>Kdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>m   s     =u!*1-us   )bytes)valuerL   s    r   decode_data_gym8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gyml   s    =u===r   s   <|endoftext|>s   <|startoftext|>)rP   r6   returnrO   )rangechrisprintableappendlenr@   decodesplittuple	enumeraterO   jsonloadsitemspop)vocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashrK   rank_to_intbytenvocab_bpe_contents	merge_str
bpe_mergesrQ   i	bpe_ranksfirstsecondr]   encoder_jsonkvencoder_json_loadedrL   s                       @r   data_gym_to_mergeable_bpe_ranksrr   V   s    #(+X+QQ1C1C1Eq#a&TW-q+OX0?@1SVQY@	A4[#""1%34!#dQh-0FA	 
 4''' *.IPPR<N<T<TUY<Z[\]_<`a<`y%	)*<`Ja> ,5_+EF+E41sQ+EIFIA#FG/%(?6+BBC	Q $ 
 ::./@TUL=I=O=O=QR=QTQ?1-q0=QR,d3.5++++M Y@ b G Ss(   "G*G*G*G/-#G4(G9#G?c                p    SS K nUR                  US5       n[        U R	                  5       S S9 HJ  u  pVUR                  [        R                  " U5      S-   [        U5      R                  5       -   S-   5        ML     S S S 5        g ! [         a  n[        S5      UeS nAff = f! , (       d  f       g = f)Nr   r   r&   c                    U S   $ )NrD   rI   )xs    r   <lambda>#dump_tiktoken_bpe.<locals>.<lambda>   s    1Q4r   )key       
)
r   r	   r
   sortedr_   r8   base64	b64encoder6   r.   )rk   tiktoken_bpe_filer   r   r   tokenranks          r   dump_tiktoken_bper      s    
 
		,d	3q!)//"3HKEGGF$$U+d2SY5E5E5GG%OP I 
4	3	  ]
	 
4	3s#   B	 A(B'	
B$BB$'
B5c                   [        X5      n0 nUR                  5        HA  nU(       d  M   UR                  5       u  pV[        U5      U[        R
                  " U5      '   MC     U$ ! [         a  n[        SU< SU  35      UeS nAff = f)NzError parsing line z in )r@   
splitlinesrZ   intr|   	b64decode	Exceptionr3   )r~   r   r>   retliner   r   r   s           r   load_tiktoken_bper      s     1AH
C##%	[**,KE+.t9C  '( & J  	[24($?P>QRSYZZ	[s   4A%%
B/BB)r   r6   rS   rO   )r   rO   r   r6   rS   boolr   )r   r6   r   
str | NonerS   rO   )NN)
ra   r6   rb   r6   rc   r   rd   r   rS   dict[bytes, int])rk   r   r~   r6   rS   None)r~   r6   r   r   rS   r   )
__future__r   r|   r   r'   r   r   r@   rr   r   r   rI   r   r   <module>r      sh    "   	&(
3r "&$(	--- - "	-
 -`	Qr   