
    	 @gT                    r    d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZddZ	ddZ
ddZddZddZdS )    )annotationsNblobpathstrreturnbytesc                   |                      d          s|                      d          sj	 dd l}n"# t          $ r}t          d          |d }~ww xY w|                    | d          5 }|                                cd d d            S # 1 swxY w Y   t          j        |           }|                                 |j        S )Nzhttp://zhttps://r   Oblobfile is not installed. Please install it by running `pip install blobfile`.rb)	
startswithblobfileImportErrorBlobFilereadrequestsgetraise_for_statuscontent)r   r   efresps        A/var/www/piapp/venv/lib/python3.11/site-packages/tiktoken/load.py	read_filer      s   y)) (2E2Ej2Q2Q 	OOOO 	 	 	a 	 x.. 	!6688	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 <!!D<s&   1 
AAA)B

BBc                   dt           j        v rt           j        d         }nRdt           j        v rt           j        d         }n1t           j                            t	          j                    d          }|dk    rt          |           S t          j        | 	                                          
                                }t           j                            ||          }t           j                            |          r<t          |d          5 }|                                cd d d            S # 1 swxY w Y   t          |           }t          j        |d           |dz   t          t!          j                              z   d	z   }t          |d
          5 }|                    |           d d d            n# 1 swxY w Y   t          j        ||           |S )NTIKTOKEN_CACHE_DIRDATA_GYM_CACHE_DIRzdata-gym-cache r
   T)exist_ok.z.tmpwb)osenvironpathjointempfile
gettempdirr   hashlibsha1encode	hexdigestexistsopenr   makedirsr   uuiduuid4writerename)r   	cache_dir	cache_key
cache_pathr   contentstmp_filenames          r   read_file_cachedr6      s   rz))J34				+	+J34		GLL!4!6!68HII	B"""X__..//99;;Ii33J	w~~j!! *d## 	q6688	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ""HK	D))))#c$*,,&7&77&@L	lD	!	! Q	              IlJ'''Os$   D11D58D5F<<G G vocab_bpe_fileencoder_json_filedict[bytes, int]c                0   d t          d          D             }d |D             d}t          d          D ]5}||vr/|                    |           |t          d|z             <   |dz  }6t          |          dk    sJ t	          |                                           }d |                    d          dd         D             }dfdd t          |          D             }t          |          }|D ]$\  }}	|| |           |	          z   <   |dz  }%t          j	        t	          |                    }
fd|

                                D             }|                    dd            |                    dd            ||k    sJ |S )Nc                z    g | ]8}t          |                                          #t          |          d k    6|9S ) )chrisprintable.0bs     r   
<listcomp>z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>?   s;    XXXQQ1C1C1E1EX#a&&TW--q---       c                .    i | ]}t          |          |S  )r=   r?   s     r   
<dictcomp>z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>A   s     @@@1SVVQ@@@rC   r      c                P    g | ]#}t          |                                          $S rF   )tuplesplit)r@   	merge_strs     r   rB   z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>L   s*    aaay%	))**aaarC   
valuer   r   r   c                :    t          fd| D                       S )Nc              3  (   K   | ]}|         V  d S NrF   )r@   rA   data_gym_byte_to_bytes     r   	<genexpr>zKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>O   s)      ==!*1-======rC   r   )rO   rS   s    r   decode_data_gymz8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymN   s&    ====u======rC   c                6    i | ]\  }}t          |g          |S rF   rU   )r@   irA   s      r   rG   z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>R   s&    FFF41asQFFFrC   c                .    i | ]\  }} |          |S rF   rF   )r@   kvrV   s      r   rG   z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>]   s)    RRRTQ??1--qRRRrC   s   <|endoftext|>s   <|startoftext|>)rO   r   r   r   )rangeappendr=   lenr6   decoderK   	enumeratejsonloadsitemspop)r7   r8   rank_to_intbytenrA   vocab_bpe_contents
bpe_merges	bpe_ranksfirstsecondencoder_jsonencoder_json_loadedrS   rV   s               @@r   data_gym_to_mergeable_bpe_ranksrn   ;   s    YX%++XXXO@@@@@	A4[[  O##""1%%%34!#dQh--0FA4'''' *.99@@BBaa<N<T<TUY<Z<Z[\]_[_<`aaaJ> > > > > > GF9_+E+EFFFIIA#  vFG	//%((??6+B+BBC	Q
 :./@AABBLRRRR\=O=O=Q=QRRR,d333.555+++++rC   ri   tiktoken_bpe_fileNonec                   	 dd l }n"# t          $ r}t          d          |d }~ww xY w|                    |d          5 }t          |                                 d           D ]T\  }}|                    t          j        |          dz   t          |          	                                z   dz              U	 d d d            d S # 1 swxY w Y   d S )Nr   r	   r   c                    | d         S )NrH   rF   )xs    r   <lambda>z#dump_tiktoken_bpe.<locals>.<lambda>n   s
    1Q4 rC   )key       
)
r   r   r   sortedrc   r/   base64	b64encoder   r(   )ri   ro   r   r   r   tokenranks          r   dump_tiktoken_bper}   f   sK      ]
 
	 
		,d	3	3 Qq!)//"3"3HHH 	Q 	QKE4GGF$U++d2SYY5E5E5G5GG%OPPPP	QQ Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Qs"    
&!&A:CCCc                p    t          |           }d d |                                D             D             S )Nc                X    i | ]'\  }}t          j        |          t          |          (S rF   )ry   	b64decodeint)r@   r{   r|   s      r   rG   z%load_tiktoken_bpe.<locals>.<dictcomp>u   s?       E4 	T  rC   c              3  B   K   | ]}||                                 V  d S rR   )rK   )r@   lines     r   rT   z$load_tiktoken_bpe.<locals>.<genexpr>w   s/      SSTdSDJJLLSSSSSSrC   )r6   
splitlines)ro   r4   s     r   load_tiktoken_bper   r   sK     122H SSX5H5H5J5JSSS   rC   )r   r   r   r   )r7   r   r8   r   r   r9   )ri   r9   ro   r   r   rp   )ro   r   r   r9   )
__future__r   ry   r&   ra   r    r$   r-   r   r   r6   rn   r}   r   rF   rC   r   <module>r      s    " " " " " "    				          <( ( ( (V	Q 	Q 	Q 	Q     rC   