
    	 @g6                     X    d dl mZmZ dZdZdZdZdZd Zd Z	d	 Z
d
 Zd Zee	e
eedZdS )    )data_gym_to_mergeable_bpe_ranksload_tiktoken_bpez<|endoftext|>z<|fim_prefix|>z<|fim_middle|>z<|fim_suffix|>z<|endofprompt|>c                  B    t          dd          } ddd| t          didS )	NzIhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpezLhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json)vocab_bpe_fileencoder_json_filegpt2Q  J's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+P  nameexplicit_n_vocabpat_strmergeable_ranksspecial_tokens)r   	ENDOFTEXTr   s    N/var/www/piapp/venv/lib/python3.11/site-packages/tiktoken_ext/openai_public.pyr   r   
   sA    5bh  O
 !d*$e,      c                  >    t          d          } ddd| t          didS )NzGhttps://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken	r50k_baser	   r
   r   r   r   r   r   s    r   r   r      9    'Q O !d*$e,  r   c                  >    t          d          } ddd| t          didS )NGhttps://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken	p50k_basei  r
   r   r   r   r   s    r   r   r   %   r   r   c                  j    t          d          } t          dt          dt          dt          di}dd| |dS )	Nr   r   r   ij  ik  	p50k_editr
   r   r   r   r   )r   r   
FIM_PREFIX
FIM_MIDDLE
FIM_SUFFIXr   r   s     r   r   r   2   sH    'Q O  
E:ujZ_`Nd*(	  r   c            
      x    t          d          } t          dt          dt          dt          dt
          di}dd| |d	S )
NzIhttps://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktokeni i i i i cl100k_basezs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+r    )r   r   r!   r"   r#   ENDOFPROMPTr$   s     r   r&   r&   ?   sV    'S O 	6FFFVN  N*(	  r   )r   r   r   r   r&   N)tiktoken.loadr   r   r   r!   r"   r#   r'   r   r   r   r   r&   ENCODING_CONSTRUCTORS r   r   <module>r+      s    L L L L L L L L	


  
 
 

 
 

 
 
  (     r   