




"""simple docstring"""

import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a							      =     logging.get_logger(__name__)

a							      =     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

a							      =     {
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

a							      =     {
    '''ctrl''': 256,
}

a							      =     {
    '''Pregnancy''': 168_629,
    '''Christianity''': 7_675,
    '''Explain''': 106_423,
    '''Fitness''': 63_440,
    '''Saving''': 63_163,
    '''Ask''': 27_171,
    '''Ass''': 95_985,
    '''Joke''': 163_509,
    '''Questions''': 45_622,
    '''Thoughts''': 49_605,
    '''Retail''': 52_342,
    '''Feminism''': 164_338,
    '''Writing''': 11_992,
    '''Atheism''': 192_263,
    '''Netflix''': 48_616,
    '''Computing''': 39_639,
    '''Opinion''': 43_213,
    '''Alone''': 44_967,
    '''Funny''': 58_917,
    '''Gaming''': 40_358,
    '''Human''': 4_088,
    '''India''': 1_331,
    '''Joker''': 77_138,
    '''Diet''': 36_206,
    '''Legal''': 11_859,
    '''Norman''': 4_939,
    '''Tip''': 72_689,
    '''Weight''': 52_343,
    '''Movies''': 46_273,
    '''Running''': 23_425,
    '''Science''': 2_090,
    '''Horror''': 37_793,
    '''Confession''': 60_572,
    '''Finance''': 12_250,
    '''Politics''': 16_360,
    '''Scary''': 191_985,
    '''Support''': 12_654,
    '''Technologies''': 32_516,
    '''Teenage''': 66_160,
    '''Event''': 32_769,
    '''Learned''': 67_460,
    '''Notion''': 182_770,
    '''Wikipedia''': 37_583,
    '''Books''': 6_665,
    '''Extract''': 76_050,
    '''Confessions''': 102_701,
    '''Conspiracy''': 75_932,
    '''Links''': 63_674,
    '''Narcissus''': 150_425,
    '''Relationship''': 54_766,
    '''Relationships''': 134_796,
    '''Reviews''': 41_671,
    '''News''': 4_256,
    '''Translation''': 26_820,
    '''multilingual''': 128_406,
}






def 		_snake_case	(   _snake_case				:   Optional[Any]				)       ->   Any:






      '''simple docstring'''


      _A     =  set()
      _A     =  word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)				)
            _A     =  char

      _A     =  set(_snake_case				)
      return pairs







class 							lowercase_			(	__lowerCAmelCase				):



 '''simple docstring'''
 UpperCAmelCase       :	int     =      VOCAB_FILES_NAMES
 UpperCAmelCase       :	Optional[Any]     =      PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase       :	List[Any]     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase       :	Optional[int]     =      CONTROL_CODES

 def __init__(					self			:			Optional[int]       ,				_UpperCAmelCase			:			List[Any]       ,				_UpperCAmelCase			:			int       ,				_UpperCAmelCase			:			Tuple="<unk>"       ,				**_UpperCAmelCase			:			Any						):
       super().__init__(unk_token=_UpperCAmelCase       ,				**_UpperCAmelCase						)

       with open(_UpperCAmelCase       ,				encoding='utf-8'						) as vocab_handle:
             _A     =  json.load(_UpperCAmelCase						)
       _A     =  {v: k for k, v in self.encoder.items()}
       with open(_UpperCAmelCase       ,				encoding='utf-8'						) as merges_handle:
             _A     =  merges_handle.read().split('\n'						)[1:-1]
       _A     =  [tuple(merge.split()						) for merge in merges]
       _A     =  dict(zip(_UpperCAmelCase       ,				range(len(_UpperCAmelCase						)						)						)						)
       _A     =  {}

 @property
 def      lowerCAmelCase_				(					self			:			Dict						):
       return len(self.encoder						)

 def      lowerCAmelCase_				(					self			:			Any						):
       return dict(self.encoder       ,				**self.added_tokens_encoder						)

 def      lowerCAmelCase_				(					self			:			Optional[int]       ,				_UpperCAmelCase			:			List[str]						):
       if token in self.cache:
             return self.cache[token]
       _A     =  tuple(_UpperCAmelCase						)
       _A     =  tuple(list(word[:-1]						) + [word[-1] + '</w>']						)
       _A     =  get_pairs(_UpperCAmelCase						)

       if not pairs:
             return token

       while True:
             _A     =  min(_UpperCAmelCase       ,				key=lambda _UpperCAmelCase						: self.bpe_ranks.get(_UpperCAmelCase       ,				float('inf'						)						)						)
             if bigram not in self.bpe_ranks:
                   break
             _A  ,       _A     =  bigram
             _A     =  []
             _A     =  0
             while i < len(_UpperCAmelCase						):
                   try:
                         _A     =  word.index(_UpperCAmelCase       ,				_UpperCAmelCase						)
                   except ValueError:
                         new_word.extend(word[i:]						)
                         break
                   else:
                         new_word.extend(word[i:j]						)
                         _A     =  j

                   if word[i] == first and i < len(_UpperCAmelCase						) - 1 and word[i + 1] == second:
                         new_word.append(first + second						)
                         i += 2
                   else:
                         new_word.append(word[i]						)
                         i += 1
             _A     =  tuple(_UpperCAmelCase						)
             _A     =  new_word
             if len(_UpperCAmelCase						) == 1:
                   break
             else:
                   _A     =  get_pairs(_UpperCAmelCase						)
       _A     =  '@@ '.join(_UpperCAmelCase						)
       _A     =  word[:-4]
       _A     =  word
       return word

 def      lowerCAmelCase_				(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			Dict						):
       _A     =  []

       _A     =  re.findall(r'\S+\n?'       ,				_UpperCAmelCase						)

       for token in words:
             split_tokens.extend(list(self.bpe(_UpperCAmelCase						).split(' '						)						)						)
       return split_tokens

 def      lowerCAmelCase_				(					self			:			str       ,				_UpperCAmelCase			:			Dict						):
       return self.encoder.get(_UpperCAmelCase       ,				self.encoder.get(self.unk_token						)						)

 def      lowerCAmelCase_				(					self			:			Optional[int]       ,				_UpperCAmelCase			:			Tuple						):
       return self.decoder.get(_UpperCAmelCase       ,				self.unk_token						)

 def      lowerCAmelCase_				(					self			:			Optional[Any]       ,				_UpperCAmelCase			:			Tuple						):
       _A     =  ' '.join(_UpperCAmelCase						).replace('@@ '       ,				''						).strip()
       return out_string



 def      lowerCAmelCase_				(					self			:			Optional[Any]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Optional[str] = None						):
       if not os.path.isdir(_UpperCAmelCase						):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
             return
       _A     =  os.path.join(
           _UpperCAmelCase       ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
       _A     =  os.path.join(
           _UpperCAmelCase       ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

       with open(_UpperCAmelCase       ,				'w'       ,				encoding='utf-8'						) as f:
             f.write(json.dumps(self.encoder       ,				indent=2       ,				sort_keys=_UpperCAmelCase       ,				ensure_ascii=_UpperCAmelCase						) + '\n'						)

       _A     =  0
       with open(_UpperCAmelCase       ,				'w'       ,				encoding='utf-8'						) as writer:
             writer.write('#version: 0.2\n'						)
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,				key=lambda _UpperCAmelCase						: kv[1]						):
                   if index != token_index:
                         logger.warning(
                             F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                             ' Please check that the tokenizer is not corrupted!'						)
                         _A     =  token_index
                   writer.write(' '.join(_UpperCAmelCase						) + '\n'						)
                   index += 1

       return vocab_file, merge_file

 # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
 #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
 #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
 #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
 #     return ''.join(tokens_generated_so_far)







"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a							      =     logging.get_logger(__name__)

a							      =     {
    '''bigcode/gpt_bigcode-santacoder''': '''https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json''',
}





class 							lowercase_			(	__lowerCAmelCase				):



 '''simple docstring'''
 UpperCAmelCase       :	int     =      '''gpt_bigcode'''
 UpperCAmelCase       :	str     =      ['''past_key_values''']
 UpperCAmelCase       :	Dict     =      {
     '''hidden_size''': '''n_embd''',
     '''max_position_embeddings''': '''n_positions''',
     '''num_attention_heads''': '''n_head''',
     '''num_hidden_layers''': '''n_layer''',
 }

 def __init__(					self			:			Tuple       ,				_UpperCAmelCase			:			Dict=50_257       ,				_UpperCAmelCase			:			List[Any]=1_024       ,				_UpperCAmelCase			:			Any=768       ,				_UpperCAmelCase			:			int=12       ,				_UpperCAmelCase			:			Any=12       ,				_UpperCAmelCase			:			Tuple=None       ,				_UpperCAmelCase			:			str="gelu_pytorch_tanh"       ,				_UpperCAmelCase			:			str=0.1       ,				_UpperCAmelCase			:			Optional[Any]=0.1       ,				_UpperCAmelCase			:			Tuple=0.1       ,				_UpperCAmelCase			:			List[Any]=1E-5       ,				_UpperCAmelCase			:			List[Any]=0.02       ,				_UpperCAmelCase			:			Optional[Any]=True       ,				_UpperCAmelCase			:			Dict=True       ,				_UpperCAmelCase			:			List[Any]=50_256       ,				_UpperCAmelCase			:			Dict=50_256       ,				_UpperCAmelCase			:			int=True       ,				_UpperCAmelCase			:			Union[str, Any]=True       ,				_UpperCAmelCase			:			Any=True       ,				**_UpperCAmelCase			:			Any       ,				):
       _A     =  vocab_size
       _A     =  n_positions
       _A     =  n_embd
       _A     =  n_layer
       _A     =  n_head
       _A     =  n_inner
       _A     =  activation_function
       _A     =  resid_pdrop
       _A     =  embd_pdrop
       _A     =  attn_pdrop
       _A     =  layer_norm_epsilon
       _A     =  initializer_range
       _A     =  scale_attn_weights
       _A     =  use_cache
       _A     =  attention_softmax_in_fpaa
       _A     =  scale_attention_softmax_in_fpaa
       _A     =  multi_query

       _A     =  bos_token_id
       _A     =  eos_token_id

       super().__init__(bos_token_id=_UpperCAmelCase       ,				eos_token_id=_UpperCAmelCase       ,				**_UpperCAmelCase						)

