





'''simple docstring'''


import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE									=				logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE									=				{"vocab_file": "vocab.json"}

_SCREAMING_SNAKE_CASE									=				{
    "vocab_file": {
        "mgp-str": "https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json",
    }
}

_SCREAMING_SNAKE_CASE									=				{"mgp-str": 27}

class  lowerCAmelCase_		(		__magic_name__		):
 __lowerCamelCase    :				List[Any]								=       VOCAB_FILES_NAMES
 __lowerCamelCase    :				Tuple								=       PRETRAINED_VOCAB_FILES_MAP
 __lowerCamelCase    :				List[str]								=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




 def __init__(						self      ,  _lowerCAmelCase      ,  _lowerCAmelCase="[GO]"      ,  _lowerCAmelCase="[GO]"      ,  _lowerCAmelCase="[s]"      ,  _lowerCAmelCase="[GO]"      ,  **_lowerCAmelCase		)     ->				Union[str, Any]:
     super().__init__(
         unk_token=_lowerCAmelCase      ,  bos_token=_lowerCAmelCase      ,  eos_token=_lowerCAmelCase      ,  pad_token=_lowerCAmelCase      ,  **_lowerCAmelCase      ,  )

     with open(_lowerCAmelCase      ,  encoding="utf-8"		) as vocab_handle:
         _lowerCAmelCase						      =   json.load(_lowerCAmelCase		)
     _lowerCAmelCase						      =   {v: k for k, v in self.vocab.items()}




 @property
 def 				_snake_case (						self		)     ->				Union[str, Any]:
     return len(self.vocab		)




 def 				_snake_case (						self		)     ->				int:
     return dict(self.vocab      ,  **self.added_tokens_encoder		)




 def 				_snake_case (						self      ,  _lowerCAmelCase		)     ->				Union[str, Any]:
     _lowerCAmelCase						      =   []
     for s in text:
         char_tokens.extend(_lowerCAmelCase		)
     return char_tokens




 def 				_snake_case (						self      ,  _lowerCAmelCase		)     ->				Tuple:
     return self.vocab.get(_lowerCAmelCase      ,  self.vocab.get(self.unk_token		)		)




 def 				_snake_case (						self      ,  _lowerCAmelCase		)     ->				str:
     return self.decoder.get(_lowerCAmelCase		)





 def 				_snake_case (						self      ,  _lowerCAmelCase      ,  _lowerCAmelCase = None		)     ->				Tuple[str]:
     if not os.path.isdir(_lowerCAmelCase		):
         logger.error("Vocabulary path ({}) should be a directory".format(_lowerCAmelCase		)		)
         return
     _lowerCAmelCase						      =   os.path.join(
         _lowerCAmelCase      ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

     with open(_lowerCAmelCase      ,  "w"      ,  encoding="utf-8"		) as f:
         f.write(json.dumps(self.vocab      ,  indent=2      ,  sort_keys=_lowerCAmelCase      ,  ensure_ascii=_lowerCAmelCase		) + "\n"		)

     return (vocab_file,)









'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE									=				logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE									=				{
    "bigcode/gpt_bigcode-santacoder": "https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json",
}

class  lowerCAmelCase_		(		__magic_name__		):
 __lowerCamelCase    :				str								=       "gpt_bigcode"
 __lowerCamelCase    :				Optional[int]								=       ["past_key_values"]
 __lowerCamelCase    :				List[str]								=       {
     "hidden_size": "n_embd",
     "max_position_embeddings": "n_positions",
     "num_attention_heads": "n_head",
     "num_hidden_layers": "n_layer",
 }




 def __init__(						self      ,  _lowerCAmelCase=50257      ,  _lowerCAmelCase=1024      ,  _lowerCAmelCase=768      ,  _lowerCAmelCase=12      ,  _lowerCAmelCase=12      ,  _lowerCAmelCase=None      ,  _lowerCAmelCase="gelu_pytorch_tanh"      ,  _lowerCAmelCase=0.1      ,  _lowerCAmelCase=0.1      ,  _lowerCAmelCase=0.1      ,  _lowerCAmelCase=1E-5      ,  _lowerCAmelCase=0.02      ,  _lowerCAmelCase=True      ,  _lowerCAmelCase=True      ,  _lowerCAmelCase=50256      ,  _lowerCAmelCase=50256      ,  _lowerCAmelCase=True      ,  _lowerCAmelCase=True      ,  _lowerCAmelCase=True      ,  **_lowerCAmelCase      ,  )     ->				List[Any]:
     _lowerCAmelCase						      =   vocab_size
     _lowerCAmelCase						      =   n_positions
     _lowerCAmelCase						      =   n_embd
     _lowerCAmelCase						      =   n_layer
     _lowerCAmelCase						      =   n_head
     _lowerCAmelCase						      =   n_inner
     _lowerCAmelCase						      =   activation_function
     _lowerCAmelCase						      =   resid_pdrop
     _lowerCAmelCase						      =   embd_pdrop
     _lowerCAmelCase						      =   attn_pdrop
     _lowerCAmelCase						      =   layer_norm_epsilon
     _lowerCAmelCase						      =   initializer_range
     _lowerCAmelCase						      =   scale_attn_weights
     _lowerCAmelCase						      =   use_cache
     _lowerCAmelCase						      =   attention_softmax_in_fpaa
     _lowerCAmelCase						      =   scale_attention_softmax_in_fpaa
     _lowerCAmelCase						      =   multi_query

     _lowerCAmelCase						      =   bos_token_id
     _lowerCAmelCase						      =   eos_token_id

     super().__init__(bos_token_id=_lowerCAmelCase      ,  eos_token_id=_lowerCAmelCase      ,  **_lowerCAmelCase		)


