


'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase												=						logging.get_logger(__name__)

__lowerCamelCase												=						{'''vocab_file''': '''sentencepiece.model'''}

__lowerCamelCase												=						{
    '''vocab_file''': {
        '''google/rembert''': '''https://huggingface.co/google/rembert/resolve/main/sentencepiece.model''',
    },
}

__lowerCamelCase												=						{
    '''google/rembert''': 256,
}




class 				UpperCAmelCase			(			_snake_case						):
 UpperCAmelCase				=					VOCAB_FILES_NAMES
 UpperCAmelCase				=					PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase				=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


 def __init__(      self							:	Optional[Any]			,       __lowerCamelCase							:	int			,       __lowerCamelCase							:	Tuple=False			,       __lowerCamelCase							:	Optional[int]=True			,       __lowerCamelCase							:	Optional[int]=True			,       __lowerCamelCase							:	List[Any]="[CLS]"			,       __lowerCamelCase							:	Union[str, Any]="[SEP]"			,       __lowerCamelCase							:	int="[UNK]"			,       __lowerCamelCase							:	Tuple="[SEP]"			,       __lowerCamelCase							:	str="[PAD]"			,       __lowerCamelCase							:	List[str]="[CLS]"			,       __lowerCamelCase							:	Tuple="[MASK]"			,       **__lowerCamelCase							:	Union[str, Any]			,       ):
    super().__init__(
        do_lower_case=UpperCamelCase__			,       remove_space=UpperCamelCase__			,       keep_accents=UpperCamelCase__			,       bos_token=UpperCamelCase__			,       eos_token=UpperCamelCase__			,       unk_token=UpperCamelCase__			,       sep_token=UpperCamelCase__			,       pad_token=UpperCamelCase__			,       cls_token=UpperCamelCase__			,       mask_token=UpperCamelCase__			,       **UpperCamelCase__			,       )

    UpperCAmelCase__	:List[Any]      =      do_lower_case
    UpperCAmelCase__	:int      =      remove_space
    UpperCAmelCase__	:Optional[int]      =      keep_accents
    UpperCAmelCase__	:List[str]      =      vocab_file

    UpperCAmelCase__	:str      =      spm.SentencePieceProcessor()
    self.sp_model.Load(UpperCamelCase__ )


 @property
 def        __SCREAMING_SNAKE_CASE					(      self							:	List[str] ):
    return len(self.sp_model )


 def        __SCREAMING_SNAKE_CASE					(      self							:	List[Any] ):
    UpperCAmelCase__	:str      =      {self.convert_ids_to_tokens(UpperCamelCase__ ): i for i in range(self.vocab_size )}
    vocab.update(self.added_tokens_encoder )
    return vocab


 def __getstate__(      self							:	Tuple ):
    UpperCAmelCase__	:Any      =      self.__dict__.copy()
    UpperCAmelCase__	:Optional[int]      =      None
    return state


 def __setstate__(      self							:	int			,       __lowerCamelCase							:	Dict ):
    UpperCAmelCase__	:Optional[int]      =      d
    UpperCAmelCase__	:List[str]      =      spm.SentencePieceProcessor()
    self.sp_model.Load(self.vocab_file )


 def        __SCREAMING_SNAKE_CASE					(      self							:	List[Any]			,       __lowerCamelCase							:	Tuple			,       __lowerCamelCase							:	str=False ):
    UpperCAmelCase__	:List[str]      =      self.sp_model.EncodeAsPieces(UpperCamelCase__ )
    return pieces


 def        __SCREAMING_SNAKE_CASE					(      self							:	Optional[Any]			,       __lowerCamelCase							:	Any ):
    return self.sp_model.PieceToId(UpperCamelCase__ )


 def        __SCREAMING_SNAKE_CASE					(      self							:	Optional[Any]			,       __lowerCamelCase							:	Union[str, Any] ):
    return self.sp_model.IdToPiece(UpperCamelCase__ )


 def        __SCREAMING_SNAKE_CASE					(      self							:	Any			,       __lowerCamelCase							:	Any ):
    UpperCAmelCase__	:Optional[int]      =      self.sp_model.decode_pieces(UpperCamelCase__ )
    return out_string


 def        __SCREAMING_SNAKE_CASE					(      self							:	str			,       __lowerCamelCase							:	Tuple			,       __lowerCamelCase							:	Dict = None ):
    UpperCAmelCase__	:Any      =      [self.sep_token_id]
    UpperCAmelCase__	:Dict      =      [self.cls_token_id]
    if token_ids_a is None:
       return cls + token_ids_a + sep
    return cls + token_ids_a + sep + token_ids_a + sep


 def        __SCREAMING_SNAKE_CASE					(      self							:	Tuple			,       __lowerCamelCase							:	Dict			,       __lowerCamelCase							:	str = None			,       __lowerCamelCase							:	List[str] = False ):

    if already_has_special_tokens:
       if token_ids_a is not None:
          raise ValueError(
              '''You should not supply a second sequence if the provided sequence of '''
              '''ids is already formatted with special tokens for the model.''' )
       return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

    if token_ids_a is not None:
       return [1] + ([0] * len(UpperCamelCase__ )) + [1] + ([0] * len(UpperCamelCase__ )) + [1]
    return [1] + ([0] * len(UpperCamelCase__ )) + [1]


 def        __SCREAMING_SNAKE_CASE					(      self							:	Tuple			,       __lowerCamelCase							:	str			,       __lowerCamelCase							:	Optional[int] = None ):
    UpperCAmelCase__	:int      =      [self.sep_token_id]
    UpperCAmelCase__	:List[str]      =      [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep ) * [0]
    return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





 def        __SCREAMING_SNAKE_CASE					(      self							:	Any			,       __lowerCamelCase							:	Optional[Any]			,       __lowerCamelCase							:	Any = None ):
    if not os.path.isdir(UpperCamelCase__ ):
       logger.error('''Vocabulary path ({}) should be a directory'''.format(UpperCamelCase__ ) )
       return
    UpperCAmelCase__	:List[str]      =      os.path.join(
        UpperCamelCase__			,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

    if os.path.abspath(self.vocab_file ) != os.path.abspath(UpperCamelCase__ ):
       copyfile(self.vocab_file			,       UpperCamelCase__ )

    return (out_vocab_file,)






'''simple docstring'''







from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


__lowerCamelCase									=     logging.get_logger(__name__)


__lowerCamelCase									=     {
    '''Salesforce/codegen-350M-nl''': '''https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json''',
    '''Salesforce/codegen-350M-multi''': '''https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json''',
    '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json''',
    '''Salesforce/codegen-2B-nl''': '''https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-2B-multi''': '''https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-2B-mono''': '''https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-6B-nl''': '''https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-6B-multi''': '''https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-6B-mono''': '''https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-16B-nl''': '''https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-16B-multi''': '''https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-16B-mono''': '''https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json''',
}


class        A__   (   _snake_case      ):
    lowercase						       =		"codegen"
    lowercase						       =		{
        "max_position_embeddings": "n_positions",
        "hidden_size": "n_embd",
        "num_attention_heads": "n_head",
        "num_hidden_layers": "n_layer",
    }



    def __init__(       self		,					UpperCamelCase__=50400		,					UpperCamelCase__=2048		,					UpperCamelCase__=2048		,					UpperCamelCase__=4096		,					UpperCamelCase__=28		,					UpperCamelCase__=16		,					UpperCamelCase__=64		,					UpperCamelCase__=None		,					UpperCamelCase__="gelu_new"		,					UpperCamelCase__=0.0		,					UpperCamelCase__=0.0		,					UpperCamelCase__=0.0		,					UpperCamelCase__=1e-5		,					UpperCamelCase__=0.02		,					UpperCamelCase__=True		,					UpperCamelCase__=50256		,					UpperCamelCase__=50256		,					UpperCamelCase__=False		,					**UpperCamelCase__		,					)    ->   Tuple:





           '''simple docstring'''
           A_						       =   vocab_size
           A_						       =   n_ctx
           A_						       =   n_positions
           A_						       =   n_embd
           A_						       =   n_layer
           A_						       =   n_head
           A_						       =   n_inner
           A_						       =   rotary_dim
           A_						       =   activation_function
           A_						       =   resid_pdrop
           A_						       =   embd_pdrop
           A_						       =   attn_pdrop
           A_						       =   layer_norm_epsilon
           A_						       =   initializer_range
           A_						       =   use_cache

           A_						       =   bos_token_id
           A_						       =   eos_token_id

           super().__init__(
               bos_token_id=UpperCamelCase__		,					eos_token_id=UpperCamelCase__		,					tie_word_embeddings=UpperCamelCase__		,					**UpperCamelCase__							)


class        A__   (   _snake_case      ):



    def __init__(       self		,					UpperCamelCase__		,					UpperCamelCase__ = "default"		,					UpperCamelCase__ = None		,					UpperCamelCase__ = False		,					)    ->   Tuple:





           '''simple docstring'''
           super().__init__(UpperCamelCase__		,					task=UpperCamelCase__		,					patching_specs=UpperCamelCase__		,					use_past=UpperCamelCase__							)
           if not getattr(self._config		,					"""pad_token_id"""		,					UpperCamelCase__							):
                  # TODO: how to do that better?
                  A_						       =   0



    @property
    def        snake_case_      (       self							)    ->   Mapping[str, Mapping[int, str]]:





           '''simple docstring'''
           A_						       =   OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}							)
           if self.use_past:
                  self.fill_with_past_key_values_(UpperCamelCase__		,					direction="""inputs"""							)
                  A_						       =   {0: """batch""", 1: """past_sequence + sequence"""}
           else:
                  A_						       =   {0: """batch""", 1: """sequence"""}

           return common_inputs



    @property
    def        snake_case_      (       self							)    ->   int:





           '''simple docstring'''
           return self._config.n_layer



    @property
    def        snake_case_      (       self							)    ->   int:





           '''simple docstring'''
           return self._config.n_head



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__ = -1		,					UpperCamelCase__ = -1		,					UpperCamelCase__ = False		,					UpperCamelCase__ = None		,					)    ->   Mapping[str, Any]:





           '''simple docstring'''
           A_						       =   super(UpperCamelCase__		,					self							).generate_dummy_inputs(
               UpperCamelCase__		,					batch_size=UpperCamelCase__		,					seq_length=UpperCamelCase__		,					is_pair=UpperCamelCase__		,					framework=UpperCamelCase__							)

           # We need to order the input in the way they appears in the forward()
           A_						       =   OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}							)

           # Need to add the past_keys
           if self.use_past:
                  if not is_torch_available():
                         raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""							)
                  else:
                         import torch

                         A_       ,      A_						       =   common_inputs["""input_ids"""].shape
                         # Not using the same length for past_key_values
                         A_						       =   seqlen + 2
                         A_						       =   (
                             batch,
                             self.num_attention_heads,
                             past_key_values_length,
                             self._config.hidden_size // self.num_attention_heads,
                         )
                         A_						       =   [
                             (torch.zeros(UpperCamelCase__							), torch.zeros(UpperCamelCase__							)) for _ in range(self.num_layers							)
                         ]

           A_						       =   common_inputs["""attention_mask"""]
           if self.use_past:
                  A_						       =   ordered_inputs["""attention_mask"""].dtype
                  A_						       =   torch.cat(
                      [ordered_inputs["""attention_mask"""], torch.ones(UpperCamelCase__		,					UpperCamelCase__		,					dtype=UpperCamelCase__							)]		,					dim=1							)

           return ordered_inputs







    @property
    def        snake_case_      (       self							)    ->   int:





           '''simple docstring'''
           return 13
