




'''simple docstring'''





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase_		:     str          =		logging.get_logger(__name__)

UpperCamelCase_		:     Union[str, Any]          =		{'''vocab_file''': '''sentencepiece.bpe.model'''}

UpperCamelCase_		:     Optional[Any]          =		{
    '''vocab_file''': {
        '''camembert-base''': '''https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model''',
    }
}

UpperCamelCase_		:     Optional[Any]          =		{
    '''camembert-base''': 512,
}

UpperCamelCase_		:     List[str]          =		'''▁'''

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       Union[str, Any]       							=		VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE_				:       Any       							=		["""input_ids""", """attention_mask"""]






     def __init__(					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE="<s>"   ,_SCREAMING_SNAKE_CASE="</s>"   ,_SCREAMING_SNAKE_CASE="</s>"   ,_SCREAMING_SNAKE_CASE="<s>"   ,_SCREAMING_SNAKE_CASE="<unk>"   ,_SCREAMING_SNAKE_CASE="<pad>"   ,_SCREAMING_SNAKE_CASE="<mask>"   ,_SCREAMING_SNAKE_CASE=["<s>NOTUSED", "</s>NOTUSED"]   ,_SCREAMING_SNAKE_CASE = None   ,**_SCREAMING_SNAKE_CASE   ,)				->     None:
          # Mask token behave like a normal word, i.e. include the space before it
          _snake_case				= AddedToken(_SCREAMING_SNAKE_CASE   ,lstrip=_SCREAMING_SNAKE_CASE   ,rstrip=_SCREAMING_SNAKE_CASE       ) if isinstance(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       ) else mask_token

          _snake_case				= {} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              bos_token=_SCREAMING_SNAKE_CASE   ,eos_token=_SCREAMING_SNAKE_CASE   ,unk_token=_SCREAMING_SNAKE_CASE   ,sep_token=_SCREAMING_SNAKE_CASE   ,cls_token=_SCREAMING_SNAKE_CASE   ,pad_token=_SCREAMING_SNAKE_CASE   ,mask_token=_SCREAMING_SNAKE_CASE   ,additional_special_tokens=_SCREAMING_SNAKE_CASE   ,sp_model_kwargs=self.sp_model_kwargs   ,**_SCREAMING_SNAKE_CASE   ,)
          _snake_case				= spm.SentencePieceProcessor(**self.sp_model_kwargs       )
          self.sp_model.Load(str(_SCREAMING_SNAKE_CASE       )       )
          _snake_case				= vocab_file
          # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
          # sentencepiece vocabulary (this is the case for <s> and </s>
          _snake_case				= {"<s>NOTUSED": 0, "<pad>": 1, "</s>NOTUSED": 2, "<unk>": 3}
          _snake_case				= len(self.fairseq_tokens_to_ids       )
          _snake_case				= len(self.sp_model       ) + len(self.fairseq_tokens_to_ids       )
          _snake_case				= {v: k for k, v in self.fairseq_tokens_to_ids.items()}






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None       )				->     List[int]:

          if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          _snake_case				= [self.cls_token_id]
          _snake_case				= [self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None   ,_SCREAMING_SNAKE_CASE = False       )				->     List[int]:
          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=_SCREAMING_SNAKE_CASE   ,token_ids_a=_SCREAMING_SNAKE_CASE   ,already_has_special_tokens=_SCREAMING_SNAKE_CASE       )

          if token_ids_a is None:
               return [1] + ([0] * len(_SCREAMING_SNAKE_CASE       )) + [1]
          return [1] + ([0] * len(_SCREAMING_SNAKE_CASE       )) + [1, 1] + ([0] * len(_SCREAMING_SNAKE_CASE       )) + [1]






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None       )				->     List[int]:
          _snake_case				= [self.sep_token_id]
          _snake_case				= [self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep       ) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]






     @property
     def 							_lowercase      (					self       )				->     Optional[Any]:
          return len(self.fairseq_tokens_to_ids       ) + len(self.sp_model       )






     def 							_lowercase      (					self       )				->     Optional[int]:
          _snake_case				= {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE       ): i for i in range(self.vocab_size       )}
          vocab.update(self.added_tokens_encoder       )
          return vocab






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     List[str]:
          return self.sp_model.encode(_SCREAMING_SNAKE_CASE   ,out_type=_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     int:
          if token in self.fairseq_tokens_to_ids:
               return self.fairseq_tokens_to_ids[token]
          elif self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE       ) == 0:
               # Convert sentence piece unk token to fairseq unk token index
               return self.unk_token_id
          return self.fairseq_offset + self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     Tuple:
          if index in self.fairseq_ids_to_tokens:
               return self.fairseq_ids_to_tokens[index]
          return self.sp_model.IdToPiece(index - self.fairseq_offset       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     Union[str, Any]:
          _snake_case				= []
          _snake_case				= ""
          _snake_case				= False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    if not prev_is_special:
                         out_string += " "
                    out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE       ) + token
                    _snake_case				= True
                    _snake_case				= []
               else:
                    current_sub_tokens.append(_SCREAMING_SNAKE_CASE       )
                    _snake_case				= False
          out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE       )
          return out_string.strip()






     def __getstate__(					self       )				->     Optional[Any]:
          _snake_case				= self.__dict__.copy()
          _snake_case				= None
          return state






     def __setstate__(					self   ,_SCREAMING_SNAKE_CASE       )				->     Tuple:
          _snake_case				= d

          # for backward compatibility
          if not hasattr(self   ,"sp_model_kwargs"       ):
               _snake_case				= {}

          _snake_case				= spm.SentencePieceProcessor(**self.sp_model_kwargs       )
          self.sp_model.Load(self.vocab_file       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None       )				->     Tuple[str]:
          if not os.path.isdir(_SCREAMING_SNAKE_CASE       ):
               logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
               return
          _snake_case				= os.path.join(
              _SCREAMING_SNAKE_CASE   ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

          if os.path.abspath(self.vocab_file       ) != os.path.abspath(_SCREAMING_SNAKE_CASE       ) and os.path.isfile(self.vocab_file       ):
               copyfile(self.vocab_file   ,_SCREAMING_SNAKE_CASE       )
          elif not os.path.isfile(self.vocab_file       ):
               with open(_SCREAMING_SNAKE_CASE   ,"wb"       ) as fi:
                    _snake_case				= self.sp_model.serialized_model_proto()
                    fi.write(_SCREAMING_SNAKE_CASE       )

          return (out_vocab_file,)








'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_		:     int          =		logging.get_logger(__name__)

UpperCamelCase_		:     Any          =		{
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		"""falcon"""
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		["""past_key_values"""]






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=65_024   ,_SCREAMING_SNAKE_CASE=4_544   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=71   ,_SCREAMING_SNAKE_CASE=1e-5   ,_SCREAMING_SNAKE_CASE=0.0_2   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=11   ,_SCREAMING_SNAKE_CASE=11   ,**_SCREAMING_SNAKE_CASE   ,)				->     Dict:
          _snake_case				= vocab_size
          # Backward compatibility with n_embed kwarg
          _snake_case				= kwargs.pop("n_embed"   ,_SCREAMING_SNAKE_CASE       )
          _snake_case				= hidden_size if n_embed is None else n_embed
          _snake_case				= num_hidden_layers
          _snake_case				= num_attention_heads
          _snake_case				= layer_norm_epsilon
          _snake_case				= initializer_range
          _snake_case				= use_cache
          _snake_case				= hidden_dropout
          _snake_case				= attention_dropout

          _snake_case				= bos_token_id
          _snake_case				= eos_token_id
          _snake_case				= num_attention_heads if num_kv_heads is None else num_kv_heads
          _snake_case				= alibi
          _snake_case				= new_decoder_architecture
          _snake_case				= multi_query  # Ignored when new_decoder_architecture is True
          _snake_case				= parallel_attn
          _snake_case				= bias

          super().__init__(bos_token_id=_SCREAMING_SNAKE_CASE   ,eos_token_id=_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE       )






     @property
     def 							_lowercase      (					self       )				->     Union[str, Any]:
          return self.hidden_size // self.num_attention_heads






     @property
     def 							_lowercase      (					self       )				->     Optional[int]:
          return not self.alibi


