


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_mbart import MBartTokenizer
else:
 SCREAMING_SNAKE_CASE			: List[str]				   =					None


SCREAMING_SNAKE_CASE			: Tuple				   =					logging.get_logger(__name__)


SCREAMING_SNAKE_CASE			: List[Any]				   =					{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE			: Dict				   =					{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

SCREAMING_SNAKE_CASE			: str				   =					{
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
SCREAMING_SNAKE_CASE			: List[str]				   =					['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']






class _lowerCamelCase(		_snake_case  ):
       lowercase_					:      Any       					=     VOCAB_FILES_NAMES
       lowercase_					:      List[str]       					=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowercase_					:      Any       					=     PRETRAINED_VOCAB_FILES_MAP
       lowercase_					:      Tuple       					=     ["""input_ids""", """attention_mask"""]
       lowercase_					:      Dict       					=     MBartTokenizer

       lowercase_					:      List[int]       					=     []
       lowercase_					:      List[int]       					=     []
       def __init__(   self, lowerCamelCase=None, lowerCamelCase=None, lowerCamelCase="<s>", lowerCamelCase="</s>", lowerCamelCase="</s>", lowerCamelCase="<s>", lowerCamelCase="<unk>", lowerCamelCase="<pad>", lowerCamelCase="<mask>", lowerCamelCase=None, lowerCamelCase=None, lowerCamelCase=None, **lowerCamelCase, )			->  List[Any]:




           """simple docstring"""

           _lowercase			:				List[Any] 						=						AddedToken(__snake_case, lstrip=__snake_case, rstrip=__snake_case) if isinstance(__snake_case, __snake_case) else mask_token

           super().__init__(
               vocab_file=__snake_case, tokenizer_file=__snake_case, bos_token=__snake_case, eos_token=__snake_case, sep_token=__snake_case, cls_token=__snake_case, unk_token=__snake_case, pad_token=__snake_case, mask_token=__snake_case, src_lang=__snake_case, tgt_lang=__snake_case, additional_special_tokens=__snake_case, **__snake_case, )

           _lowercase			:				List[str] 						=						vocab_file
           _lowercase			:				Optional[int] 						=						False if not self.vocab_file else True

           _lowercase			:				int 						=						FAIRSEQ_LANGUAGE_CODES.copy()

           if additional_special_tokens is not None:
               # Only add those special tokens if they are not already there.
               _additional_special_tokens.extend(
                   [t for t in additional_special_tokens if t not in _additional_special_tokens])

           self.add_special_tokens({'additional_special_tokens': _additional_special_tokens})
           _lowercase			:				str 						=						{
               lang_code: self.convert_tokens_to_ids(__snake_case) for lang_code in FAIRSEQ_LANGUAGE_CODES
           }

           _lowercase			:				Optional[Any] 						=						src_lang if src_lang is not None else """en_XX"""
           _lowercase			:				str 						=						self.convert_tokens_to_ids(self._src_lang)
           _lowercase			:				Any 						=						tgt_lang
           self.set_src_lang_special_tokens(self._src_lang)
       @property
       def      UpperCamelCase	(   self)			->  Tuple:




           """simple docstring"""

           return self._src_lang
       @src_lang.setter
       def      UpperCamelCase	(   self, lowerCamelCase)			->  Optional[Any]:




           """simple docstring"""

           _lowercase			:				Any 						=						new_src_lang
           self.set_src_lang_special_tokens(self._src_lang)
       def      UpperCamelCase	(   self, lowerCamelCase, lowerCamelCase = None)			->  Any:




           """simple docstring"""

           if token_ids_a is None:
               return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
       def      UpperCamelCase	(   self, lowerCamelCase, lowerCamelCase = None)			->  Optional[Any]:




           """simple docstring"""

           _lowercase			:				List[str] 						=						[self.sep_token_id]
           _lowercase			:				Union[str, Any] 						=						[self.cls_token_id]

           if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]
       def      UpperCamelCase	(   self, lowerCamelCase, lowerCamelCase, lowerCamelCase, lowerCamelCase, **lowerCamelCase)			->  int:




           """simple docstring"""

           if src_lang is None or tgt_lang is None:
               raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model')
           _lowercase			:				Dict 						=						src_lang
           _lowercase			:				List[Any] 						=						self(__snake_case, add_special_tokens=__snake_case, return_tensors=__snake_case, **__snake_case)
           _lowercase			:				Dict 						=						self.convert_tokens_to_ids(__snake_case)
           _lowercase			:				Union[str, Any] 						=						tgt_lang_id
           return inputs
       def      UpperCamelCase	(   self, lowerCamelCase, lowerCamelCase = "en_XX", lowerCamelCase = None, lowerCamelCase = "ro_RO", **lowerCamelCase, )			->  List[str]:




           """simple docstring"""

           _lowercase			:				Optional[Any] 						=						src_lang
           _lowercase			:				Union[str, Any] 						=						tgt_lang
           return super().prepare_seqaseq_batch(__snake_case, __snake_case, **__snake_case)
       def      UpperCamelCase	(   self)			->  Any:




           """simple docstring"""

           return self.set_src_lang_special_tokens(self.src_lang)
       def      UpperCamelCase	(   self)			->  Union[str, Any]:




           """simple docstring"""

           return self.set_tgt_lang_special_tokens(self.tgt_lang)
       def      UpperCamelCase	(   self, lowerCamelCase)			->  int:




           """simple docstring"""

           _lowercase			:				List[str] 						=						self.convert_tokens_to_ids(__snake_case)
           _lowercase			:				str 						=						[]
           _lowercase			:				Optional[Any] 						=						[self.eos_token_id, self.cur_lang_code]

           _lowercase			:				int 						=						self.convert_ids_to_tokens(self.prefix_tokens)
           _lowercase			:				Optional[Any] 						=						self.convert_ids_to_tokens(self.suffix_tokens)

           _lowercase			:				Optional[int] 						=						processors.TemplateProcessing(
               single=prefix_tokens_str + ['$A'] + suffix_tokens_str, pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str, special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str, self.prefix_tokens + self.suffix_tokens)), )
       def      UpperCamelCase	(   self, lowerCamelCase)			->  Optional[Any]:




           """simple docstring"""

           _lowercase			:				List[str] 						=						self.convert_tokens_to_ids(__snake_case)
           _lowercase			:				List[str] 						=						[]
           _lowercase			:				Tuple 						=						[self.eos_token_id, self.cur_lang_code]

           _lowercase			:				Union[str, Any] 						=						self.convert_ids_to_tokens(self.prefix_tokens)
           _lowercase			:				str 						=						self.convert_ids_to_tokens(self.suffix_tokens)

           _lowercase			:				Union[str, Any] 						=						processors.TemplateProcessing(
               single=prefix_tokens_str + ['$A'] + suffix_tokens_str, pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str, special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str, self.prefix_tokens + self.suffix_tokens)), )




       def      UpperCamelCase	(   self, lowerCamelCase, lowerCamelCase = None)			->  int:




           """simple docstring"""

           if not self.can_save_slow_tokenizer:
               raise ValueError(
                   'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                   'tokenizer.')

           if not os.path.isdir(__snake_case):
               logger.error(F'''Vocabulary path ({save_directory}) should be a directory.''')
               return
           _lowercase			:				List[str] 						=						os.path.join(
               __snake_case, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

           if os.path.abspath(self.vocab_file) != os.path.abspath(__snake_case):
               copyfile(self.vocab_file, __snake_case)

           return (out_vocab_file,)






'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_						:  List[Any]				  =						logging.get_logger(__name__)

UpperCAmelCase_						:  Any				  =						{
    'bigcode/gpt_bigcode-santacoder': 'https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json',
}





class        lowercase__    (		_snake_case   ):





							'''simple docstring'''


							A_				: Dict       =    """gpt_bigcode"""
							A_				: str       =    ["""past_key_values"""]
							A_				: Tuple       =    {
							    """hidden_size""": """n_embd""",
							    """max_position_embeddings""": """n_positions""",
							    """num_attention_heads""": """n_head""",
							    """num_hidden_layers""": """n_layer""",
							}






							def __init__(       self       ,   __snake_case=5_0257       ,   __snake_case=1024       ,   __snake_case=768       ,   __snake_case=12       ,   __snake_case=12       ,   __snake_case=None       ,   __snake_case="gelu_pytorch_tanh"       ,   __snake_case=0.1       ,   __snake_case=0.1       ,   __snake_case=0.1       ,   __snake_case=1e-5       ,   __snake_case=0.02       ,   __snake_case=True       ,   __snake_case=True       ,   __snake_case=5_0256       ,   __snake_case=5_0256       ,   __snake_case=True       ,   __snake_case=True       ,   __snake_case=True       ,   **__snake_case       ,   ):
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= vocab_size
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= n_positions
											_SCREAMING_SNAKE_CASE					:			List[Any]							= n_embd
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= n_layer
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= n_head
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= n_inner
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= activation_function
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= resid_pdrop
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= embd_pdrop
											_SCREAMING_SNAKE_CASE					:			List[Any]							= attn_pdrop
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= layer_norm_epsilon
											_SCREAMING_SNAKE_CASE					:			Any							= initializer_range
											_SCREAMING_SNAKE_CASE					:			str							= scale_attn_weights
											_SCREAMING_SNAKE_CASE					:			Tuple							= use_cache
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= attention_softmax_in_fpaa
											_SCREAMING_SNAKE_CASE					:			List[Any]							= scale_attention_softmax_in_fpaa
											_SCREAMING_SNAKE_CASE					:			Any							= multi_query

											_SCREAMING_SNAKE_CASE					:			Optional[int]							= bos_token_id
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= eos_token_id

											super().__init__(bos_token_id=__snake_case       ,   eos_token_id=__snake_case       ,   **__snake_case    )




