





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase							:       Tuple					 =	logging.get_logger(__name__)

lowerCAmelCase							:       List[Any]					 =	"""▁"""

lowerCAmelCase							:       Optional[int]					 =	{"""vocab_file""": """sentencepiece.bpe.model"""}

lowerCAmelCase							:       Optional[int]					 =	{
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

lowerCAmelCase							:       Tuple					 =	{
    """facebook/mbart-large-50-one-to-many-mmt""": 1024,
}

# fmt: off
lowerCAmelCase							:       Optional[Any]					 =	["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]





class 				a			(      __lowercase			):
				SCREAMING_SNAKE_CASE__     :   List[str]		      =     VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__     :   List[str]		      =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__     :   Any		      =     PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__     :   List[Any]		      =     ['''input_ids''', '''attention_mask''']

				SCREAMING_SNAKE_CASE__     :   List[int]		      =     []
				SCREAMING_SNAKE_CASE__     :   List[int]		      =     []






				def __init__(  self			, _lowerCAmelCase			, _lowerCAmelCase=None			, _lowerCAmelCase=None			, _lowerCAmelCase="</s>"			, _lowerCAmelCase="</s>"			, _lowerCAmelCase="<s>"			, _lowerCAmelCase="<unk>"			, _lowerCAmelCase="<pad>"			, _lowerCAmelCase="<mask>"			, _lowerCAmelCase = None			, **_lowerCAmelCase			, ):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   AddedToken(_lowerCAmelCase			, lstrip=_lowerCAmelCase			, rstrip=_lowerCAmelCase					) if isinstance(_lowerCAmelCase			, _lowerCAmelCase					) else mask_token

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   {} if sp_model_kwargs is None else sp_model_kwargs

										__SCREAMING_SNAKE_CASE:				int           =   kwargs.get('''additional_special_tokens'''			, []					)
										kwargs["additional_special_tokens"] += [
										    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
										]

										super().__init__(
										    src_lang=_lowerCAmelCase			, tgt_lang=_lowerCAmelCase			, eos_token=_lowerCAmelCase			, unk_token=_lowerCAmelCase			, sep_token=_lowerCAmelCase			, cls_token=_lowerCAmelCase			, pad_token=_lowerCAmelCase			, mask_token=_lowerCAmelCase			, sp_model_kwargs=self.sp_model_kwargs			, **_lowerCAmelCase			, )

										__SCREAMING_SNAKE_CASE:				Dict           =   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
										self.sp_model.Load(str(_lowerCAmelCase					)					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   vocab_file

										# Original fairseq vocab and spm vocab must be "aligned":
										# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
										# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
										# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
										# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

										# Mimic fairseq token-to-id alignment for the first 4 token
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

										# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   1

										__SCREAMING_SNAKE_CASE:				List[Any]           =   len(self.sp_model					)
										__SCREAMING_SNAKE_CASE:				Any           =   {
										    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_lowerCAmelCase					)
										}
										__SCREAMING_SNAKE_CASE:				List[str]           =   {v: k for k, v in self.lang_code_to_id.items()}
										__SCREAMING_SNAKE_CASE:				Any           =   len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset

										self.fairseq_tokens_to_ids.update(self.lang_code_to_id					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   {v: k for k, v in self.fairseq_tokens_to_ids.items()}

										__SCREAMING_SNAKE_CASE:				Dict           =   src_lang if src_lang is not None else '''en_XX'''
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.lang_code_to_id[self._src_lang]
										__SCREAMING_SNAKE_CASE:				Dict           =   tgt_lang
										self.set_src_lang_special_tokens(self._src_lang					)






				@property
				def 			snake_case_    (  self					):

										"""simple docstring"""




										return len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset + 1  # Plus 1 for the mask token






				@property
				def 			snake_case_    (  self					):

										"""simple docstring"""




										return self._src_lang






				@src_lang.setter
				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   new_src_lang
										self.set_src_lang_special_tokens(self._src_lang					)






				def __getstate__(  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   self.__dict__.copy()
										__SCREAMING_SNAKE_CASE:				Dict           =   None
										return state






				def __setstate__(  self			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   d

										# for backward compatibility
										if not hasattr(self			, '''sp_model_kwargs'''					):
																__SCREAMING_SNAKE_CASE:				int           =   {}

										__SCREAMING_SNAKE_CASE:				Tuple           =   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
										self.sp_model.Load(self.vocab_file					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   {self.convert_ids_to_tokens(_lowerCAmelCase					): i for i in range(self.vocab_size					)}
										vocab.update(self.added_tokens_encoder					)
										return vocab






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										return self.sp_model.encode(_lowerCAmelCase			, out_type=_lowerCAmelCase					)






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										if token in self.fairseq_tokens_to_ids:
																return self.fairseq_tokens_to_ids[token]
										__SCREAMING_SNAKE_CASE:				Any           =   self.sp_model.PieceToId(_lowerCAmelCase					)

										# Need to return unknown token if the SP model returned 0
										return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										if index in self.fairseq_ids_to_tokens:
																return self.fairseq_ids_to_tokens[index]
										return self.sp_model.IdToPiece(index - self.fairseq_offset					)






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   []
										__SCREAMING_SNAKE_CASE:				str           =   ''''''
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   False
										for token in tokens:
																# make sure that special tokens are not decoded using sentencepiece model
																if token in self.all_special_tokens:
																						if not prev_is_special:
																												out_string += " "
																						out_string += self.sp_model.decode(_lowerCAmelCase					) + token
																						__SCREAMING_SNAKE_CASE:				int           =   True
																						__SCREAMING_SNAKE_CASE:				List[str]           =   []
																else:
																						current_sub_tokens.append(_lowerCAmelCase					)
																						__SCREAMING_SNAKE_CASE:				Tuple           =   False
										out_string += self.sp_model.decode(_lowerCAmelCase					)
										return out_string.strip()






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase = None					):

										"""simple docstring"""




										if not os.path.isdir(_lowerCAmelCase					):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
																return
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   os.path.join(
										    _lowerCAmelCase			, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

										if os.path.abspath(self.vocab_file					) != os.path.abspath(_lowerCAmelCase					) and os.path.isfile(self.vocab_file					):
																copyfile(self.vocab_file			, _lowerCAmelCase					)
										elif not os.path.isfile(self.vocab_file					):
																with open(_lowerCAmelCase			, '''wb'''					) as fi:
																						__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   self.sp_model.serialized_model_proto()
																						fi.write(_lowerCAmelCase					)

										return (out_vocab_file,)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase = None			, _lowerCAmelCase = False					):

										"""simple docstring"""




										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_lowerCAmelCase			, token_ids_a=_lowerCAmelCase			, already_has_special_tokens=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Tuple           =   [1] * len(self.prefix_tokens					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   [1] * len(self.suffix_tokens					)
										if token_ids_a is None:
																return prefix_ones + ([0] * len(_lowerCAmelCase					)) + suffix_ones
										return prefix_ones + ([0] * len(_lowerCAmelCase					)) + ([0] * len(_lowerCAmelCase					)) + suffix_ones






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase = None					):

										"""simple docstring"""




										if token_ids_a is None:
																return self.prefix_tokens + token_ids_a + self.suffix_tokens
										# We don't expect to process pairs, but leave the pair logic for API consistency
										return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, **_lowerCAmelCase					):

										"""simple docstring"""




										if src_lang is None or tgt_lang is None:
																raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''					)
										__SCREAMING_SNAKE_CASE:				int           =   src_lang
										__SCREAMING_SNAKE_CASE:				str           =   self(_lowerCAmelCase			, add_special_tokens=_lowerCAmelCase			, return_tensors=_lowerCAmelCase			, **_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Any           =   self.convert_tokens_to_ids(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Any           =   tgt_lang_id
										return inputs






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase = "en_XX"			, _lowerCAmelCase = None			, _lowerCAmelCase = "ro_RO"			, **_lowerCAmelCase			, ):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   src_lang
										__SCREAMING_SNAKE_CASE:				Dict           =   tgt_lang
										return super().prepare_seqaseq_batch(_lowerCAmelCase			, _lowerCAmelCase			, **_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										return self.set_src_lang_special_tokens(self.src_lang					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										return self.set_tgt_lang_special_tokens(self.tgt_lang					)






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.lang_code_to_id[src_lang]
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   [self.cur_lang_code_id]
										__SCREAMING_SNAKE_CASE:				str           =   [self.eos_token_id]






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   self.lang_code_to_id[tgt_lang]
										__SCREAMING_SNAKE_CASE:				Dict           =   [self.cur_lang_code_id]
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   [self.eos_token_id]










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowerCAmelCase							:       List[str]					 =	{
    """configuration_gpt_bigcode""": ["""GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """GPTBigCodeConfig"""],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase							:       str					 =	[
							    """GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """GPTBigCodeForSequenceClassification""",
							    """GPTBigCodeForTokenClassification""",
							    """GPTBigCodeForCausalLM""",
							    """GPTBigCodeModel""",
							    """GPTBigCodePreTrainedModel""",
							]

if TYPE_CHECKING:
							from .configuration_gpt_bigcode import GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTBigCodeConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_gpt_bigcode import (
														    GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST,
														    GPTBigCodeForCausalLM,
														    GPTBigCodeForSequenceClassification,
														    GPTBigCodeForTokenClassification,
														    GPTBigCodeModel,
														    GPTBigCodePreTrainedModel,
														)


else:
							import sys

							lowerCAmelCase							:       List[str]					 =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



