




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE 						="""▁"""

__SCREAMING_SNAKE_CASE 						={"""vocab_file""": """sentencepiece.bpe.model"""}

__SCREAMING_SNAKE_CASE 						={
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

__SCREAMING_SNAKE_CASE 						={
    """facebook/mbart-large-en-ro""": 1_024,
    """facebook/mbart-large-cc25""": 1_024,
}

# fmt: off
__SCREAMING_SNAKE_CASE 						=["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							Optional[Any]	      =					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__					:							Any	      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__					:							Dict	      =					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__					:							Union[str, Any]	      =					["input_ids", "attention_mask"]

				SCREAMING_SNAKE_CASE__					:							List[int]	      =					[]
				SCREAMING_SNAKE_CASE__					:							List[int]	      =					[]


				def __init__(							self:   Dict		,					_lowerCamelCase:   int		,					_lowerCamelCase:   List[str]="<s>"		,					_lowerCamelCase:   List[Any]="</s>"		,					_lowerCamelCase:   Optional[int]="</s>"		,					_lowerCamelCase:   int="<s>"		,					_lowerCamelCase:   Tuple="<unk>"		,					_lowerCamelCase:   Dict="<pad>"		,					_lowerCamelCase:   int="<mask>"		,					_lowerCamelCase:   Tuple=None		,					_lowerCamelCase:   Union[str, Any]=None		,					_lowerCamelCase:   Optional[int]=None		,					_lowerCamelCase:   Optional[Dict[str, Any]] = None		,					_lowerCamelCase:   Optional[Any]=None		,					**_lowerCamelCase:   Union[str, Any]		,					):
											# Mask token behave like a normal word, i.e. include the space before it
											SCREAMING_SNAKE_CASE_   	=    AddedToken(_lowerCamelCase		,					lstrip=_lowerCamelCase		,					rstrip=_lowerCamelCase		) if isinstance(_lowerCamelCase		,					_lowerCamelCase		) else mask_token

											SCREAMING_SNAKE_CASE_   	=    {} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    bos_token=_lowerCamelCase		,					eos_token=_lowerCamelCase		,					unk_token=_lowerCamelCase		,					sep_token=_lowerCamelCase		,					cls_token=_lowerCamelCase		,					pad_token=_lowerCamelCase		,					mask_token=_lowerCamelCase		,					tokenizer_file=_lowerCamelCase		,					src_lang=_lowerCamelCase		,					tgt_lang=_lowerCamelCase		,					additional_special_tokens=_lowerCamelCase		,					sp_model_kwargs=self.sp_model_kwargs		,					**_lowerCamelCase		,					)

											SCREAMING_SNAKE_CASE_   	=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(str(_lowerCamelCase		)		)
											SCREAMING_SNAKE_CASE_   	=    vocab_file

											# Original fairseq vocab and spm vocab must be "aligned":
											# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
											# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
											# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
											# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

											# Mimic fairseq token-to-id alignment for the first 4 token
											SCREAMING_SNAKE_CASE_   	=    {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

											# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
											SCREAMING_SNAKE_CASE_   	=    1

											SCREAMING_SNAKE_CASE_   	=    len(self.sp_model		)
											SCREAMING_SNAKE_CASE_   	=    {
											    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_lowerCamelCase		)
											}
											SCREAMING_SNAKE_CASE_   	=    {v: k for k, v in self.lang_code_to_id.items()}
											SCREAMING_SNAKE_CASE_   	=    len(self.sp_model		) + len(self.lang_code_to_id		) + self.fairseq_offset

											self.fairseq_tokens_to_ids.update(self.lang_code_to_id		)
											SCREAMING_SNAKE_CASE_   	=    {v: k for k, v in self.fairseq_tokens_to_ids.items()}
											SCREAMING_SNAKE_CASE_   	=    list(self.lang_code_to_id.keys()		)

											if additional_special_tokens is not None:
																		# Only add those special tokens if they are not already there.
																		self._additional_special_tokens.extend(
																		    [t for t in additional_special_tokens if t not in self._additional_special_tokens]		)

											SCREAMING_SNAKE_CASE_   	=    src_lang if src_lang is not None else '''en_XX'''
											SCREAMING_SNAKE_CASE_   	=    self.lang_code_to_id[self._src_lang]
											SCREAMING_SNAKE_CASE_   	=    tgt_lang
											self.set_src_lang_special_tokens(self._src_lang		)


				def __getstate__(							self:   Optional[Any]		):
											SCREAMING_SNAKE_CASE_   	=    self.__dict__.copy()
											SCREAMING_SNAKE_CASE_   	=    None
											SCREAMING_SNAKE_CASE_   	=    self.sp_model.serialized_model_proto()
											return state


				def __setstate__(							self:   str		,					_lowerCamelCase:   List[Any]		):
											SCREAMING_SNAKE_CASE_   	=    d

											# for backward compatibility
											if not hasattr(self		,					'''sp_model_kwargs'''		):
																		SCREAMING_SNAKE_CASE_   	=    {}

											SCREAMING_SNAKE_CASE_   	=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)


				@property
				def     _A							(							self:   Optional[int]		):
											return len(self.sp_model		) + len(self.lang_code_to_id		) + self.fairseq_offset + 1  # Plus 1 for the mask token


				@property
				def     _A							(							self:   Dict		):
											return self._src_lang


				@src_lang.setter
				def     _A							(							self:   Optional[int]		,					_lowerCamelCase:   str		):
											SCREAMING_SNAKE_CASE_   	=    new_src_lang
											self.set_src_lang_special_tokens(self._src_lang		)


				def     _A							(							self:   int		,					_lowerCamelCase:   List[int]		,					_lowerCamelCase:   Optional[List[int]] = None		,					_lowerCamelCase:   bool = False		):

											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=_lowerCamelCase		,					token_ids_a=_lowerCamelCase		,					already_has_special_tokens=_lowerCamelCase		)

											SCREAMING_SNAKE_CASE_   	=    [1] * len(self.prefix_tokens		)
											SCREAMING_SNAKE_CASE_   	=    [1] * len(self.suffix_tokens		)
											if token_ids_a is None:
																		return prefix_ones + ([0] * len(_lowerCamelCase		)) + suffix_ones
											return prefix_ones + ([0] * len(_lowerCamelCase		)) + ([0] * len(_lowerCamelCase		)) + suffix_ones


				def     _A							(							self:   List[Any]		,					_lowerCamelCase:   List[int]		,					_lowerCamelCase:   Optional[List[int]] = None		):
											if token_ids_a is None:
																		return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


				def     _A							(							self:   List[str]		,					_lowerCamelCase:   List[int]		,					_lowerCamelCase:   Optional[List[int]] = None		):
											SCREAMING_SNAKE_CASE_   	=    [self.sep_token_id]
											SCREAMING_SNAKE_CASE_   	=    [self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]


				def     _A							(							self:   List[Any]		,					_lowerCamelCase:   Optional[Any]		,					_lowerCamelCase:   str		,					_lowerCamelCase:   Optional[str]		,					_lowerCamelCase:   Optional[str]		,					**_lowerCamelCase:   Union[str, Any]		):
											if src_lang is None or tgt_lang is None:
																		raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''		)
											SCREAMING_SNAKE_CASE_   	=    src_lang
											SCREAMING_SNAKE_CASE_   	=    self(_lowerCamelCase		,					add_special_tokens=_lowerCamelCase		,					return_tensors=_lowerCamelCase		,					**_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self.convert_tokens_to_ids(_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    tgt_lang_id
											return inputs


				def     _A							(							self:   Optional[int]		):
											SCREAMING_SNAKE_CASE_   	=    {self.convert_ids_to_tokens(_lowerCamelCase		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab


				def     _A							(							self:   Union[str, Any]		,					_lowerCamelCase:   str		):
											return self.sp_model.encode(_lowerCamelCase		,					out_type=_lowerCamelCase		)


				def     _A							(							self:   str		,					_lowerCamelCase:   Union[str, Any]		):
											if token in self.fairseq_tokens_to_ids:
																		return self.fairseq_tokens_to_ids[token]
											SCREAMING_SNAKE_CASE_   	=    self.sp_model.PieceToId(_lowerCamelCase		)

											# Need to return unknown token if the SP model returned 0
											return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


				def     _A							(							self:   List[str]		,					_lowerCamelCase:   Any		):
											if index in self.fairseq_ids_to_tokens:
																		return self.fairseq_ids_to_tokens[index]
											return self.sp_model.IdToPiece(index - self.fairseq_offset		)


				def     _A							(							self:   List[Any]		,					_lowerCamelCase:   Union[str, Any]		):
											SCREAMING_SNAKE_CASE_   	=    ''''''.join(_lowerCamelCase		).replace(_lowerCamelCase		,					''' '''		).strip()
											return out_string


				def     _A							(							self:   int		,					_lowerCamelCase:   str		,					_lowerCamelCase:   Optional[str] = None		):
											if not os.path.isdir(_lowerCamelCase		):
																		logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
																		return
											SCREAMING_SNAKE_CASE_   	=    os.path.join(
											    _lowerCamelCase		,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(_lowerCamelCase		) and os.path.isfile(self.vocab_file		):
																		copyfile(self.vocab_file		,					_lowerCamelCase		)
											elif not os.path.isfile(self.vocab_file		):
																		with open(_lowerCamelCase		,					'''wb'''		) as fi:
																									SCREAMING_SNAKE_CASE_   	=    self.sp_model.serialized_model_proto()
																									fi.write(_lowerCamelCase		)

											return (out_vocab_file,)


				def     _A							(							self:   List[Any]		,					_lowerCamelCase:   List[str]		,					_lowerCamelCase:   str = "en_XX"		,					_lowerCamelCase:   Optional[List[str]] = None		,					_lowerCamelCase:   str = "ro_RO"		,					**_lowerCamelCase:   str		,					):
											SCREAMING_SNAKE_CASE_   	=    src_lang
											SCREAMING_SNAKE_CASE_   	=    tgt_lang
											return super().prepare_seqaseq_batch(_lowerCamelCase		,					_lowerCamelCase		,					**_lowerCamelCase		)


				def     _A							(							self:   Dict		):
											return self.set_src_lang_special_tokens(self.src_lang		)


				def     _A							(							self:   List[str]		):
											return self.set_tgt_lang_special_tokens(self.tgt_lang		)


				def     _A							(							self:   Any		,					_lowerCamelCase:   Optional[int]		):
											SCREAMING_SNAKE_CASE_   	=    self.lang_code_to_id[src_lang]
											SCREAMING_SNAKE_CASE_   	=    []
											SCREAMING_SNAKE_CASE_   	=    [self.eos_token_id, self.cur_lang_code]





				def     _A							(							self:   Optional[int]		,					_lowerCamelCase:   str		):
											SCREAMING_SNAKE_CASE_   	=    self.lang_code_to_id[lang]
											SCREAMING_SNAKE_CASE_   	=    []
											SCREAMING_SNAKE_CASE_   	=    [self.eos_token_id, self.cur_lang_code]








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__SCREAMING_SNAKE_CASE 						={
    """configuration_xlm_roberta_xl""": [
        """XLM_ROBERTA_XL_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """XLMRobertaXLConfig""",
        """XLMRobertaXLOnnxConfig""",
    ],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__SCREAMING_SNAKE_CASE 						=[
			    """XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST""",
			    """XLMRobertaXLForCausalLM""",
			    """XLMRobertaXLForMaskedLM""",
			    """XLMRobertaXLForMultipleChoice""",
			    """XLMRobertaXLForQuestionAnswering""",
			    """XLMRobertaXLForSequenceClassification""",
			    """XLMRobertaXLForTokenClassification""",
			    """XLMRobertaXLModel""",
			    """XLMRobertaXLPreTrainedModel""",
			]

if TYPE_CHECKING:
			from .configuration_xlm_roberta_xl import (
			    XLM_ROBERTA_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,
			    XLMRobertaXLConfig,
			    XLMRobertaXLOnnxConfig,
			)

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_xlm_roberta_xl import (
						    XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    XLMRobertaXLForCausalLM,
						    XLMRobertaXLForMaskedLM,
						    XLMRobertaXLForMultipleChoice,
						    XLMRobertaXLForQuestionAnswering,
						    XLMRobertaXLForSequenceClassification,
						    XLMRobertaXLForTokenClassification,
						    XLMRobertaXLModel,
						    XLMRobertaXLPreTrainedModel,
						)

else:
			import sys

			__SCREAMING_SNAKE_CASE 						=_LazyModule(__name__, globals()["""__file__"""], _import_structure)


