



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_mbart import MBartTokenizer
else:
	UpperCAmelCase							 =					None


UpperCAmelCase							 =					logging.get_logger(__name__)


UpperCAmelCase							 =					{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase							 =					{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

UpperCAmelCase							 =					{
    """facebook/mbart-large-en-ro""": 1_024,
    """facebook/mbart-large-cc25""": 1_024,
}

# fmt: off
UpperCAmelCase							 =					["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]

class  __snake_case      (   SCREAMING_SNAKE_CASE):







							'''simple docstring'''






							UpperCamelCase__				:      Optional[Any]         =					VOCAB_FILES_NAMES
							UpperCamelCase__				:      Optional[int]         =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__				:      Any         =					PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__				:      List[str]         =					["""input_ids""", """attention_mask"""]
							UpperCamelCase__				:      Union[str, Any]         =					MBartTokenizer

							UpperCamelCase__				:      List[str]         =					[]
							UpperCamelCase__				:      List[Any]         =					[]


							def __init__(							self  ,  a_=None  ,  a_=None  ,  a_="<s>"  ,  a_="</s>"  ,  a_="</s>"  ,  a_="<s>"  ,  a_="<unk>"  ,  a_="<pad>"  ,  a_="<mask>"  ,  a_=None  ,  a_=None  ,  a_=None  ,  **a_  ,  ):
											a__				      =		AddedToken(a_  ,  lstrip=a_  ,  rstrip=a_		) if isinstance(a_  ,  a_		) else mask_token

											super().__init__(
											    vocab_file=a_  ,  tokenizer_file=a_  ,  bos_token=a_  ,  eos_token=a_  ,  sep_token=a_  ,  cls_token=a_  ,  unk_token=a_  ,  pad_token=a_  ,  mask_token=a_  ,  src_lang=a_  ,  tgt_lang=a_  ,  additional_special_tokens=a_  ,  **a_  ,  )

											a__				      =		vocab_file
											a__				      =		False if not self.vocab_file else True

											a__				      =		FAIRSEQ_LANGUAGE_CODES.copy()

											if additional_special_tokens is not None:
															# Only add those special tokens if they are not already there.
															_additional_special_tokens.extend(
															    [t for t in additional_special_tokens if t not in _additional_special_tokens]		)

											self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}		)
											a__				      =		{
											    lang_code: self.convert_tokens_to_ids(a_		) for lang_code in FAIRSEQ_LANGUAGE_CODES
											}

											a__				      =		src_lang if src_lang is not None else 'en_XX'
											a__				      =		self.convert_tokens_to_ids(self._src_lang		)
											a__				      =		tgt_lang
											self.set_src_lang_special_tokens(self._src_lang		)


							@property
							def _a      (							self		):
											return self._src_lang


							@src_lang.setter
							def _a      (							self  ,  a_		):
											a__				      =		new_src_lang
											self.set_src_lang_special_tokens(self._src_lang		)


							def _a      (							self  ,  a_  ,  a_ = None		):







											if token_ids_a is None:
															return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


							def _a      (							self  ,  a_  ,  a_ = None		):
											a__				      =		[self.sep_token_id]
											a__				      =		[self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]


							def _a      (							self  ,  a_  ,  a_  ,  a_  ,  a_  ,  **a_		):







											if src_lang is None or tgt_lang is None:
															raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""		)
											a__				      =		src_lang
											a__				      =		self(a_  ,  add_special_tokens=a_  ,  return_tensors=a_  ,  **a_		)
											a__				      =		self.convert_tokens_to_ids(a_		)
											a__				      =		tgt_lang_id
											return inputs


							def _a      (							self  ,  a_  ,  a_ = "en_XX"  ,  a_ = None  ,  a_ = "ro_RO"  ,  **a_  ,  ):
											a__				      =		src_lang
											a__				      =		tgt_lang
											return super().prepare_seqaseq_batch(a_  ,  a_  ,  **a_		)


							def _a      (							self		):
											return self.set_src_lang_special_tokens(self.src_lang		)


							def _a      (							self		):
											return self.set_tgt_lang_special_tokens(self.tgt_lang		)


							def _a      (							self  ,  a_		):
											a__				      =		self.convert_tokens_to_ids(a_		)
											a__				      =		[]
											a__				      =		[self.eos_token_id, self.cur_lang_code]

											a__				      =		self.convert_ids_to_tokens(self.prefix_tokens		)
											a__				      =		self.convert_ids_to_tokens(self.suffix_tokens		)

											a__				      =		processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str  ,  pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str  ,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str  ,  self.prefix_tokens + self.suffix_tokens		)		)  ,  )


							def _a      (							self  ,  a_		):
											a__				      =		self.convert_tokens_to_ids(a_		)
											a__				      =		[]
											a__				      =		[self.eos_token_id, self.cur_lang_code]

											a__				      =		self.convert_ids_to_tokens(self.prefix_tokens		)
											a__				      =		self.convert_ids_to_tokens(self.suffix_tokens		)

											a__				      =		processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str  ,  pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str  ,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str  ,  self.prefix_tokens + self.suffix_tokens		)		)  ,  )







							def _a      (							self  ,  a_  ,  a_ = None		):







											if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer."""		)

											if not os.path.isdir(a_		):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''		)
															return
											a__				      =		os.path.join(
											    a_  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(a_		):
															copyfile(self.vocab_file  ,  a_		)

											return (out_vocab_file,)






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase							 =					{
    """configuration_xlm_roberta""": [
        """XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """XLMRobertaConfig""",
        """XLMRobertaOnnxConfig""",
    ],
}

try:
	if not is_sentencepiece_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	UpperCAmelCase							 =					["""XLMRobertaTokenizer"""]

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	UpperCAmelCase							 =					["""XLMRobertaTokenizerFast"""]

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	UpperCAmelCase							 =					[
	    """XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """XLMRobertaForCausalLM""",
	    """XLMRobertaForMaskedLM""",
	    """XLMRobertaForMultipleChoice""",
	    """XLMRobertaForQuestionAnswering""",
	    """XLMRobertaForSequenceClassification""",
	    """XLMRobertaForTokenClassification""",
	    """XLMRobertaModel""",
	    """XLMRobertaPreTrainedModel""",
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	UpperCAmelCase							 =					[
	    """TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """TFXLMRobertaForCausalLM""",
	    """TFXLMRobertaForMaskedLM""",
	    """TFXLMRobertaForMultipleChoice""",
	    """TFXLMRobertaForQuestionAnswering""",
	    """TFXLMRobertaForSequenceClassification""",
	    """TFXLMRobertaForTokenClassification""",
	    """TFXLMRobertaModel""",
	    """TFXLMRobertaPreTrainedModel""",
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	UpperCAmelCase							 =					[
	    """FLAX_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """FlaxXLMRobertaForMaskedLM""",
	    """FlaxXLMRobertaForCausalLM""",
	    """FlaxXLMRobertaForMultipleChoice""",
	    """FlaxXLMRobertaForQuestionAnswering""",
	    """FlaxXLMRobertaForSequenceClassification""",
	    """FlaxXLMRobertaForTokenClassification""",
	    """FlaxXLMRobertaModel""",
	    """FlaxXLMRobertaPreTrainedModel""",
	]

if TYPE_CHECKING:
	from .configuration_xlm_roberta import (
	    XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    XLMRobertaConfig,
	    XLMRobertaOnnxConfig,
	)

	try:
		if not is_sentencepiece_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_xlm_roberta import XLMRobertaTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_xlm_roberta_fast import XLMRobertaTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_xlm_roberta import (
		    XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
		    XLMRobertaForCausalLM,
		    XLMRobertaForMaskedLM,
		    XLMRobertaForMultipleChoice,
		    XLMRobertaForQuestionAnswering,
		    XLMRobertaForSequenceClassification,
		    XLMRobertaForTokenClassification,
		    XLMRobertaModel,
		    XLMRobertaPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_xlm_roberta import (
		    TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFXLMRobertaForCausalLM,
		    TFXLMRobertaForMaskedLM,
		    TFXLMRobertaForMultipleChoice,
		    TFXLMRobertaForQuestionAnswering,
		    TFXLMRobertaForSequenceClassification,
		    TFXLMRobertaForTokenClassification,
		    TFXLMRobertaModel,
		    TFXLMRobertaPreTrainedModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_xlm_roberta import (
		    FLAX_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
		    FlaxXLMRobertaForCausalLM,
		    FlaxXLMRobertaForMaskedLM,
		    FlaxXLMRobertaForMultipleChoice,
		    FlaxXLMRobertaForQuestionAnswering,
		    FlaxXLMRobertaForSequenceClassification,
		    FlaxXLMRobertaForTokenClassification,
		    FlaxXLMRobertaModel,
		    FlaxXLMRobertaPreTrainedModel,
		)

else:
	import sys

	UpperCAmelCase							 =					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

