




from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TFConvBertForMaskedLM,
			    TFConvBertForMultipleChoice,
			    TFConvBertForQuestionAnswering,
			    TFConvBertForSequenceClassification,
			    TFConvBertForTokenClassification,
			    TFConvBertModel,
			)







class a :
						"""simple docstring"""

						def __init__( self :     List[str]				,							lowerCamelCase__ :     Tuple				,							lowerCamelCase__ :     str=13				,							lowerCamelCase__ :     Tuple=7				,							lowerCamelCase__ :     Optional[int]=True				,							lowerCamelCase__ :     int=True				,							lowerCamelCase__ :     Union[str, Any]=True				,							lowerCamelCase__ :     Dict=True				,							lowerCamelCase__ :     List[Any]=99				,							lowerCamelCase__ :     Optional[int]=32				,							lowerCamelCase__ :     int=2				,							lowerCamelCase__ :     str=4				,							lowerCamelCase__ :     Optional[Any]=37				,							lowerCamelCase__ :     str="gelu"				,							lowerCamelCase__ :     str=0.1				,							lowerCamelCase__ :     Union[str, Any]=0.1				,							lowerCamelCase__ :     Union[str, Any]=512				,							lowerCamelCase__ :     List[Any]=16				,							lowerCamelCase__ :     Optional[Any]=2				,							lowerCamelCase__ :     Any=0.0_2				,							lowerCamelCase__ :     Any=3				,							lowerCamelCase__ :     int=4				,							lowerCamelCase__ :     Any=None				,							)    ->						List[str]:
													"""simple docstring"""


													__lowercase      =    parent
													__lowercase      =    13
													__lowercase      =    7
													__lowercase      =    True
													__lowercase      =    True
													__lowercase      =    True
													__lowercase      =    True
													__lowercase      =    99
													__lowercase      =    384
													__lowercase      =    2
													__lowercase      =    4
													__lowercase      =    37
													__lowercase      =    '''gelu'''
													__lowercase      =    0.1
													__lowercase      =    0.1
													__lowercase      =    512
													__lowercase      =    16
													__lowercase      =    2
													__lowercase      =    0.0_2
													__lowercase      =    3
													__lowercase      =    4
													__lowercase      =    128
													__lowercase      =    2
													__lowercase      =    9
													__lowercase      =    1
													__lowercase      =    None






						def     UpperCAmelCase_      ( self :     List[str]				)    ->						Optional[Any]:
													"""simple docstring"""


													__lowercase      =    ids_tensor([self.batch_size, self.seq_length]				,							self.vocab_size				)

													__lowercase      =    None
													if self.use_input_mask:
																				__lowercase      =    random_attention_mask([self.batch_size, self.seq_length]				)

													__lowercase      =    None
													if self.use_token_type_ids:
																				__lowercase      =    ids_tensor([self.batch_size, self.seq_length]				,							self.type_vocab_size				)

													__lowercase      =    None
													__lowercase      =    None
													__lowercase      =    None
													if self.use_labels:
																				__lowercase      =    ids_tensor([self.batch_size]				,							self.type_sequence_label_size				)
																				__lowercase      =    ids_tensor([self.batch_size, self.seq_length]				,							self.num_labels				)
																				__lowercase      =    ids_tensor([self.batch_size]				,							self.num_choices				)

													__lowercase      =    ConvBertConfig(
													    vocab_size=self.vocab_size				,							hidden_size=self.hidden_size				,							num_hidden_layers=self.num_hidden_layers				,							num_attention_heads=self.num_attention_heads				,							intermediate_size=self.intermediate_size				,							hidden_act=self.hidden_act				,							hidden_dropout_prob=self.hidden_dropout_prob				,							attention_probs_dropout_prob=self.attention_probs_dropout_prob				,							max_position_embeddings=self.max_position_embeddings				,							type_vocab_size=self.type_vocab_size				,							initializer_range=self.initializer_range				,							return_dict=lowerCamelCase__				,							)

													return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def     UpperCAmelCase_      ( self :     Any				,							lowerCamelCase__ :     Tuple				,							lowerCamelCase__ :     Union[str, Any]				,							lowerCamelCase__ :     Any				,							lowerCamelCase__ :     Optional[Any]				,							lowerCamelCase__ :     int				,							lowerCamelCase__ :     Optional[int]				,							lowerCamelCase__ :     List[Any]				)    ->						int:
													"""simple docstring"""


													__lowercase      =    TFConvBertModel(config=lowerCamelCase__				)
													__lowercase      =    {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

													__lowercase      =    [input_ids, input_mask]
													__lowercase      =    model(lowerCamelCase__				)

													__lowercase      =    model(lowerCamelCase__				)

													self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)				)






						def     UpperCAmelCase_      ( self :     int				,							lowerCamelCase__ :     Union[str, Any]				,							lowerCamelCase__ :     Optional[int]				,							lowerCamelCase__ :     Any				,							lowerCamelCase__ :     Tuple				,							lowerCamelCase__ :     Dict				,							lowerCamelCase__ :     Union[str, Any]				,							lowerCamelCase__ :     Dict				)    ->						str:
													"""simple docstring"""


													__lowercase      =    TFConvBertForMaskedLM(config=lowerCamelCase__				)
													__lowercase      =    {
													    '''input_ids''': input_ids,
													    '''attention_mask''': input_mask,
													    '''token_type_ids''': token_type_ids,
													}
													__lowercase      =    model(lowerCamelCase__				)
													self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.vocab_size)				)






						def     UpperCAmelCase_      ( self :     Tuple				,							lowerCamelCase__ :     Optional[int]				,							lowerCamelCase__ :     str				,							lowerCamelCase__ :     Union[str, Any]				,							lowerCamelCase__ :     List[str]				,							lowerCamelCase__ :     Optional[Any]				,							lowerCamelCase__ :     Union[str, Any]				,							lowerCamelCase__ :     List[str]				)    ->						int:
													"""simple docstring"""


													__lowercase      =    self.num_labels
													__lowercase      =    TFConvBertForSequenceClassification(config=lowerCamelCase__				)
													__lowercase      =    {
													    '''input_ids''': input_ids,
													    '''attention_mask''': input_mask,
													    '''token_type_ids''': token_type_ids,
													}

													__lowercase      =    model(lowerCamelCase__				)
													self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.num_labels)				)






						def     UpperCAmelCase_      ( self :     Optional[int]				,							lowerCamelCase__ :     List[str]				,							lowerCamelCase__ :     Any				,							lowerCamelCase__ :     str				,							lowerCamelCase__ :     Any				,							lowerCamelCase__ :     Optional[Any]				,							lowerCamelCase__ :     Dict				,							lowerCamelCase__ :     Union[str, Any]				)    ->						Optional[Any]:
													"""simple docstring"""


													__lowercase      =    self.num_choices
													__lowercase      =    TFConvBertForMultipleChoice(config=lowerCamelCase__				)
													__lowercase      =    tf.tile(tf.expand_dims(lowerCamelCase__				,							1				)				,							(1, self.num_choices, 1)				)
													__lowercase      =    tf.tile(tf.expand_dims(lowerCamelCase__				,							1				)				,							(1, self.num_choices, 1)				)
													__lowercase      =    tf.tile(tf.expand_dims(lowerCamelCase__				,							1				)				,							(1, self.num_choices, 1)				)
													__lowercase      =    {
													    '''input_ids''': multiple_choice_inputs_ids,
													    '''attention_mask''': multiple_choice_input_mask,
													    '''token_type_ids''': multiple_choice_token_type_ids,
													}
													__lowercase      =    model(lowerCamelCase__				)
													self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.num_choices)				)






						def     UpperCAmelCase_      ( self :     Optional[int]				,							lowerCamelCase__ :     List[str]				,							lowerCamelCase__ :     Union[str, Any]				,							lowerCamelCase__ :     int				,							lowerCamelCase__ :     Optional[int]				,							lowerCamelCase__ :     Union[str, Any]				,							lowerCamelCase__ :     Dict				,							lowerCamelCase__ :     str				)    ->						Optional[int]:
													"""simple docstring"""


													__lowercase      =    self.num_labels
													__lowercase      =    TFConvBertForTokenClassification(config=lowerCamelCase__				)
													__lowercase      =    {
													    '''input_ids''': input_ids,
													    '''attention_mask''': input_mask,
													    '''token_type_ids''': token_type_ids,
													}
													__lowercase      =    model(lowerCamelCase__				)
													self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.num_labels)				)






						def     UpperCAmelCase_      ( self :     List[str]				,							lowerCamelCase__ :     int				,							lowerCamelCase__ :     List[Any]				,							lowerCamelCase__ :     List[Any]				,							lowerCamelCase__ :     List[str]				,							lowerCamelCase__ :     Optional[int]				,							lowerCamelCase__ :     Any				,							lowerCamelCase__ :     Any				)    ->						str:
													"""simple docstring"""


													__lowercase      =    TFConvBertForQuestionAnswering(config=lowerCamelCase__				)
													__lowercase      =    {
													    '''input_ids''': input_ids,
													    '''attention_mask''': input_mask,
													    '''token_type_ids''': token_type_ids,
													}

													__lowercase      =    model(lowerCamelCase__				)
													self.parent.assertEqual(result.start_logits.shape				,							(self.batch_size, self.seq_length)				)
													self.parent.assertEqual(result.end_logits.shape				,							(self.batch_size, self.seq_length)				)






						def     UpperCAmelCase_      ( self :     int				)    ->						Optional[Any]:
													"""simple docstring"""


													__lowercase      =    self.prepare_config_and_inputs()
													(
													    __lowercase				
													)      =    config_and_inputs
													__lowercase      =    {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
													return config, inputs_dict







@require_tf
class a (     __lowerCAmelCase       ,						__lowerCAmelCase       ,						unittest.TestCase  ):
						"""simple docstring"""

						UpperCamelCase_	:       Dict       	=		(
						    (
						        TFConvBertModel,
						        TFConvBertForMaskedLM,
						        TFConvBertForQuestionAnswering,
						        TFConvBertForSequenceClassification,
						        TFConvBertForTokenClassification,
						        TFConvBertForMultipleChoice,
						    )
						    if is_tf_available()
						    else ()
						)
						UpperCamelCase_	:       List[Any]       	=		(
						    {
						        "feature-extraction": TFConvBertModel,
						        "fill-mask": TFConvBertForMaskedLM,
						        "question-answering": TFConvBertForQuestionAnswering,
						        "text-classification": TFConvBertForSequenceClassification,
						        "token-classification": TFConvBertForTokenClassification,
						        "zero-shot": TFConvBertForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						UpperCamelCase_	:       List[str]       	=		False
						UpperCamelCase_	:       int       	=		False
						UpperCamelCase_	:       List[Any]       	=		False






						def     UpperCAmelCase_      ( self :     Optional[Any]				)    ->						Dict:
													"""simple docstring"""


													__lowercase      =    TFConvBertModelTester(self				)
													__lowercase      =    ConfigTester(self				,							config_class=lowerCamelCase__				,							hidden_size=37				)






						def     UpperCAmelCase_      ( self :     List[Any]				)    ->						Optional[int]:
													"""simple docstring"""


													self.config_tester.run_common_tests()






						def     UpperCAmelCase_      ( self :     str				)    ->						str:
													"""simple docstring"""


													__lowercase      =    self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     Dict				)    ->						Dict:
													"""simple docstring"""


													__lowercase      =    self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     str				)    ->						int:
													"""simple docstring"""


													__lowercase      =    self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     Optional[int]				)    ->						int:
													"""simple docstring"""


													__lowercase      =    self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_question_answering(*lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     Union[str, Any]				)    ->						List[Any]:
													"""simple docstring"""


													__lowercase      =    self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     List[Any]				)    ->						Union[str, Any]:
													"""simple docstring"""


													__lowercase      =    self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_token_classification(*lowerCamelCase__				)






						@slow
						def     UpperCAmelCase_      ( self :     Any				)    ->						Tuple:
													"""simple docstring"""


													__lowercase      =    self.model_tester.prepare_config_and_inputs_for_common()
													__lowercase      =    True
													__lowercase      =    True

													if hasattr(lowerCamelCase__				,							'''use_cache'''				):
																				__lowercase      =    True

													__lowercase      =    getattr(self.model_tester				,							'''encoder_seq_length'''				,							self.model_tester.seq_length				)
													__lowercase      =    getattr(self.model_tester				,							'''key_length'''				,							lowerCamelCase__				)

													for model_class in self.all_model_classes:
																				__lowercase      =    self._prepare_for_class(lowerCamelCase__				,							lowerCamelCase__				)
																				__lowercase      =    model_class(lowerCamelCase__				)
																				__lowercase      =    len(model(lowerCamelCase__				)				)

																				with tempfile.TemporaryDirectory() as tmpdirname:
																											model.save_pretrained(lowerCamelCase__				,							saved_model=lowerCamelCase__				)
																											__lowercase      =    os.path.join(lowerCamelCase__				,							'''saved_model'''				,							'''1'''				)
																											__lowercase      =    tf.keras.models.load_model(lowerCamelCase__				)
																											__lowercase      =    model(lowerCamelCase__				)

																											if self.is_encoder_decoder:
																																		__lowercase      =    outputs['''encoder_hidden_states''']
																																		__lowercase      =    outputs['''encoder_attentions''']
																											else:
																																		__lowercase      =    outputs['''hidden_states''']
																																		__lowercase      =    outputs['''attentions''']

																											self.assertEqual(len(lowerCamelCase__				)				,							lowerCamelCase__				)

																											__lowercase      =    getattr(
																											    self.model_tester				,							'''expected_num_hidden_layers'''				,							self.model_tester.num_hidden_layers + 1				)

																											self.assertEqual(len(lowerCamelCase__				)				,							lowerCamelCase__				)
																											self.assertListEqual(
																											    list(output_hidden_states[0].shape[-2:]				)				,							[self.model_tester.seq_length, self.model_tester.hidden_size]				,							)

																											self.assertEqual(len(lowerCamelCase__				)				,							self.model_tester.num_hidden_layers				)
																											self.assertListEqual(
																											    list(output_attentions[0].shape[-3:]				)				,							[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]				,							)






						@slow
						def     UpperCAmelCase_      ( self :     Tuple				)    ->						Union[str, Any]:
													"""simple docstring"""


													__lowercase      =    TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''				)
													self.assertIsNotNone(lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     int				)    ->						int:
													"""simple docstring"""


													__lowercase      =    self.model_tester.prepare_config_and_inputs_for_common()
													__lowercase      =    True
													__lowercase      =    getattr(self.model_tester				,							'''decoder_seq_length'''				,							self.model_tester.seq_length				)
													__lowercase      =    getattr(self.model_tester				,							'''encoder_seq_length'''				,							self.model_tester.seq_length				)
													__lowercase      =    getattr(self.model_tester				,							'''key_length'''				,							lowerCamelCase__				)
													__lowercase      =    getattr(self.model_tester				,							'''key_length'''				,							lowerCamelCase__				)

													def check_decoder_attentions_output(lowerCamelCase__ :     Any				):
																				__lowercase      =    len(lowerCamelCase__				)
																				self.assertEqual(out_len % 2				,							0				)
																				__lowercase      =    outputs.decoder_attentions
																				self.assertEqual(len(lowerCamelCase__				)				,							self.model_tester.num_hidden_layers				)
																				self.assertListEqual(
																				    list(decoder_attentions[0].shape[-3:]				)				,							[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]				,							)

													def check_encoder_attentions_output(lowerCamelCase__ :     Tuple				):
																				__lowercase      =    [
																				    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
																				]
																				self.assertEqual(len(lowerCamelCase__				)				,							self.model_tester.num_hidden_layers				)
																				self.assertListEqual(
																				    list(attentions[0].shape[-3:]				)				,							[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]				,							)

													for model_class in self.all_model_classes:
																				__lowercase      =    True
																				__lowercase      =    False
																				__lowercase      =    model_class(lowerCamelCase__				)
																				__lowercase      =    model(self._prepare_for_class(lowerCamelCase__				,							lowerCamelCase__				)				)
																				__lowercase      =    len(lowerCamelCase__				)
																				self.assertEqual(config.output_hidden_states				,							lowerCamelCase__				)
																				check_encoder_attentions_output(lowerCamelCase__				)

																				if self.is_encoder_decoder:
																											__lowercase      =    model_class(lowerCamelCase__				)
																											__lowercase      =    model(self._prepare_for_class(lowerCamelCase__				,							lowerCamelCase__				)				)
																											self.assertEqual(config.output_hidden_states				,							lowerCamelCase__				)
																											check_decoder_attentions_output(lowerCamelCase__				)

																				# Check that output attentions can also be changed via the config
																				del inputs_dict["output_attentions"]
																				__lowercase      =    True
																				__lowercase      =    model_class(lowerCamelCase__				)
																				__lowercase      =    model(self._prepare_for_class(lowerCamelCase__				,							lowerCamelCase__				)				)
																				self.assertEqual(config.output_hidden_states				,							lowerCamelCase__				)
																				check_encoder_attentions_output(lowerCamelCase__				)

																				# Check attention is always last and order is fine
																				__lowercase      =    True
																				__lowercase      =    True
																				__lowercase      =    model_class(lowerCamelCase__				)
																				__lowercase      =    model(self._prepare_for_class(lowerCamelCase__				,							lowerCamelCase__				)				)

																				self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)				,							len(lowerCamelCase__				)				)
																				self.assertEqual(model.config.output_hidden_states				,							lowerCamelCase__				)
																				check_encoder_attentions_output(lowerCamelCase__				)







@require_tf
class a (     unittest.TestCase  ):
						"""simple docstring"""

						@slow
						def     UpperCAmelCase_      ( self :     Optional[Any]				)    ->						Optional[Any]:
													"""simple docstring"""


													__lowercase      =    TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''				)
													__lowercase      =    tf.constant([[0, 1, 2, 3, 4, 5]]				)
													__lowercase      =    model(lowerCamelCase__				)[0]

													__lowercase      =    [1, 6, 768]
													self.assertEqual(output.shape				,							lowerCamelCase__				)

													__lowercase      =    tf.constant(
													    [
													        [
													            [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
													            [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
													            [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
													        ]
													    ]				)
													tf.debugging.assert_near(output[:, :3, :3]				,							lowerCamelCase__				,							atol=1e-4				)








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_mbart import MBartTokenizer
else:
			UpperCAmelCase__				     = None


UpperCAmelCase__				     = logging.get_logger(__name__)


UpperCAmelCase__				     = {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__				     = {
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

UpperCAmelCase__				     = {
    "facebook/mbart-large-en-ro": 1024,
    "facebook/mbart-large-cc25": 1024,
}

# fmt: off
UpperCAmelCase__				     = ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]







class a (     __SCREAMING_SNAKE_CASE  ):
						"""simple docstring"""

						UpperCamelCase_	:       int       	=		VOCAB_FILES_NAMES
						UpperCamelCase_	:       List[str]       	=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase_	:       int       	=		PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase_	:       Optional[int]       	=		['input_ids', 'attention_mask']
						UpperCamelCase_	:       Optional[int]       	=		MBartTokenizer

						UpperCamelCase_	:       List[int]       	=		[]
						UpperCamelCase_	:       List[int]       	=		[]






						def __init__( self :     Tuple				,							lowerCamelCase__ :     str=None				,							lowerCamelCase__ :     Dict=None				,							lowerCamelCase__ :     Optional[Any]="<s>"				,							lowerCamelCase__ :     int="</s>"				,							lowerCamelCase__ :     int="</s>"				,							lowerCamelCase__ :     List[Any]="<s>"				,							lowerCamelCase__ :     Union[str, Any]="<unk>"				,							lowerCamelCase__ :     Union[str, Any]="<pad>"				,							lowerCamelCase__ :     List[Any]="<mask>"				,							lowerCamelCase__ :     Optional[Any]=None				,							lowerCamelCase__ :     Any=None				,							lowerCamelCase__ :     Dict=None				,							**lowerCamelCase__ :     List[str]				,							)    ->						Tuple:
													"""simple docstring"""


													__lowercase      =    AddedToken(lowerCamelCase__				,							lstrip=lowerCamelCase__				,							rstrip=lowerCamelCase__				) if isinstance(lowerCamelCase__				,							lowerCamelCase__				) else mask_token

													super().__init__(
													    vocab_file=lowerCamelCase__				,							tokenizer_file=lowerCamelCase__				,							bos_token=lowerCamelCase__				,							eos_token=lowerCamelCase__				,							sep_token=lowerCamelCase__				,							cls_token=lowerCamelCase__				,							unk_token=lowerCamelCase__				,							pad_token=lowerCamelCase__				,							mask_token=lowerCamelCase__				,							src_lang=lowerCamelCase__				,							tgt_lang=lowerCamelCase__				,							additional_special_tokens=lowerCamelCase__				,							**lowerCamelCase__				,							)

													__lowercase      =    vocab_file
													__lowercase      =    False if not self.vocab_file else True

													__lowercase      =    FAIRSEQ_LANGUAGE_CODES.copy()

													if additional_special_tokens is not None:
																				# Only add those special tokens if they are not already there.
																				_additional_special_tokens.extend(
																				    [t for t in additional_special_tokens if t not in _additional_special_tokens]				)

													self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}				)
													__lowercase      =    {
													    lang_code: self.convert_tokens_to_ids(lowerCamelCase__				) for lang_code in FAIRSEQ_LANGUAGE_CODES
													}

													__lowercase      =    src_lang if src_lang is not None else '''en_XX'''
													__lowercase      =    self.convert_tokens_to_ids(self._src_lang				)
													__lowercase      =    tgt_lang
													self.set_src_lang_special_tokens(self._src_lang				)






						@property
						def     UpperCAmelCase_      ( self :     Any				)    ->						str:
													"""simple docstring"""


													return self._src_lang






						@src_lang.setter
						def     UpperCAmelCase_      ( self :     Optional[Any]				,							lowerCamelCase__ :     str				)    ->						None:
													"""simple docstring"""


													__lowercase      =    new_src_lang
													self.set_src_lang_special_tokens(self._src_lang				)






						def     UpperCAmelCase_      ( self :     Any				,							lowerCamelCase__ :     List[int]				,							lowerCamelCase__ :     Optional[List[int]] = None				)    ->						List[int]:
													"""simple docstring"""


													if token_ids_a is None:
																				return self.prefix_tokens + token_ids_a + self.suffix_tokens
													# We don't expect to process pairs, but leave the pair logic for API consistency
													return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






						def     UpperCAmelCase_      ( self :     Optional[int]				,							lowerCamelCase__ :     List[int]				,							lowerCamelCase__ :     Optional[List[int]] = None				)    ->						List[int]:
													"""simple docstring"""


													__lowercase      =    [self.sep_token_id]
													__lowercase      =    [self.cls_token_id]

													if token_ids_a is None:
																				return len(cls + token_ids_a + sep				) * [0]
													return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]






						def     UpperCAmelCase_      ( self :     List[str]				,							lowerCamelCase__ :     str				,							lowerCamelCase__ :     str				,							lowerCamelCase__ :     Optional[str]				,							lowerCamelCase__ :     Optional[str]				,							**lowerCamelCase__ :     List[str]				)    ->						Optional[int]:
													"""simple docstring"""


													if src_lang is None or tgt_lang is None:
																				raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''				)
													__lowercase      =    src_lang
													__lowercase      =    self(lowerCamelCase__				,							add_special_tokens=lowerCamelCase__				,							return_tensors=lowerCamelCase__				,							**lowerCamelCase__				)
													__lowercase      =    self.convert_tokens_to_ids(lowerCamelCase__				)
													__lowercase      =    tgt_lang_id
													return inputs






						def     UpperCAmelCase_      ( self :     List[str]				,							lowerCamelCase__ :     List[str]				,							lowerCamelCase__ :     str = "en_XX"				,							lowerCamelCase__ :     Optional[List[str]] = None				,							lowerCamelCase__ :     str = "ro_RO"				,							**lowerCamelCase__ :     Union[str, Any]				,							)    ->						BatchEncoding:
													"""simple docstring"""


													__lowercase      =    src_lang
													__lowercase      =    tgt_lang
													return super().prepare_seqaseq_batch(lowerCamelCase__				,							lowerCamelCase__				,							**lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     int				)    ->						Dict:
													"""simple docstring"""


													return self.set_src_lang_special_tokens(self.src_lang				)






						def     UpperCAmelCase_      ( self :     Union[str, Any]				)    ->						List[str]:
													"""simple docstring"""


													return self.set_tgt_lang_special_tokens(self.tgt_lang				)






						def     UpperCAmelCase_      ( self :     Optional[int]				,							lowerCamelCase__ :     Optional[Any]				)    ->						None:
													"""simple docstring"""


													__lowercase      =    self.convert_tokens_to_ids(lowerCamelCase__				)
													__lowercase      =    []
													__lowercase      =    [self.eos_token_id, self.cur_lang_code]

													__lowercase      =    self.convert_ids_to_tokens(self.prefix_tokens				)
													__lowercase      =    self.convert_ids_to_tokens(self.suffix_tokens				)

													__lowercase      =    processors.TemplateProcessing(
													    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,							pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,							self.prefix_tokens + self.suffix_tokens				)				)				,							)






						def     UpperCAmelCase_      ( self :     Tuple				,							lowerCamelCase__ :     str				)    ->						None:
													"""simple docstring"""


													__lowercase      =    self.convert_tokens_to_ids(lowerCamelCase__				)
													__lowercase      =    []
													__lowercase      =    [self.eos_token_id, self.cur_lang_code]

													__lowercase      =    self.convert_ids_to_tokens(self.prefix_tokens				)
													__lowercase      =    self.convert_ids_to_tokens(self.suffix_tokens				)

													__lowercase      =    processors.TemplateProcessing(
													    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,							pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,							self.prefix_tokens + self.suffix_tokens				)				)				,							)






						def     UpperCAmelCase_      ( self :     Optional[int]				,							lowerCamelCase__ :     str				,							lowerCamelCase__ :     Optional[str] = None				)    ->						Tuple[str]:
													"""simple docstring"""


													if not self.can_save_slow_tokenizer:
																				raise ValueError(
																				    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																				    '''tokenizer.'''				)

													if not os.path.isdir(lowerCamelCase__				):
																				logger.error(f'Vocabulary path ({save_directory}) should be a directory.'				)
																				return
													__lowercase      =    os.path.join(
													    lowerCamelCase__				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

													if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCamelCase__				):
																				copyfile(self.vocab_file				,							lowerCamelCase__				)

													return (out_vocab_file,)






