


'''simple docstring'''


from __future__ import annotations

import unittest

import numpy as np

from transformers import LayoutLMConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers.models.layoutlm.modeling_tf_layoutlm import (
				    TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFLayoutLMForMaskedLM,
				    TFLayoutLMForQuestionAnswering,
				    TFLayoutLMForSequenceClassification,
				    TFLayoutLMForTokenClassification,
				    TFLayoutLMModel,
				)





class        lowercase__    :





							'''simple docstring'''


							def __init__(       self       ,   __snake_case       ,   __snake_case=13       ,   __snake_case=7       ,   __snake_case=True       ,   __snake_case=True       ,   __snake_case=True       ,   __snake_case=True       ,   __snake_case=99       ,   __snake_case=32       ,   __snake_case=2       ,   __snake_case=4       ,   __snake_case=37       ,   __snake_case="gelu"       ,   __snake_case=0.1       ,   __snake_case=0.1       ,   __snake_case=512       ,   __snake_case=16       ,   __snake_case=2       ,   __snake_case=0.02       ,   __snake_case=3       ,   __snake_case=4       ,   __snake_case=None       ,   __snake_case=1000       ,   ):
											_SCREAMING_SNAKE_CASE					:			int							= parent
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= batch_size
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= seq_length
											_SCREAMING_SNAKE_CASE					:			Dict							= is_training
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= use_input_mask
											_SCREAMING_SNAKE_CASE					:			List[str]							= use_token_type_ids
											_SCREAMING_SNAKE_CASE					:			List[str]							= use_labels
											_SCREAMING_SNAKE_CASE					:			List[Any]							= vocab_size
											_SCREAMING_SNAKE_CASE					:			str							= hidden_size
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= num_hidden_layers
											_SCREAMING_SNAKE_CASE					:			Any							= num_attention_heads
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= intermediate_size
											_SCREAMING_SNAKE_CASE					:			int							= hidden_act
											_SCREAMING_SNAKE_CASE					:			List[str]							= hidden_dropout_prob
											_SCREAMING_SNAKE_CASE					:			List[Any]							= attention_probs_dropout_prob
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= max_position_embeddings
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= type_vocab_size
											_SCREAMING_SNAKE_CASE					:			List[str]							= type_sequence_label_size
											_SCREAMING_SNAKE_CASE					:			List[Any]							= initializer_range
											_SCREAMING_SNAKE_CASE					:			str							= num_labels
											_SCREAMING_SNAKE_CASE					:			Any							= num_choices
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= scope
											_SCREAMING_SNAKE_CASE					:			int							= range_bbox






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			str							= ids_tensor([self.batch_size, self.seq_length]       ,   self.vocab_size    )

											# convert bbox to numpy since TF does not support item assignment
											_SCREAMING_SNAKE_CASE					:			Any							= ids_tensor([self.batch_size, self.seq_length, 4]       ,   self.range_bbox    ).numpy()
											# Ensure that bbox is legal
											for i in range(bbox.shape[0]    ):
															for j in range(bbox.shape[1]    ):
																			if bbox[i, j, 3] < bbox[i, j, 1]:
																							_SCREAMING_SNAKE_CASE					:			int							= bbox[i, j, 3]
																							_SCREAMING_SNAKE_CASE					:			int							= bbox[i, j, 1]
																							_SCREAMING_SNAKE_CASE					:			Dict							= t
																			if bbox[i, j, 2] < bbox[i, j, 0]:
																							_SCREAMING_SNAKE_CASE					:			Optional[int]							= bbox[i, j, 2]
																							_SCREAMING_SNAKE_CASE					:			Optional[Any]							= bbox[i, j, 0]
																							_SCREAMING_SNAKE_CASE					:			Any							= t
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= tf.convert_to_tensor(__snake_case    )

											_SCREAMING_SNAKE_CASE					:			List[Any]							= None
											if self.use_input_mask:
															_SCREAMING_SNAKE_CASE					:			Dict							= random_attention_mask([self.batch_size, self.seq_length]    )

											_SCREAMING_SNAKE_CASE					:			str							= None
											if self.use_token_type_ids:
															_SCREAMING_SNAKE_CASE					:			Optional[int]							= ids_tensor([self.batch_size, self.seq_length]       ,   self.type_vocab_size    )

											_SCREAMING_SNAKE_CASE					:			Dict							= None
											_SCREAMING_SNAKE_CASE					:			Tuple							= None
											_SCREAMING_SNAKE_CASE					:			List[str]							= None
											if self.use_labels:
															_SCREAMING_SNAKE_CASE					:			Dict							= ids_tensor([self.batch_size]       ,   self.type_sequence_label_size    )
															_SCREAMING_SNAKE_CASE					:			Optional[Any]							= ids_tensor([self.batch_size, self.seq_length]       ,   self.num_labels    )
															_SCREAMING_SNAKE_CASE					:			Any							= ids_tensor([self.batch_size]       ,   self.num_choices    )

											_SCREAMING_SNAKE_CASE					:			List[Any]							= LayoutLMConfig(
											    vocab_size=self.vocab_size       ,   hidden_size=self.hidden_size       ,   num_hidden_layers=self.num_hidden_layers       ,   num_attention_heads=self.num_attention_heads       ,   intermediate_size=self.intermediate_size       ,   hidden_act=self.hidden_act       ,   hidden_dropout_prob=self.hidden_dropout_prob       ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,   max_position_embeddings=self.max_position_embeddings       ,   type_vocab_size=self.type_vocab_size       ,   initializer_range=self.initializer_range       ,   )

											return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			List[Any]							= TFLayoutLMModel(config=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			Tuple							= model(__snake_case       ,   __snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case    )
											_SCREAMING_SNAKE_CASE					:			str							= model(__snake_case       ,   __snake_case       ,   token_type_ids=__snake_case    )
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= model(__snake_case       ,   __snake_case    )
											self.parent.assertEqual(result.last_hidden_state.shape       ,   (self.batch_size, self.seq_length, self.hidden_size)    )
											self.parent.assertEqual(result.pooler_output.shape       ,   (self.batch_size, self.hidden_size)    )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			Any							= TFLayoutLMForMaskedLM(config=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			str							= model(__snake_case       ,   __snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case       ,   labels=__snake_case    )
											self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.seq_length, self.vocab_size)    )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			int							= self.num_labels
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= TFLayoutLMForSequenceClassification(config=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= model(__snake_case       ,   __snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case    )
											self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.num_labels)    )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			Any							= self.num_labels
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= TFLayoutLMForTokenClassification(config=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			Dict							= model(__snake_case       ,   __snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case       ,   labels=__snake_case    )
											self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.seq_length, self.num_labels)    )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			Tuple							= TFLayoutLMForQuestionAnswering(config=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			Tuple							= model(__snake_case       ,   __snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case    )
											self.parent.assertEqual(result.start_logits.shape       ,   (self.batch_size, self.seq_length)    )
											self.parent.assertEqual(result.end_logits.shape       ,   (self.batch_size, self.seq_length)    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			str							= self.prepare_config_and_inputs()
											(
											    (
											    _SCREAMING_SNAKE_CASE
											)						,			(
											    _SCREAMING_SNAKE_CASE
											)						,			(
											    _SCREAMING_SNAKE_CASE
											)						,			(
											    _SCREAMING_SNAKE_CASE
											)						,			(
											    _SCREAMING_SNAKE_CASE
											)						,			(
											    _SCREAMING_SNAKE_CASE
											)						,			(
											    _SCREAMING_SNAKE_CASE
											)						,			(
											    _SCREAMING_SNAKE_CASE
											)						,			
											)					:			Union[str, Any]							= config_and_inputs
											_SCREAMING_SNAKE_CASE					:			int							= {
											    """input_ids""": input_ids,
											    """bbox""": bbox,
											    """token_type_ids""": token_type_ids,
											    """attention_mask""": input_mask,
											}
											return config, inputs_dict





@require_tf
class        lowercase__    (		_snake_case	,	_snake_case	,	unittest.TestCase   ):





							'''simple docstring'''


							A_				: List[str]       =    (
							    (
							        TFLayoutLMModel,
							        TFLayoutLMForMaskedLM,
							        TFLayoutLMForTokenClassification,
							        TFLayoutLMForSequenceClassification,
							        TFLayoutLMForQuestionAnswering,
							    )
							    if is_tf_available()
							    else ()
							)
							A_				: List[Any]       =    (
							    {
							        """feature-extraction""": TFLayoutLMModel,
							        """fill-mask""": TFLayoutLMForMaskedLM,
							        """text-classification""": TFLayoutLMForSequenceClassification,
							        """token-classification""": TFLayoutLMForTokenClassification,
							        """zero-shot""": TFLayoutLMForSequenceClassification,
							    }
							    if is_tf_available()
							    else {}
							)
							A_				: Optional[int]       =    False
							A_				: List[str]       =    True
							A_				: List[Any]       =    10






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Tuple							= TFLayoutLMModelTester(self    )
											_SCREAMING_SNAKE_CASE					:			Any							= ConfigTester(self       ,   config_class=__snake_case       ,   hidden_size=37    )






							def UpperCAmelCase_      (       self    ):
											self.config_tester.run_common_tests()






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Any							= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__snake_case    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*__snake_case    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			int							= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*__snake_case    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*__snake_case    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*__snake_case    )






							@slow
							def UpperCAmelCase_      (       self    ):
											for model_name in TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															_SCREAMING_SNAKE_CASE					:			List[str]							= TFLayoutLMModel.from_pretrained(__snake_case    )
															self.assertIsNotNone(__snake_case    )






							@unittest.skip("""Onnx compliancy broke with TF 2.10"""    )
							def UpperCAmelCase_      (       self    ):
											pass







def      snake_case_  (    ):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE					:			List[str]							= tf.convert_to_tensor([[101,1019,1014,1016,1037,1_2849,4747,1004,1_4246,2278,5439,4524,5002,2930,2193,2930,4341,3208,1005,1055,2171,2848,1_1300,3531,102],[101,4070,4034,7020,1024,3058,1015,1013,2861,1013,6070,1_9274,2772,6205,2_7814,1_6147,1_6147,4343,2047,1_0283,1_0969,1_4389,1012,2338,102]]	)  # noqa: E231
				_SCREAMING_SNAKE_CASE					:			Tuple							= tf.convert_to_tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]	)  # noqa: E231
				_SCREAMING_SNAKE_CASE					:			Tuple							= tf.convert_to_tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1000,1000,1000,1000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1000,1000,1000,1000]]]	)  # noqa: E231
				_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= tf.convert_to_tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]	)  # noqa: E231
				# these are sequence labels (i.e. at the token level)
				_SCREAMING_SNAKE_CASE					:			int							= tf.convert_to_tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]]	)  # noqa: E231
				# fmt: on

				return input_ids, attention_mask, bbox, token_type_ids, labels







@require_tf
class        lowercase__    (		unittest.TestCase   ):





							'''simple docstring'''


							@slow
							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			int							= TFLayoutLMModel.from_pretrained("""microsoft/layoutlm-base-uncased"""    )

											_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE					:			Optional[Any]							= prepare_layoutlm_batch_inputs()

											# forward pass
											_SCREAMING_SNAKE_CASE					:			Tuple							= model(input_ids=__snake_case       ,   bbox=__snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case    )

											# test the sequence output on [0, :3, :3]
											_SCREAMING_SNAKE_CASE					:			Dict							= tf.convert_to_tensor(
											    [[0.1785, -0.1947, -0.0425], [-0.3254, -0.2807, 0.2553], [-0.5391, -0.3322, 0.3364]]       ,   )

											self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]       ,   __snake_case       ,   atol=1e-3    )    )

											# test the pooled output on [1, :3]
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= tf.convert_to_tensor([-0.6580, -0.0214, 0.8552]    )

											self.assertTrue(np.allclose(outputs.pooler_output[1, :3]       ,   __snake_case       ,   atol=1e-3    )    )






							@slow
							def UpperCAmelCase_      (       self    ):
											# initialize model with randomly initialized sequence classification head
											_SCREAMING_SNAKE_CASE					:			Any							= TFLayoutLMForSequenceClassification.from_pretrained("""microsoft/layoutlm-base-uncased"""       ,   num_labels=2    )

											_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE					:			Optional[int]							= prepare_layoutlm_batch_inputs()

											# forward pass
											_SCREAMING_SNAKE_CASE					:			List[str]							= model(
											    input_ids=__snake_case       ,   bbox=__snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case       ,   labels=tf.convert_to_tensor([1, 1]    )       ,   )

											# test whether we get a loss as a scalar
											_SCREAMING_SNAKE_CASE					:			Any							= outputs.loss
											_SCREAMING_SNAKE_CASE					:			str							= (2,)
											self.assertEqual(loss.shape       ,   __snake_case    )

											# test the shape of the logits
											_SCREAMING_SNAKE_CASE					:			Dict							= outputs.logits
											_SCREAMING_SNAKE_CASE					:			Tuple							= (2, 2)
											self.assertEqual(logits.shape       ,   __snake_case    )






							@slow
							def UpperCAmelCase_      (       self    ):
											# initialize model with randomly initialized token classification head
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= TFLayoutLMForTokenClassification.from_pretrained("""microsoft/layoutlm-base-uncased"""       ,   num_labels=13    )

											_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE					:			List[Any]							= prepare_layoutlm_batch_inputs()

											# forward pass
											_SCREAMING_SNAKE_CASE					:			Dict							= model(
											    input_ids=__snake_case       ,   bbox=__snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case       ,   labels=__snake_case    )

											# test the shape of the logits
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= outputs.logits
											_SCREAMING_SNAKE_CASE					:			Any							= tf.convert_to_tensor((2, 25, 13)    )
											self.assertEqual(logits.shape       ,   __snake_case    )






							@slow
							def UpperCAmelCase_      (       self    ):
											# initialize model with randomly initialized token classification head
											_SCREAMING_SNAKE_CASE					:			int							= TFLayoutLMForQuestionAnswering.from_pretrained("""microsoft/layoutlm-base-uncased"""    )

											_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE						,			_SCREAMING_SNAKE_CASE					:			List[str]							= prepare_layoutlm_batch_inputs()

											# forward pass
											_SCREAMING_SNAKE_CASE					:			List[Any]							= model(input_ids=__snake_case       ,   bbox=__snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case    )

											# test the shape of the logits
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= tf.convert_to_tensor((2, 25)    )
											self.assertEqual(outputs.start_logits.shape       ,   __snake_case    )
											self.assertEqual(outputs.end_logits.shape       ,   __snake_case    )








'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_mbart import MBartTokenizer
else:
				UpperCAmelCase_						:  List[str]				  =						None


UpperCAmelCase_						:  Tuple				  =						logging.get_logger(__name__)


UpperCAmelCase_						:  List[Any]				  =						{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_						:  Dict				  =						{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

UpperCAmelCase_						:  str				  =						{
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
UpperCAmelCase_						:  List[str]				  =						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']





class        lowercase__    (		_snake_case   ):





							'''simple docstring'''


							A_				: Any       =    VOCAB_FILES_NAMES
							A_				: List[str]       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_				: Any       =    PRETRAINED_VOCAB_FILES_MAP
							A_				: Tuple       =    ["""input_ids""", """attention_mask"""]
							A_				: Dict       =    MBartTokenizer

							A_				: List[int]       =    []
							A_				: List[int]       =    []






							def __init__(       self       ,   __snake_case=None       ,   __snake_case=None       ,   __snake_case="<s>"       ,   __snake_case="</s>"       ,   __snake_case="</s>"       ,   __snake_case="<s>"       ,   __snake_case="<unk>"       ,   __snake_case="<pad>"       ,   __snake_case="<mask>"       ,   __snake_case=None       ,   __snake_case=None       ,   __snake_case=None       ,   **__snake_case       ,   ):
											# Mask token behave like a normal word, i.e. include the space before it
											_SCREAMING_SNAKE_CASE					:			List[Any]							= AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case    ) if isinstance(__snake_case       ,   __snake_case    ) else mask_token

											super().__init__(
											    vocab_file=__snake_case       ,   tokenizer_file=__snake_case       ,   bos_token=__snake_case       ,   eos_token=__snake_case       ,   sep_token=__snake_case       ,   cls_token=__snake_case       ,   unk_token=__snake_case       ,   pad_token=__snake_case       ,   mask_token=__snake_case       ,   src_lang=__snake_case       ,   tgt_lang=__snake_case       ,   additional_special_tokens=__snake_case       ,   **__snake_case       ,   )

											_SCREAMING_SNAKE_CASE					:			List[str]							= vocab_file
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= False if not self.vocab_file else True

											_SCREAMING_SNAKE_CASE					:			int							= FAIRSEQ_LANGUAGE_CODES.copy()

											if additional_special_tokens is not None:
															# Only add those special tokens if they are not already there.
															_additional_special_tokens.extend(
															    [t for t in additional_special_tokens if t not in _additional_special_tokens]    )

											self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}    )
											_SCREAMING_SNAKE_CASE					:			str							= {
											    lang_code: self.convert_tokens_to_ids(__snake_case    ) for lang_code in FAIRSEQ_LANGUAGE_CODES
											}

											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= src_lang if src_lang is not None else """en_XX"""
											_SCREAMING_SNAKE_CASE					:			str							= self.convert_tokens_to_ids(self._src_lang    )
											_SCREAMING_SNAKE_CASE					:			Any							= tgt_lang
											self.set_src_lang_special_tokens(self._src_lang    )






							@property
							def UpperCAmelCase_      (       self    ):
											return self._src_lang






							@src_lang.setter
							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			Any							= new_src_lang
											self.set_src_lang_special_tokens(self._src_lang    )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											if token_ids_a is None:
															return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= [self.sep_token_id]
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= [self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   **__snake_case    ):
											if src_lang is None or tgt_lang is None:
															raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""    )
											_SCREAMING_SNAKE_CASE					:			Dict							= src_lang
											_SCREAMING_SNAKE_CASE					:			List[Any]							= self(__snake_case       ,   add_special_tokens=__snake_case       ,   return_tensors=__snake_case       ,   **__snake_case    )
											_SCREAMING_SNAKE_CASE					:			Dict							= self.convert_tokens_to_ids(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= tgt_lang_id
											return inputs






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = "en_XX"       ,   __snake_case = None       ,   __snake_case = "ro_RO"       ,   **__snake_case       ,   ):
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= src_lang
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= tgt_lang
											return super().prepare_seqaseq_batch(__snake_case       ,   __snake_case       ,   **__snake_case    )






							def UpperCAmelCase_      (       self    ):
											return self.set_src_lang_special_tokens(self.src_lang    )






							def UpperCAmelCase_      (       self    ):
											return self.set_tgt_lang_special_tokens(self.tgt_lang    )






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.convert_tokens_to_ids(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			str							= []
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= [self.eos_token_id, self.cur_lang_code]

											_SCREAMING_SNAKE_CASE					:			int							= self.convert_ids_to_tokens(self.prefix_tokens    )
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= self.convert_ids_to_tokens(self.suffix_tokens    )

											_SCREAMING_SNAKE_CASE					:			Optional[int]							= processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,   pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,   self.prefix_tokens + self.suffix_tokens    )    )       ,   )






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.convert_tokens_to_ids(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			List[str]							= []
											_SCREAMING_SNAKE_CASE					:			Tuple							= [self.eos_token_id, self.cur_lang_code]

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= self.convert_ids_to_tokens(self.prefix_tokens    )
											_SCREAMING_SNAKE_CASE					:			str							= self.convert_ids_to_tokens(self.suffix_tokens    )

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,   pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,   self.prefix_tokens + self.suffix_tokens    )    )       ,   )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer."""    )

											if not os.path.isdir(__snake_case    ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""    )
															return
											_SCREAMING_SNAKE_CASE					:			List[str]							= os.path.join(
											    __snake_case       ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(__snake_case    ):
															copyfile(self.vocab_file       ,   __snake_case    )

											return (out_vocab_file,)




