




import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase   :			List[str]          = get_tests_dir("fixtures/test_sentencepiece.model")

if is_torch_available():
		from transformers.models.mbart.modeling_mbart import shift_tokens_right

__UpperCamelCase   :			str          = 250_004
__UpperCamelCase   :			Dict          = 250_020

@require_sentencepiece
@require_tokenizers
class 						__lowerCAmelCase      (      __magic_name__ ,			unittest.TestCase					):
							UpperCamelCase__   	=    MBartaaTokenizer
							UpperCamelCase__   	=    MBartaaTokenizerFast
							UpperCamelCase__   	=    True
							UpperCamelCase__   	=    True
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								super().setUp()

								# We have a SentencePiece fixture for testing
								a									=				MBartaaTokenizer(__magic_name__						, src_lang="""en_XX"""						, tgt_lang="""ro_RO"""						, keep_accents=__magic_name__							)
								tokenizer.save_pretrained(self.tmpdirname							)
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				"""<s>"""
								a									=				0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(__magic_name__							)						, __magic_name__							)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(__magic_name__							)						, __magic_name__							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								a									=				list(self.get_tokenizer().get_vocab().keys()							)

								self.assertEqual(vocab_keys[0]						, """<s>"""							)
								self.assertEqual(vocab_keys[1]						, """<pad>"""							)
								self.assertEqual(vocab_keys[-1]						, """<mask>"""							)
								self.assertEqual(len(__magic_name__							)						, 1054							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								self.assertEqual(self.get_tokenizer().vocab_size						, 1054							)
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				MBartaaTokenizer(__magic_name__						, src_lang="""en_XX"""						, tgt_lang="""ro_RO"""						, keep_accents=__magic_name__							)

								a									=				tokenizer.tokenize("""This is a test"""							)
								self.assertListEqual(__magic_name__						, ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]							)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(__magic_name__							)						, [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						, )

								a									=				tokenizer.tokenize("""I was born in 92000, and this is falsé."""							)
								self.assertListEqual(
								    __magic_name__						, [SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]						, )
								a									=				tokenizer.convert_tokens_to_ids(__magic_name__							)
								self.assertListEqual(
								    __magic_name__						, [
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								    ]						, )

								a									=				tokenizer.convert_ids_to_tokens(__magic_name__							)
								self.assertListEqual(
								    __magic_name__						, [SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]						, )
							@slow
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								a									=				{"""input_ids""": [[25_0004, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 5428, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 1_2399, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 12_2009, 11_5774, 23, 805, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [25_0004, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_0004, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=__magic_name__						, model_name="""facebook/mbart-large-50"""						, revision="""d3913889c59cd5c9e456b269c376325eabad57e2"""						, )





							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								if not self.test_slow_tokenizer:
									# as we don't have a slow version, we can't compare the outputs between slow and fast versions
									return

								a									=				(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-mbart50""", {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
										a									=				self.rust_tokenizer_class.from_pretrained(__magic_name__						, **__magic_name__							)
										a									=				self.tokenizer_class.from_pretrained(__magic_name__						, **__magic_name__							)

										a									=				tempfile.mkdtemp()

										a									=				tokenizer_r.save_pretrained(__magic_name__							)
										a									=				tokenizer_p.save_pretrained(__magic_name__							)

										# Checks it save with the same files + the tokenizer.json file for the fast one
										self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files							)							)
										a									=				tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f							)
										self.assertSequenceEqual(__magic_name__						, __magic_name__							)

										# Checks everything loads correctly in the same way
										a									=				tokenizer_r.from_pretrained(__magic_name__							)
										a									=				tokenizer_p.from_pretrained(__magic_name__							)

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__magic_name__						, __magic_name__							)							)
											# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
											# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

										shutil.rmtree(__magic_name__							)

										# Save tokenizer rust, legacy_format=True
										a									=				tempfile.mkdtemp()

										a									=				tokenizer_r.save_pretrained(__magic_name__						, legacy_format=__magic_name__							)
										a									=				tokenizer_p.save_pretrained(__magic_name__							)

										# Checks it save with the same files
										self.assertSequenceEqual(__magic_name__						, __magic_name__							)

										# Checks everything loads correctly in the same way
										a									=				tokenizer_r.from_pretrained(__magic_name__							)
										a									=				tokenizer_p.from_pretrained(__magic_name__							)

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__magic_name__						, __magic_name__							)							)

										shutil.rmtree(__magic_name__							)

										# Save tokenizer rust, legacy_format=False
										a									=				tempfile.mkdtemp()

										a									=				tokenizer_r.save_pretrained(__magic_name__						, legacy_format=__magic_name__							)
										a									=				tokenizer_p.save_pretrained(__magic_name__							)

										# Checks it saved the tokenizer.json file
										self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files							)							)

										# Checks everything loads correctly in the same way
										a									=				tokenizer_r.from_pretrained(__magic_name__							)
										a									=				tokenizer_p.from_pretrained(__magic_name__							)

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__magic_name__						, __magic_name__							)							)

										shutil.rmtree(__magic_name__							)







@require_torch
@require_sentencepiece
@require_tokenizers
class 						__lowerCAmelCase      (      unittest.TestCase					):
							UpperCamelCase__   	=    '''facebook/mbart-large-50-one-to-many-mmt'''
							UpperCamelCase__   	=    [
							    ''' UN Chief Says There Is No Military Solution in Syria''',
							    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
							]
							UpperCamelCase__   	=    [
							    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
							    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
							    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
							    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
							]
							UpperCamelCase__   	=    [EN_CODE, 8274, 127873, 25916, 7, 8622, 2071, 438, 67485, 53, 187895, 23, 51712, 2]
							@classmethod
							def  lowerCamelCase__				(		cls		:str							):







								'''simple docstring'''







								a									=				MBartaaTokenizer.from_pretrained(
								    cls.checkpoint_name						, src_lang="""en_XX"""						, tgt_lang="""ro_RO"""							)
								a									=				1
								return cls
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]						, 25_0001							)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]						, 25_0004							)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]						, 25_0020							)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""mr_IN"""]						, 25_0038							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								a									=				self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
								self.assertListEqual(self.expected_src_tokens						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								self.assertIn(__magic_name__						, self.tokenizer.all_special_ids							)
								a									=				[RO_CODE, 884, 9019, 96, 9, 916, 8_6792, 36, 1_8743, 1_5596, 5, 2]
								a									=				self.tokenizer.decode(__magic_name__						, skip_special_tokens=__magic_name__							)
								a									=				self.tokenizer.decode(generated_ids[1:]						, skip_special_tokens=__magic_name__							)
								self.assertEqual(__magic_name__						, __magic_name__							)
								self.assertNotIn(self.tokenizer.eos_token						, __magic_name__							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				["""this is gunna be a long sentence """ * 20]
								assert isinstance(src_text[0]						, __magic_name__							)
								a									=				10
								a									=				self.tokenizer(__magic_name__						, max_length=__magic_name__						, truncation=__magic_name__							).input_ids[0]
								self.assertEqual(ids[0]						, __magic_name__							)
								self.assertEqual(ids[-1]						, 2							)
								self.assertEqual(len(__magic_name__							)						, __magic_name__							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]							)						, [25_0053, 25_0001]							)
							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				tempfile.mkdtemp()
								a									=				self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(__magic_name__							)
								a									=				MBartaaTokenizer.from_pretrained(__magic_name__							)
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids						, __magic_name__							)
							@require_torch
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				self.tokenizer(self.src_text						, text_target=self.tgt_text						, padding=__magic_name__						, return_tensors="""pt"""							)
								a									=				shift_tokens_right(batch["""labels"""]						, self.tokenizer.pad_token_id							)

								# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
								assert batch.input_ids[1][0] == EN_CODE
								assert batch.input_ids[1][-1] == 2
								assert batch.labels[1][0] == RO_CODE
								assert batch.labels[1][-1] == 2
								assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]
							@require_torch
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				self.tokenizer(
								    self.src_text						, text_target=self.tgt_text						, padding=__magic_name__						, truncation=__magic_name__						, max_length=len(self.expected_src_tokens							)						, return_tensors="""pt"""						, )
								a									=				shift_tokens_right(batch["""labels"""]						, self.tokenizer.pad_token_id							)

								self.assertIsInstance(__magic_name__						, __magic_name__							)

								self.assertEqual((2, 14)						, batch.input_ids.shape							)
								self.assertEqual((2, 14)						, batch.attention_mask.shape							)
								a									=				batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens						, __magic_name__							)
								self.assertEqual(2						, batch.decoder_input_ids[0, 0]							)  # decoder_start_token_id
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens						, [EN_CODE]							)
								self.assertEqual(self.tokenizer.suffix_tokens						, [self.tokenizer.eos_token_id]							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				self.tokenizer(self.src_text						, padding=__magic_name__						, truncation=__magic_name__						, max_length=3						, return_tensors="""pt"""							)
								a									=				self.tokenizer(
								    text_target=self.tgt_text						, padding=__magic_name__						, truncation=__magic_name__						, max_length=10						, return_tensors="""pt"""							)
								a									=				targets["""input_ids"""]
								a									=				shift_tokens_right(__magic_name__						, self.tokenizer.pad_token_id							)

								self.assertEqual(batch.input_ids.shape[1]						, 3							)
								self.assertEqual(batch.decoder_input_ids.shape[1]						, 10							)





							@require_torch
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				self.tokenizer._build_translation_inputs(
								    """A test"""						, return_tensors="""pt"""						, src_lang="""en_XX"""						, tgt_lang="""ar_AR"""							)

								self.assertEqual(
								    nested_simplify(__magic_name__							)						, {
								        # en_XX, A, test, EOS
								        """input_ids""": [[25_0004, 62, 3034, 2]],
								        """attention_mask""": [[1, 1, 1, 1]],
								        # ar_AR
								        """forced_bos_token_id""": 25_0001,
								    }						, )







import warnings

from ...utils import logging
from .image_processing_layoutlmva import LayoutLMvaImageProcessor


__UpperCamelCase   :			Optional[int]          = logging.get_logger(__name__)

class 						__lowerCAmelCase      (      __magic_name__					):
							def __init__(		self		:Union[str, Any]						, *__magic_name__		:int						, **__magic_name__		:str							):







								'''simple docstring'''







								warnings.warn(
								    """The class LayoutLMv2FeatureExtractor is deprecated and will be removed in version 5 of Transformers."""
								    """ Please use LayoutLMv2ImageProcessor instead."""						, __magic_name__						, )
								super().__init__(*__magic_name__						, **__magic_name__							)

