


"""simple docstring"""






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase     =  get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
	from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_lowerCamelCase     =  256047
_lowerCamelCase     =  256145


@require_sentencepiece
@require_tokenizers
class 			lowerCamelCase_		(			lowercase       ,  unittest.TestCase   ):





				"""simple docstring"""







				_lowerCAmelCase      :      Dict      							=					NllbTokenizer
				_lowerCAmelCase      :      Dict      							=					NllbTokenizerFast
				_lowerCAmelCase      :      int      							=					True
				_lowerCAmelCase      :      Union[str, Any]      							=					True
				_lowerCAmelCase      :      Optional[int]      							=					{}



				def   lowerCAmelCase__		(						self     ):
								super().setUp()

								# We have a SentencePiece fixture for testing
								SCREAMING_SNAKE_CASE__     							=				NllbTokenizer(UpperCAmelCase__       ,		keep_accents=UpperCAmelCase__     )
								tokenizer.save_pretrained(self.tmpdirname     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				NllbTokenizer(UpperCAmelCase__       ,		keep_accents=UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				tokenizer.tokenize("This is a test"     )
								self.assertListEqual(UpperCAmelCase__       ,		["▁This", "▁is", "▁a", "▁t", "est"]     )

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(UpperCAmelCase__     )       ,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,		)

								SCREAMING_SNAKE_CASE__     							=				tokenizer.tokenize("I was born in 92000, and this is falsé."     )
								self.assertListEqual(
								    UpperCAmelCase__       ,		[
								        SPIECE_UNDERLINE + "I",
								        SPIECE_UNDERLINE + "was",
								        SPIECE_UNDERLINE + "b",
								        "or",
								        "n",
								        SPIECE_UNDERLINE + "in",
								        SPIECE_UNDERLINE + "",
								        "9",
								        "2",
								        "0",
								        "0",
								        "0",
								        ",",
								        SPIECE_UNDERLINE + "and",
								        SPIECE_UNDERLINE + "this",
								        SPIECE_UNDERLINE + "is",
								        SPIECE_UNDERLINE + "f",
								        "al",
								        "s",
								        "é",
								        ".",
								    ]       ,		)
								SCREAMING_SNAKE_CASE__     							=				tokenizer.convert_tokens_to_ids(UpperCAmelCase__     )
								self.assertListEqual(
								    UpperCAmelCase__       ,		[
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								    ]       ,		)

								SCREAMING_SNAKE_CASE__     							=				tokenizer.convert_ids_to_tokens(UpperCAmelCase__     )
								self.assertListEqual(
								    UpperCAmelCase__       ,		[
								        SPIECE_UNDERLINE + "I",
								        SPIECE_UNDERLINE + "was",
								        SPIECE_UNDERLINE + "b",
								        "or",
								        "n",
								        SPIECE_UNDERLINE + "in",
								        SPIECE_UNDERLINE + "",
								        "<unk>",
								        "2",
								        "0",
								        "0",
								        "0",
								        ",",
								        SPIECE_UNDERLINE + "and",
								        SPIECE_UNDERLINE + "this",
								        SPIECE_UNDERLINE + "is",
								        SPIECE_UNDERLINE + "f",
								        "al",
								        "s",
								        "<unk>",
								        ".",
								    ]       ,		)



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				(self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
																SCREAMING_SNAKE_CASE__     							=				self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__       ,		**UpperCAmelCase__     )
																SCREAMING_SNAKE_CASE__     							=				self.tokenizer_class.from_pretrained(UpperCAmelCase__       ,		**UpperCAmelCase__     )

																SCREAMING_SNAKE_CASE__     							=				tempfile.mkdtemp()

																SCREAMING_SNAKE_CASE__     							=				tokenizer_r.save_pretrained(UpperCAmelCase__     )
																SCREAMING_SNAKE_CASE__     							=				tokenizer_p.save_pretrained(UpperCAmelCase__     )

																# Checks it save with the same files + the tokenizer.json file for the fast one
																self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files     )     )
																SCREAMING_SNAKE_CASE__     							=				tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f     )
																self.assertSequenceEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )

																# Checks everything loads correctly in the same way
																SCREAMING_SNAKE_CASE__     							=				tokenizer_r.from_pretrained(UpperCAmelCase__     )
																SCREAMING_SNAKE_CASE__     							=				tokenizer_p.from_pretrained(UpperCAmelCase__     )

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																				self.assertTrue(hasattr(UpperCAmelCase__       ,		UpperCAmelCase__     )     )

																shutil.rmtree(UpperCAmelCase__     )

																# Save tokenizer rust, legacy_format=True
																SCREAMING_SNAKE_CASE__     							=				tempfile.mkdtemp()

																SCREAMING_SNAKE_CASE__     							=				tokenizer_r.save_pretrained(UpperCAmelCase__       ,		legacy_format=UpperCAmelCase__     )
																SCREAMING_SNAKE_CASE__     							=				tokenizer_p.save_pretrained(UpperCAmelCase__     )

																# Checks it save with the same files
																self.assertSequenceEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )

																# Checks everything loads correctly in the same way
																SCREAMING_SNAKE_CASE__     							=				tokenizer_r.from_pretrained(UpperCAmelCase__     )
																SCREAMING_SNAKE_CASE__     							=				tokenizer_p.from_pretrained(UpperCAmelCase__     )

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																				self.assertTrue(hasattr(UpperCAmelCase__       ,		UpperCAmelCase__     )     )

																shutil.rmtree(UpperCAmelCase__     )

																# Save tokenizer rust, legacy_format=False
																SCREAMING_SNAKE_CASE__     							=				tempfile.mkdtemp()

																SCREAMING_SNAKE_CASE__     							=				tokenizer_r.save_pretrained(UpperCAmelCase__       ,		legacy_format=UpperCAmelCase__     )
																SCREAMING_SNAKE_CASE__     							=				tokenizer_p.save_pretrained(UpperCAmelCase__     )

																# Checks it saved the tokenizer.json file
																self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files     )     )

																# Checks everything loads correctly in the same way
																SCREAMING_SNAKE_CASE__     							=				tokenizer_r.from_pretrained(UpperCAmelCase__     )
																SCREAMING_SNAKE_CASE__     							=				tokenizer_p.from_pretrained(UpperCAmelCase__     )

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																				self.assertTrue(hasattr(UpperCAmelCase__       ,		UpperCAmelCase__     )     )

																shutil.rmtree(UpperCAmelCase__     )



				@require_torch
				def   lowerCAmelCase__		(						self     ):
								if not self.test_seqaseq:
												return

								SCREAMING_SNAKE_CASE__     							=				self.get_tokenizers()
								for tokenizer in tokenizers:
												with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																# Longer text that will definitely require truncation.
																SCREAMING_SNAKE_CASE__     							=				[
																    " UN Chief Says There Is No Military Solution in Syria",
																    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"
																    " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"
																    " will only worsen the violence and misery for millions of people.",
																]
																SCREAMING_SNAKE_CASE__     							=				[
																    "Şeful ONU declară că nu există o soluţie militară în Siria",
																    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
																    " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
																    " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
																]
																try:
																				SCREAMING_SNAKE_CASE__     							=				tokenizer.prepare_seqaseq_batch(
																				    src_texts=UpperCAmelCase__       ,		tgt_texts=UpperCAmelCase__       ,		max_length=3       ,		max_target_length=10       ,		return_tensors="pt"       ,		src_lang="eng_Latn"       ,		tgt_lang="ron_Latn"       ,		)
																except NotImplementedError:
																				return
																self.assertEqual(batch.input_ids.shape[1]       ,		3     )
																self.assertEqual(batch.labels.shape[1]       ,		10     )
																# max_target_length will default to max_length if not specified
																SCREAMING_SNAKE_CASE__     							=				tokenizer.prepare_seqaseq_batch(
																    UpperCAmelCase__       ,		tgt_texts=UpperCAmelCase__       ,		max_length=3       ,		return_tensors="pt"     )
																self.assertEqual(batch.input_ids.shape[1]       ,		3     )
																self.assertEqual(batch.labels.shape[1]       ,		3     )

																SCREAMING_SNAKE_CASE__     							=				tokenizer.prepare_seqaseq_batch(
																    src_texts=UpperCAmelCase__       ,		max_length=3       ,		max_target_length=10       ,		return_tensors="pt"     )
																self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,		3     )
																self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,		3     )
																self.assertNotIn("decoder_input_ids"       ,		UpperCAmelCase__     )



				@unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece."     )
				def   lowerCAmelCase__		(						self     ):
								pass






				def   lowerCAmelCase__		(						self     ):
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
																SCREAMING_SNAKE_CASE__     							=				[AddedToken("<special>"       ,		lstrip=UpperCAmelCase__     )]

																SCREAMING_SNAKE_CASE__     							=				self.rust_tokenizer_class.from_pretrained(
																    UpperCAmelCase__       ,		additional_special_tokens=UpperCAmelCase__       ,		**UpperCAmelCase__     )
																SCREAMING_SNAKE_CASE__     							=				tokenizer_r.encode("Hey this is a <special> token"     )

																SCREAMING_SNAKE_CASE__     							=				tokenizer_r.encode("<special>"       ,		add_special_tokens=UpperCAmelCase__     )[0]

																self.assertTrue(special_token_id in r_output     )

																if self.test_slow_tokenizer:
																				SCREAMING_SNAKE_CASE__     							=				self.rust_tokenizer_class.from_pretrained(
																				    UpperCAmelCase__       ,		additional_special_tokens=UpperCAmelCase__       ,		**UpperCAmelCase__       ,		)
																				SCREAMING_SNAKE_CASE__     							=				self.tokenizer_class.from_pretrained(
																				    UpperCAmelCase__       ,		additional_special_tokens=UpperCAmelCase__       ,		**UpperCAmelCase__     )

																				SCREAMING_SNAKE_CASE__     							=				tokenizer_p.encode("Hey this is a <special> token"     )

																				SCREAMING_SNAKE_CASE__     							=				tokenizer_cr.encode("Hey this is a <special> token"     )

																				self.assertEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )
																				self.assertEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )
																				self.assertTrue(special_token_id in p_output     )
																				self.assertTrue(special_token_id in cr_output     )


@require_torch
@require_sentencepiece
@require_tokenizers
class 			lowerCamelCase_		(			unittest.TestCase   ):





				"""simple docstring"""







				_lowerCAmelCase      :      Dict      							=					"facebook/nllb-200-distilled-600M"
				_lowerCAmelCase      :      Dict      							=					[
				    " UN Chief Says There Is No Military Solution in Syria",
				    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
				]
				_lowerCAmelCase      :      List[str]      							=					[
				    "Şeful ONU declară că nu există o soluţie militară în Siria",
				    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
				    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
				    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
				]
				_lowerCAmelCase      :      Optional[Any]      							=					[
				    256_047,
				    16_297,
				    134_408,
				    8_165,
				    248_066,
				    14_734,
				    950,
				    1_135,
				    105_721,
				    3_573,
				    83,
				    27_352,
				    108,
				    49_486,
				    2,
				]



				@classmethod
				def   lowerCAmelCase__		(						cls     ):
								SCREAMING_SNAKE_CASE__     							=				NllbTokenizer.from_pretrained(
								    cls.checkpoint_name       ,		src_lang="eng_Latn"       ,		tgt_lang="ron_Latn"     )
								SCREAMING_SNAKE_CASE__     							=				1
								return cls



				def   lowerCAmelCase__		(						self     ):
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]       ,		25_6001     )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]       ,		25_6002     )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]       ,		25_6057     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer.batch_encode_plus(self.src_text     ).input_ids[0]
								self.assertListEqual(self.expected_src_tokens       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								self.assertIn(UpperCAmelCase__       ,		self.tokenizer.all_special_ids     )
								# fmt: off
								SCREAMING_SNAKE_CASE__     							=				[RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
								# fmt: on

								SCREAMING_SNAKE_CASE__     							=				self.tokenizer.decode(UpperCAmelCase__       ,		skip_special_tokens=UpperCAmelCase__     )
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer.decode(generated_ids[1:]       ,		skip_special_tokens=UpperCAmelCase__     )
								self.assertEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )
								self.assertNotIn(self.tokenizer.eos_token       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				["this is gunna be a long sentence " * 20]
								assert isinstance(src_text[0]       ,		UpperCAmelCase__     )
								SCREAMING_SNAKE_CASE__     							=				10
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer(UpperCAmelCase__       ,		max_length=UpperCAmelCase__       ,		truncation=UpperCAmelCase__     ).input_ids[0]
								self.assertEqual(ids[-1]       ,		2     )
								self.assertEqual(ids[0]       ,		UpperCAmelCase__     )
								self.assertEqual(len(UpperCAmelCase__     )       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]     )       ,		[25_6203, 3]     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				tempfile.mkdtemp()
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(UpperCAmelCase__     )
								SCREAMING_SNAKE_CASE__     							=				NllbTokenizer.from_pretrained(UpperCAmelCase__     )
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,		UpperCAmelCase__     )



				@require_torch
				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer(
								    self.src_text       ,		text_target=self.tgt_text       ,		padding=UpperCAmelCase__       ,		truncation=UpperCAmelCase__       ,		max_length=len(self.expected_src_tokens     )       ,		return_tensors="pt"       ,		)
								SCREAMING_SNAKE_CASE__     							=				shift_tokens_right(
								    batch["labels"]       ,		self.tokenizer.pad_token_id       ,		self.tokenizer.lang_code_to_id["ron_Latn"]     )

								self.assertIsInstance(UpperCAmelCase__       ,		UpperCAmelCase__     )

								self.assertEqual((2, 15)       ,		batch.input_ids.shape     )
								self.assertEqual((2, 15)       ,		batch.attention_mask.shape     )
								SCREAMING_SNAKE_CASE__     							=				batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens       ,		UpperCAmelCase__     )
								self.assertEqual(UpperCAmelCase__       ,		batch.decoder_input_ids[0, 0]     )  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens       ,		[EN_CODE]     )
								self.assertEqual(self.tokenizer.suffix_tokens       ,		[self.tokenizer.eos_token_id]     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer(self.src_text       ,		padding=UpperCAmelCase__       ,		truncation=UpperCAmelCase__       ,		max_length=3       ,		return_tensors="pt"     )
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer(
								    text_target=self.tgt_text       ,		padding=UpperCAmelCase__       ,		truncation=UpperCAmelCase__       ,		max_length=10       ,		return_tensors="pt"     )
								SCREAMING_SNAKE_CASE__     							=				targets["input_ids"]
								SCREAMING_SNAKE_CASE__     							=				shift_tokens_right(
								    UpperCAmelCase__       ,		self.tokenizer.pad_token_id       ,		decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,		)

								self.assertEqual(batch.input_ids.shape[1]       ,		3     )
								self.assertEqual(batch.decoder_input_ids.shape[1]       ,		10     )



				@require_torch
				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer._build_translation_inputs(
								    "A test"       ,		return_tensors="pt"       ,		src_lang="eng_Latn"       ,		tgt_lang="fra_Latn"     )

								self.assertEqual(
								    nested_simplify(UpperCAmelCase__     )       ,		{
								        # A, test, EOS, en_XX
								        "input_ids": [[25_6047, 70, 7356, 2]],
								        "attention_mask": [[1, 1, 1, 1]],
								        # ar_AR
								        "forced_bos_token_id": 25_6057,
								    }       ,		)






				@require_torch
				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				True
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer(
								    "UN Chief says there is no military solution in Syria"       ,		src_lang="eng_Latn"       ,		tgt_lang="fra_Latn"     )
								self.assertEqual(
								    inputs.input_ids       ,		[1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]     )

								SCREAMING_SNAKE_CASE__     							=				False
								SCREAMING_SNAKE_CASE__     							=				self.tokenizer(
								    "UN Chief says there is no military solution in Syria"       ,		src_lang="eng_Latn"       ,		tgt_lang="fra_Latn"     )
								self.assertEqual(
								    inputs.input_ids       ,		[25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]     )




"""simple docstring"""






from transformers import BertTokenizerFast

from .custom_tokenization import CustomTokenizer


class 			lowerCamelCase_		(			lowercase   ):





				"""simple docstring"""







				_lowerCAmelCase      :      List[Any]      							=					CustomTokenizer
				pass
