

"""simple docstring"""






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__					:	Optional[Any]				    =	get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
						from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCamelCase__					:	Any				    =	25_60_47
lowerCamelCase__					:	List[str]				    =	25_61_45






@require_sentencepiece
@require_tokenizers
class      _UpperCAmelCase      (		__a     ,  unittest.TestCase):
			__a			:							List[str]					       =				NllbTokenizer
			__a			:							int					       =				NllbTokenizerFast
			__a			:							List[Any]					       =				True
			__a			:							List[Any]					       =				True
			__a			:							int					       =				{}
			def 				__snake_case			(		self  )     ->     int:






				'''simple docstring'''







				super().setUp()

				# We have a SentencePiece fixture for testing
				_UpperCAmelCase     :				List[str]          =       NllbTokenizer(__lowerCAmelCase   ,		keep_accents=__lowerCAmelCase  )
				tokenizer.save_pretrained(self.tmpdirname  )
			def 				__snake_case			(		self  )     ->     Any:






				'''simple docstring'''







				_UpperCAmelCase     :				Optional[int]          =       NllbTokenizer(__lowerCAmelCase   ,		keep_accents=__lowerCAmelCase  )

				_UpperCAmelCase     :				List[str]          =       tokenizer.tokenize("""This is a test"""  )
				self.assertListEqual(__lowerCAmelCase   ,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(__lowerCAmelCase  )   ,		[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]   ,		)

				_UpperCAmelCase     :				Optional[Any]          =       tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
				self.assertListEqual(
				    __lowerCAmelCase   ,		[
				        SPIECE_UNDERLINE + """I""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """9""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """s""",
				        """é""",
				        """.""",
				    ]   ,		)
				_UpperCAmelCase     :				Dict          =       tokenizer.convert_tokens_to_ids(__lowerCAmelCase  )
				self.assertListEqual(
				    __lowerCAmelCase   ,		[
				        value + tokenizer.fairseq_offset
				        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
				    ]   ,		)

				_UpperCAmelCase     :				Any          =       tokenizer.convert_ids_to_tokens(__lowerCAmelCase  )
				self.assertListEqual(
				    __lowerCAmelCase   ,		[
				        SPIECE_UNDERLINE + """I""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """<unk>""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """s""",
				        """<unk>""",
				        """.""",
				    ]   ,		)
			def 				__snake_case			(		self  )     ->     str:






				'''simple docstring'''







				_UpperCAmelCase     :				Tuple          =       (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
						_UpperCAmelCase     :				int          =       self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase   ,		**__lowerCAmelCase  )
						_UpperCAmelCase     :				Optional[int]          =       self.tokenizer_class.from_pretrained(__lowerCAmelCase   ,		**__lowerCAmelCase  )

						_UpperCAmelCase     :				Tuple          =       tempfile.mkdtemp()

						_UpperCAmelCase     :				List[str]          =       tokenizer_r.save_pretrained(__lowerCAmelCase  )
						_UpperCAmelCase     :				Dict          =       tokenizer_p.save_pretrained(__lowerCAmelCase  )

						# Checks it save with the same files + the tokenizer.json file for the fast one
						self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files  )  )
						_UpperCAmelCase     :				Union[str, Any]          =       tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f  )
						self.assertSequenceEqual(__lowerCAmelCase   ,		__lowerCAmelCase  )

						# Checks everything loads correctly in the same way
						_UpperCAmelCase     :				Any          =       tokenizer_r.from_pretrained(__lowerCAmelCase  )
						_UpperCAmelCase     :				int          =       tokenizer_p.from_pretrained(__lowerCAmelCase  )

						# Check special tokens are set accordingly on Rust and Python
						for key in tokenizer_pp.special_tokens_map:
							self.assertTrue(hasattr(__lowerCAmelCase   ,		__lowerCAmelCase  )  )

						shutil.rmtree(__lowerCAmelCase  )

						# Save tokenizer rust, legacy_format=True
						_UpperCAmelCase     :				Tuple          =       tempfile.mkdtemp()

						_UpperCAmelCase     :				Any          =       tokenizer_r.save_pretrained(__lowerCAmelCase   ,		legacy_format=__lowerCAmelCase  )
						_UpperCAmelCase     :				Union[str, Any]          =       tokenizer_p.save_pretrained(__lowerCAmelCase  )

						# Checks it save with the same files
						self.assertSequenceEqual(__lowerCAmelCase   ,		__lowerCAmelCase  )

						# Checks everything loads correctly in the same way
						_UpperCAmelCase     :				Any          =       tokenizer_r.from_pretrained(__lowerCAmelCase  )
						_UpperCAmelCase     :				Optional[Any]          =       tokenizer_p.from_pretrained(__lowerCAmelCase  )

						# Check special tokens are set accordingly on Rust and Python
						for key in tokenizer_pp.special_tokens_map:
							self.assertTrue(hasattr(__lowerCAmelCase   ,		__lowerCAmelCase  )  )

						shutil.rmtree(__lowerCAmelCase  )

						# Save tokenizer rust, legacy_format=False
						_UpperCAmelCase     :				List[str]          =       tempfile.mkdtemp()

						_UpperCAmelCase     :				str          =       tokenizer_r.save_pretrained(__lowerCAmelCase   ,		legacy_format=__lowerCAmelCase  )
						_UpperCAmelCase     :				Union[str, Any]          =       tokenizer_p.save_pretrained(__lowerCAmelCase  )

						# Checks it saved the tokenizer.json file
						self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files  )  )

						# Checks everything loads correctly in the same way
						_UpperCAmelCase     :				List[Any]          =       tokenizer_r.from_pretrained(__lowerCAmelCase  )
						_UpperCAmelCase     :				Union[str, Any]          =       tokenizer_p.from_pretrained(__lowerCAmelCase  )

						# Check special tokens are set accordingly on Rust and Python
						for key in tokenizer_pp.special_tokens_map:
							self.assertTrue(hasattr(__lowerCAmelCase   ,		__lowerCAmelCase  )  )

						shutil.rmtree(__lowerCAmelCase  )
			@require_torch
			def 				__snake_case			(		self  )     ->     Tuple:






				'''simple docstring'''







				if not self.test_seqaseq:
					return

				_UpperCAmelCase     :				Any          =       self.get_tokenizers()
				for tokenizer in tokenizers:
					with self.subTest(f'''{tokenizer.__class__.__name__}'''  ):
						# Longer text that will definitely require truncation.
						_UpperCAmelCase     :				Union[str, Any]          =       [
						    """ UN Chief Says There Is No Military Solution in Syria""",
						    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
						    """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
						    """ will only worsen the violence and misery for millions of people.""",
						]
						_UpperCAmelCase     :				Any          =       [
						    """Şeful ONU declară că nu există o soluţie militară în Siria""",
						    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
						    """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
						    """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
						]
						try:
							_UpperCAmelCase     :				str          =       tokenizer.prepare_seqaseq_batch(
							    src_texts=__lowerCAmelCase   ,		tgt_texts=__lowerCAmelCase   ,		max_length=3   ,		max_target_length=10   ,		return_tensors="""pt"""   ,		src_lang="""eng_Latn"""   ,		tgt_lang="""ron_Latn"""   ,		)
						except NotImplementedError:
							return
						self.assertEqual(batch.input_ids.shape[1]   ,		3  )
						self.assertEqual(batch.labels.shape[1]   ,		10  )
						# max_target_length will default to max_length if not specified
						_UpperCAmelCase     :				Union[str, Any]          =       tokenizer.prepare_seqaseq_batch(
						    __lowerCAmelCase   ,		tgt_texts=__lowerCAmelCase   ,		max_length=3   ,		return_tensors="""pt"""  )
						self.assertEqual(batch.input_ids.shape[1]   ,		3  )
						self.assertEqual(batch.labels.shape[1]   ,		3  )

						_UpperCAmelCase     :				Optional[Any]          =       tokenizer.prepare_seqaseq_batch(
						    src_texts=__lowerCAmelCase   ,		max_length=3   ,		max_target_length=10   ,		return_tensors="""pt"""  )
						self.assertEqual(batch_encoder_only.input_ids.shape[1]   ,		3  )
						self.assertEqual(batch_encoder_only.attention_mask.shape[1]   ,		3  )
						self.assertNotIn("""decoder_input_ids"""   ,		__lowerCAmelCase  )
			@unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""  )
			def 				__snake_case			(		self  )     ->     List[Any]:






				'''simple docstring'''







				pass
			def 				__snake_case			(		self  )     ->     int:






				'''simple docstring'''







				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
						_UpperCAmelCase     :				Optional[int]          =       [AddedToken("""<special>"""   ,		lstrip=__lowerCAmelCase  )]

						_UpperCAmelCase     :				List[Any]          =       self.rust_tokenizer_class.from_pretrained(
						    __lowerCAmelCase   ,		additional_special_tokens=__lowerCAmelCase   ,		**__lowerCAmelCase  )
						_UpperCAmelCase     :				str          =       tokenizer_r.encode("""Hey this is a <special> token"""  )

						_UpperCAmelCase     :				Union[str, Any]          =       tokenizer_r.encode("""<special>"""   ,		add_special_tokens=__lowerCAmelCase  )[0]

						self.assertTrue(special_token_id in r_output  )

						if self.test_slow_tokenizer:
							_UpperCAmelCase     :				Optional[Any]          =       self.rust_tokenizer_class.from_pretrained(
							    __lowerCAmelCase   ,		additional_special_tokens=__lowerCAmelCase   ,		**__lowerCAmelCase   ,		)
							_UpperCAmelCase     :				List[Any]          =       self.tokenizer_class.from_pretrained(
							    __lowerCAmelCase   ,		additional_special_tokens=__lowerCAmelCase   ,		**__lowerCAmelCase  )

							_UpperCAmelCase     :				List[Any]          =       tokenizer_p.encode("""Hey this is a <special> token"""  )

							_UpperCAmelCase     :				int          =       tokenizer_cr.encode("""Hey this is a <special> token"""  )

							self.assertEqual(__lowerCAmelCase   ,		__lowerCAmelCase  )
							self.assertEqual(__lowerCAmelCase   ,		__lowerCAmelCase  )
							self.assertTrue(special_token_id in p_output  )
							self.assertTrue(special_token_id in cr_output  )






@require_torch
@require_sentencepiece
@require_tokenizers
class      _UpperCAmelCase      (		unittest.TestCase):
			__a			:							Union[str, Any]					       =				'''facebook/nllb-200-distilled-600M'''
			__a			:							Optional[Any]					       =				[
			    ''' UN Chief Says There Is No Military Solution in Syria''',
			    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
			]
			__a			:							List[str]					       =				[
			    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
			    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
			    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
			    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
			]
			__a			:							List[Any]					       =				[
			    2_5_6_0_4_7,
			    1_6_2_9_7,
			    1_3_4_4_0_8,
			    8_1_6_5,
			    2_4_8_0_6_6,
			    1_4_7_3_4,
			    9_5_0,
			    1_1_3_5,
			    1_0_5_7_2_1,
			    3_5_7_3,
			    8_3,
			    2_7_3_5_2,
			    1_0_8,
			    4_9_4_8_6,
			    2,
			]
			@classmethod
			def 				__snake_case			(		cls  )     ->     Union[str, Any]:






				'''simple docstring'''







				_UpperCAmelCase     :				Optional[int]          =       NllbTokenizer.from_pretrained(
				    cls.checkpoint_name   ,		src_lang="""eng_Latn"""   ,		tgt_lang="""ron_Latn"""  )
				_UpperCAmelCase     :				Optional[Any]          =       1
				return cls
			def 				__snake_case			(		self  )     ->     Optional[Any]:






				'''simple docstring'''







				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]   ,		25_60_01  )
				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]   ,		25_60_02  )
				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]   ,		25_60_57  )
			def 				__snake_case			(		self  )     ->     Tuple:






				'''simple docstring'''







				_UpperCAmelCase     :				Tuple          =       self.tokenizer.batch_encode_plus(self.src_text  ).input_ids[0]
				self.assertListEqual(self.expected_src_tokens   ,		__lowerCAmelCase  )
			def 				__snake_case			(		self  )     ->     Optional[Any]:






				'''simple docstring'''







				self.assertIn(__lowerCAmelCase   ,		self.tokenizer.all_special_ids  )
				# fmt: off
				_UpperCAmelCase     :				Dict          =       [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
				# fmt: on

				_UpperCAmelCase     :				Any          =       self.tokenizer.decode(__lowerCAmelCase   ,		skip_special_tokens=__lowerCAmelCase  )
				_UpperCAmelCase     :				Optional[int]          =       self.tokenizer.decode(generated_ids[1:]   ,		skip_special_tokens=__lowerCAmelCase  )
				self.assertEqual(__lowerCAmelCase   ,		__lowerCAmelCase  )
				self.assertNotIn(self.tokenizer.eos_token   ,		__lowerCAmelCase  )
			def 				__snake_case			(		self  )     ->     Tuple:






				'''simple docstring'''







				_UpperCAmelCase     :				Tuple          =       ["""this is gunna be a long sentence """ * 20]
				assert isinstance(src_text[0]   ,		__lowerCAmelCase  )
				_UpperCAmelCase     :				List[Any]          =       10
				_UpperCAmelCase     :				Any          =       self.tokenizer(__lowerCAmelCase   ,		max_length=__lowerCAmelCase   ,		truncation=__lowerCAmelCase  ).input_ids[0]
				self.assertEqual(ids[-1]   ,		2  )
				self.assertEqual(ids[0]   ,		__lowerCAmelCase  )
				self.assertEqual(len(__lowerCAmelCase  )   ,		__lowerCAmelCase  )
			def 				__snake_case			(		self  )     ->     int:






				'''simple docstring'''







				self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]  )   ,		[25_62_03, 3]  )
			def 				__snake_case			(		self  )     ->     List[str]:






				'''simple docstring'''







				_UpperCAmelCase     :				Optional[Any]          =       tempfile.mkdtemp()
				_UpperCAmelCase     :				Optional[Any]          =       self.tokenizer.fairseq_tokens_to_ids
				self.tokenizer.save_pretrained(__lowerCAmelCase  )
				_UpperCAmelCase     :				List[Any]          =       NllbTokenizer.from_pretrained(__lowerCAmelCase  )
				self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,		__lowerCAmelCase  )
			@require_torch
			def 				__snake_case			(		self  )     ->     List[str]:






				'''simple docstring'''







				_UpperCAmelCase     :				Optional[Any]          =       self.tokenizer(
				    self.src_text   ,		text_target=self.tgt_text   ,		padding=__lowerCAmelCase   ,		truncation=__lowerCAmelCase   ,		max_length=len(self.expected_src_tokens  )   ,		return_tensors="""pt"""   ,		)
				_UpperCAmelCase     :				int          =       shift_tokens_right(
				    batch["""labels"""]   ,		self.tokenizer.pad_token_id   ,		self.tokenizer.lang_code_to_id["""ron_Latn"""]  )

				self.assertIsInstance(__lowerCAmelCase   ,		__lowerCAmelCase  )

				self.assertEqual((2, 15)   ,		batch.input_ids.shape  )
				self.assertEqual((2, 15)   ,		batch.attention_mask.shape  )
				_UpperCAmelCase     :				Optional[Any]          =       batch.input_ids.tolist()[0]
				self.assertListEqual(self.expected_src_tokens   ,		__lowerCAmelCase  )
				self.assertEqual(__lowerCAmelCase   ,		batch.decoder_input_ids[0, 0]  )  # EOS
				# Test that special tokens are reset
				self.assertEqual(self.tokenizer.prefix_tokens   ,		[EN_CODE]  )
				self.assertEqual(self.tokenizer.suffix_tokens   ,		[self.tokenizer.eos_token_id]  )
			def 				__snake_case			(		self  )     ->     Tuple:






				'''simple docstring'''







				_UpperCAmelCase     :				Any          =       self.tokenizer(self.src_text   ,		padding=__lowerCAmelCase   ,		truncation=__lowerCAmelCase   ,		max_length=3   ,		return_tensors="""pt"""  )
				_UpperCAmelCase     :				str          =       self.tokenizer(
				    text_target=self.tgt_text   ,		padding=__lowerCAmelCase   ,		truncation=__lowerCAmelCase   ,		max_length=10   ,		return_tensors="""pt"""  )
				_UpperCAmelCase     :				Tuple          =       targets["""input_ids"""]
				_UpperCAmelCase     :				Tuple          =       shift_tokens_right(
				    __lowerCAmelCase   ,		self.tokenizer.pad_token_id   ,		decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]   ,		)

				self.assertEqual(batch.input_ids.shape[1]   ,		3  )
				self.assertEqual(batch.decoder_input_ids.shape[1]   ,		10  )
			@require_torch
			def 				__snake_case			(		self  )     ->     Any:






				'''simple docstring'''







				_UpperCAmelCase     :				Optional[int]          =       self.tokenizer._build_translation_inputs(
				    """A test"""   ,		return_tensors="""pt"""   ,		src_lang="""eng_Latn"""   ,		tgt_lang="""fra_Latn"""  )

				self.assertEqual(
				    nested_simplify(__lowerCAmelCase  )   ,		{
				        # A, test, EOS, en_XX
				        """input_ids""": [[25_60_47, 70, 73_56, 2]],
				        """attention_mask""": [[1, 1, 1, 1]],
				        # ar_AR
				        """forced_bos_token_id""": 25_60_57,
				    }   ,		)
			@require_torch
			def 				__snake_case			(		self  )     ->     Optional[Any]:






				'''simple docstring'''







				_UpperCAmelCase     :				Any          =       True
				_UpperCAmelCase     :				Optional[Any]          =       self.tokenizer(
				    """UN Chief says there is no military solution in Syria"""   ,		src_lang="""eng_Latn"""   ,		tgt_lang="""fra_Latn"""  )
				self.assertEqual(
				    inputs.input_ids   ,		[1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]  )

				_UpperCAmelCase     :				List[Any]          =       False
				_UpperCAmelCase     :				int          =       self.tokenizer(
				    """UN Chief says there is no military solution in Syria"""   ,		src_lang="""eng_Latn"""   ,		tgt_lang="""fra_Latn"""  )
				self.assertEqual(
				    inputs.input_ids   ,		[25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]  )












import numpy as np







def   __lowerCamelCase   (				__a  :np.array       )						->							np.array:






		"""simple docstring"""

		return 1 / (1 + np.exp(-vector       ))







def   __lowerCamelCase   (				__a  :np.array       )						->							np.array:






		"""simple docstring"""

		return vector * sigmoid(1.702 * vector       )


if __name__ == "__main__":
					import doctest

					doctest.testmod()



