





import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


A       :   Optional[Any]					= get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
					from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

A       :   Any					= 2_5_6_0_4_7
A       :   List[str]					= 2_5_6_1_4_5


@require_sentencepiece
@require_tokenizers
class     A     (SCREAMING_SNAKE_CASE  ,	unittest.TestCase       ):
			'''simple docstring'''




			__lowerCamelCase      :			List[str]							     =       NllbTokenizer
			__lowerCamelCase      :			int							     =       NllbTokenizerFast
			__lowerCamelCase      :			List[Any]							     =       True
			__lowerCamelCase      :			List[Any]							     =       True
			__lowerCamelCase      :			int							     =       {}






			def   a_			(     self  :       Optional[int]  )      ->      int:







					"""simple docstring"""






					super().setUp()

					# We have a SentencePiece fixture for testing
					A__     =  NllbTokenizer(__lowerCAmelCase				,      keep_accents=__lowerCAmelCase  )
					tokenizer.save_pretrained(self.tmpdirname  )






			def   a_			(     self  :       Optional[int]  )      ->      Any:







					"""simple docstring"""






					A__     =  NllbTokenizer(__lowerCAmelCase				,      keep_accents=__lowerCAmelCase  )

					A__     =  tokenizer.tokenize("""This is a test"""  )
					self.assertListEqual(__lowerCAmelCase				,      ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(__lowerCAmelCase  )				,      [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]				,      )

					A__     =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
					self.assertListEqual(
					    __lowerCAmelCase				,      [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """9""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """é""",
					        """.""",
					    ]				,      )
					A__     =  tokenizer.convert_tokens_to_ids(__lowerCAmelCase  )
					self.assertListEqual(
					    __lowerCAmelCase				,      [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					    ]				,      )

					A__     =  tokenizer.convert_ids_to_tokens(__lowerCAmelCase  )
					self.assertListEqual(
					    __lowerCAmelCase				,      [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """<unk>""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """<unk>""",
					        """.""",
					    ]				,      )






			def   a_			(     self  :       List[str]  )      ->      str:







					"""simple docstring"""






					A__     =  (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
									A__     =  self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase				,      **__lowerCAmelCase  )
									A__     =  self.tokenizer_class.from_pretrained(__lowerCAmelCase				,      **__lowerCAmelCase  )

									A__     =  tempfile.mkdtemp()

									A__     =  tokenizer_r.save_pretrained(__lowerCAmelCase  )
									A__     =  tokenizer_p.save_pretrained(__lowerCAmelCase  )

									# Checks it save with the same files + the tokenizer.json file for the fast one
									self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files  )  )
									A__     =  tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f  )
									self.assertSequenceEqual(__lowerCAmelCase				,      __lowerCAmelCase  )

									# Checks everything loads correctly in the same way
									A__     =  tokenizer_r.from_pretrained(__lowerCAmelCase  )
									A__     =  tokenizer_p.from_pretrained(__lowerCAmelCase  )

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase				,      __lowerCAmelCase  )  )

									shutil.rmtree(__lowerCAmelCase  )

									# Save tokenizer rust, legacy_format=True
									A__     =  tempfile.mkdtemp()

									A__     =  tokenizer_r.save_pretrained(__lowerCAmelCase				,      legacy_format=__lowerCAmelCase  )
									A__     =  tokenizer_p.save_pretrained(__lowerCAmelCase  )

									# Checks it save with the same files
									self.assertSequenceEqual(__lowerCAmelCase				,      __lowerCAmelCase  )

									# Checks everything loads correctly in the same way
									A__     =  tokenizer_r.from_pretrained(__lowerCAmelCase  )
									A__     =  tokenizer_p.from_pretrained(__lowerCAmelCase  )

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase				,      __lowerCAmelCase  )  )

									shutil.rmtree(__lowerCAmelCase  )

									# Save tokenizer rust, legacy_format=False
									A__     =  tempfile.mkdtemp()

									A__     =  tokenizer_r.save_pretrained(__lowerCAmelCase				,      legacy_format=__lowerCAmelCase  )
									A__     =  tokenizer_p.save_pretrained(__lowerCAmelCase  )

									# Checks it saved the tokenizer.json file
									self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files  )  )

									# Checks everything loads correctly in the same way
									A__     =  tokenizer_r.from_pretrained(__lowerCAmelCase  )
									A__     =  tokenizer_p.from_pretrained(__lowerCAmelCase  )

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase				,      __lowerCAmelCase  )  )

									shutil.rmtree(__lowerCAmelCase  )






			@require_torch
			def   a_			(     self  :       Tuple  )      ->      Tuple:







					"""simple docstring"""






					if not self.test_seqaseq:
							return

					A__     =  self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(f'{tokenizer.__class__.__name__}'  ):
									# Longer text that will definitely require truncation.
									A__     =  [
									    """ UN Chief Says There Is No Military Solution in Syria""",
									    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
									    """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
									    """ will only worsen the violence and misery for millions of people.""",
									]
									A__     =  [
									    """Şeful ONU declară că nu există o soluţie militară în Siria""",
									    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
									    """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
									    """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
									]
									try:
											A__     =  tokenizer.prepare_seqaseq_batch(
											    src_texts=__lowerCAmelCase				,      tgt_texts=__lowerCAmelCase				,      max_length=3				,      max_target_length=10				,      return_tensors="""pt"""				,      src_lang="""eng_Latn"""				,      tgt_lang="""ron_Latn"""				,      )
									except NotImplementedError:
											return
									self.assertEqual(batch.input_ids.shape[1]				,      3  )
									self.assertEqual(batch.labels.shape[1]				,      10  )
									# max_target_length will default to max_length if not specified
									A__     =  tokenizer.prepare_seqaseq_batch(
									    __lowerCAmelCase				,      tgt_texts=__lowerCAmelCase				,      max_length=3				,      return_tensors="""pt"""  )
									self.assertEqual(batch.input_ids.shape[1]				,      3  )
									self.assertEqual(batch.labels.shape[1]				,      3  )

									A__     =  tokenizer.prepare_seqaseq_batch(
									    src_texts=__lowerCAmelCase				,      max_length=3				,      max_target_length=10				,      return_tensors="""pt"""  )
									self.assertEqual(batch_encoder_only.input_ids.shape[1]				,      3  )
									self.assertEqual(batch_encoder_only.attention_mask.shape[1]				,      3  )
									self.assertNotIn("""decoder_input_ids"""				,      __lowerCAmelCase  )






			@unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""  )
			def   a_			(     self  :       Union[str, Any]  )      ->      List[Any]:







					"""simple docstring"""






					pass







			def   a_			(     self  :       str  )      ->      int:







					"""simple docstring"""






					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
									A__     =  [AddedToken("""<special>"""				,      lstrip=__lowerCAmelCase  )]

									A__     =  self.rust_tokenizer_class.from_pretrained(
									    __lowerCAmelCase				,      additional_special_tokens=__lowerCAmelCase				,      **__lowerCAmelCase  )
									A__     =  tokenizer_r.encode("""Hey this is a <special> token"""  )

									A__     =  tokenizer_r.encode("""<special>"""				,      add_special_tokens=__lowerCAmelCase  )[0]

									self.assertTrue(special_token_id in r_output  )

									if self.test_slow_tokenizer:
											A__     =  self.rust_tokenizer_class.from_pretrained(
											    __lowerCAmelCase				,      additional_special_tokens=__lowerCAmelCase				,      **__lowerCAmelCase				,      )
											A__     =  self.tokenizer_class.from_pretrained(
											    __lowerCAmelCase				,      additional_special_tokens=__lowerCAmelCase				,      **__lowerCAmelCase  )

											A__     =  tokenizer_p.encode("""Hey this is a <special> token"""  )

											A__     =  tokenizer_cr.encode("""Hey this is a <special> token"""  )

											self.assertEqual(__lowerCAmelCase				,      __lowerCAmelCase  )
											self.assertEqual(__lowerCAmelCase				,      __lowerCAmelCase  )
											self.assertTrue(special_token_id in p_output  )
											self.assertTrue(special_token_id in cr_output  )


@require_torch
@require_sentencepiece
@require_tokenizers
class     A     (unittest.TestCase       ):
			'''simple docstring'''




			__lowerCamelCase      :			Union[str, Any]							     =       '''facebook/nllb-200-distilled-600M'''
			__lowerCamelCase      :			Optional[Any]							     =       [
			    ''' UN Chief Says There Is No Military Solution in Syria''',
			    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
			]
			__lowerCamelCase      :			List[str]							     =       [
			    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
			    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
			    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
			    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
			]
			__lowerCamelCase      :			List[Any]							     =       [
			    256047,
			    16297,
			    134408,
			    8165,
			    248066,
			    14734,
			    950,
			    1135,
			    105721,
			    3573,
			    83,
			    27352,
			    108,
			    49486,
			    2,
			]






			@classmethod
			def   a_			(     cls  :       Dict  )      ->      Union[str, Any]:







					"""simple docstring"""






					A__     =  NllbTokenizer.from_pretrained(
					    cls.checkpoint_name				,      src_lang="""eng_Latn"""				,      tgt_lang="""ron_Latn"""  )
					A__     =  1
					return cls






			def   a_			(     self  :       Tuple  )      ->      Optional[Any]:







					"""simple docstring"""






					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]				,      25_60_01  )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]				,      25_60_02  )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]				,      25_60_57  )






			def   a_			(     self  :       Dict  )      ->      Tuple:







					"""simple docstring"""






					A__     =  self.tokenizer.batch_encode_plus(self.src_text  ).input_ids[0]
					self.assertListEqual(self.expected_src_tokens				,      __lowerCAmelCase  )






			def   a_			(     self  :       int  )      ->      Optional[Any]:







					"""simple docstring"""






					self.assertIn(__lowerCAmelCase				,      self.tokenizer.all_special_ids  )
					# fmt: off
					A__     =  [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
					# fmt: on

					A__     =  self.tokenizer.decode(__lowerCAmelCase				,      skip_special_tokens=__lowerCAmelCase  )
					A__     =  self.tokenizer.decode(generated_ids[1:]				,      skip_special_tokens=__lowerCAmelCase  )
					self.assertEqual(__lowerCAmelCase				,      __lowerCAmelCase  )
					self.assertNotIn(self.tokenizer.eos_token				,      __lowerCAmelCase  )






			def   a_			(     self  :       Any  )      ->      Tuple:







					"""simple docstring"""






					A__     =  ["""this is gunna be a long sentence """ * 20]
					assert isinstance(src_text[0]				,      __lowerCAmelCase  )
					A__     =  10
					A__     =  self.tokenizer(__lowerCAmelCase				,      max_length=__lowerCAmelCase				,      truncation=__lowerCAmelCase  ).input_ids[0]
					self.assertEqual(ids[-1]				,      2  )
					self.assertEqual(ids[0]				,      __lowerCAmelCase  )
					self.assertEqual(len(__lowerCAmelCase  )				,      __lowerCAmelCase  )






			def   a_			(     self  :       str  )      ->      int:







					"""simple docstring"""






					self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]  )				,      [25_62_03, 3]  )






			def   a_			(     self  :       Union[str, Any]  )      ->      List[str]:







					"""simple docstring"""






					A__     =  tempfile.mkdtemp()
					A__     =  self.tokenizer.fairseq_tokens_to_ids
					self.tokenizer.save_pretrained(__lowerCAmelCase  )
					A__     =  NllbTokenizer.from_pretrained(__lowerCAmelCase  )
					self.assertDictEqual(new_tok.fairseq_tokens_to_ids				,      __lowerCAmelCase  )






			@require_torch
			def   a_			(     self  :       Tuple  )      ->      List[str]:







					"""simple docstring"""






					A__     =  self.tokenizer(
					    self.src_text				,      text_target=self.tgt_text				,      padding=__lowerCAmelCase				,      truncation=__lowerCAmelCase				,      max_length=len(self.expected_src_tokens  )				,      return_tensors="""pt"""				,      )
					A__     =  shift_tokens_right(
					    batch["""labels"""]				,      self.tokenizer.pad_token_id				,      self.tokenizer.lang_code_to_id["""ron_Latn"""]  )

					self.assertIsInstance(__lowerCAmelCase				,      __lowerCAmelCase  )

					self.assertEqual((2, 15)				,      batch.input_ids.shape  )
					self.assertEqual((2, 15)				,      batch.attention_mask.shape  )
					A__     =  batch.input_ids.tolist()[0]
					self.assertListEqual(self.expected_src_tokens				,      __lowerCAmelCase  )
					self.assertEqual(__lowerCAmelCase				,      batch.decoder_input_ids[0, 0]  )  # EOS
					# Test that special tokens are reset
					self.assertEqual(self.tokenizer.prefix_tokens				,      [EN_CODE]  )
					self.assertEqual(self.tokenizer.suffix_tokens				,      [self.tokenizer.eos_token_id]  )






			def   a_			(     self  :       Tuple  )      ->      Tuple:







					"""simple docstring"""






					A__     =  self.tokenizer(self.src_text				,      padding=__lowerCAmelCase				,      truncation=__lowerCAmelCase				,      max_length=3				,      return_tensors="""pt"""  )
					A__     =  self.tokenizer(
					    text_target=self.tgt_text				,      padding=__lowerCAmelCase				,      truncation=__lowerCAmelCase				,      max_length=10				,      return_tensors="""pt"""  )
					A__     =  targets["""input_ids"""]
					A__     =  shift_tokens_right(
					    __lowerCAmelCase				,      self.tokenizer.pad_token_id				,      decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]				,      )

					self.assertEqual(batch.input_ids.shape[1]				,      3  )
					self.assertEqual(batch.decoder_input_ids.shape[1]				,      10  )






			@require_torch
			def   a_			(     self  :       Optional[Any]  )      ->      Any:







					"""simple docstring"""






					A__     =  self.tokenizer._build_translation_inputs(
					    """A test"""				,      return_tensors="""pt"""				,      src_lang="""eng_Latn"""				,      tgt_lang="""fra_Latn"""  )

					self.assertEqual(
					    nested_simplify(__lowerCAmelCase  )				,      {
					        # A, test, EOS, en_XX
					        """input_ids""": [[25_60_47, 70, 73_56, 2]],
					        """attention_mask""": [[1, 1, 1, 1]],
					        # ar_AR
					        """forced_bos_token_id""": 25_60_57,
					    }				,      )







			@require_torch
			def   a_			(     self  :       Union[str, Any]  )      ->      Optional[Any]:







					"""simple docstring"""






					A__     =  True
					A__     =  self.tokenizer(
					    """UN Chief says there is no military solution in Syria"""				,      src_lang="""eng_Latn"""				,      tgt_lang="""fra_Latn"""  )
					self.assertEqual(
					    inputs.input_ids				,      [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]  )

					A__     =  False
					A__     =  self.tokenizer(
					    """UN Chief says there is no military solution in Syria"""				,      src_lang="""eng_Latn"""				,      tgt_lang="""fra_Latn"""  )
					self.assertEqual(
					    inputs.input_ids				,      [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]  )










def   __lowerCamelCase   (				__a  :str       )						->							bool:






		"""simple docstring"""

		A__     =  0
		for ch in input_str:
				A__     =  ord(__a       )
				A__     =  pow(2       ,	__a       )

				# If we already turned on bit for current character's unicode
				if bitmap >> ch_unicode & 1 == 1:
						return False
				bitmap |= ch_bit_index_on
		return True


if __name__ == "__main__":
					import doctest

					doctest.testmod()



