
"""simple docstring"""






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase        =				get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
					from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase        =				2_5_6_0_4_7
UpperCAmelCase        =				2_5_6_1_4_5



@require_sentencepiece
@require_tokenizers
class 			lowercase					( lowercase__   ,unittest.TestCase		):
			lowercase        =     NllbTokenizer
			lowercase        =     NllbTokenizerFast
			lowercase        =     True
			lowercase        =     True
			lowercase        =     {}






			def 					UpperCAmelCase							(self  :	Any			)       ->	Optional[int]:





				"""simple docstring"""


				super().setUp()

				# We have a SentencePiece fixture for testing
				lowerCAmelCase												=  NllbTokenizer(SCREAMING_SNAKE_CASE_      ,keep_accents=SCREAMING_SNAKE_CASE_			)
				tokenizer.save_pretrained(self.tmpdirname			)






			def 					UpperCAmelCase							(self  :	List[Any]			)       ->	Tuple:





				"""simple docstring"""


				lowerCAmelCase												=  NllbTokenizer(SCREAMING_SNAKE_CASE_      ,keep_accents=SCREAMING_SNAKE_CASE_			)

				lowerCAmelCase												=  tokenizer.tokenize('''This is a test'''			)
				self.assertListEqual(SCREAMING_SNAKE_CASE_      ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']			)

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			)      ,[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]      ,)

				lowerCAmelCase												=  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''			)
				self.assertListEqual(
				    SCREAMING_SNAKE_CASE_      ,[
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''9''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''é''',
				        '''.''',
				    ]      ,)
				lowerCAmelCase												=  tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			)
				self.assertListEqual(
				    SCREAMING_SNAKE_CASE_      ,[
				        value + tokenizer.fairseq_offset
				        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
				    ]      ,)

				lowerCAmelCase												=  tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_			)
				self.assertListEqual(
				    SCREAMING_SNAKE_CASE_      ,[
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''<unk>''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''<unk>''',
				        '''.''',
				    ]      ,)






			def 					UpperCAmelCase							(self  :	Tuple			)       ->	Dict:





				"""simple docstring"""


				lowerCAmelCase												=  (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
						lowerCAmelCase												=  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_			)
						lowerCAmelCase												=  self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_			)

						lowerCAmelCase												=  tempfile.mkdtemp()

						lowerCAmelCase												=  tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_			)
						lowerCAmelCase												=  tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_			)

						# Checks it save with the same files + the tokenizer.json file for the fast one
						self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files			)			)
						lowerCAmelCase												=  tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f			)
						self.assertSequenceEqual(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)

						# Checks everything loads correctly in the same way
						lowerCAmelCase												=  tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_			)
						lowerCAmelCase												=  tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_			)

						# Check special tokens are set accordingly on Rust and Python
						for key in tokenizer_pp.special_tokens_map:
							self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)			)

						shutil.rmtree(SCREAMING_SNAKE_CASE_			)

						# Save tokenizer rust, legacy_format=True
						lowerCAmelCase												=  tempfile.mkdtemp()

						lowerCAmelCase												=  tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_      ,legacy_format=SCREAMING_SNAKE_CASE_			)
						lowerCAmelCase												=  tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_			)

						# Checks it save with the same files
						self.assertSequenceEqual(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)

						# Checks everything loads correctly in the same way
						lowerCAmelCase												=  tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_			)
						lowerCAmelCase												=  tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_			)

						# Check special tokens are set accordingly on Rust and Python
						for key in tokenizer_pp.special_tokens_map:
							self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)			)

						shutil.rmtree(SCREAMING_SNAKE_CASE_			)

						# Save tokenizer rust, legacy_format=False
						lowerCAmelCase												=  tempfile.mkdtemp()

						lowerCAmelCase												=  tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_      ,legacy_format=SCREAMING_SNAKE_CASE_			)
						lowerCAmelCase												=  tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_			)

						# Checks it saved the tokenizer.json file
						self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files			)			)

						# Checks everything loads correctly in the same way
						lowerCAmelCase												=  tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_			)
						lowerCAmelCase												=  tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_			)

						# Check special tokens are set accordingly on Rust and Python
						for key in tokenizer_pp.special_tokens_map:
							self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)			)

						shutil.rmtree(SCREAMING_SNAKE_CASE_			)






			@require_torch
			def 					UpperCAmelCase							(self  :	List[str]			)       ->	List[Any]:





				"""simple docstring"""


				if not self.test_seqaseq:
					return

				lowerCAmelCase												=  self.get_tokenizers()
				for tokenizer in tokenizers:
					with self.subTest(F"""{tokenizer.__class__.__name__}"""			):
						# Longer text that will definitely require truncation.
						lowerCAmelCase												=  [
						    ''' UN Chief Says There Is No Military Solution in Syria''',
						    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
						    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
						    ''' will only worsen the violence and misery for millions of people.''',
						]
						lowerCAmelCase												=  [
						    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
						    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
						    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
						    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
						]
						try:
							lowerCAmelCase												=  tokenizer.prepare_seqaseq_batch(
							    src_texts=SCREAMING_SNAKE_CASE_      ,tgt_texts=SCREAMING_SNAKE_CASE_      ,max_length=3      ,max_target_length=10      ,return_tensors='''pt'''      ,src_lang='''eng_Latn'''      ,tgt_lang='''ron_Latn'''      ,)
						except NotImplementedError:
							return
						self.assertEqual(batch.input_ids.shape[1]      ,3			)
						self.assertEqual(batch.labels.shape[1]      ,10			)
						# max_target_length will default to max_length if not specified
						lowerCAmelCase												=  tokenizer.prepare_seqaseq_batch(
						    SCREAMING_SNAKE_CASE_      ,tgt_texts=SCREAMING_SNAKE_CASE_      ,max_length=3      ,return_tensors='''pt'''			)
						self.assertEqual(batch.input_ids.shape[1]      ,3			)
						self.assertEqual(batch.labels.shape[1]      ,3			)

						lowerCAmelCase												=  tokenizer.prepare_seqaseq_batch(
						    src_texts=SCREAMING_SNAKE_CASE_      ,max_length=3      ,max_target_length=10      ,return_tensors='''pt'''			)
						self.assertEqual(batch_encoder_only.input_ids.shape[1]      ,3			)
						self.assertEqual(batch_encoder_only.attention_mask.shape[1]      ,3			)
						self.assertNotIn('''decoder_input_ids'''      ,SCREAMING_SNAKE_CASE_			)






			@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''			)
			def 					UpperCAmelCase							(self  :	List[Any]			)       ->	Union[str, Any]:





				"""simple docstring"""


				pass






			def 					UpperCAmelCase							(self  :	Optional[Any]			)       ->	List[str]:





				"""simple docstring"""


				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
						lowerCAmelCase												=  [AddedToken('''<special>'''      ,lstrip=SCREAMING_SNAKE_CASE_			)]

						lowerCAmelCase												=  self.rust_tokenizer_class.from_pretrained(
						    SCREAMING_SNAKE_CASE_      ,additional_special_tokens=SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_			)
						lowerCAmelCase												=  tokenizer_r.encode('''Hey this is a <special> token'''			)

						lowerCAmelCase												=  tokenizer_r.encode('''<special>'''      ,add_special_tokens=SCREAMING_SNAKE_CASE_			)[0]

						self.assertTrue(special_token_id in r_output			)

						if self.test_slow_tokenizer:
							lowerCAmelCase												=  self.rust_tokenizer_class.from_pretrained(
							    SCREAMING_SNAKE_CASE_      ,additional_special_tokens=SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_      ,)
							lowerCAmelCase												=  self.tokenizer_class.from_pretrained(
							    SCREAMING_SNAKE_CASE_      ,additional_special_tokens=SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_			)

							lowerCAmelCase												=  tokenizer_p.encode('''Hey this is a <special> token'''			)

							lowerCAmelCase												=  tokenizer_cr.encode('''Hey this is a <special> token'''			)

							self.assertEqual(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)
							self.assertEqual(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)
							self.assertTrue(special_token_id in p_output			)
							self.assertTrue(special_token_id in cr_output			)





@require_torch
@require_sentencepiece
@require_tokenizers
class 			lowercase					( unittest.TestCase		):
			lowercase        =     '''facebook/nllb-200-distilled-600M'''
			lowercase        =     [
			    ''' UN Chief Says There Is No Military Solution in Syria''',
			    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
			]
			lowercase        =     [
			    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
			    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
			    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
			    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
			]
			lowercase        =     [
			    256_047,
			    16_297,
			    134_408,
			    8_165,
			    248_066,
			    14_734,
			    950,
			    1_135,
			    105_721,
			    3_573,
			    83,
			    27_352,
			    108,
			    49_486,
			    2,
			]






			@classmethod
			def 					UpperCAmelCase							(cls  :	Optional[Any]			)       ->	str:





				"""simple docstring"""


				lowerCAmelCase												=  NllbTokenizer.from_pretrained(
				    cls.checkpoint_name      ,src_lang='''eng_Latn'''      ,tgt_lang='''ron_Latn'''			)
				lowerCAmelCase												=  1
				return cls






			def 					UpperCAmelCase							(self  :	Union[str, Any]			)       ->	Tuple:





				"""simple docstring"""


				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']      ,256_001			)
				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']      ,256_002			)
				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']      ,256_057			)






			def 					UpperCAmelCase							(self  :	List[Any]			)       ->	Optional[Any]:





				"""simple docstring"""


				lowerCAmelCase												=  self.tokenizer.batch_encode_plus(self.src_text			).input_ids[0]
				self.assertListEqual(self.expected_src_tokens      ,SCREAMING_SNAKE_CASE_			)






			def 					UpperCAmelCase							(self  :	List[str]			)       ->	Dict:





				"""simple docstring"""


				self.assertIn(SCREAMING_SNAKE_CASE_      ,self.tokenizer.all_special_ids			)
				# fmt: off
				lowerCAmelCase												=  [RO_CODE, 4_254, 98_068, 112_923, 39_072, 3_909, 713, 102_767, 26, 17_314, 35_642, 14_683, 33_118, 2_022, 66_987, 2, 256_047]
				# fmt: on

				lowerCAmelCase												=  self.tokenizer.decode(SCREAMING_SNAKE_CASE_      ,skip_special_tokens=SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  self.tokenizer.decode(generated_ids[1:]      ,skip_special_tokens=SCREAMING_SNAKE_CASE_			)
				self.assertEqual(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)
				self.assertNotIn(self.tokenizer.eos_token      ,SCREAMING_SNAKE_CASE_			)






			def 					UpperCAmelCase							(self  :	str			)       ->	Tuple:





				"""simple docstring"""


				lowerCAmelCase												=  ['''this is gunna be a long sentence ''' * 20]
				assert isinstance(src_text[0]      ,SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  10
				lowerCAmelCase												=  self.tokenizer(SCREAMING_SNAKE_CASE_      ,max_length=SCREAMING_SNAKE_CASE_      ,truncation=SCREAMING_SNAKE_CASE_			).input_ids[0]
				self.assertEqual(ids[-1]      ,2			)
				self.assertEqual(ids[0]      ,SCREAMING_SNAKE_CASE_			)
				self.assertEqual(len(SCREAMING_SNAKE_CASE_			)      ,SCREAMING_SNAKE_CASE_			)






			def 					UpperCAmelCase							(self  :	List[Any]			)       ->	str:





				"""simple docstring"""


				self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']			)      ,[256_203, 3]			)






			def 					UpperCAmelCase							(self  :	str			)       ->	str:





				"""simple docstring"""


				lowerCAmelCase												=  tempfile.mkdtemp()
				lowerCAmelCase												=  self.tokenizer.fairseq_tokens_to_ids
				self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  NllbTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
				self.assertDictEqual(new_tok.fairseq_tokens_to_ids      ,SCREAMING_SNAKE_CASE_			)






			@require_torch
			def 					UpperCAmelCase							(self  :	Tuple			)       ->	int:





				"""simple docstring"""


				lowerCAmelCase												=  self.tokenizer(
				    self.src_text      ,text_target=self.tgt_text      ,padding=SCREAMING_SNAKE_CASE_      ,truncation=SCREAMING_SNAKE_CASE_      ,max_length=len(self.expected_src_tokens			)      ,return_tensors='''pt'''      ,)
				lowerCAmelCase												=  shift_tokens_right(
				    batch['''labels''']      ,self.tokenizer.pad_token_id      ,self.tokenizer.lang_code_to_id['''ron_Latn''']			)

				self.assertIsInstance(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)

				self.assertEqual((2, 15)      ,batch.input_ids.shape			)
				self.assertEqual((2, 15)      ,batch.attention_mask.shape			)
				lowerCAmelCase												=  batch.input_ids.tolist()[0]
				self.assertListEqual(self.expected_src_tokens      ,SCREAMING_SNAKE_CASE_			)
				self.assertEqual(SCREAMING_SNAKE_CASE_      ,batch.decoder_input_ids[0, 0]			)  # EOS
				# Test that special tokens are reset
				self.assertEqual(self.tokenizer.prefix_tokens      ,[EN_CODE]			)
				self.assertEqual(self.tokenizer.suffix_tokens      ,[self.tokenizer.eos_token_id]			)






			def 					UpperCAmelCase							(self  :	Any			)       ->	Union[str, Any]:





				"""simple docstring"""


				lowerCAmelCase												=  self.tokenizer(self.src_text      ,padding=SCREAMING_SNAKE_CASE_      ,truncation=SCREAMING_SNAKE_CASE_      ,max_length=3      ,return_tensors='''pt'''			)
				lowerCAmelCase												=  self.tokenizer(
				    text_target=self.tgt_text      ,padding=SCREAMING_SNAKE_CASE_      ,truncation=SCREAMING_SNAKE_CASE_      ,max_length=10      ,return_tensors='''pt'''			)
				lowerCAmelCase												=  targets['''input_ids''']
				lowerCAmelCase												=  shift_tokens_right(
				    SCREAMING_SNAKE_CASE_      ,self.tokenizer.pad_token_id      ,decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]      ,)

				self.assertEqual(batch.input_ids.shape[1]      ,3			)
				self.assertEqual(batch.decoder_input_ids.shape[1]      ,10			)






			@require_torch
			def 					UpperCAmelCase							(self  :	Tuple			)       ->	Tuple:





				"""simple docstring"""


				lowerCAmelCase												=  self.tokenizer._build_translation_inputs(
				    '''A test'''      ,return_tensors='''pt'''      ,src_lang='''eng_Latn'''      ,tgt_lang='''fra_Latn'''			)

				self.assertEqual(
				    nested_simplify(SCREAMING_SNAKE_CASE_			)      ,{
				        # A, test, EOS, en_XX
				        '''input_ids''': [[256_047, 70, 7_356, 2]],
				        '''attention_mask''': [[1, 1, 1, 1]],
				        # ar_AR
				        '''forced_bos_token_id''': 256_057,
				    }      ,)






			@require_torch
			def 					UpperCAmelCase							(self  :	Tuple			)       ->	List[str]:





				"""simple docstring"""


				lowerCAmelCase												=  True
				lowerCAmelCase												=  self.tokenizer(
				    '''UN Chief says there is no military solution in Syria'''      ,src_lang='''eng_Latn'''      ,tgt_lang='''fra_Latn'''			)
				self.assertEqual(
				    inputs.input_ids      ,[16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2, 256_047]			)

				lowerCAmelCase												=  False
				lowerCAmelCase												=  self.tokenizer(
				    '''UN Chief says there is no military solution in Syria'''      ,src_lang='''eng_Latn'''      ,tgt_lang='''fra_Latn'''			)
				self.assertEqual(
				    inputs.input_ids      ,[256_047, 16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2]			)




"""simple docstring"""






def __magic_name__	(				_lowerCamelCase:		Tuple,					_lowerCamelCase:		Any			)					->						Optional[Any]:

	'''simple docstring'''




	lowerCAmelCase												=  [1]
	for i in range(2,					_lowerCamelCase			):
		factorials.append(factorials[-1] * i			)
	assert 0 <= k < factorials[-1] * n, "k out of bounds"

	lowerCAmelCase												=  []
	lowerCAmelCase												=  list(range(_lowerCamelCase			)			)

	# Find permutation
	while factorials:
		lowerCAmelCase												=  factorials.pop()
		lowerCAmelCase							,       lowerCAmelCase												=  divmod(_lowerCamelCase,					_lowerCamelCase			)
		permutation.append(elements[number]			)
		elements.remove(elements[number]			)
	permutation.append(elements[0]			)

	return permutation


if __name__ == "__main__":
					import doctest

					doctest.testmod()


