


'''simple docstring'''



import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase     :						Tuple	=     get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
	from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__lowercase     :						Optional[int]	=     25_60_47
__lowercase     :						Tuple	=     25_61_45





@require_sentencepiece
@require_tokenizers
class 							__UpperCamelCase    (		lowerCAmelCase_		,				unittest.TestCase ):
	A_           =   NllbTokenizer
	A_           =   NllbTokenizerFast
	A_           =   True
	A_           =   True
	A_           =   {}




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						super().setUp()

						# We have a SentencePiece fixture for testing
						__a      :	str    =  NllbTokenizer(__a						,    keep_accents=__a							)
						tokenizer.save_pretrained(self.tmpdirname							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	int    =  NllbTokenizer(__a						,    keep_accents=__a							)

						__a      :	List[Any]    =  tokenizer.tokenize('This is a test'							)
						self.assertListEqual(__a						,    ['▁This', '▁is', '▁a', '▁t', 'est']							)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__a							)						,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,    )

						__a      :	Union[str, Any]    =  tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
						self.assertListEqual(
						    __a						,    [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]						,    )
						__a      :	List[Any]    =  tokenizer.convert_tokens_to_ids(__a							)
						self.assertListEqual(
						    __a						,    [
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
						    ]						,    )

						__a      :	Tuple    =  tokenizer.convert_ids_to_tokens(__a							)
						self.assertListEqual(
						    __a						,    [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '<unk>',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '<unk>',
						        '.',
						    ]						,    )




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	int    =  (self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																__a      :	str    =  self.rust_tokenizer_class.from_pretrained(__a						,    **__a							)
																__a      :	str    =  self.tokenizer_class.from_pretrained(__a						,    **__a							)

																__a      :	List[str]    =  tempfile.mkdtemp()

																__a      :	Dict    =  tokenizer_r.save_pretrained(__a							)
																__a      :	List[Any]    =  tokenizer_p.save_pretrained(__a							)

																# Checks it save with the same files + the tokenizer.json file for the fast one
																self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files							)							)
																__a      :	List[str]    =  tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f							)
																self.assertSequenceEqual(__a						,    __a							)

																# Checks everything loads correctly in the same way
																__a      :	str    =  tokenizer_r.from_pretrained(__a							)
																__a      :	Dict    =  tokenizer_p.from_pretrained(__a							)

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																					self.assertTrue(hasattr(__a						,    __a							)							)

																shutil.rmtree(__a							)

																# Save tokenizer rust, legacy_format=True
																__a      :	Optional[int]    =  tempfile.mkdtemp()

																__a      :	Union[str, Any]    =  tokenizer_r.save_pretrained(__a						,    legacy_format=__a							)
																__a      :	List[str]    =  tokenizer_p.save_pretrained(__a							)

																# Checks it save with the same files
																self.assertSequenceEqual(__a						,    __a							)

																# Checks everything loads correctly in the same way
																__a      :	Tuple    =  tokenizer_r.from_pretrained(__a							)
																__a      :	str    =  tokenizer_p.from_pretrained(__a							)

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																					self.assertTrue(hasattr(__a						,    __a							)							)

																shutil.rmtree(__a							)

																# Save tokenizer rust, legacy_format=False
																__a      :	Tuple    =  tempfile.mkdtemp()

																__a      :	Union[str, Any]    =  tokenizer_r.save_pretrained(__a						,    legacy_format=__a							)
																__a      :	Union[str, Any]    =  tokenizer_p.save_pretrained(__a							)

																# Checks it saved the tokenizer.json file
																self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files							)							)

																# Checks everything loads correctly in the same way
																__a      :	Tuple    =  tokenizer_r.from_pretrained(__a							)
																__a      :	Dict    =  tokenizer_p.from_pretrained(__a							)

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																					self.assertTrue(hasattr(__a						,    __a							)							)

																shutil.rmtree(__a							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						if not self.test_seqaseq:
											return

						__a      :	str    =  self.get_tokenizers()
						for tokenizer in tokenizers:
											with self.subTest(f"""{tokenizer.__class__.__name__}"""							):
																# Longer text that will definitely require truncation.
																__a      :	Dict    =  [
																    ' UN Chief Says There Is No Military Solution in Syria',
																    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
																    ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
																    ' will only worsen the violence and misery for millions of people.',
																]
																__a      :	Dict    =  [
																    'Şeful ONU declară că nu există o soluţie militară în Siria',
																    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
																    ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
																    ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
																]
																try:
																					__a      :	Optional[Any]    =  tokenizer.prepare_seqaseq_batch(
																					    src_texts=__a						,    tgt_texts=__a						,    max_length=3						,    max_target_length=10						,    return_tensors='pt'						,    src_lang='eng_Latn'						,    tgt_lang='ron_Latn'						,    )
																except NotImplementedError:
																					return
																self.assertEqual(batch.input_ids.shape[1]						,    3							)
																self.assertEqual(batch.labels.shape[1]						,    10							)
																# max_target_length will default to max_length if not specified
																__a      :	Tuple    =  tokenizer.prepare_seqaseq_batch(
																    __a						,    tgt_texts=__a						,    max_length=3						,    return_tensors='pt'							)
																self.assertEqual(batch.input_ids.shape[1]						,    3							)
																self.assertEqual(batch.labels.shape[1]						,    3							)

																__a      :	Any    =  tokenizer.prepare_seqaseq_batch(
																    src_texts=__a						,    max_length=3						,    max_target_length=10						,    return_tensors='pt'							)
																self.assertEqual(batch_encoder_only.input_ids.shape[1]						,    3							)
																self.assertEqual(batch_encoder_only.attention_mask.shape[1]						,    3							)
																self.assertNotIn('decoder_input_ids'						,    __a							)




	@unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'							)
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						pass





	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																__a      :	Union[str, Any]    =  [AddedToken('<special>'						,    lstrip=__a							)]

																__a      :	Dict    =  self.rust_tokenizer_class.from_pretrained(
																    __a						,    additional_special_tokens=__a						,    **__a							)
																__a      :	Dict    =  tokenizer_r.encode('Hey this is a <special> token'							)

																__a      :	Optional[Any]    =  tokenizer_r.encode('<special>'						,    add_special_tokens=__a							)[0]

																self.assertTrue(special_token_id in r_output							)

																if self.test_slow_tokenizer:
																					__a      :	int    =  self.rust_tokenizer_class.from_pretrained(
																					    __a						,    additional_special_tokens=__a						,    **__a						,    )
																					__a      :	List[str]    =  self.tokenizer_class.from_pretrained(
																					    __a						,    additional_special_tokens=__a						,    **__a							)

																					__a      :	List[Any]    =  tokenizer_p.encode('Hey this is a <special> token'							)

																					__a      :	Tuple    =  tokenizer_cr.encode('Hey this is a <special> token'							)

																					self.assertEqual(__a						,    __a							)
																					self.assertEqual(__a						,    __a							)
																					self.assertTrue(special_token_id in p_output							)
																					self.assertTrue(special_token_id in cr_output							)





@require_torch
@require_sentencepiece
@require_tokenizers
class 							__UpperCamelCase    (		unittest.TestCase ):
	A_           =   "facebook/nllb-200-distilled-600M"
	A_           =   [
	    " UN Chief Says There Is No Military Solution in Syria",
	    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
	]
	A_           =   [
	    "Şeful ONU declară că nu există o soluţie militară în Siria",
	    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
	    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
	    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
	]
	A_           =   [
	    256047,
	    16297,
	    134408,
	    8165,
	    248066,
	    14734,
	    950,
	    1135,
	    105721,
	    3573,
	    83,
	    27352,
	    108,
	    49486,
	    2,
	]




	@classmethod
	def 				__UpperCAmelCase							(       cls							):

						'''simple docstring'''





						__a      :	NllbTokenizer    =  NllbTokenizer.from_pretrained(
						    cls.checkpoint_name						,    src_lang='eng_Latn'						,    tgt_lang='ron_Latn'							)
						__a      :	List[str]    =  1
						return cls




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']						,    25_6001							)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']						,    25_6002							)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']						,    25_6057							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	int    =  self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
						self.assertListEqual(self.expected_src_tokens						,    __a							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						self.assertIn(__a						,    self.tokenizer.all_special_ids							)
						# fmt: off
						__a      :	List[Any]    =  [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
						# fmt: on

						__a      :	Dict    =  self.tokenizer.decode(__a						,    skip_special_tokens=__a							)
						__a      :	Dict    =  self.tokenizer.decode(generated_ids[1:]						,    skip_special_tokens=__a							)
						self.assertEqual(__a						,    __a							)
						self.assertNotIn(self.tokenizer.eos_token						,    __a							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Any    =  ['this is gunna be a long sentence ' * 20]
						assert isinstance(src_text[0]						,    __a							)
						__a      :	int    =  10
						__a      :	List[Any]    =  self.tokenizer(__a						,    max_length=__a						,    truncation=__a							).input_ids[0]
						self.assertEqual(ids[-1]						,    2							)
						self.assertEqual(ids[0]						,    __a							)
						self.assertEqual(len(__a							)						,    __a							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']							)						,    [25_6203, 3]							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Union[str, Any]    =  tempfile.mkdtemp()
						__a      :	str    =  self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(__a							)
						__a      :	Dict    =  NllbTokenizer.from_pretrained(__a							)
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,    __a							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Tuple    =  self.tokenizer(
						    self.src_text						,    text_target=self.tgt_text						,    padding=__a						,    truncation=__a						,    max_length=len(self.expected_src_tokens							)						,    return_tensors='pt'						,    )
						__a      :	str    =  shift_tokens_right(
						    batch['labels']						,    self.tokenizer.pad_token_id						,    self.tokenizer.lang_code_to_id['ron_Latn']							)

						self.assertIsInstance(__a						,    __a							)

						self.assertEqual((2, 15)						,    batch.input_ids.shape							)
						self.assertEqual((2, 15)						,    batch.attention_mask.shape							)
						__a      :	int    =  batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens						,    __a							)
						self.assertEqual(__a						,    batch.decoder_input_ids[0, 0]							)  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens						,    [EN_CODE]							)
						self.assertEqual(self.tokenizer.suffix_tokens						,    [self.tokenizer.eos_token_id]							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Tuple    =  self.tokenizer(self.src_text						,    padding=__a						,    truncation=__a						,    max_length=3						,    return_tensors='pt'							)
						__a      :	List[Any]    =  self.tokenizer(
						    text_target=self.tgt_text						,    padding=__a						,    truncation=__a						,    max_length=10						,    return_tensors='pt'							)
						__a      :	List[Any]    =  targets['input_ids']
						__a      :	Dict    =  shift_tokens_right(
						    __a						,    self.tokenizer.pad_token_id						,    decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]						,    )

						self.assertEqual(batch.input_ids.shape[1]						,    3							)
						self.assertEqual(batch.decoder_input_ids.shape[1]						,    10							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  self.tokenizer._build_translation_inputs(
						    'A test'						,    return_tensors='pt'						,    src_lang='eng_Latn'						,    tgt_lang='fra_Latn'							)

						self.assertEqual(
						    nested_simplify(__a							)						,    {
						        # A, test, EOS, en_XX
						        'input_ids': [[25_6047, 70, 7356, 2]],
						        'attention_mask': [[1, 1, 1, 1]],
						        # ar_AR
						        'forced_bos_token_id': 25_6057,
						    }						,    )





	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  True
						__a      :	Any    =  self.tokenizer(
						    'UN Chief says there is no military solution in Syria'						,    src_lang='eng_Latn'						,    tgt_lang='fra_Latn'							)
						self.assertEqual(
						    inputs.input_ids						,    [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]							)

						__a      :	List[str]    =  False
						__a      :	Any    =  self.tokenizer(
						    'UN Chief says there is no military solution in Syria'						,    src_lang='eng_Latn'						,    tgt_lang='fra_Latn'							)
						self.assertEqual(
						    inputs.input_ids						,    [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]							)







'''simple docstring'''



def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       int       ):
					return number & 1 == 0


if __name__ == "__main__":
	import doctest

	doctest.testmod()



