
"""simple docstring"""




import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


A__   :					Optional[int]							=   get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

A__   :					List[Any]							=   2_5_6_0_4_7
A__   :					Optional[Any]							=   2_5_6_1_4_5


@require_sentencepiece
@require_tokenizers
class 			__magic_name__       (       __A   ,					unittest.TestCase ):
       UpperCamelCase_   =		NllbTokenizer
       UpperCamelCase_   =		NllbTokenizerFast
       UpperCamelCase_   =		True
       UpperCamelCase_   =		True
       UpperCamelCase_   =		{}
       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         super().setUp()

         # We have a SentencePiece fixture for testing
         _lowercase:			Union[str, Any]       =     NllbTokenizer(UpperCamelCase__ ,				keep_accents=UpperCamelCase__       )
         tokenizer.save_pretrained(self.tmpdirname       )
       def 							lowercase_							(     self       )     ->	str:





         """simple docstring"""




         _lowercase:			Union[str, Any]       =     NllbTokenizer(UpperCamelCase__ ,				keep_accents=UpperCamelCase__       )

         _lowercase:			str       =     tokenizer.tokenize('''This is a test'''       )
         self.assertListEqual(UpperCamelCase__ ,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']       )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(UpperCamelCase__       ) ,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]] ,				)

         _lowercase:			List[Any]       =     tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
         self.assertListEqual(
             UpperCamelCase__ ,				[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''9''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''é''',
                 '''.''',
             ] ,				)
         _lowercase:			Union[str, Any]       =     tokenizer.convert_tokens_to_ids(UpperCamelCase__       )
         self.assertListEqual(
             UpperCamelCase__ ,				[
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
             ] ,				)

         _lowercase:			Optional[Any]       =     tokenizer.convert_ids_to_tokens(UpperCamelCase__       )
         self.assertListEqual(
             UpperCamelCase__ ,				[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''<unk>''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''<unk>''',
                 '''.''',
             ] ,				)
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         _lowercase:			List[str]       =     (self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
             _lowercase:			str       =     self.rust_tokenizer_class.from_pretrained(UpperCamelCase__ ,				**UpperCamelCase__       )
             _lowercase:			List[Any]       =     self.tokenizer_class.from_pretrained(UpperCamelCase__ ,				**UpperCamelCase__       )

             _lowercase:			Optional[int]       =     tempfile.mkdtemp()

             _lowercase:			Optional[Any]       =     tokenizer_r.save_pretrained(UpperCamelCase__       )
             _lowercase:			Optional[Any]       =     tokenizer_p.save_pretrained(UpperCamelCase__       )

             # Checks it save with the same files + the tokenizer.json file for the fast one
             self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files       )       )
             _lowercase:			Tuple       =     tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f       )
             self.assertSequenceEqual(UpperCamelCase__ ,				UpperCamelCase__       )

             # Checks everything loads correctly in the same way
             _lowercase:			List[Any]       =     tokenizer_r.from_pretrained(UpperCamelCase__       )
             _lowercase:			Tuple       =     tokenizer_p.from_pretrained(UpperCamelCase__       )

             # Check special tokens are set accordingly on Rust and Python
             for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(UpperCamelCase__ ,				UpperCamelCase__       )       )

             shutil.rmtree(UpperCamelCase__       )

             # Save tokenizer rust, legacy_format=True
             _lowercase:			int       =     tempfile.mkdtemp()

             _lowercase:			Optional[int]       =     tokenizer_r.save_pretrained(UpperCamelCase__ ,				legacy_format=UpperCamelCase__       )
             _lowercase:			List[str]       =     tokenizer_p.save_pretrained(UpperCamelCase__       )

             # Checks it save with the same files
             self.assertSequenceEqual(UpperCamelCase__ ,				UpperCamelCase__       )

             # Checks everything loads correctly in the same way
             _lowercase:			List[Any]       =     tokenizer_r.from_pretrained(UpperCamelCase__       )
             _lowercase:			Tuple       =     tokenizer_p.from_pretrained(UpperCamelCase__       )

             # Check special tokens are set accordingly on Rust and Python
             for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(UpperCamelCase__ ,				UpperCamelCase__       )       )

             shutil.rmtree(UpperCamelCase__       )

             # Save tokenizer rust, legacy_format=False
             _lowercase:			Dict       =     tempfile.mkdtemp()

             _lowercase:			Dict       =     tokenizer_r.save_pretrained(UpperCamelCase__ ,				legacy_format=UpperCamelCase__       )
             _lowercase:			str       =     tokenizer_p.save_pretrained(UpperCamelCase__       )

             # Checks it saved the tokenizer.json file
             self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files       )       )

             # Checks everything loads correctly in the same way
             _lowercase:			str       =     tokenizer_r.from_pretrained(UpperCamelCase__       )
             _lowercase:			Tuple       =     tokenizer_p.from_pretrained(UpperCamelCase__       )

             # Check special tokens are set accordingly on Rust and Python
             for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(UpperCamelCase__ ,				UpperCamelCase__       )       )

             shutil.rmtree(UpperCamelCase__       )
       @require_torch
       def 							lowercase_							(     self       )     ->	Tuple:





         """simple docstring"""




         if not self.test_seqaseq:
           return

         _lowercase:			Union[str, Any]       =     self.get_tokenizers()
         for tokenizer in tokenizers:
           with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
             # Longer text that will definitely require truncation.
             _lowercase:			List[str]       =     [
                 ' UN Chief Says There Is No Military Solution in Syria',
                 ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                 ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                 ' will only worsen the violence and misery for millions of people.',
             ]
             _lowercase:			int       =     [
                 'Şeful ONU declară că nu există o soluţie militară în Siria',
                 'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                 ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
                 ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
             ]
             try:
               _lowercase:			Union[str, Any]       =     tokenizer.prepare_seqaseq_batch(
                   src_texts=UpperCamelCase__ ,				tgt_texts=UpperCamelCase__ ,				max_length=3 ,				max_target_length=10 ,				return_tensors='''pt''' ,				src_lang='''eng_Latn''' ,				tgt_lang='''ron_Latn''' ,				)
             except NotImplementedError:
               return
             self.assertEqual(batch.input_ids.shape[1] ,				3       )
             self.assertEqual(batch.labels.shape[1] ,				10       )
             # max_target_length will default to max_length if not specified
             _lowercase:			Optional[int]       =     tokenizer.prepare_seqaseq_batch(
                 UpperCamelCase__ ,				tgt_texts=UpperCamelCase__ ,				max_length=3 ,				return_tensors='''pt'''       )
             self.assertEqual(batch.input_ids.shape[1] ,				3       )
             self.assertEqual(batch.labels.shape[1] ,				3       )

             _lowercase:			Any       =     tokenizer.prepare_seqaseq_batch(
                 src_texts=UpperCamelCase__ ,				max_length=3 ,				max_target_length=10 ,				return_tensors='''pt'''       )
             self.assertEqual(batch_encoder_only.input_ids.shape[1] ,				3       )
             self.assertEqual(batch_encoder_only.attention_mask.shape[1] ,				3       )
             self.assertNotIn('''decoder_input_ids''' ,				UpperCamelCase__       )
       @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''       )
       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         pass



       def 							lowercase_							(     self       )     ->	Dict:





         """simple docstring"""




         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
             _lowercase:			str       =     [AddedToken('''<special>''' ,				lstrip=UpperCamelCase__       )]

             _lowercase:			Tuple       =     self.rust_tokenizer_class.from_pretrained(
                 UpperCamelCase__ ,				additional_special_tokens=UpperCamelCase__ ,				**UpperCamelCase__       )
             _lowercase:			Dict       =     tokenizer_r.encode('''Hey this is a <special> token'''       )

             _lowercase:			int       =     tokenizer_r.encode('''<special>''' ,				add_special_tokens=UpperCamelCase__       )[0]

             self.assertTrue(special_token_id in r_output       )

             if self.test_slow_tokenizer:
               _lowercase:			Optional[int]       =     self.rust_tokenizer_class.from_pretrained(
                   UpperCamelCase__ ,				additional_special_tokens=UpperCamelCase__ ,				**UpperCamelCase__ ,				)
               _lowercase:			int       =     self.tokenizer_class.from_pretrained(
                   UpperCamelCase__ ,				additional_special_tokens=UpperCamelCase__ ,				**UpperCamelCase__       )

               _lowercase:			Dict       =     tokenizer_p.encode('''Hey this is a <special> token'''       )

               _lowercase:			str       =     tokenizer_cr.encode('''Hey this is a <special> token'''       )

               self.assertEqual(UpperCamelCase__ ,				UpperCamelCase__       )
               self.assertEqual(UpperCamelCase__ ,				UpperCamelCase__       )
               self.assertTrue(special_token_id in p_output       )
               self.assertTrue(special_token_id in cr_output       )


@require_torch
@require_sentencepiece
@require_tokenizers
class 			__magic_name__       (       unittest.TestCase ):
       UpperCamelCase_   =		"""facebook/nllb-200-distilled-600M"""
       UpperCamelCase_   =		[
           """ UN Chief Says There Is No Military Solution in Syria""",
           """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
       ]
       UpperCamelCase_   =		[
           """Şeful ONU declară că nu există o soluţie militară în Siria""",
           """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
           """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
           """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
       ]
       UpperCamelCase_   =		[
           25_6047,
           1_6297,
           13_4408,
           8165,
           24_8066,
           1_4734,
           950,
           1135,
           10_5721,
           3573,
           83,
           2_7352,
           108,
           4_9486,
           2,
       ]
       @classmethod
       def 							lowercase_							(     cls       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			NllbTokenizer       =     NllbTokenizer.from_pretrained(
             cls.checkpoint_name ,				src_lang='''eng_Latn''' ,				tgt_lang='''ron_Latn'''       )
         _lowercase:			List[str]       =     1
         return cls
       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab'''] ,				25_6001       )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn'''] ,				25_6002       )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn'''] ,				25_6057       )
       def 							lowercase_							(     self       )     ->	Optional[Any]:





         """simple docstring"""




         _lowercase:			Tuple       =     self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
         self.assertListEqual(self.expected_src_tokens ,				UpperCamelCase__       )
       def 							lowercase_							(     self       )     ->	List[Any]:





         """simple docstring"""




         self.assertIn(UpperCamelCase__ ,				self.tokenizer.all_special_ids       )
         # fmt: off
         _lowercase:			str       =     [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
         # fmt: on

         _lowercase:			Dict       =     self.tokenizer.decode(UpperCamelCase__ ,				skip_special_tokens=UpperCamelCase__       )
         _lowercase:			str       =     self.tokenizer.decode(generated_ids[1:] ,				skip_special_tokens=UpperCamelCase__       )
         self.assertEqual(UpperCamelCase__ ,				UpperCamelCase__       )
         self.assertNotIn(self.tokenizer.eos_token ,				UpperCamelCase__       )
       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         _lowercase:			str       =     ['this is gunna be a long sentence ' * 20]
         assert isinstance(src_text[0] ,				UpperCamelCase__       )
         _lowercase:			Optional[int]       =     10
         _lowercase:			Dict       =     self.tokenizer(UpperCamelCase__ ,				max_length=UpperCamelCase__ ,				truncation=UpperCamelCase__       ).input_ids[0]
         self.assertEqual(ids[-1] ,				2       )
         self.assertEqual(ids[0] ,				UpperCamelCase__       )
         self.assertEqual(len(UpperCamelCase__       ) ,				UpperCamelCase__       )
       def 							lowercase_							(     self       )     ->	Tuple:





         """simple docstring"""




         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']       ) ,				[25_6203, 3]       )
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         _lowercase:			str       =     tempfile.mkdtemp()
         _lowercase:			List[Any]       =     self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(UpperCamelCase__       )
         _lowercase:			Optional[Any]       =     NllbTokenizer.from_pretrained(UpperCamelCase__       )
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids ,				UpperCamelCase__       )
       @require_torch
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         _lowercase:			Tuple       =     self.tokenizer(
             self.src_text ,				text_target=self.tgt_text ,				padding=UpperCamelCase__ ,				truncation=UpperCamelCase__ ,				max_length=len(self.expected_src_tokens       ) ,				return_tensors='''pt''' ,				)
         _lowercase:			int       =     shift_tokens_right(
             batch['''labels'''] ,				self.tokenizer.pad_token_id ,				self.tokenizer.lang_code_to_id['''ron_Latn''']       )

         self.assertIsInstance(UpperCamelCase__ ,				UpperCamelCase__       )

         self.assertEqual((2, 15) ,				batch.input_ids.shape       )
         self.assertEqual((2, 15) ,				batch.attention_mask.shape       )
         _lowercase:			int       =     batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens ,				UpperCamelCase__       )
         self.assertEqual(UpperCamelCase__ ,				batch.decoder_input_ids[0, 0]       )  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens ,				[EN_CODE]       )
         self.assertEqual(self.tokenizer.suffix_tokens ,				[self.tokenizer.eos_token_id]       )
       def 							lowercase_							(     self       )     ->	List[Any]:





         """simple docstring"""




         _lowercase:			Tuple       =     self.tokenizer(self.src_text ,				padding=UpperCamelCase__ ,				truncation=UpperCamelCase__ ,				max_length=3 ,				return_tensors='''pt'''       )
         _lowercase:			Dict       =     self.tokenizer(
             text_target=self.tgt_text ,				padding=UpperCamelCase__ ,				truncation=UpperCamelCase__ ,				max_length=10 ,				return_tensors='''pt'''       )
         _lowercase:			Union[str, Any]       =     targets['input_ids']
         _lowercase:			int       =     shift_tokens_right(
             UpperCamelCase__ ,				self.tokenizer.pad_token_id ,				decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang] ,				)

         self.assertEqual(batch.input_ids.shape[1] ,				3       )
         self.assertEqual(batch.decoder_input_ids.shape[1] ,				10       )
       @require_torch
       def 							lowercase_							(     self       )     ->	Tuple:





         """simple docstring"""




         _lowercase:			List[str]       =     self.tokenizer._build_translation_inputs(
             '''A test''' ,				return_tensors='''pt''' ,				src_lang='''eng_Latn''' ,				tgt_lang='''fra_Latn'''       )

         self.assertEqual(
             nested_simplify(UpperCamelCase__       ) ,				{
                 # A, test, EOS, en_XX
                 '''input_ids''': [[25_6047, 70, 7356, 2]],
                 '''attention_mask''': [[1, 1, 1, 1]],
                 # ar_AR
                 '''forced_bos_token_id''': 25_6057,
             } ,				)



       @require_torch
       def 							lowercase_							(     self       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			Dict       =     True
         _lowercase:			Any       =     self.tokenizer(
             '''UN Chief says there is no military solution in Syria''' ,				src_lang='''eng_Latn''' ,				tgt_lang='''fra_Latn'''       )
         self.assertEqual(
             inputs.input_ids ,				[1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]       )

         _lowercase:			Any       =     False
         _lowercase:			Any       =     self.tokenizer(
             '''UN Chief says there is no military solution in Syria''' ,				src_lang='''eng_Latn''' ,				tgt_lang='''fra_Latn'''       )
         self.assertEqual(
             inputs.input_ids ,				[25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]       )


"""simple docstring"""




def  _lowerCAmelCase		(		_UpperCamelCase						):






  """simple docstring"""



  if not isinstance(_UpperCamelCase ,				_UpperCamelCase						):
    raise ValueError('''Input series is not valid, valid series - [2, 4, 6]'''						)
  if len(_UpperCamelCase						) == 0:
    raise ValueError('''Input list must be a non empty list'''						)
  if len(_UpperCamelCase						) == 1:
    return True
  _lowercase:			Any       =     series[1] - series[0]
  for index in range(len(_UpperCamelCase						) - 1						):
    if series[index + 1] - series[index] != common_diff:
      return False
  return True




def  _lowerCAmelCase		(		_UpperCamelCase						):






  """simple docstring"""



  if not isinstance(_UpperCamelCase ,				_UpperCamelCase						):
    raise ValueError('''Input series is not valid, valid series - [2, 4, 6]'''						)
  if len(_UpperCamelCase						) == 0:
    raise ValueError('''Input list must be a non empty list'''						)
  _lowercase:			Tuple       =     0
  for val in series:
    answer += val
  return answer / len(_UpperCamelCase						)


if __name__ == "__main__":
  import doctest

  doctest.testmod()
