


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__a	 =      get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__a	 =      25_6047
__a	 =      25_6145






@require_sentencepiece
@require_tokenizers
class 	__lowercase					(	__snake_case			,       unittest.TestCase	):
      UpperCamelCase										=							NllbTokenizer
      UpperCamelCase										=							NllbTokenizerFast
      UpperCamelCase										=							True
      UpperCamelCase										=							True
      UpperCamelCase										=							{}





      def 							_lowercase					(		self    :       Any							)     -> Tuple:






         """simple docstring"""







         super().setUp()

         # We have a SentencePiece fixture for testing
         UpperCAmelCase      					=  NllbTokenizer(__lowerCamelCase       ,							keep_accents=__lowerCamelCase							)
         tokenizer.save_pretrained(self.tmpdirname							)





      def 							_lowercase					(		self    :       Optional[int]							)     -> Optional[int]:






         """simple docstring"""







         UpperCAmelCase      					=  NllbTokenizer(__lowerCamelCase       ,							keep_accents=__lowerCamelCase							)

         UpperCAmelCase      					=  tokenizer.tokenize("""This is a test"""							)
         self.assertListEqual(__lowerCamelCase       ,							["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]							)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(__lowerCamelCase							)       ,							[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]       ,							)

         UpperCAmelCase      					=  tokenizer.tokenize("""I was born in 92000, and this is falsé."""							)
         self.assertListEqual(
             __lowerCamelCase       ,							[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """é""",
                 """.""",
             ]       ,							)
         UpperCAmelCase      					=  tokenizer.convert_tokens_to_ids(__lowerCamelCase							)
         self.assertListEqual(
             __lowerCamelCase       ,							[
                 value + tokenizer.fairseq_offset
                 for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
             ]       ,							)

         UpperCAmelCase      					=  tokenizer.convert_ids_to_tokens(__lowerCamelCase							)
         self.assertListEqual(
             __lowerCamelCase       ,							[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """<unk>""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """<unk>""",
                 """.""",
             ]       ,							)





      def 							_lowercase					(		self    :       Dict							)     -> Dict:






         """simple docstring"""







         UpperCAmelCase      					=  (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
               UpperCAmelCase      					=  self.rust_tokenizer_class.from_pretrained(__lowerCamelCase       ,							**__lowerCamelCase							)
               UpperCAmelCase      					=  self.tokenizer_class.from_pretrained(__lowerCamelCase       ,							**__lowerCamelCase							)

               UpperCAmelCase      					=  tempfile.mkdtemp()

               UpperCAmelCase      					=  tokenizer_r.save_pretrained(__lowerCamelCase							)
               UpperCAmelCase      					=  tokenizer_p.save_pretrained(__lowerCamelCase							)

               # Checks it save with the same files + the tokenizer.json file for the fast one
               self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files							)							)
               UpperCAmelCase      					=  tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f							)
               self.assertSequenceEqual(__lowerCamelCase       ,							__lowerCamelCase							)

               # Checks everything loads correctly in the same way
               UpperCAmelCase      					=  tokenizer_r.from_pretrained(__lowerCamelCase							)
               UpperCAmelCase      					=  tokenizer_p.from_pretrained(__lowerCamelCase							)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(__lowerCamelCase       ,							__lowerCamelCase							)							)

               shutil.rmtree(__lowerCamelCase							)

               # Save tokenizer rust, legacy_format=True
               UpperCAmelCase      					=  tempfile.mkdtemp()

               UpperCAmelCase      					=  tokenizer_r.save_pretrained(__lowerCamelCase       ,							legacy_format=__lowerCamelCase							)
               UpperCAmelCase      					=  tokenizer_p.save_pretrained(__lowerCamelCase							)

               # Checks it save with the same files
               self.assertSequenceEqual(__lowerCamelCase       ,							__lowerCamelCase							)

               # Checks everything loads correctly in the same way
               UpperCAmelCase      					=  tokenizer_r.from_pretrained(__lowerCamelCase							)
               UpperCAmelCase      					=  tokenizer_p.from_pretrained(__lowerCamelCase							)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(__lowerCamelCase       ,							__lowerCamelCase							)							)

               shutil.rmtree(__lowerCamelCase							)

               # Save tokenizer rust, legacy_format=False
               UpperCAmelCase      					=  tempfile.mkdtemp()

               UpperCAmelCase      					=  tokenizer_r.save_pretrained(__lowerCamelCase       ,							legacy_format=__lowerCamelCase							)
               UpperCAmelCase      					=  tokenizer_p.save_pretrained(__lowerCamelCase							)

               # Checks it saved the tokenizer.json file
               self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files							)							)

               # Checks everything loads correctly in the same way
               UpperCAmelCase      					=  tokenizer_r.from_pretrained(__lowerCamelCase							)
               UpperCAmelCase      					=  tokenizer_p.from_pretrained(__lowerCamelCase							)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(__lowerCamelCase       ,							__lowerCamelCase							)							)

               shutil.rmtree(__lowerCamelCase							)





      @require_torch
      def 							_lowercase					(		self    :       Dict							)     -> List[str]:






         """simple docstring"""







         if not self.test_seqaseq:
            return

         UpperCAmelCase      					=  self.get_tokenizers()
         for tokenizer in tokenizers:
            with self.subTest(F"""{tokenizer.__class__.__name__}"""							):
               # Longer text that will definitely require truncation.
               UpperCAmelCase      					=  [
                   """ UN Chief Says There Is No Military Solution in Syria""",
                   """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
                   """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
                   """ will only worsen the violence and misery for millions of people.""",
               ]
               UpperCAmelCase      					=  [
                   """Şeful ONU declară că nu există o soluţie militară în Siria""",
                   """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
                   """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
                   """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
               ]
               try:
                  UpperCAmelCase      					=  tokenizer.prepare_seqaseq_batch(
                      src_texts=__lowerCamelCase       ,							tgt_texts=__lowerCamelCase       ,							max_length=3       ,							max_target_length=1_0       ,							return_tensors="""pt"""       ,							src_lang="""eng_Latn"""       ,							tgt_lang="""ron_Latn"""       ,							)
               except NotImplementedError:
                  return
               self.assertEqual(batch.input_ids.shape[1]       ,							3							)
               self.assertEqual(batch.labels.shape[1]       ,							1_0							)
               # max_target_length will default to max_length if not specified
               UpperCAmelCase      					=  tokenizer.prepare_seqaseq_batch(
                   __lowerCamelCase       ,							tgt_texts=__lowerCamelCase       ,							max_length=3       ,							return_tensors="""pt"""							)
               self.assertEqual(batch.input_ids.shape[1]       ,							3							)
               self.assertEqual(batch.labels.shape[1]       ,							3							)

               UpperCAmelCase      					=  tokenizer.prepare_seqaseq_batch(
                   src_texts=__lowerCamelCase       ,							max_length=3       ,							max_target_length=1_0       ,							return_tensors="""pt"""							)
               self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,							3							)
               self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,							3							)
               self.assertNotIn("""decoder_input_ids"""       ,							__lowerCamelCase							)





      @unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""							)
      def 							_lowercase					(		self    :       str							)     -> List[str]:






         """simple docstring"""







         pass





      def 							_lowercase					(		self    :       Union[str, Any]							)     -> Any:






         """simple docstring"""







         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
               UpperCAmelCase      					=  [AddedToken("""<special>"""       ,							lstrip=__lowerCamelCase							)]

               UpperCAmelCase      					=  self.rust_tokenizer_class.from_pretrained(
                   __lowerCamelCase       ,							additional_special_tokens=__lowerCamelCase       ,							**__lowerCamelCase							)
               UpperCAmelCase      					=  tokenizer_r.encode("""Hey this is a <special> token"""							)

               UpperCAmelCase      					=  tokenizer_r.encode("""<special>"""       ,							add_special_tokens=__lowerCamelCase							)[0]

               self.assertTrue(special_token_id in r_output							)

               if self.test_slow_tokenizer:
                  UpperCAmelCase      					=  self.rust_tokenizer_class.from_pretrained(
                      __lowerCamelCase       ,							additional_special_tokens=__lowerCamelCase       ,							**__lowerCamelCase       ,							)
                  UpperCAmelCase      					=  self.tokenizer_class.from_pretrained(
                      __lowerCamelCase       ,							additional_special_tokens=__lowerCamelCase       ,							**__lowerCamelCase							)

                  UpperCAmelCase      					=  tokenizer_p.encode("""Hey this is a <special> token"""							)

                  UpperCAmelCase      					=  tokenizer_cr.encode("""Hey this is a <special> token"""							)

                  self.assertEqual(__lowerCamelCase       ,							__lowerCamelCase							)
                  self.assertEqual(__lowerCamelCase       ,							__lowerCamelCase							)
                  self.assertTrue(special_token_id in p_output							)
                  self.assertTrue(special_token_id in cr_output							)






@require_torch
@require_sentencepiece
@require_tokenizers
class 	__lowercase					(	unittest.TestCase	):
      UpperCamelCase										=							'''facebook/nllb-200-distilled-600M'''
      UpperCamelCase										=							[
          ''' UN Chief Says There Is No Military Solution in Syria''',
          ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
      ]
      UpperCamelCase										=							[
          '''Şeful ONU declară că nu există o soluţie militară în Siria''',
          '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
          ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
          ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
      ]
      UpperCamelCase										=							[
          25_60_47,
          1_62_97,
          13_44_08,
          81_65,
          24_80_66,
          1_47_34,
          9_50,
          11_35,
          10_57_21,
          35_73,
          83,
          2_73_52,
          1_08,
          4_94_86,
          2,
      ]





      @classmethod
      def 							_lowercase					(		cls    :       int							)     -> Tuple:






         """simple docstring"""







         UpperCAmelCase      					=  NllbTokenizer.from_pretrained(
             cls.checkpoint_name       ,							src_lang="""eng_Latn"""       ,							tgt_lang="""ron_Latn"""							)
         UpperCAmelCase      					=  1
         return cls





      def 							_lowercase					(		self    :       Dict							)     -> Optional[int]:






         """simple docstring"""







         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]       ,							2_5_6_0_0_1							)
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]       ,							2_5_6_0_0_2							)
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]       ,							2_5_6_0_5_7							)





      def 							_lowercase					(		self    :       Union[str, Any]							)     -> Tuple:






         """simple docstring"""







         UpperCAmelCase      					=  self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
         self.assertListEqual(self.expected_src_tokens       ,							__lowerCamelCase							)





      def 							_lowercase					(		self    :       str							)     -> List[str]:






         """simple docstring"""







         self.assertIn(__lowerCamelCase       ,							self.tokenizer.all_special_ids							)
         # fmt: off
         UpperCAmelCase      					=  [RO_CODE, 4_2_5_4, 9_8_0_6_8, 1_1_2_9_2_3, 3_9_0_7_2, 3_9_0_9, 7_1_3, 1_0_2_7_6_7, 2_6, 1_7_3_1_4, 3_5_6_4_2, 1_4_6_8_3, 3_3_1_1_8, 2_0_2_2, 6_6_9_8_7, 2, 2_5_6_0_4_7]
         # fmt: on

         UpperCAmelCase      					=  self.tokenizer.decode(__lowerCamelCase       ,							skip_special_tokens=__lowerCamelCase							)
         UpperCAmelCase      					=  self.tokenizer.decode(generated_ids[1:]       ,							skip_special_tokens=__lowerCamelCase							)
         self.assertEqual(__lowerCamelCase       ,							__lowerCamelCase							)
         self.assertNotIn(self.tokenizer.eos_token       ,							__lowerCamelCase							)





      def 							_lowercase					(		self    :       List[str]							)     -> Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase      					=  ["""this is gunna be a long sentence """ * 2_0]
         assert isinstance(src_text[0]       ,							__lowerCamelCase							)
         UpperCAmelCase      					=  1_0
         UpperCAmelCase      					=  self.tokenizer(__lowerCamelCase       ,							max_length=__lowerCamelCase       ,							truncation=__lowerCamelCase							).input_ids[0]
         self.assertEqual(ids[-1]       ,							2							)
         self.assertEqual(ids[0]       ,							__lowerCamelCase							)
         self.assertEqual(len(__lowerCamelCase							)       ,							__lowerCamelCase							)





      def 							_lowercase					(		self    :       List[str]							)     -> List[Any]:






         """simple docstring"""







         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]							)       ,							[2_5_6_2_0_3, 3]							)





      def 							_lowercase					(		self    :       List[str]							)     -> Dict:






         """simple docstring"""







         UpperCAmelCase      					=  tempfile.mkdtemp()
         UpperCAmelCase      					=  self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(__lowerCamelCase							)
         UpperCAmelCase      					=  NllbTokenizer.from_pretrained(__lowerCamelCase							)
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,							__lowerCamelCase							)





      @require_torch
      def 							_lowercase					(		self    :       str							)     -> Tuple:






         """simple docstring"""







         UpperCAmelCase      					=  self.tokenizer(
             self.src_text       ,							text_target=self.tgt_text       ,							padding=__lowerCamelCase       ,							truncation=__lowerCamelCase       ,							max_length=len(self.expected_src_tokens							)       ,							return_tensors="""pt"""       ,							)
         UpperCAmelCase      					=  shift_tokens_right(
             batch["""labels"""]       ,							self.tokenizer.pad_token_id       ,							self.tokenizer.lang_code_to_id["""ron_Latn"""]							)

         self.assertIsInstance(__lowerCamelCase       ,							__lowerCamelCase							)

         self.assertEqual((2, 1_5)       ,							batch.input_ids.shape							)
         self.assertEqual((2, 1_5)       ,							batch.attention_mask.shape							)
         UpperCAmelCase      					=  batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens       ,							__lowerCamelCase							)
         self.assertEqual(__lowerCamelCase       ,							batch.decoder_input_ids[0, 0]							)  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens       ,							[EN_CODE]							)
         self.assertEqual(self.tokenizer.suffix_tokens       ,							[self.tokenizer.eos_token_id]							)





      def 							_lowercase					(		self    :       str							)     -> Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase      					=  self.tokenizer(self.src_text       ,							padding=__lowerCamelCase       ,							truncation=__lowerCamelCase       ,							max_length=3       ,							return_tensors="""pt"""							)
         UpperCAmelCase      					=  self.tokenizer(
             text_target=self.tgt_text       ,							padding=__lowerCamelCase       ,							truncation=__lowerCamelCase       ,							max_length=1_0       ,							return_tensors="""pt"""							)
         UpperCAmelCase      					=  targets["""input_ids"""]
         UpperCAmelCase      					=  shift_tokens_right(
             __lowerCamelCase       ,							self.tokenizer.pad_token_id       ,							decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,							)

         self.assertEqual(batch.input_ids.shape[1]       ,							3							)
         self.assertEqual(batch.decoder_input_ids.shape[1]       ,							1_0							)





      @require_torch
      def 							_lowercase					(		self    :       Any							)     -> Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase      					=  self.tokenizer._build_translation_inputs(
             """A test"""       ,							return_tensors="""pt"""       ,							src_lang="""eng_Latn"""       ,							tgt_lang="""fra_Latn"""							)

         self.assertEqual(
             nested_simplify(__lowerCamelCase							)       ,							{
                 # A, test, EOS, en_XX
                 """input_ids""": [[2_5_6_0_4_7, 7_0, 7_3_5_6, 2]],
                 """attention_mask""": [[1, 1, 1, 1]],
                 # ar_AR
                 """forced_bos_token_id""": 2_5_6_0_5_7,
             }       ,							)





      @require_torch
      def 							_lowercase					(		self    :       Optional[int]							)     -> Tuple:






         """simple docstring"""







         UpperCAmelCase      					=  True
         UpperCAmelCase      					=  self.tokenizer(
             """UN Chief says there is no military solution in Syria"""       ,							src_lang="""eng_Latn"""       ,							tgt_lang="""fra_Latn"""							)
         self.assertEqual(
             inputs.input_ids       ,							[1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2, 2_5_6_0_4_7]							)

         UpperCAmelCase      					=  False
         UpperCAmelCase      					=  self.tokenizer(
             """UN Chief says there is no military solution in Syria"""       ,							src_lang="""eng_Latn"""       ,							tgt_lang="""fra_Latn"""							)
         self.assertEqual(
             inputs.input_ids       ,							[2_5_6_0_4_7, 1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2]							)







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__a	 =      {
    """configuration_pegasus_x""": ["""PEGASUS_X_PRETRAINED_CONFIG_ARCHIVE_MAP""", """PegasusXConfig"""],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __a	 =      [
     """PEGASUS_X_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """PegasusXForConditionalGeneration""",
     """PegasusXModel""",
     """PegasusXPreTrainedModel""",
 ]


if TYPE_CHECKING:
 from .configuration_pegasus_x import PEGASUS_X_PRETRAINED_CONFIG_ARCHIVE_MAP, PegasusXConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_pegasus_x import (
      PEGASUS_X_PRETRAINED_MODEL_ARCHIVE_LIST,
      PegasusXForConditionalGeneration,
      PegasusXModel,
      PegasusXPreTrainedModel,
  )


else:
 import sys

 __a	 =      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



