






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase      :				str							=get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCamelCase      :				Tuple							=256047
lowerCamelCase      :				int							=256145





@require_sentencepiece
@require_tokenizers
class      __a    (       snake_case_			,  unittest.TestCase			):
 _lowerCAmelCase			:			Dict								=					NllbTokenizer
 _lowerCAmelCase			:			List[str]								=					NllbTokenizerFast
 _lowerCAmelCase			:			Optional[Any]								=					True
 _lowerCAmelCase			:			Optional[int]								=					True
 _lowerCAmelCase			:			Optional[int]								=					{}







 def    __lowercase   (						self   :  str		):


      '''simple docstring'''






      super().setUp()

      # We have a SentencePiece fixture for testing
      UpperCamelCase__     :  List[Any]										=    NllbTokenizer(SCREAMING_SNAKE_CASE				,  keep_accents=SCREAMING_SNAKE_CASE		)
      tokenizer.save_pretrained(self.tmpdirname		)







 def    __lowercase   (						self   :  Any		):


      '''simple docstring'''






      UpperCamelCase__     :  Any										=    NllbTokenizer(SCREAMING_SNAKE_CASE				,  keep_accents=SCREAMING_SNAKE_CASE		)

      UpperCamelCase__     :  int										=    tokenizer.tokenize("This is a test"		)
      self.assertListEqual(SCREAMING_SNAKE_CASE				,  ["▁This", "▁is", "▁a", "▁t", "est"]		)

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE		)				,  [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]				,  )

      UpperCamelCase__     :  Optional[Any]										=    tokenizer.tokenize("I was born in 92000, and this is falsé."		)
      self.assertListEqual(
          SCREAMING_SNAKE_CASE				,  [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "é",
              ".",
          ]				,  )
      UpperCamelCase__     :  Optional[Any]										=    tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE		)
      self.assertListEqual(
          SCREAMING_SNAKE_CASE				,  [
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
          ]				,  )

      UpperCamelCase__     :  str										=    tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE		)
      self.assertListEqual(
          SCREAMING_SNAKE_CASE				,  [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "<unk>",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "<unk>",
              ".",
          ]				,  )







 def    __lowercase   (						self   :  int		):


      '''simple docstring'''






      UpperCamelCase__     :  str										=    (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
                UpperCamelCase__     :  List[Any]										=    self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE				,  **SCREAMING_SNAKE_CASE		)
                UpperCamelCase__     :  int										=    self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE				,  **SCREAMING_SNAKE_CASE		)

                UpperCamelCase__     :  Optional[int]										=    tempfile.mkdtemp()

                UpperCamelCase__     :  Tuple										=    tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE		)
                UpperCamelCase__     :  Dict										=    tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE		)

                # Checks it save with the same files + the tokenizer.json file for the fast one
                self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files		)		)
                UpperCamelCase__     :  int										=    tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f		)
                self.assertSequenceEqual(SCREAMING_SNAKE_CASE				,  SCREAMING_SNAKE_CASE		)

                # Checks everything loads correctly in the same way
                UpperCamelCase__     :  int										=    tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE		)
                UpperCamelCase__     :  Dict										=    tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE		)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(SCREAMING_SNAKE_CASE				,  SCREAMING_SNAKE_CASE		)		)

                shutil.rmtree(SCREAMING_SNAKE_CASE		)

                # Save tokenizer rust, legacy_format=True
                UpperCamelCase__     :  List[str]										=    tempfile.mkdtemp()

                UpperCamelCase__     :  Tuple										=    tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE				,  legacy_format=SCREAMING_SNAKE_CASE		)
                UpperCamelCase__     :  Union[str, Any]										=    tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE		)

                # Checks it save with the same files
                self.assertSequenceEqual(SCREAMING_SNAKE_CASE				,  SCREAMING_SNAKE_CASE		)

                # Checks everything loads correctly in the same way
                UpperCamelCase__     :  int										=    tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE		)
                UpperCamelCase__     :  Optional[int]										=    tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE		)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(SCREAMING_SNAKE_CASE				,  SCREAMING_SNAKE_CASE		)		)

                shutil.rmtree(SCREAMING_SNAKE_CASE		)

                # Save tokenizer rust, legacy_format=False
                UpperCamelCase__     :  List[Any]										=    tempfile.mkdtemp()

                UpperCamelCase__     :  Tuple										=    tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE				,  legacy_format=SCREAMING_SNAKE_CASE		)
                UpperCamelCase__     :  Tuple										=    tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE		)

                # Checks it saved the tokenizer.json file
                self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files		)		)

                # Checks everything loads correctly in the same way
                UpperCamelCase__     :  Optional[Any]										=    tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE		)
                UpperCamelCase__     :  Union[str, Any]										=    tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE		)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(SCREAMING_SNAKE_CASE				,  SCREAMING_SNAKE_CASE		)		)

                shutil.rmtree(SCREAMING_SNAKE_CASE		)







 @require_torch
 def    __lowercase   (						self   :  List[str]		):


      '''simple docstring'''






      if not self.test_seqaseq:
           return

      UpperCamelCase__     :  Any										=    self.get_tokenizers()
      for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'		):
                # Longer text that will definitely require truncation.
                UpperCamelCase__     :  Dict										=    [
                    " UN Chief Says There Is No Military Solution in Syria",
                    " Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for"
                    " Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons"
                    " will only worsen the violence and misery for millions of people.",
                ]
                UpperCamelCase__     :  Any										=    [
                    "Şeful ONU declară că nu există o soluţie militară în Siria",
                    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
                    " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
                    " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
                ]
                try:
                     UpperCamelCase__     :  Any										=    tokenizer.prepare_seqaseq_batch(
                         src_texts=SCREAMING_SNAKE_CASE				,  tgt_texts=SCREAMING_SNAKE_CASE				,  max_length=3				,  max_target_length=10				,  return_tensors="pt"				,  src_lang="eng_Latn"				,  tgt_lang="ron_Latn"				,  )
                except NotImplementedError:
                     return
                self.assertEqual(batch.input_ids.shape[1]				,  3		)
                self.assertEqual(batch.labels.shape[1]				,  10		)
                # max_target_length will default to max_length if not specified
                UpperCamelCase__     :  str										=    tokenizer.prepare_seqaseq_batch(
                    SCREAMING_SNAKE_CASE				,  tgt_texts=SCREAMING_SNAKE_CASE				,  max_length=3				,  return_tensors="pt"		)
                self.assertEqual(batch.input_ids.shape[1]				,  3		)
                self.assertEqual(batch.labels.shape[1]				,  3		)

                UpperCamelCase__     :  List[Any]										=    tokenizer.prepare_seqaseq_batch(
                    src_texts=SCREAMING_SNAKE_CASE				,  max_length=3				,  max_target_length=10				,  return_tensors="pt"		)
                self.assertEqual(batch_encoder_only.input_ids.shape[1]				,  3		)
                self.assertEqual(batch_encoder_only.attention_mask.shape[1]				,  3		)
                self.assertNotIn("decoder_input_ids"				,  SCREAMING_SNAKE_CASE		)







 @unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece."		)
 def    __lowercase   (						self   :  str		):


      '''simple docstring'''






      pass







 def    __lowercase   (						self   :  Optional[int]		):


      '''simple docstring'''






      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
                UpperCamelCase__     :  Optional[int]										=    [AddedToken("<special>"				,  lstrip=SCREAMING_SNAKE_CASE		)]

                UpperCamelCase__     :  Any										=    self.rust_tokenizer_class.from_pretrained(
                    SCREAMING_SNAKE_CASE				,  additional_special_tokens=SCREAMING_SNAKE_CASE				,  **SCREAMING_SNAKE_CASE		)
                UpperCamelCase__     :  Tuple										=    tokenizer_r.encode("Hey this is a <special> token"		)

                UpperCamelCase__     :  Union[str, Any]										=    tokenizer_r.encode("<special>"				,  add_special_tokens=SCREAMING_SNAKE_CASE		)[0]

                self.assertTrue(special_token_id in r_output		)

                if self.test_slow_tokenizer:
                     UpperCamelCase__     :  Optional[int]										=    self.rust_tokenizer_class.from_pretrained(
                         SCREAMING_SNAKE_CASE				,  additional_special_tokens=SCREAMING_SNAKE_CASE				,  **SCREAMING_SNAKE_CASE				,  )
                     UpperCamelCase__     :  Dict										=    self.tokenizer_class.from_pretrained(
                         SCREAMING_SNAKE_CASE				,  additional_special_tokens=SCREAMING_SNAKE_CASE				,  **SCREAMING_SNAKE_CASE		)

                     UpperCamelCase__     :  Tuple										=    tokenizer_p.encode("Hey this is a <special> token"		)

                     UpperCamelCase__     :  Optional[Any]										=    tokenizer_cr.encode("Hey this is a <special> token"		)

                     self.assertEqual(SCREAMING_SNAKE_CASE				,  SCREAMING_SNAKE_CASE		)
                     self.assertEqual(SCREAMING_SNAKE_CASE				,  SCREAMING_SNAKE_CASE		)
                     self.assertTrue(special_token_id in p_output		)
                     self.assertTrue(special_token_id in cr_output		)







@require_torch
@require_sentencepiece
@require_tokenizers
class      __a    (       unittest.TestCase			):
 _lowerCAmelCase			:			str								=					'''facebook/nllb-200-distilled-600M'''
 _lowerCAmelCase			:			Optional[Any]								=					[
     ''' UN Chief Says There Is No Military Solution in Syria''',
     ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
 ]
 _lowerCAmelCase			:			Optional[Any]								=					[
     '''Şeful ONU declară că nu există o soluţie militară în Siria''',
     '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
     ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
     ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
 ]
 _lowerCAmelCase			:			List[Any]								=					[
     2_5_6_0_4_7,
     1_6_2_9_7,
     1_3_4_4_0_8,
     8_1_6_5,
     2_4_8_0_6_6,
     1_4_7_3_4,
     9_5_0,
     1_1_3_5,
     1_0_5_7_2_1,
     3_5_7_3,
     8_3,
     2_7_3_5_2,
     1_0_8,
     4_9_4_8_6,
     2,
 ]







 @classmethod
 def    __lowercase   (						cls   :  Optional[Any]		):


      '''simple docstring'''






      UpperCamelCase__     :  Union[str, Any]										=    NllbTokenizer.from_pretrained(
          cls.checkpoint_name				,  src_lang="eng_Latn"				,  tgt_lang="ron_Latn"		)
      UpperCamelCase__     :  Dict										=    1
      return cls







 def    __lowercase   (						self   :  Union[str, Any]		):


      '''simple docstring'''






      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]				,  25_60_01		)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]				,  25_60_02		)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]				,  25_60_57		)







 def    __lowercase   (						self   :  List[Any]		):


      '''simple docstring'''






      UpperCamelCase__     :  List[Any]										=    self.tokenizer.batch_encode_plus(self.src_text		).input_ids[0]
      self.assertListEqual(self.expected_src_tokens				,  SCREAMING_SNAKE_CASE		)







 def    __lowercase   (						self   :  List[Any]		):


      '''simple docstring'''






      self.assertIn(SCREAMING_SNAKE_CASE				,  self.tokenizer.all_special_ids		)
      # fmt: off
      UpperCamelCase__     :  int										=    [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
      # fmt: on

      UpperCamelCase__     :  List[str]										=    self.tokenizer.decode(SCREAMING_SNAKE_CASE				,  skip_special_tokens=SCREAMING_SNAKE_CASE		)
      UpperCamelCase__     :  int										=    self.tokenizer.decode(generated_ids[1:]				,  skip_special_tokens=SCREAMING_SNAKE_CASE		)
      self.assertEqual(SCREAMING_SNAKE_CASE				,  SCREAMING_SNAKE_CASE		)
      self.assertNotIn(self.tokenizer.eos_token				,  SCREAMING_SNAKE_CASE		)







 def    __lowercase   (						self   :  Dict		):


      '''simple docstring'''






      UpperCamelCase__     :  Optional[int]										=    ["this is gunna be a long sentence " * 20]
      assert isinstance(src_text[0]				,  SCREAMING_SNAKE_CASE		)
      UpperCamelCase__     :  Union[str, Any]										=    10
      UpperCamelCase__     :  Tuple										=    self.tokenizer(SCREAMING_SNAKE_CASE				,  max_length=SCREAMING_SNAKE_CASE				,  truncation=SCREAMING_SNAKE_CASE		).input_ids[0]
      self.assertEqual(ids[-1]				,  2		)
      self.assertEqual(ids[0]				,  SCREAMING_SNAKE_CASE		)
      self.assertEqual(len(SCREAMING_SNAKE_CASE		)				,  SCREAMING_SNAKE_CASE		)







 def    __lowercase   (						self   :  Dict		):


      '''simple docstring'''






      self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]		)				,  [25_62_03, 3]		)







 def    __lowercase   (						self   :  List[Any]		):


      '''simple docstring'''






      UpperCamelCase__     :  str										=    tempfile.mkdtemp()
      UpperCamelCase__     :  Tuple										=    self.tokenizer.fairseq_tokens_to_ids
      self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE		)
      UpperCamelCase__     :  Optional[Any]										=    NllbTokenizer.from_pretrained(SCREAMING_SNAKE_CASE		)
      self.assertDictEqual(new_tok.fairseq_tokens_to_ids				,  SCREAMING_SNAKE_CASE		)







 @require_torch
 def    __lowercase   (						self   :  List[Any]		):


      '''simple docstring'''






      UpperCamelCase__     :  Tuple										=    self.tokenizer(
          self.src_text				,  text_target=self.tgt_text				,  padding=SCREAMING_SNAKE_CASE				,  truncation=SCREAMING_SNAKE_CASE				,  max_length=len(self.expected_src_tokens		)				,  return_tensors="pt"				,  )
      UpperCamelCase__     :  List[str]										=    shift_tokens_right(
          batch["labels"]				,  self.tokenizer.pad_token_id				,  self.tokenizer.lang_code_to_id["ron_Latn"]		)

      self.assertIsInstance(SCREAMING_SNAKE_CASE				,  SCREAMING_SNAKE_CASE		)

      self.assertEqual((2, 15)				,  batch.input_ids.shape		)
      self.assertEqual((2, 15)				,  batch.attention_mask.shape		)
      UpperCamelCase__     :  str										=    batch.input_ids.tolist()[0]
      self.assertListEqual(self.expected_src_tokens				,  SCREAMING_SNAKE_CASE		)
      self.assertEqual(SCREAMING_SNAKE_CASE				,  batch.decoder_input_ids[0, 0]		)  # EOS
      # Test that special tokens are reset
      self.assertEqual(self.tokenizer.prefix_tokens				,  [EN_CODE]		)
      self.assertEqual(self.tokenizer.suffix_tokens				,  [self.tokenizer.eos_token_id]		)







 def    __lowercase   (						self   :  Union[str, Any]		):


      '''simple docstring'''






      UpperCamelCase__     :  int										=    self.tokenizer(self.src_text				,  padding=SCREAMING_SNAKE_CASE				,  truncation=SCREAMING_SNAKE_CASE				,  max_length=3				,  return_tensors="pt"		)
      UpperCamelCase__     :  Any										=    self.tokenizer(
          text_target=self.tgt_text				,  padding=SCREAMING_SNAKE_CASE				,  truncation=SCREAMING_SNAKE_CASE				,  max_length=10				,  return_tensors="pt"		)
      UpperCamelCase__     :  Dict										=    targets["input_ids"]
      UpperCamelCase__     :  int										=    shift_tokens_right(
          SCREAMING_SNAKE_CASE				,  self.tokenizer.pad_token_id				,  decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]				,  )

      self.assertEqual(batch.input_ids.shape[1]				,  3		)
      self.assertEqual(batch.decoder_input_ids.shape[1]				,  10		)







 @require_torch
 def    __lowercase   (						self   :  Dict		):


      '''simple docstring'''






      UpperCamelCase__     :  Tuple										=    self.tokenizer._build_translation_inputs(
          "A test"				,  return_tensors="pt"				,  src_lang="eng_Latn"				,  tgt_lang="fra_Latn"		)

      self.assertEqual(
          nested_simplify(SCREAMING_SNAKE_CASE		)				,  {
              # A, test, EOS, en_XX
              "input_ids": [[25_60_47, 70, 73_56, 2]],
              "attention_mask": [[1, 1, 1, 1]],
              # ar_AR
              "forced_bos_token_id": 25_60_57,
          }				,  )







 @require_torch
 def    __lowercase   (						self   :  Optional[Any]		):


      '''simple docstring'''






      UpperCamelCase__     :  List[Any]										=    True
      UpperCamelCase__     :  str										=    self.tokenizer(
          "UN Chief says there is no military solution in Syria"				,  src_lang="eng_Latn"				,  tgt_lang="fra_Latn"		)
      self.assertEqual(
          inputs.input_ids				,  [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]		)

      UpperCamelCase__     :  str										=    False
      UpperCamelCase__     :  Any										=    self.tokenizer(
          "UN Chief says there is no military solution in Syria"				,  src_lang="eng_Latn"				,  tgt_lang="fra_Latn"		)
      self.assertEqual(
          inputs.input_ids				,  [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]		)







import numpy as np





def 		__UpperCAmelCase     (	a_):
							return 1 / (1 + np.exp(-vector))


if __name__ == "__main__":
	import doctest

	doctest.testmod()





