


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


snake_case__			 =						get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

snake_case__			 =						25_6047
snake_case__			 =						25_6145


@require_sentencepiece
@require_tokenizers
class 					lowerCAmelCase_ (       _a			,unittest.TestCase):
      lowerCamelCase_         =     NllbTokenizer
      lowerCamelCase_         =     NllbTokenizerFast
      lowerCamelCase_         =     True
      lowerCamelCase_         =     True
      lowerCamelCase_         =     {}




      def  _snake_case					(	self						:   List[Any]						)      ->Tuple:






         """simple docstring"""






         super().setUp()

         # We have a SentencePiece fixture for testing
         a__ :List[Any]						  =    NllbTokenizer(__A   ,     keep_accents=__A						)
         tokenizer.save_pretrained(self.tmpdirname						)




      def  _snake_case					(	self						:   Dict						)      ->Dict:






         """simple docstring"""






         a__ :Any						  =    NllbTokenizer(__A   ,     keep_accents=__A						)

         a__ :Any						  =    tokenizer.tokenize("This is a test"						)
         self.assertListEqual(__A   ,     ["▁This", "▁is", "▁a", "▁t", "est"]						)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(__A						)   ,     [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]   ,     )

         a__ :Optional[Any]						  =    tokenizer.tokenize("I was born in 92000, and this is falsé."						)
         self.assertListEqual(
             __A   ,     [
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "é",
                 ".",
             ]   ,     )
         a__ :Dict						  =    tokenizer.convert_tokens_to_ids(__A						)
         self.assertListEqual(
             __A   ,     [
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
             ]   ,     )

         a__ :int						  =    tokenizer.convert_ids_to_tokens(__A						)
         self.assertListEqual(
             __A   ,     [
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "<unk>",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "<unk>",
                 ".",
             ]   ,     )




      def  _snake_case					(	self						:   int						)      ->str:






         """simple docstring"""






         a__ :Optional[int]						  =    (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
               a__ :int						  =    self.rust_tokenizer_class.from_pretrained(__A   ,     **__A						)
               a__ :str						  =    self.tokenizer_class.from_pretrained(__A   ,     **__A						)

               a__ :List[Any]						  =    tempfile.mkdtemp()

               a__ :Optional[int]						  =    tokenizer_r.save_pretrained(__A						)
               a__ :List[str]						  =    tokenizer_p.save_pretrained(__A						)

               # Checks it save with the same files + the tokenizer.json file for the fast one
               self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files						)						)
               a__ :Dict						  =    tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f						)
               self.assertSequenceEqual(__A   ,     __A						)

               # Checks everything loads correctly in the same way
               a__ :int						  =    tokenizer_r.from_pretrained(__A						)
               a__ :str						  =    tokenizer_p.from_pretrained(__A						)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(__A   ,     __A						)						)

               shutil.rmtree(__A						)

               # Save tokenizer rust, legacy_format=True
               a__ :Dict						  =    tempfile.mkdtemp()

               a__ :str						  =    tokenizer_r.save_pretrained(__A   ,     legacy_format=__A						)
               a__ :Optional[int]						  =    tokenizer_p.save_pretrained(__A						)

               # Checks it save with the same files
               self.assertSequenceEqual(__A   ,     __A						)

               # Checks everything loads correctly in the same way
               a__ :Optional[Any]						  =    tokenizer_r.from_pretrained(__A						)
               a__ :str						  =    tokenizer_p.from_pretrained(__A						)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(__A   ,     __A						)						)

               shutil.rmtree(__A						)

               # Save tokenizer rust, legacy_format=False
               a__ :Optional[int]						  =    tempfile.mkdtemp()

               a__ :Optional[int]						  =    tokenizer_r.save_pretrained(__A   ,     legacy_format=__A						)
               a__ :Optional[int]						  =    tokenizer_p.save_pretrained(__A						)

               # Checks it saved the tokenizer.json file
               self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files						)						)

               # Checks everything loads correctly in the same way
               a__ :Union[str, Any]						  =    tokenizer_r.from_pretrained(__A						)
               a__ :Union[str, Any]						  =    tokenizer_p.from_pretrained(__A						)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(__A   ,     __A						)						)

               shutil.rmtree(__A						)




      @require_torch
      def  _snake_case					(	self						:   List[Any]						)      ->str:






         """simple docstring"""






         if not self.test_seqaseq:
            return

         a__ :Dict						  =    self.get_tokenizers()
         for tokenizer in tokenizers:
            with self.subTest(F'''{tokenizer.__class__.__name__}'''						):
               # Longer text that will definitely require truncation.
               a__ :Optional[int]						  =    [
                   " UN Chief Says There Is No Military Solution in Syria",
                   " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"
                   " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"
                   " will only worsen the violence and misery for millions of people.",
               ]
               a__ :List[Any]						  =    [
                   "Şeful ONU declară că nu există o soluţie militară în Siria",
                   "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
                   " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
                   " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
               ]
               try:
                  a__ :str						  =    tokenizer.prepare_seqaseq_batch(
                      src_texts=__A   ,     tgt_texts=__A   ,     max_length=3   ,     max_target_length=10   ,     return_tensors="pt"   ,     src_lang="eng_Latn"   ,     tgt_lang="ron_Latn"   ,     )
               except NotImplementedError:
                  return
               self.assertEqual(batch.input_ids.shape[1]   ,     3						)
               self.assertEqual(batch.labels.shape[1]   ,     10						)
               # max_target_length will default to max_length if not specified
               a__ :Optional[int]						  =    tokenizer.prepare_seqaseq_batch(
                   __A   ,     tgt_texts=__A   ,     max_length=3   ,     return_tensors="pt"						)
               self.assertEqual(batch.input_ids.shape[1]   ,     3						)
               self.assertEqual(batch.labels.shape[1]   ,     3						)

               a__ :List[Any]						  =    tokenizer.prepare_seqaseq_batch(
                   src_texts=__A   ,     max_length=3   ,     max_target_length=10   ,     return_tensors="pt"						)
               self.assertEqual(batch_encoder_only.input_ids.shape[1]   ,     3						)
               self.assertEqual(batch_encoder_only.attention_mask.shape[1]   ,     3						)
               self.assertNotIn("decoder_input_ids"   ,     __A						)




      @unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece."						)
      def  _snake_case					(	self						:   List[str]						)      ->Tuple:






         """simple docstring"""






         pass






      def  _snake_case					(	self						:   int						)      ->Optional[int]:






         """simple docstring"""






         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
               a__ :Dict						  =    [AddedToken("<special>"   ,     lstrip=__A						)]

               a__ :Dict						  =    self.rust_tokenizer_class.from_pretrained(
                   __A   ,     additional_special_tokens=__A   ,     **__A						)
               a__ :Tuple						  =    tokenizer_r.encode("Hey this is a <special> token"						)

               a__ :str						  =    tokenizer_r.encode("<special>"   ,     add_special_tokens=__A						)[0]

               self.assertTrue(special_token_id in r_output						)

               if self.test_slow_tokenizer:
                  a__ :List[Any]						  =    self.rust_tokenizer_class.from_pretrained(
                      __A   ,     additional_special_tokens=__A   ,     **__A   ,     )
                  a__ :Optional[Any]						  =    self.tokenizer_class.from_pretrained(
                      __A   ,     additional_special_tokens=__A   ,     **__A						)

                  a__ :Any						  =    tokenizer_p.encode("Hey this is a <special> token"						)

                  a__ :Optional[int]						  =    tokenizer_cr.encode("Hey this is a <special> token"						)

                  self.assertEqual(__A   ,     __A						)
                  self.assertEqual(__A   ,     __A						)
                  self.assertTrue(special_token_id in p_output						)
                  self.assertTrue(special_token_id in cr_output						)






@require_torch
@require_sentencepiece
@require_tokenizers
class 					lowerCAmelCase_ (       unittest.TestCase):
      lowerCamelCase_         =     'facebook/nllb-200-distilled-600M'
      lowerCamelCase_         =     [
          ' UN Chief Says There Is No Military Solution in Syria',
          ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
      ]
      lowerCamelCase_         =     [
          'Şeful ONU declară că nu există o soluţie militară în Siria',
          'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
          ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
          ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
      ]
      lowerCamelCase_         =     [
          256_047,
          16_297,
          134_408,
          8_165,
          248_066,
          14_734,
          950,
          1_135,
          105_721,
          3_573,
          83,
          27_352,
          108,
          49_486,
          2,
      ]




      @classmethod
      def  _snake_case					(	cls						:   Optional[Any]						)      ->Optional[Any]:






         """simple docstring"""






         a__ :NllbTokenizer						  =    NllbTokenizer.from_pretrained(
             cls.checkpoint_name   ,     src_lang="eng_Latn"   ,     tgt_lang="ron_Latn"						)
         a__ :Union[str, Any]						  =    1
         return cls




      def  _snake_case					(	self						:   Tuple						)      ->Dict:






         """simple docstring"""






         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]   ,     256001						)
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]   ,     256002						)
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]   ,     256057						)




      def  _snake_case					(	self						:   List[str]						)      ->Any:






         """simple docstring"""






         a__ :Tuple						  =    self.tokenizer.batch_encode_plus(self.src_text						).input_ids[0]
         self.assertListEqual(self.expected_src_tokens   ,     __A						)




      def  _snake_case					(	self						:   int						)      ->Dict:






         """simple docstring"""






         self.assertIn(__A   ,     self.tokenizer.all_special_ids						)
         # fmt: off
         a__ :Optional[int]						  =    [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
         # fmt: on

         a__ :Optional[int]						  =    self.tokenizer.decode(__A   ,     skip_special_tokens=__A						)
         a__ :List[str]						  =    self.tokenizer.decode(generated_ids[1:]   ,     skip_special_tokens=__A						)
         self.assertEqual(__A   ,     __A						)
         self.assertNotIn(self.tokenizer.eos_token   ,     __A						)




      def  _snake_case					(	self						:   List[str]						)      ->Optional[Any]:






         """simple docstring"""






         a__ :Union[str, Any]						  =    ["this is gunna be a long sentence " * 20]
         assert isinstance(src_text[0]   ,     __A						)
         a__ :Optional[Any]						  =    10
         a__ :str						  =    self.tokenizer(__A   ,     max_length=__A   ,     truncation=__A						).input_ids[0]
         self.assertEqual(ids[-1]   ,     2						)
         self.assertEqual(ids[0]   ,     __A						)
         self.assertEqual(len(__A						)   ,     __A						)




      def  _snake_case					(	self						:   str						)      ->str:






         """simple docstring"""






         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]						)   ,     [256203, 3]						)




      def  _snake_case					(	self						:   Dict						)      ->List[str]:






         """simple docstring"""






         a__ :Optional[Any]						  =    tempfile.mkdtemp()
         a__ :List[str]						  =    self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(__A						)
         a__ :Tuple						  =    NllbTokenizer.from_pretrained(__A						)
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,     __A						)




      @require_torch
      def  _snake_case					(	self						:   List[str]						)      ->Any:






         """simple docstring"""






         a__ :Dict						  =    self.tokenizer(
             self.src_text   ,     text_target=self.tgt_text   ,     padding=__A   ,     truncation=__A   ,     max_length=len(self.expected_src_tokens						)   ,     return_tensors="pt"   ,     )
         a__ :Optional[int]						  =    shift_tokens_right(
             batch["labels"]   ,     self.tokenizer.pad_token_id   ,     self.tokenizer.lang_code_to_id["ron_Latn"]						)

         self.assertIsInstance(__A   ,     __A						)

         self.assertEqual((2, 15)   ,     batch.input_ids.shape						)
         self.assertEqual((2, 15)   ,     batch.attention_mask.shape						)
         a__ :List[str]						  =    batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens   ,     __A						)
         self.assertEqual(__A   ,     batch.decoder_input_ids[0, 0]						)  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens   ,     [EN_CODE]						)
         self.assertEqual(self.tokenizer.suffix_tokens   ,     [self.tokenizer.eos_token_id]						)




      def  _snake_case					(	self						:   Union[str, Any]						)      ->Tuple:






         """simple docstring"""






         a__ :Tuple						  =    self.tokenizer(self.src_text   ,     padding=__A   ,     truncation=__A   ,     max_length=3   ,     return_tensors="pt"						)
         a__ :str						  =    self.tokenizer(
             text_target=self.tgt_text   ,     padding=__A   ,     truncation=__A   ,     max_length=10   ,     return_tensors="pt"						)
         a__ :Tuple						  =    targets["input_ids"]
         a__ :Dict						  =    shift_tokens_right(
             __A   ,     self.tokenizer.pad_token_id   ,     decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]   ,     )

         self.assertEqual(batch.input_ids.shape[1]   ,     3						)
         self.assertEqual(batch.decoder_input_ids.shape[1]   ,     10						)




      @require_torch
      def  _snake_case					(	self						:   Dict						)      ->Optional[int]:






         """simple docstring"""






         a__ :Any						  =    self.tokenizer._build_translation_inputs(
             "A test"   ,     return_tensors="pt"   ,     src_lang="eng_Latn"   ,     tgt_lang="fra_Latn"						)

         self.assertEqual(
             nested_simplify(__A						)   ,     {
                 # A, test, EOS, en_XX
                 "input_ids": [[256047, 70, 7356, 2]],
                 "attention_mask": [[1, 1, 1, 1]],
                 # ar_AR
                 "forced_bos_token_id": 256057,
             }   ,     )






      @require_torch
      def  _snake_case					(	self						:   Optional[int]						)      ->Dict:






         """simple docstring"""






         a__ :str						  =    True
         a__ :Optional[Any]						  =    self.tokenizer(
             "UN Chief says there is no military solution in Syria"   ,     src_lang="eng_Latn"   ,     tgt_lang="fra_Latn"						)
         self.assertEqual(
             inputs.input_ids   ,     [16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]						)

         a__ :List[Any]						  =    False
         a__ :str						  =    self.tokenizer(
             "UN Chief says there is no military solution in Syria"   ,     src_lang="eng_Latn"   ,     tgt_lang="fra_Latn"						)
         self.assertEqual(
             inputs.input_ids   ,     [256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]						)






def 				lowerCamelCase__	(      a	:   int	,  a	:   int				)    ->	Any:





			"""simple docstring"""
			# "extended trapezoidal rule"
			# int(f) = dx/2 * (f1 + 2f2 + ... + fn)
			a__ :Optional[int]						  =    (boundary[1] - boundary[0]) / steps
			a__ :str						  =    boundary[0]
			a__ :Optional[int]						  =    boundary[1]
			a__ :str						  =    make_points(a	,  a	,  a				)
			a__ :Any						  =    0.0
			y += (h / 2.0) * f(a				)
			for i in x_i:
						# print(i)
						y += h * f(a				)
			y += (h / 2.0) * f(a				)
			return y


def 				lowerCamelCase__	(      a	:   Any	,  a	:   str	,  a	:   Dict				)    ->	int:





			"""simple docstring"""
			a__ :Union[str, Any]						  =    a + h
			while x < (b - h):
						yield x
						a__ :str						  =    x + h


def 				lowerCamelCase__	(      a	:   Optional[int]				)    ->	List[Any]:  # enter your function here





			"""simple docstring"""
			a__ :Tuple						  =    (x - 0) * (x - 0)
			return y


def 				lowerCamelCase__	(      )    ->	List[str]:





			"""simple docstring"""
			a__ :Optional[Any]						  =    0.0  # Lower bound of integration
			a__ :Optional[Any]						  =    1.0  # Upper bound of integration
			a__ :List[str]						  =    1_0.0  # define number of steps or resolution
			a__ :int						  =    [a, b]  # define boundary of integration
			a__ :str						  =    method_a(a	,  a				)
			print(F'''y = {y}'''				)


if __name__ == "__main__":
		main()


