
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase					:			Optional[int]			      =		get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__lowerCAmelCase					:			List[Any]			      =		256_047
__lowerCAmelCase					:			Dict			      =		256_145


@require_sentencepiece
@require_tokenizers
class 						__lowerCAmelCase	(	lowerCAmelCase_ , unittest.TestCase      ):



      """simple docstring"""







      A__	:			Optional[Any]        =    NllbTokenizer
      A__	:			List[str]        =    NllbTokenizerFast
      A__	:			int        =    True
      A__	:			Any        =    True
      A__	:			int        =    {}




      def 		snake_case_			(      self		:       str				):
          super().setUp()

          # We have a SentencePiece fixture for testing
          __lowercase		:			List[str]	    =       NllbTokenizer(_lowerCAmelCase				,       keep_accents=_lowerCAmelCase				)
          tokenizer.save_pretrained(self.tmpdirname				)




      def 		snake_case_			(      self		:       List[Any]				):
          __lowercase		:			List[str]	    =       NllbTokenizer(_lowerCAmelCase				,       keep_accents=_lowerCAmelCase				)

          __lowercase		:			Dict	    =       tokenizer.tokenize('''This is a test'''				)
          self.assertListEqual(_lowerCAmelCase				,       ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(_lowerCAmelCase				)				,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]				,       )

          __lowercase		:			int	    =       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
          self.assertListEqual(
              _lowerCAmelCase				,       [
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''9''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''é''',
                  '''.''',
              ]				,       )
          __lowercase		:			Any	    =       tokenizer.convert_tokens_to_ids(_lowerCAmelCase				)
          self.assertListEqual(
              _lowerCAmelCase				,       [
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ]				,       )

          __lowercase		:			Optional[int]	    =       tokenizer.convert_ids_to_tokens(_lowerCAmelCase				)
          self.assertListEqual(
              _lowerCAmelCase				,       [
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''<unk>''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''<unk>''',
                  '''.''',
              ]				,       )




      def 		snake_case_			(      self		:       Optional[Any]				):
          __lowercase		:			Union[str, Any]	    =       (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
                  __lowercase		:			Union[str, Any]	    =       self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase				,       **_lowerCAmelCase				)
                  __lowercase		:			Optional[Any]	    =       self.tokenizer_class.from_pretrained(_lowerCAmelCase				,       **_lowerCAmelCase				)

                  __lowercase		:			Any	    =       tempfile.mkdtemp()

                  __lowercase		:			int	    =       tokenizer_r.save_pretrained(_lowerCAmelCase				)
                  __lowercase		:			Any	    =       tokenizer_p.save_pretrained(_lowerCAmelCase				)

                  # Checks it save with the same files + the tokenizer.json file for the fast one
                  self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)
                  __lowercase		:			Dict	    =       tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f				)
                  self.assertSequenceEqual(_lowerCAmelCase				,       _lowerCAmelCase				)

                  # Checks everything loads correctly in the same way
                  __lowercase		:			int	    =       tokenizer_r.from_pretrained(_lowerCAmelCase				)
                  __lowercase		:			List[Any]	    =       tokenizer_p.from_pretrained(_lowerCAmelCase				)

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                      self.assertTrue(hasattr(_lowerCAmelCase				,       _lowerCAmelCase				)				)

                  shutil.rmtree(_lowerCAmelCase				)

                  # Save tokenizer rust, legacy_format=True
                  __lowercase		:			str	    =       tempfile.mkdtemp()

                  __lowercase		:			List[str]	    =       tokenizer_r.save_pretrained(_lowerCAmelCase				,       legacy_format=_lowerCAmelCase				)
                  __lowercase		:			str	    =       tokenizer_p.save_pretrained(_lowerCAmelCase				)

                  # Checks it save with the same files
                  self.assertSequenceEqual(_lowerCAmelCase				,       _lowerCAmelCase				)

                  # Checks everything loads correctly in the same way
                  __lowercase		:			Optional[int]	    =       tokenizer_r.from_pretrained(_lowerCAmelCase				)
                  __lowercase		:			List[str]	    =       tokenizer_p.from_pretrained(_lowerCAmelCase				)

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                      self.assertTrue(hasattr(_lowerCAmelCase				,       _lowerCAmelCase				)				)

                  shutil.rmtree(_lowerCAmelCase				)

                  # Save tokenizer rust, legacy_format=False
                  __lowercase		:			int	    =       tempfile.mkdtemp()

                  __lowercase		:			Any	    =       tokenizer_r.save_pretrained(_lowerCAmelCase				,       legacy_format=_lowerCAmelCase				)
                  __lowercase		:			Any	    =       tokenizer_p.save_pretrained(_lowerCAmelCase				)

                  # Checks it saved the tokenizer.json file
                  self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)

                  # Checks everything loads correctly in the same way
                  __lowercase		:			Any	    =       tokenizer_r.from_pretrained(_lowerCAmelCase				)
                  __lowercase		:			List[str]	    =       tokenizer_p.from_pretrained(_lowerCAmelCase				)

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                      self.assertTrue(hasattr(_lowerCAmelCase				,       _lowerCAmelCase				)				)

                  shutil.rmtree(_lowerCAmelCase				)




      @require_torch
      def 		snake_case_			(      self		:       Union[str, Any]				):
          if not self.test_seqaseq:
              return

          __lowercase		:			Optional[int]	    =       self.get_tokenizers()
          for tokenizer in tokenizers:
              with self.subTest(F'{tokenizer.__class__.__name__}'				):
                  # Longer text that will definitely require truncation.
                  __lowercase		:			Optional[Any]	    =       [
                      ''' UN Chief Says There Is No Military Solution in Syria''',
                      ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                      ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                      ''' will only worsen the violence and misery for millions of people.''',
                  ]
                  __lowercase		:			str	    =       [
                      '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                      '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                      ''' Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi'''
                      ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
                  ]
                  try:
                      __lowercase		:			str	    =       tokenizer.prepare_seqaseq_batch(
                          src_texts=_lowerCAmelCase				,       tgt_texts=_lowerCAmelCase				,       max_length=3				,       max_target_length=10				,       return_tensors='''pt'''				,       src_lang='''eng_Latn'''				,       tgt_lang='''ron_Latn'''				,       )
                  except NotImplementedError:
                      return
                  self.assertEqual(batch.input_ids.shape[1]				,       3				)
                  self.assertEqual(batch.labels.shape[1]				,       10				)
                  # max_target_length will default to max_length if not specified
                  __lowercase		:			List[Any]	    =       tokenizer.prepare_seqaseq_batch(
                      _lowerCAmelCase				,       tgt_texts=_lowerCAmelCase				,       max_length=3				,       return_tensors='''pt'''				)
                  self.assertEqual(batch.input_ids.shape[1]				,       3				)
                  self.assertEqual(batch.labels.shape[1]				,       3				)

                  __lowercase		:			List[Any]	    =       tokenizer.prepare_seqaseq_batch(
                      src_texts=_lowerCAmelCase				,       max_length=3				,       max_target_length=10				,       return_tensors='''pt'''				)
                  self.assertEqual(batch_encoder_only.input_ids.shape[1]				,       3				)
                  self.assertEqual(batch_encoder_only.attention_mask.shape[1]				,       3				)
                  self.assertNotIn('''decoder_input_ids'''				,       _lowerCAmelCase				)




      @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''				)
      def 		snake_case_			(      self		:       int				):
          pass




      def 		snake_case_			(      self		:       str				):
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
                  __lowercase		:			List[Any]	    =       [AddedToken('''<special>'''				,       lstrip=_lowerCAmelCase				)]

                  __lowercase		:			Tuple	    =       self.rust_tokenizer_class.from_pretrained(
                      _lowerCAmelCase				,       additional_special_tokens=_lowerCAmelCase				,       **_lowerCAmelCase				)
                  __lowercase		:			Tuple	    =       tokenizer_r.encode('''Hey this is a <special> token'''				)

                  __lowercase		:			Optional[Any]	    =       tokenizer_r.encode('''<special>'''				,       add_special_tokens=_lowerCAmelCase				)[0]

                  self.assertTrue(special_token_id in r_output				)

                  if self.test_slow_tokenizer:
                      __lowercase		:			Tuple	    =       self.rust_tokenizer_class.from_pretrained(
                          _lowerCAmelCase				,       additional_special_tokens=_lowerCAmelCase				,       **_lowerCAmelCase				,       )
                      __lowercase		:			str	    =       self.tokenizer_class.from_pretrained(
                          _lowerCAmelCase				,       additional_special_tokens=_lowerCAmelCase				,       **_lowerCAmelCase				)

                      __lowercase		:			Any	    =       tokenizer_p.encode('''Hey this is a <special> token'''				)

                      __lowercase		:			List[Any]	    =       tokenizer_cr.encode('''Hey this is a <special> token'''				)

                      self.assertEqual(_lowerCAmelCase				,       _lowerCAmelCase				)
                      self.assertEqual(_lowerCAmelCase				,       _lowerCAmelCase				)
                      self.assertTrue(special_token_id in p_output				)
                      self.assertTrue(special_token_id in cr_output				)






@require_torch
@require_sentencepiece
@require_tokenizers
class 						__lowerCAmelCase	(	unittest.TestCase      ):



      """simple docstring"""







      A__	:			int        =    '''facebook/nllb-200-distilled-600M'''
      A__	:			Dict        =    [
          ''' UN Chief Says There Is No Military Solution in Syria''',
          ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
      ]
      A__	:			int        =    [
          '''Şeful ONU declară că nu există o soluţie militară în Siria''',
          '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
          ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
          ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
      ]
      A__	:			Optional[Any]        =    [
          2_5_6_0_4_7,
          1_6_2_9_7,
          1_3_4_4_0_8,
          8_1_6_5,
          2_4_8_0_6_6,
          1_4_7_3_4,
          9_5_0,
          1_1_3_5,
          1_0_5_7_2_1,
          3_5_7_3,
          8_3,
          2_7_3_5_2,
          1_0_8,
          4_9_4_8_6,
          2,
      ]




      @classmethod
      def 		snake_case_			(      cls		:       Dict				):
          __lowercase		:			str	    =       NllbTokenizer.from_pretrained(
              cls.checkpoint_name				,       src_lang='''eng_Latn'''				,       tgt_lang='''ron_Latn'''				)
          __lowercase		:			Any	    =       1
          return cls




      def 		snake_case_			(      self		:       List[str]				):
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']				,       25_6001				)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']				,       25_6002				)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']				,       25_6057				)




      def 		snake_case_			(      self		:       Optional[Any]				):
          __lowercase		:			Union[str, Any]	    =       self.tokenizer.batch_encode_plus(self.src_text				).input_ids[0]
          self.assertListEqual(self.expected_src_tokens				,       _lowerCAmelCase				)




      def 		snake_case_			(      self		:       Optional[int]				):
          self.assertIn(_lowerCAmelCase				,       self.tokenizer.all_special_ids				)
          # fmt: off
          __lowercase		:			Optional[int]	    =       [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
          # fmt: on

          __lowercase		:			Any	    =       self.tokenizer.decode(_lowerCAmelCase				,       skip_special_tokens=_lowerCAmelCase				)
          __lowercase		:			Optional[int]	    =       self.tokenizer.decode(generated_ids[1:]				,       skip_special_tokens=_lowerCAmelCase				)
          self.assertEqual(_lowerCAmelCase				,       _lowerCAmelCase				)
          self.assertNotIn(self.tokenizer.eos_token				,       _lowerCAmelCase				)




      def 		snake_case_			(      self		:       Any				):
          __lowercase		:			Optional[int]	    =       ['''this is gunna be a long sentence ''' * 20]
          assert isinstance(src_text[0]				,       _lowerCAmelCase				)
          __lowercase		:			List[Any]	    =       10
          __lowercase		:			List[Any]	    =       self.tokenizer(_lowerCAmelCase				,       max_length=_lowerCAmelCase				,       truncation=_lowerCAmelCase				).input_ids[0]
          self.assertEqual(ids[-1]				,       2				)
          self.assertEqual(ids[0]				,       _lowerCAmelCase				)
          self.assertEqual(len(_lowerCAmelCase				)				,       _lowerCAmelCase				)




      def 		snake_case_			(      self		:       List[Any]				):
          self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']				)				,       [25_6203, 3]				)




      def 		snake_case_			(      self		:       List[Any]				):
          __lowercase		:			List[Any]	    =       tempfile.mkdtemp()
          __lowercase		:			List[str]	    =       self.tokenizer.fairseq_tokens_to_ids
          self.tokenizer.save_pretrained(_lowerCAmelCase				)
          __lowercase		:			Union[str, Any]	    =       NllbTokenizer.from_pretrained(_lowerCAmelCase				)
          self.assertDictEqual(new_tok.fairseq_tokens_to_ids				,       _lowerCAmelCase				)




      @require_torch
      def 		snake_case_			(      self		:       List[Any]				):
          __lowercase		:			int	    =       self.tokenizer(
              self.src_text				,       text_target=self.tgt_text				,       padding=_lowerCAmelCase				,       truncation=_lowerCAmelCase				,       max_length=len(self.expected_src_tokens				)				,       return_tensors='''pt'''				,       )
          __lowercase		:			Optional[Any]	    =       shift_tokens_right(
              batch['''labels''']				,       self.tokenizer.pad_token_id				,       self.tokenizer.lang_code_to_id['''ron_Latn''']				)

          self.assertIsInstance(_lowerCAmelCase				,       _lowerCAmelCase				)

          self.assertEqual((2, 15)				,       batch.input_ids.shape				)
          self.assertEqual((2, 15)				,       batch.attention_mask.shape				)
          __lowercase		:			Tuple	    =       batch.input_ids.tolist()[0]
          self.assertListEqual(self.expected_src_tokens				,       _lowerCAmelCase				)
          self.assertEqual(_lowerCAmelCase				,       batch.decoder_input_ids[0, 0]				)  # EOS
          # Test that special tokens are reset
          self.assertEqual(self.tokenizer.prefix_tokens				,       [EN_CODE]				)
          self.assertEqual(self.tokenizer.suffix_tokens				,       [self.tokenizer.eos_token_id]				)




      def 		snake_case_			(      self		:       Optional[int]				):
          __lowercase		:			Tuple	    =       self.tokenizer(self.src_text				,       padding=_lowerCAmelCase				,       truncation=_lowerCAmelCase				,       max_length=3				,       return_tensors='''pt'''				)
          __lowercase		:			List[str]	    =       self.tokenizer(
              text_target=self.tgt_text				,       padding=_lowerCAmelCase				,       truncation=_lowerCAmelCase				,       max_length=10				,       return_tensors='''pt'''				)
          __lowercase		:			List[Any]	    =       targets['''input_ids''']
          __lowercase		:			Tuple	    =       shift_tokens_right(
              _lowerCAmelCase				,       self.tokenizer.pad_token_id				,       decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]				,       )

          self.assertEqual(batch.input_ids.shape[1]				,       3				)
          self.assertEqual(batch.decoder_input_ids.shape[1]				,       10				)




      @require_torch
      def 		snake_case_			(      self		:       Any				):
          __lowercase		:			Optional[int]	    =       self.tokenizer._build_translation_inputs(
              '''A test'''				,       return_tensors='''pt'''				,       src_lang='''eng_Latn'''				,       tgt_lang='''fra_Latn'''				)

          self.assertEqual(
              nested_simplify(_lowerCAmelCase				)				,       {
                  # A, test, EOS, en_XX
                  '''input_ids''': [[25_6047, 70, 7356, 2]],
                  '''attention_mask''': [[1, 1, 1, 1]],
                  # ar_AR
                  '''forced_bos_token_id''': 25_6057,
              }				,       )




      @require_torch
      def 		snake_case_			(      self		:       str				):
          __lowercase		:			str	    =       True
          __lowercase		:			Dict	    =       self.tokenizer(
              '''UN Chief says there is no military solution in Syria'''				,       src_lang='''eng_Latn'''				,       tgt_lang='''fra_Latn'''				)
          self.assertEqual(
              inputs.input_ids				,       [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]				)

          __lowercase		:			Any	    =       False
          __lowercase		:			Any	    =       self.tokenizer(
              '''UN Chief says there is no military solution in Syria'''				,       src_lang='''eng_Latn'''				,       tgt_lang='''fra_Latn'''				)
          self.assertEqual(
              inputs.input_ids				,       [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]				)




from __future__ import annotations


def     UpperCAmelCase_  (						__lowerCAmelCase	) ->						int:
    if not nums:
        return 0
    __lowercase		:			List[Any]	    =       nums[0]
    __lowercase		:			Union[str, Any]	    =       0
    for num in nums[1:]:
        __lowercase    ,      __lowercase		:			Dict	    =       (
            max_excluding + num,
            max(__lowerCAmelCase	,					__lowerCAmelCase	),
        )
    return max(__lowerCAmelCase	,					__lowerCAmelCase	)


if __name__ == "__main__":
 import doctest

 doctest.testmod()


