






"""simple docstring"""






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_     :				List[Any]         =				get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase_     :				List[Any]         =				256047
UpperCAmelCase_     :				Dict         =				256145




@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase__       (    UpperCAmelCase__       ,	unittest.TestCase				):






      '''simple docstring'''

      __UpperCamelCase							=						NllbTokenizer
      __UpperCamelCase							=						NllbTokenizerFast
      __UpperCamelCase							=						True
      __UpperCamelCase							=						True
      __UpperCamelCase							=						{}
      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]):




          '''simple docstring'''


          super().setUp()

          # We have a SentencePiece fixture for testing
          SCREAMING_SNAKE_CASE_   :						int							=	NllbTokenizer(lowercase_		,			keep_accents=lowercase_)
          tokenizer.save_pretrained(self.tmpdirname)
      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	NllbTokenizer(lowercase_		,			keep_accents=lowercase_)

          SCREAMING_SNAKE_CASE_   :						int							=	tokenizer.tokenize('''This is a test''')
          self.assertListEqual(lowercase_		,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(lowercase_)		,			[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]		,			)

          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
          self.assertListEqual(
              lowercase_		,			[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''9''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''é''',
                  '''.''',
              ]		,			)
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	tokenizer.convert_tokens_to_ids(lowercase_)
          self.assertListEqual(
              lowercase_		,			[
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ]		,			)

          SCREAMING_SNAKE_CASE_   :						List[Any]							=	tokenizer.convert_ids_to_tokens(lowercase_)
          self.assertListEqual(
              lowercase_		,			[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''<unk>''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''<unk>''',
                  '''.''',
              ]		,			)
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                  SCREAMING_SNAKE_CASE_   :						Tuple							=	self.rust_tokenizer_class.from_pretrained(lowercase_		,			**lowercase_)
                  SCREAMING_SNAKE_CASE_   :						List[Any]							=	self.tokenizer_class.from_pretrained(lowercase_		,			**lowercase_)

                  SCREAMING_SNAKE_CASE_   :						Optional[int]							=	tempfile.mkdtemp()

                  SCREAMING_SNAKE_CASE_   :						Tuple							=	tokenizer_r.save_pretrained(lowercase_)
                  SCREAMING_SNAKE_CASE_   :						Dict							=	tokenizer_p.save_pretrained(lowercase_)

                  # Checks it save with the same files + the tokenizer.json file for the fast one
                  self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files))
                  SCREAMING_SNAKE_CASE_   :						Tuple							=	tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f)
                  self.assertSequenceEqual(lowercase_		,			lowercase_)

                  # Checks everything loads correctly in the same way
                  SCREAMING_SNAKE_CASE_   :						str							=	tokenizer_r.from_pretrained(lowercase_)
                  SCREAMING_SNAKE_CASE_   :						Tuple							=	tokenizer_p.from_pretrained(lowercase_)

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                      self.assertTrue(hasattr(lowercase_		,			lowercase_))

                  shutil.rmtree(lowercase_)

                  # Save tokenizer rust, legacy_format=True
                  SCREAMING_SNAKE_CASE_   :						Optional[int]							=	tempfile.mkdtemp()

                  SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	tokenizer_r.save_pretrained(lowercase_		,			legacy_format=lowercase_)
                  SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	tokenizer_p.save_pretrained(lowercase_)

                  # Checks it save with the same files
                  self.assertSequenceEqual(lowercase_		,			lowercase_)

                  # Checks everything loads correctly in the same way
                  SCREAMING_SNAKE_CASE_   :						Optional[int]							=	tokenizer_r.from_pretrained(lowercase_)
                  SCREAMING_SNAKE_CASE_   :						Tuple							=	tokenizer_p.from_pretrained(lowercase_)

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                      self.assertTrue(hasattr(lowercase_		,			lowercase_))

                  shutil.rmtree(lowercase_)

                  # Save tokenizer rust, legacy_format=False
                  SCREAMING_SNAKE_CASE_   :						List[Any]							=	tempfile.mkdtemp()

                  SCREAMING_SNAKE_CASE_   :						Dict							=	tokenizer_r.save_pretrained(lowercase_		,			legacy_format=lowercase_)
                  SCREAMING_SNAKE_CASE_   :						Any							=	tokenizer_p.save_pretrained(lowercase_)

                  # Checks it saved the tokenizer.json file
                  self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files))

                  # Checks everything loads correctly in the same way
                  SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	tokenizer_r.from_pretrained(lowercase_)
                  SCREAMING_SNAKE_CASE_   :						Tuple							=	tokenizer_p.from_pretrained(lowercase_)

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                      self.assertTrue(hasattr(lowercase_		,			lowercase_))

                  shutil.rmtree(lowercase_)
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]):




          '''simple docstring'''


          if not self.test_seqaseq:
              return

          SCREAMING_SNAKE_CASE_   :						Any							=	self.get_tokenizers()
          for tokenizer in tokenizers:
              with self.subTest(F'{tokenizer.__class__.__name__}'):
                  # Longer text that will definitely require truncation.
                  SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	[
                      ''' UN Chief Says There Is No Military Solution in Syria''',
                      ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                      ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                      ''' will only worsen the violence and misery for millions of people.''',
                  ]
                  SCREAMING_SNAKE_CASE_   :						int							=	[
                      '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                      '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                      ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                      ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
                  ]
                  try:
                      SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	tokenizer.prepare_seqaseq_batch(
                          src_texts=lowercase_		,			tgt_texts=lowercase_		,			max_length=3		,			max_target_length=10		,			return_tensors='''pt'''		,			src_lang='''eng_Latn'''		,			tgt_lang='''ron_Latn'''		,			)
                  except NotImplementedError:
                      return
                  self.assertEqual(batch.input_ids.shape[1]		,			3)
                  self.assertEqual(batch.labels.shape[1]		,			10)
                  # max_target_length will default to max_length if not specified
                  SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	tokenizer.prepare_seqaseq_batch(
                      lowercase_		,			tgt_texts=lowercase_		,			max_length=3		,			return_tensors='''pt''')
                  self.assertEqual(batch.input_ids.shape[1]		,			3)
                  self.assertEqual(batch.labels.shape[1]		,			3)

                  SCREAMING_SNAKE_CASE_   :						int							=	tokenizer.prepare_seqaseq_batch(
                      src_texts=lowercase_		,			max_length=3		,			max_target_length=10		,			return_tensors='''pt''')
                  self.assertEqual(batch_encoder_only.input_ids.shape[1]		,			3)
                  self.assertEqual(batch_encoder_only.attention_mask.shape[1]		,			3)
                  self.assertNotIn('''decoder_input_ids'''		,			lowercase_)
      @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.''')
      def _SCREAMING_SNAKE_CASE				(		self  :						List[Any]):




          '''simple docstring'''


          pass



      def _SCREAMING_SNAKE_CASE				(		self  :						str):




          '''simple docstring'''


          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                  SCREAMING_SNAKE_CASE_   :						Optional[int]							=	[AddedToken('''<special>'''		,			lstrip=lowercase_)]

                  SCREAMING_SNAKE_CASE_   :						Dict							=	self.rust_tokenizer_class.from_pretrained(
                      lowercase_		,			additional_special_tokens=lowercase_		,			**lowercase_)
                  SCREAMING_SNAKE_CASE_   :						Tuple							=	tokenizer_r.encode('''Hey this is a <special> token''')

                  SCREAMING_SNAKE_CASE_   :						Any							=	tokenizer_r.encode('''<special>'''		,			add_special_tokens=lowercase_)[0]

                  self.assertTrue(special_token_id in r_output)

                  if self.test_slow_tokenizer:
                      SCREAMING_SNAKE_CASE_   :						List[Any]							=	self.rust_tokenizer_class.from_pretrained(
                          lowercase_		,			additional_special_tokens=lowercase_		,			**lowercase_		,			)
                      SCREAMING_SNAKE_CASE_   :						int							=	self.tokenizer_class.from_pretrained(
                          lowercase_		,			additional_special_tokens=lowercase_		,			**lowercase_)

                      SCREAMING_SNAKE_CASE_   :						Dict							=	tokenizer_p.encode('''Hey this is a <special> token''')

                      SCREAMING_SNAKE_CASE_   :						Any							=	tokenizer_cr.encode('''Hey this is a <special> token''')

                      self.assertEqual(lowercase_		,			lowercase_)
                      self.assertEqual(lowercase_		,			lowercase_)
                      self.assertTrue(special_token_id in p_output)
                      self.assertTrue(special_token_id in cr_output)




@require_torch
@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase__       (    unittest.TestCase				):






      '''simple docstring'''

      __UpperCamelCase							=						"facebook/nllb-200-distilled-600M"
      __UpperCamelCase							=						[
          " UN Chief Says There Is No Military Solution in Syria",
          " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
      ]
      __UpperCamelCase							=						[
          "Şeful ONU declară că nu există o soluţie militară în Siria",
          "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
          " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
          " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
      ]
      __UpperCamelCase							=						[
          2_5_6_0_4_7,
          1_6_2_9_7,
          1_3_4_4_0_8,
          8_1_6_5,
          2_4_8_0_6_6,
          1_4_7_3_4,
          9_5_0,
          1_1_3_5,
          1_0_5_7_2_1,
          3_5_7_3,
          8_3,
          2_7_3_5_2,
          1_0_8,
          4_9_4_8_6,
          2,
      ]
      @classmethod
      def _SCREAMING_SNAKE_CASE				(		cls  :						Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						NllbTokenizer							=	NllbTokenizer.from_pretrained(
              cls.checkpoint_name		,			src_lang='''eng_Latn'''		,			tgt_lang='''ron_Latn''')
          SCREAMING_SNAKE_CASE_   :						Any							=	1
          return cls
      def _SCREAMING_SNAKE_CASE				(		self  :						str):




          '''simple docstring'''


          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']		,			256001)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']		,			256002)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']		,			256057)
      def _SCREAMING_SNAKE_CASE				(		self  :						List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
          self.assertListEqual(self.expected_src_tokens		,			lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          self.assertIn(lowercase_		,			self.tokenizer.all_special_ids)
          # fmt: off
          SCREAMING_SNAKE_CASE_   :						List[str]							=	[RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
          # fmt: on

          SCREAMING_SNAKE_CASE_   :						Dict							=	self.tokenizer.decode(lowercase_		,			skip_special_tokens=lowercase_)
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	self.tokenizer.decode(generated_ids[1:]		,			skip_special_tokens=lowercase_)
          self.assertEqual(lowercase_		,			lowercase_)
          self.assertNotIn(self.tokenizer.eos_token		,			lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	['''this is gunna be a long sentence ''' * 20]
          assert isinstance(src_text[0]		,			lowercase_)
          SCREAMING_SNAKE_CASE_   :						int							=	10
          SCREAMING_SNAKE_CASE_   :						Tuple							=	self.tokenizer(lowercase_		,			max_length=lowercase_		,			truncation=lowercase_).input_ids[0]
          self.assertEqual(ids[-1]		,			2)
          self.assertEqual(ids[0]		,			lowercase_)
          self.assertEqual(len(lowercase_)		,			lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						int):




          '''simple docstring'''


          self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR'''])		,			[256203, 3])
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[str]							=	tempfile.mkdtemp()
          SCREAMING_SNAKE_CASE_   :						List[str]							=	self.tokenizer.fairseq_tokens_to_ids
          self.tokenizer.save_pretrained(lowercase_)
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	NllbTokenizer.from_pretrained(lowercase_)
          self.assertDictEqual(new_tok.fairseq_tokens_to_ids		,			lowercase_)
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	self.tokenizer(
              self.src_text		,			text_target=self.tgt_text		,			padding=lowercase_		,			truncation=lowercase_		,			max_length=len(self.expected_src_tokens)		,			return_tensors='''pt'''		,			)
          SCREAMING_SNAKE_CASE_   :						int							=	shift_tokens_right(
              batch['''labels''']		,			self.tokenizer.pad_token_id		,			self.tokenizer.lang_code_to_id['''ron_Latn'''])

          self.assertIsInstance(lowercase_		,			lowercase_)

          self.assertEqual((2, 15)		,			batch.input_ids.shape)
          self.assertEqual((2, 15)		,			batch.attention_mask.shape)
          SCREAMING_SNAKE_CASE_   :						Any							=	batch.input_ids.tolist()[0]
          self.assertListEqual(self.expected_src_tokens		,			lowercase_)
          self.assertEqual(lowercase_		,			batch.decoder_input_ids[0, 0])  # EOS
          # Test that special tokens are reset
          self.assertEqual(self.tokenizer.prefix_tokens		,			[EN_CODE])
          self.assertEqual(self.tokenizer.suffix_tokens		,			[self.tokenizer.eos_token_id])
      def _SCREAMING_SNAKE_CASE				(		self  :						Union[str, Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	self.tokenizer(self.src_text		,			padding=lowercase_		,			truncation=lowercase_		,			max_length=3		,			return_tensors='''pt''')
          SCREAMING_SNAKE_CASE_   :						List[str]							=	self.tokenizer(
              text_target=self.tgt_text		,			padding=lowercase_		,			truncation=lowercase_		,			max_length=10		,			return_tensors='''pt''')
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	targets['''input_ids''']
          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	shift_tokens_right(
              lowercase_		,			self.tokenizer.pad_token_id		,			decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]		,			)

          self.assertEqual(batch.input_ids.shape[1]		,			3)
          self.assertEqual(batch.decoder_input_ids.shape[1]		,			10)
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	self.tokenizer._build_translation_inputs(
              '''A test'''		,			return_tensors='''pt'''		,			src_lang='''eng_Latn'''		,			tgt_lang='''fra_Latn''')

          self.assertEqual(
              nested_simplify(lowercase_)		,			{
                  # A, test, EOS, en_XX
                  '''input_ids''': [[256047, 70, 7356, 2]],
                  '''attention_mask''': [[1, 1, 1, 1]],
                  # ar_AR
                  '''forced_bos_token_id''': 256057,
              }		,			)



      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Dict							=	True
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	self.tokenizer(
              '''UN Chief says there is no military solution in Syria'''		,			src_lang='''eng_Latn'''		,			tgt_lang='''fra_Latn''')
          self.assertEqual(
              inputs.input_ids		,			[16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047])

          SCREAMING_SNAKE_CASE_   :						Dict							=	False
          SCREAMING_SNAKE_CASE_   :						Any							=	self.tokenizer(
              '''UN Chief says there is no military solution in Syria'''		,			src_lang='''eng_Latn'''		,			tgt_lang='''fra_Latn''')
          self.assertEqual(
              inputs.input_ids		,			[256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2])










"""simple docstring"""






from __future__ import annotations

import requests



def       _A    (__a							)				-> dict:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_   :						Any							=	f'https://hacker-news.firebaseio.com/v0/item/{story_id}.json?print=pretty'
    return requests.get(__a							).json()



def       _A    (__a = 10							)				-> list[dict]:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_   :						Any							=	'''https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty'''
    SCREAMING_SNAKE_CASE_   :						Dict							=	requests.get(__a							).json()[:max_stories]
    return [get_hackernews_story(__a							) for story_id in story_ids]



def       _A    (__a = 10							)				-> str:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_   :						Tuple							=	hackernews_top_stories(__a							)
    return "\n".join('''* [{title}]({url})'''.format(**__a							) for story in stories							)


if __name__ == "__main__":
  print(hackernews_top_stories_as_markdown())


