
"""simple docstring"""




import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


A__ :       Union[str, Any]				= get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

A__ :       Union[str, Any]				= 2_5_6_0_4_7
A__ :       Optional[Any]				= 2_5_6_1_4_5


@require_sentencepiece
@require_tokenizers
class       __magic_name__							(							SCREAMING_SNAKE_CASE__ ,       unittest.TestCase			):
       UpperCamelCase_ =		NllbTokenizer
       UpperCamelCase_ =		NllbTokenizerFast
       UpperCamelCase_ =		True
       UpperCamelCase_ =		True
       UpperCamelCase_ =		{}
       def 				lowercase_				(  self							)  ->						Optional[Any]:





         """simple docstring"""




         super().setUp()

         # We have a SentencePiece fixture for testing
         _lowercase:      Optional[int]   			=  NllbTokenizer(A_			,					keep_accents=A_							)
         tokenizer.save_pretrained(self.tmpdirname							)
       def 				lowercase_				(  self							)  ->						Optional[Any]:





         """simple docstring"""




         _lowercase:      Dict   			=  NllbTokenizer(A_			,					keep_accents=A_							)

         _lowercase:      Any   			=  tokenizer.tokenize('''This is a test'''							)
         self.assertListEqual(A_			,					['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(A_							)			,					[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,					)

         _lowercase:      Union[str, Any]   			=  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
         self.assertListEqual(
             A_			,					[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''9''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''é''',
                 '''.''',
             ]			,					)
         _lowercase:      List[Any]   			=  tokenizer.convert_tokens_to_ids(A_							)
         self.assertListEqual(
             A_			,					[
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
             ]			,					)

         _lowercase:      int   			=  tokenizer.convert_ids_to_tokens(A_							)
         self.assertListEqual(
             A_			,					[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''<unk>''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''<unk>''',
                 '''.''',
             ]			,					)
       def 				lowercase_				(  self							)  ->						List[str]:





         """simple docstring"""




         _lowercase:      List[Any]   			=  (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             _lowercase:      List[Any]   			=  self.rust_tokenizer_class.from_pretrained(A_			,					**A_							)
             _lowercase:      str   			=  self.tokenizer_class.from_pretrained(A_			,					**A_							)

             _lowercase:      Dict   			=  tempfile.mkdtemp()

             _lowercase:      int   			=  tokenizer_r.save_pretrained(A_							)
             _lowercase:      int   			=  tokenizer_p.save_pretrained(A_							)

             # Checks it save with the same files + the tokenizer.json file for the fast one
             self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)
             _lowercase:      str   			=  tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f							)
             self.assertSequenceEqual(A_			,					A_							)

             # Checks everything loads correctly in the same way
             _lowercase:      List[Any]   			=  tokenizer_r.from_pretrained(A_							)
             _lowercase:      Any   			=  tokenizer_p.from_pretrained(A_							)

             # Check special tokens are set accordingly on Rust and Python
             for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(A_			,					A_							)							)

             shutil.rmtree(A_							)

             # Save tokenizer rust, legacy_format=True
             _lowercase:      Tuple   			=  tempfile.mkdtemp()

             _lowercase:      Dict   			=  tokenizer_r.save_pretrained(A_			,					legacy_format=A_							)
             _lowercase:      List[str]   			=  tokenizer_p.save_pretrained(A_							)

             # Checks it save with the same files
             self.assertSequenceEqual(A_			,					A_							)

             # Checks everything loads correctly in the same way
             _lowercase:      Optional[Any]   			=  tokenizer_r.from_pretrained(A_							)
             _lowercase:      int   			=  tokenizer_p.from_pretrained(A_							)

             # Check special tokens are set accordingly on Rust and Python
             for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(A_			,					A_							)							)

             shutil.rmtree(A_							)

             # Save tokenizer rust, legacy_format=False
             _lowercase:      Any   			=  tempfile.mkdtemp()

             _lowercase:      str   			=  tokenizer_r.save_pretrained(A_			,					legacy_format=A_							)
             _lowercase:      Any   			=  tokenizer_p.save_pretrained(A_							)

             # Checks it saved the tokenizer.json file
             self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)

             # Checks everything loads correctly in the same way
             _lowercase:      Any   			=  tokenizer_r.from_pretrained(A_							)
             _lowercase:      int   			=  tokenizer_p.from_pretrained(A_							)

             # Check special tokens are set accordingly on Rust and Python
             for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(A_			,					A_							)							)

             shutil.rmtree(A_							)
       @require_torch
       def 				lowercase_				(  self							)  ->						Optional[int]:





         """simple docstring"""




         if not self.test_seqaseq:
           return

         _lowercase:      List[str]   			=  self.get_tokenizers()
         for tokenizer in tokenizers:
           with self.subTest(f'''{tokenizer.__class__.__name__}'''							):
             # Longer text that will definitely require truncation.
             _lowercase:      Dict   			=  [
                 ''' UN Chief Says There Is No Military Solution in Syria''',
                 ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                 ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                 ''' will only worsen the violence and misery for millions of people.''',
             ]
             _lowercase:      List[str]   			=  [
                 '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                 '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                 ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                 ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
             ]
             try:
               _lowercase:      Union[str, Any]   			=  tokenizer.prepare_seqaseq_batch(
                   src_texts=A_			,					tgt_texts=A_			,					max_length=3			,					max_target_length=10			,					return_tensors='''pt'''			,					src_lang='''eng_Latn'''			,					tgt_lang='''ron_Latn'''			,					)
             except NotImplementedError:
               return
             self.assertEqual(batch.input_ids.shape[1]			,					3							)
             self.assertEqual(batch.labels.shape[1]			,					10							)
             # max_target_length will default to max_length if not specified
             _lowercase:      Optional[int]   			=  tokenizer.prepare_seqaseq_batch(
                 A_			,					tgt_texts=A_			,					max_length=3			,					return_tensors='''pt'''							)
             self.assertEqual(batch.input_ids.shape[1]			,					3							)
             self.assertEqual(batch.labels.shape[1]			,					3							)

             _lowercase:      Any   			=  tokenizer.prepare_seqaseq_batch(
                 src_texts=A_			,					max_length=3			,					max_target_length=10			,					return_tensors='''pt'''							)
             self.assertEqual(batch_encoder_only.input_ids.shape[1]			,					3							)
             self.assertEqual(batch_encoder_only.attention_mask.shape[1]			,					3							)
             self.assertNotIn('''decoder_input_ids'''			,					A_							)
       @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''							)
       def 				lowercase_				(  self							)  ->						int:





         """simple docstring"""




         pass



       def 				lowercase_				(  self							)  ->						Any:





         """simple docstring"""




         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             _lowercase:      Optional[int]   			=  [AddedToken('''<special>'''			,					lstrip=A_							)]

             _lowercase:      Union[str, Any]   			=  self.rust_tokenizer_class.from_pretrained(
                 A_			,					additional_special_tokens=A_			,					**A_							)
             _lowercase:      str   			=  tokenizer_r.encode('''Hey this is a <special> token'''							)

             _lowercase:      List[Any]   			=  tokenizer_r.encode('''<special>'''			,					add_special_tokens=A_							)[0]

             self.assertTrue(special_token_id in r_output							)

             if self.test_slow_tokenizer:
               _lowercase:      Optional[Any]   			=  self.rust_tokenizer_class.from_pretrained(
                   A_			,					additional_special_tokens=A_			,					**A_			,					)
               _lowercase:      List[str]   			=  self.tokenizer_class.from_pretrained(
                   A_			,					additional_special_tokens=A_			,					**A_							)

               _lowercase:      Any   			=  tokenizer_p.encode('''Hey this is a <special> token'''							)

               _lowercase:      Optional[Any]   			=  tokenizer_cr.encode('''Hey this is a <special> token'''							)

               self.assertEqual(A_			,					A_							)
               self.assertEqual(A_			,					A_							)
               self.assertTrue(special_token_id in p_output							)
               self.assertTrue(special_token_id in cr_output							)


@require_torch
@require_sentencepiece
@require_tokenizers
class       __magic_name__							(							unittest.TestCase			):
       UpperCamelCase_ =		'''facebook/nllb-200-distilled-600M'''
       UpperCamelCase_ =		[
           ''' UN Chief Says There Is No Military Solution in Syria''',
           ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
       ]
       UpperCamelCase_ =		[
           '''Şeful ONU declară că nu există o soluţie militară în Siria''',
           '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
           ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
           ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
       ]
       UpperCamelCase_ =		[
           25_6047,
           1_6297,
           13_4408,
           8165,
           24_8066,
           1_4734,
           950,
           1135,
           10_5721,
           3573,
           83,
           2_7352,
           108,
           4_9486,
           2,
       ]
       @classmethod
       def 				lowercase_				(  cls							)  ->						Union[str, Any]:





         """simple docstring"""




         _lowercase:      NllbTokenizer   			=  NllbTokenizer.from_pretrained(
             cls.checkpoint_name			,					src_lang='''eng_Latn'''			,					tgt_lang='''ron_Latn'''							)
         _lowercase:      List[str]   			=  1
         return cls
       def 				lowercase_				(  self							)  ->						Dict:





         """simple docstring"""




         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']			,					25_6001							)
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']			,					25_6002							)
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']			,					25_6057							)
       def 				lowercase_				(  self							)  ->						Optional[int]:





         """simple docstring"""




         _lowercase:      Tuple   			=  self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
         self.assertListEqual(self.expected_src_tokens			,					A_							)
       def 				lowercase_				(  self							)  ->						Optional[Any]:





         """simple docstring"""




         self.assertIn(A_			,					self.tokenizer.all_special_ids							)
         # fmt: off
         _lowercase:      Tuple   			=  [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
         # fmt: on

         _lowercase:      Optional[Any]   			=  self.tokenizer.decode(A_			,					skip_special_tokens=A_							)
         _lowercase:      Tuple   			=  self.tokenizer.decode(generated_ids[1:]			,					skip_special_tokens=A_							)
         self.assertEqual(A_			,					A_							)
         self.assertNotIn(self.tokenizer.eos_token			,					A_							)
       def 				lowercase_				(  self							)  ->						Tuple:





         """simple docstring"""




         _lowercase:      Optional[Any]   			=  ['''this is gunna be a long sentence ''' * 20]
         assert isinstance(src_text[0]			,					A_							)
         _lowercase:      List[str]   			=  10
         _lowercase:      int   			=  self.tokenizer(A_			,					max_length=A_			,					truncation=A_							).input_ids[0]
         self.assertEqual(ids[-1]			,					2							)
         self.assertEqual(ids[0]			,					A_							)
         self.assertEqual(len(A_							)			,					A_							)
       def 				lowercase_				(  self							)  ->						Any:





         """simple docstring"""




         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']							)			,					[25_6203, 3]							)
       def 				lowercase_				(  self							)  ->						Optional[Any]:





         """simple docstring"""




         _lowercase:      Tuple   			=  tempfile.mkdtemp()
         _lowercase:      int   			=  self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(A_							)
         _lowercase:      List[Any]   			=  NllbTokenizer.from_pretrained(A_							)
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,					A_							)
       @require_torch
       def 				lowercase_				(  self							)  ->						Union[str, Any]:





         """simple docstring"""




         _lowercase:      Union[str, Any]   			=  self.tokenizer(
             self.src_text			,					text_target=self.tgt_text			,					padding=A_			,					truncation=A_			,					max_length=len(self.expected_src_tokens							)			,					return_tensors='''pt'''			,					)
         _lowercase:      str   			=  shift_tokens_right(
             batch['''labels''']			,					self.tokenizer.pad_token_id			,					self.tokenizer.lang_code_to_id['''ron_Latn''']							)

         self.assertIsInstance(A_			,					A_							)

         self.assertEqual((2, 15)			,					batch.input_ids.shape							)
         self.assertEqual((2, 15)			,					batch.attention_mask.shape							)
         _lowercase:      str   			=  batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens			,					A_							)
         self.assertEqual(A_			,					batch.decoder_input_ids[0, 0]							)  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens			,					[EN_CODE]							)
         self.assertEqual(self.tokenizer.suffix_tokens			,					[self.tokenizer.eos_token_id]							)
       def 				lowercase_				(  self							)  ->						int:





         """simple docstring"""




         _lowercase:      Optional[int]   			=  self.tokenizer(self.src_text			,					padding=A_			,					truncation=A_			,					max_length=3			,					return_tensors='''pt'''							)
         _lowercase:      str   			=  self.tokenizer(
             text_target=self.tgt_text			,					padding=A_			,					truncation=A_			,					max_length=10			,					return_tensors='''pt'''							)
         _lowercase:      Optional[int]   			=  targets['''input_ids''']
         _lowercase:      Optional[Any]   			=  shift_tokens_right(
             A_			,					self.tokenizer.pad_token_id			,					decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]			,					)

         self.assertEqual(batch.input_ids.shape[1]			,					3							)
         self.assertEqual(batch.decoder_input_ids.shape[1]			,					10							)
       @require_torch
       def 				lowercase_				(  self							)  ->						Dict:





         """simple docstring"""




         _lowercase:      str   			=  self.tokenizer._build_translation_inputs(
             '''A test'''			,					return_tensors='''pt'''			,					src_lang='''eng_Latn'''			,					tgt_lang='''fra_Latn'''							)

         self.assertEqual(
             nested_simplify(A_							)			,					{
                 # A, test, EOS, en_XX
                 '''input_ids''': [[25_6047, 70, 7356, 2]],
                 '''attention_mask''': [[1, 1, 1, 1]],
                 # ar_AR
                 '''forced_bos_token_id''': 25_6057,
             }			,					)



       @require_torch
       def 				lowercase_				(  self							)  ->						Union[str, Any]:





         """simple docstring"""




         _lowercase:      Dict   			=  True
         _lowercase:      Tuple   			=  self.tokenizer(
             '''UN Chief says there is no military solution in Syria'''			,					src_lang='''eng_Latn'''			,					tgt_lang='''fra_Latn'''							)
         self.assertEqual(
             inputs.input_ids			,					[1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]							)

         _lowercase:      Dict   			=  False
         _lowercase:      int   			=  self.tokenizer(
             '''UN Chief says there is no military solution in Syria'''			,					src_lang='''eng_Latn'''			,					tgt_lang='''fra_Latn'''							)
         self.assertEqual(
             inputs.input_ids			,					[25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]							)


"""simple docstring"""




import math
import sys




def 			_lowerCAmelCase		(		_UpperCamelCase    ):






  """simple docstring"""



  _lowercase:      Any   			=  ''''''
  try:
    with open(_UpperCamelCase			,					'''rb'''    ) as binary_file:
      _lowercase:      Dict   			=  binary_file.read()
    for dat in data:
      _lowercase:      List[str]   			=  f'''{dat:08b}'''
      result += curr_byte
    return result
  except OSError:
    print('''File not accessible'''    )
    sys.exit()




def 			_lowerCAmelCase		(		_UpperCamelCase    ):






  """simple docstring"""



  _lowercase:      Any   			=  {'''0''': '''0''', '''1''': '''1'''}
  _lowercase			,					_lowercase:      Optional[int]   			=  '''''', ''''''
  _lowercase:      Dict   			=  len(_UpperCamelCase    )

  for i in range(len(_UpperCamelCase    )    ):
    curr_string += data_bits[i]
    if curr_string not in lexicon:
      continue

    _lowercase:      Any   			=  lexicon[curr_string]
    result += last_match_id
    _lowercase:      Optional[int]   			=  last_match_id + '''0'''

    if math.loga(_UpperCamelCase    ).is_integer():
      _lowercase:      str   			=  {}
      for curr_key in list(_UpperCamelCase    ):
        _lowercase:      List[Any]   			=  lexicon.pop(_UpperCamelCase    )
      _lowercase:      Tuple   			=  new_lex

    _lowercase:      List[Any]   			=  last_match_id + '''1'''
    index += 1
    _lowercase:      List[Any]   			=  ''''''
  return result




def 			_lowerCAmelCase		(		_UpperCamelCase			,					_UpperCamelCase    ):






  """simple docstring"""



  _lowercase:      Optional[Any]   			=  8
  try:
    with open(_UpperCamelCase			,					'''wb'''    ) as opened_file:
      _lowercase:      List[Any]   			=  [
          to_write[i : i + byte_length]
          for i in range(0			,					len(_UpperCamelCase    )			,					_UpperCamelCase    )
      ]

      if len(result_byte_array[-1]    ) % byte_length == 0:
        result_byte_array.append('''10000000'''    )
      else:
        result_byte_array[-1] += "1" + "0" * (
            byte_length - len(result_byte_array[-1]    ) - 1
        )

      for elem in result_byte_array[:-1]:
        opened_file.write(int(_UpperCamelCase			,					2    ).to_bytes(1			,					byteorder='''big'''    )    )
  except OSError:
    print('''File not accessible'''    )
    sys.exit()




def 			_lowerCAmelCase		(		_UpperCamelCase    ):






  """simple docstring"""



  _lowercase:      Any   			=  0
  for letter in data_bits:
    if letter == "1":
      break
    counter += 1

  _lowercase:      List[str]   			=  data_bits[counter:]
  _lowercase:      Union[str, Any]   			=  data_bits[counter + 1 :]
  return data_bits




def 			_lowerCAmelCase		(		_UpperCamelCase			,					_UpperCamelCase    ):






  """simple docstring"""



  _lowercase:      Tuple   			=  read_file_binary(_UpperCamelCase    )
  _lowercase:      Optional[Any]   			=  remove_prefix(_UpperCamelCase    )
  _lowercase:      Dict   			=  decompress_data(_UpperCamelCase    )
  write_file_binary(_UpperCamelCase			,					_UpperCamelCase    )


if __name__ == "__main__":
  compress(sys.argv[1], sys.argv[2])
