


import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
      from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
      __UpperCAmelCase    :Tuple				       =     get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
      from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__UpperCAmelCase    :Union[str, Any]				       =     1_2_8_0_2_2
__UpperCAmelCase    :List[str]				       =     1_2_8_0_2_8



@require_sentencepiece
class 			a       (	lowerCAmelCase__			,						unittest.TestCase ):


     """simple docstring"""






     SCREAMING_SNAKE_CASE :				Dict							     =     MaMaaaTokenizer
     SCREAMING_SNAKE_CASE :				List[str]							     =     False
     SCREAMING_SNAKE_CASE :				int							     =     False
     SCREAMING_SNAKE_CASE :				List[Any]							     =     True

     def 				lowerCamelCase__				(       self  :	Dict    )		->							Optional[Any]:
          super().setUp()

          __UpperCAmelCase			:				List[str]													=    ['''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''', '''\u0120''', '''<pad>''']
          __UpperCAmelCase			:				Optional[int]													=    dict(zip(_SCREAMING_SNAKE_CASE							,				range(len(_SCREAMING_SNAKE_CASE    )    )    )    )
          __UpperCAmelCase			:				Any													=    Path(self.tmpdirname    )
          save_json(_SCREAMING_SNAKE_CASE							,				save_dir / VOCAB_FILES_NAMES['''vocab_file''']    )
          if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
               copyfile(_SCREAMING_SNAKE_CASE							,				save_dir / VOCAB_FILES_NAMES['''spm_file''']    )

          __UpperCAmelCase			:				Any													=    MaMaaaTokenizer.from_pretrained(self.tmpdirname    )
          tokenizer.save_pretrained(self.tmpdirname    )

     def 				lowerCamelCase__				(       self  :	str							,				**snake_case  :	Tuple    )		->							Any:
          return MaMaaaTokenizer.from_pretrained(self.tmpdirname							,				**_SCREAMING_SNAKE_CASE    )

     def 				lowerCamelCase__				(       self  :	str							,				snake_case  :	Tuple    )		->							str:
          return (
              "This is a test",
              "This is a test",
          )

     def 				lowerCamelCase__				(       self  :	Tuple    )		->							Optional[Any]:
          __UpperCAmelCase			:				Optional[int]													=    '''</s>'''
          __UpperCAmelCase			:				str													=    0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE    )							,				_SCREAMING_SNAKE_CASE    )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE    )							,				_SCREAMING_SNAKE_CASE    )

     def 				lowerCamelCase__				(       self  :	Union[str, Any]    )		->							int:
          __UpperCAmelCase			:				List[Any]													=    self.get_tokenizer()
          __UpperCAmelCase			:				int													=    list(tokenizer.get_vocab().keys()    )

          self.assertEqual(vocab_keys[0]							,				'''</s>'''    )
          self.assertEqual(vocab_keys[1]							,				'''<unk>'''    )
          self.assertEqual(vocab_keys[-1]							,				'''<s>'''    )
          self.assertEqual(len(_SCREAMING_SNAKE_CASE    )							,				tokenizer.vocab_size + len(tokenizer.get_added_vocab()    )    )

     @unittest.skip('''Skip this test while all models are still to be uploaded.'''    )
     def 				lowerCamelCase__				(       self  :	Optional[Any]    )		->							List[str]:
          pass

     def 				lowerCamelCase__				(       self  :	Union[str, Any]    )		->							Any:
          __UpperCAmelCase			:				int													=    self.get_tokenizer()

          __UpperCAmelCase			:				List[str]													=    tokenizer.tokenize('''This is a test'''    )
          self.assertListEqual(_SCREAMING_SNAKE_CASE							,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']    )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE    )							,				[2, 3, 4, 5, 6]							,				)

          __UpperCAmelCase			:				Dict													=    tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]    )
          self.assertListEqual(_SCREAMING_SNAKE_CASE							,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']    )

          __UpperCAmelCase			:				Optional[int]													=    tokenizer.convert_tokens_to_string(_SCREAMING_SNAKE_CASE    )
          self.assertEqual(_SCREAMING_SNAKE_CASE							,				'''This is a test'''    )






     @slow
     def 				lowerCamelCase__				(       self  :	List[Any]    )		->							Optional[Any]:
          # fmt: off
          __UpperCAmelCase			:				List[Any]													=    {'''input_ids''': [[12_8022, 11_0108, 397, 11, 3_8272, 2247, 12_4811, 285, 1_8105, 1586, 207, 7, 3_9534, 4428, 397, 1019, 1_8105, 1586, 207, 7, 4_1337, 1_6786, 241, 7, 2_0214, 17, 12_5690, 1_0398, 7, 4_4378, 5_8069, 6_8342, 7798, 7343, 11, 299, 3_3310, 4, 158, 3_7350, 9_4077, 4569, 299, 3_3310, 90, 4, 5_2840, 290, 4, 3_1270, 112, 299, 682, 4, 5_2840, 3_9953, 1_4079, 193, 5_2519, 9_0894, 1_7894, 12_0697, 11, 4_0445, 551, 17, 1019, 5_2519, 9_0894, 1_7756, 963, 11, 4_0445, 480, 17, 9792, 1120, 5173, 1393, 6240, 1_6786, 241, 12_0996, 28, 1245, 1393, 11_8240, 1_1123, 1019, 9_3612, 2691, 1_0618, 9_8058, 12_0409, 1928, 279, 4, 4_0683, 367, 178, 207, 1019, 103, 10_3121, 506, 6_5296, 5, 2], [12_8022, 2_1217, 367, 117, 12_5450, 128, 719, 7, 7308, 40, 9_3612, 1_2669, 1116, 1_6704, 71, 1_7785, 3699, 1_5592, 35, 144, 9584, 241, 1_1943, 713, 950, 799, 2247, 8_8427, 150, 149, 11_8813, 12_0706, 1019, 10_6906, 8_1518, 28, 1224, 2_2799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [12_8022, 1658, 12_3311, 5155, 5578, 4722, 279, 1_4947, 2366, 1120, 1197, 14, 1348, 9232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_SCREAMING_SNAKE_CASE							,				model_name='''facebook/m2m100_418M'''							,				revision='''c168bae485c864188cf9aa0e4108b0b6934dc91e'''							,				)



@require_torch
@require_sentencepiece
@require_tokenizers
class 			a       (	unittest.TestCase ):


     """simple docstring"""






     SCREAMING_SNAKE_CASE :				Union[str, Any]							     =     "facebook/m2m100_418M"
     SCREAMING_SNAKE_CASE :				Optional[Any]							     =     [
         "In my opinion, there are two levels of response from the French government.",
         "NSA Affair Emphasizes Complete Lack of Debate on Intelligence",
     ]
     SCREAMING_SNAKE_CASE :				Optional[int]							     =     [
         "Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.",
         "L'affaire NSA souligne l'absence totale de débat sur le renseignement",
     ]

     # fmt: off
     SCREAMING_SNAKE_CASE :				Union[str, Any]							     =     [EN_CODE, 5_9_3, 1_9_4_9, 1_1_5_7_8_1, 4, 7_1_5_8_6, 4_2_3_4, 6_0_6_3_3, 1_2_6_2_3_3, 4_3_2, 1_2_3_8_0_8, 1_5_5_9_2, 1_1_9_7, 1_1_7_1_3_2, 1_2_0_6_1_8, 5, 2]

     @classmethod
     def 				lowerCamelCase__				(       cls  :	Optional[Any]    )		->							Union[str, Any]:
          __UpperCAmelCase			:				Union[str, Any]													=    MaMaaaTokenizer.from_pretrained(
              cls.checkpoint_name							,				src_lang='''en'''							,				tgt_lang='''fr'''    )
          __UpperCAmelCase			:				str													=    1
          return cls

     def 				lowerCamelCase__				(       self  :	Optional[int]    )		->							Tuple:
          self.assertEqual(self.tokenizer.get_lang_id('''ar'''    )							,				12_8006    )
          self.assertEqual(self.tokenizer.get_lang_id('''en'''    )							,				12_8022    )
          self.assertEqual(self.tokenizer.get_lang_id('''ro'''    )							,				12_8076    )
          self.assertEqual(self.tokenizer.get_lang_id('''mr'''    )							,				12_8063    )

     def 				lowerCamelCase__				(       self  :	int    )		->							Optional[int]:
          __UpperCAmelCase			:				Optional[int]													=    self.tokenizer.get_vocab()
          self.assertEqual(len(_SCREAMING_SNAKE_CASE    )							,				self.tokenizer.vocab_size    )
          self.assertEqual(vocab['''<unk>''']							,				3    )
          self.assertIn(self.tokenizer.get_lang_token('''en'''    )							,				_SCREAMING_SNAKE_CASE    )

     def 				lowerCamelCase__				(       self  :	Tuple    )		->							Optional[Any]:
          __UpperCAmelCase			:				Union[str, Any]													=    '''en'''
          __UpperCAmelCase			:				Union[str, Any]													=    self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
          self.assertListEqual(self.expected_src_tokens							,				_SCREAMING_SNAKE_CASE    )

     def 				lowerCamelCase__				(       self  :	int    )		->							Union[str, Any]:
          self.assertIn(_SCREAMING_SNAKE_CASE							,				self.tokenizer.all_special_ids    )
          # fmt: off
          __UpperCAmelCase			:				List[str]													=    [FR_CODE, 5364, 82, 8642, 4, 294, 47, 8, 1_4028, 136, 3286, 9706, 6, 9_0797, 6, 14_4012, 162, 8_8128, 3_0061, 5, 2]
          # fmt: on
          __UpperCAmelCase			:				str													=    self.tokenizer.decode(_SCREAMING_SNAKE_CASE							,				skip_special_tokens=_SCREAMING_SNAKE_CASE    )
          __UpperCAmelCase			:				int													=    self.tokenizer.decode(generated_ids[1:]							,				skip_special_tokens=_SCREAMING_SNAKE_CASE    )
          self.assertEqual(_SCREAMING_SNAKE_CASE							,				_SCREAMING_SNAKE_CASE    )
          self.assertNotIn(self.tokenizer.eos_token							,				_SCREAMING_SNAKE_CASE    )

     def 				lowerCamelCase__				(       self  :	int    )		->							int:
          __UpperCAmelCase			:				Optional[int]													=    tempfile.mkdtemp()
          __UpperCAmelCase			:				List[str]													=    self.tokenizer.lang_token_to_id
          self.tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )
          __UpperCAmelCase			:				Any													=    MaMaaaTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
          self.assertDictEqual(new_tok.lang_token_to_id							,				_SCREAMING_SNAKE_CASE    )

     @require_torch
     def 				lowerCamelCase__				(       self  :	Union[str, Any]    )		->							Tuple:
          __UpperCAmelCase			:				Optional[Any]													=    '''en'''
          __UpperCAmelCase			:				Optional[int]													=    '''fr'''

          __UpperCAmelCase			:				Tuple													=    self.tokenizer(self.src_text							,				text_target=self.tgt_text							,				padding=_SCREAMING_SNAKE_CASE							,				return_tensors='''pt'''    )

          __UpperCAmelCase			:				List[Any]													=    shift_tokens_right(
              batch['''labels''']							,				self.tokenizer.pad_token_id							,				self.tokenizer.eos_token_id    )

          for k in batch:
               __UpperCAmelCase			:				Union[str, Any]													=    batch[k].tolist()
          # batch = {k: v.tolist() for k,v in batch.items()}
          # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
          # batch.decoder_inputs_ids[0][0] ==
          assert batch.input_ids[1][0] == EN_CODE
          assert batch.input_ids[1][-1] == 2
          assert batch.labels[1][0] == FR_CODE
          assert batch.labels[1][-1] == 2
          assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]

     @require_torch
     def 				lowerCamelCase__				(       self  :	int    )		->							Dict:
          __UpperCAmelCase			:				Dict													=    '''mr'''
          self.assertListEqual(self.tokenizer.prefix_tokens							,				[self.tokenizer.get_lang_id('''mr'''    )]    )
          self.assertListEqual(self.tokenizer.suffix_tokens							,				[self.tokenizer.eos_token_id]    )

          __UpperCAmelCase			:				int													=    '''zh'''
          self.assertListEqual(self.tokenizer.prefix_tokens							,				[self.tokenizer.get_lang_id('''zh'''    )]    )
          self.assertListEqual(self.tokenizer.suffix_tokens							,				[self.tokenizer.eos_token_id]    )

     @require_torch
     def 				lowerCamelCase__				(       self  :	Optional[Any]    )		->							List[Any]:
          __UpperCAmelCase			:				Union[str, Any]													=    '''mr'''
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens							,				[self.tokenizer.get_lang_id('''mr'''    )]    )
          self.assertListEqual(self.tokenizer.suffix_tokens							,				[self.tokenizer.eos_token_id]    )
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens							,				[self.tokenizer.get_lang_id(self.tokenizer.src_lang    )]    )

          __UpperCAmelCase			:				Dict													=    '''zh'''
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens							,				[self.tokenizer.get_lang_id('''zh'''    )]    )
          self.assertListEqual(self.tokenizer.suffix_tokens							,				[self.tokenizer.eos_token_id]    )
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens							,				[self.tokenizer.get_lang_id(self.tokenizer.src_lang    )]    )






     @require_torch
     def 				lowerCamelCase__				(       self  :	List[str]    )		->							Any:
          __UpperCAmelCase			:				Any													=    self.tokenizer._build_translation_inputs('''A test'''							,				return_tensors='''pt'''							,				src_lang='''en'''							,				tgt_lang='''ar'''    )

          self.assertEqual(
              nested_simplify(_SCREAMING_SNAKE_CASE    )							,				{
                  # en_XX, A, test, EOS
                  '''input_ids''': [[12_8022, 58, 4183, 2]],
                  '''attention_mask''': [[1, 1, 1, 1]],
                  # ar_AR
                  '''forced_bos_token_id''': 12_8006,
              }							,				)



'''simple docstring'''



import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch



class 			a       (	unittest.TestCase ):


     """simple docstring"""






     SCREAMING_SNAKE_CASE :				Any							     =     JukeboxTokenizer
     SCREAMING_SNAKE_CASE :				Tuple							     =     {
         "artist": "Zac Brown Band",
         "genres": "Country",
         "lyrics": "I met a traveller from an antique land,\n        Who said \"Two vast and trunkless legs of stone\n        Stand in the desert. . . . Near them, on the sand,\n        Half sunk a shattered visage lies, whose frown,\n        And wrinkled lip, and sneer of cold command,\n        Tell that its sculptor well those passions read\n        Which yet survive, stamped on these lifeless things,\n        The hand that mocked them, and the heart that fed;\n        And on the pedestal, these words appear:\n        My name is Ozymandias, King of Kings;\n        Look on my Works, ye Mighty, and despair!\n        Nothing beside remains. Round the decay\n        Of that colossal Wreck, boundless and bare\n        The lone and level sands stretch far away\n        ",
     }

     @require_torch
     def 				lowerCamelCase__				(       self  :	Optional[int]    )		->							Optional[Any]:
          import torch

          __UpperCAmelCase			:				List[str]													=    JukeboxTokenizer.from_pretrained('''openai/jukebox-1b-lyrics'''    )
          __UpperCAmelCase			:				int													=    tokenizer(**self.metas    )['''input_ids''']
          # fmt: off
          __UpperCAmelCase			:				List[str]													=    [
              torch.tensor([[
                  0, 0, 0, 7169, 507, 9, 76, 39, 31, 46, 76, 27,
                  76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
                  44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
                  47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
                  76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
                  30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
                  27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
                  45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
                  41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                  19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
                  76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
                  76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
                  64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
                  30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
                  27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
                  34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
                  27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
                  41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
                  76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
                  44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
                  76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
                  32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
                  40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                  20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
                  45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
                  31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
                  45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
                  76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
                  34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
                  31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
                  40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
                  38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
                  76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
                  41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
                  27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
                  46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
                  76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
                  41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
                  46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
                  41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
                  40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
                  27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
                  76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
                  76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
                  41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
                  76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
                  27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
                  34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
                  44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
                  40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
                  46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
                  38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
                  40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
                  27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                  20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
                  76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
                  76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
                  76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
                  76, 76]]    ),
              torch.tensor([[0, 0, 0, 1069, 11]]    ),
              torch.tensor([[0, 0, 0, 1069, 11]]    ),
          ]
          # fmt: on
          self.assertTrue(torch.allclose(tokens[0]							,				EXPECTED_OUTPUT[0]    )    )
          self.assertTrue(torch.allclose(tokens[1]							,				EXPECTED_OUTPUT[1]    )    )
          self.assertTrue(torch.allclose(tokens[2]							,				EXPECTED_OUTPUT[2]    )    )






     @require_torch
     def 				lowerCamelCase__				(       self  :	Union[str, Any]    )		->							Union[str, Any]:
          import torch

          __UpperCAmelCase			:				Optional[int]													=    JukeboxTokenizer.from_pretrained('''openai/jukebox-5b-lyrics'''    )
          __UpperCAmelCase			:				List[Any]													=    tokenizer(**self.metas    )['''input_ids''']
          # fmt: off
          __UpperCAmelCase			:				List[str]													=    [
              torch.tensor([[
                  0, 0, 0, 1069, 11, -1, -1, -1, -1, 9, 77, 39,
                  31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
                  31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
                  40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
                  79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
                  77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
                  27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
                  37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
                  32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
                  77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
                  77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
                  77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
                  46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
                  77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
                  77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
                  77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
                  77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
                  64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
                  40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
                  40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
                  38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
                  31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
                  41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
                  77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
                  46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
                  41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
                  31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
                  31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                  23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
                  44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
                  31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
                  38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
                  40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
                  27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
                  31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
                  34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
                  31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                  1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
                  31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
                  45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
                  31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
                  15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
                  11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
                  45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
                  41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
                  44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
                  46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
                  27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
                  35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
                  77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
                  31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
                  41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
                  77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
                  40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
                  77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
                  77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
                  27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
                  77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
                  77, 77, 77, 77, 77, 77]]    ),
              torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]    ),
              torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]    ),
          ]
          # fmt: on
          self.assertTrue(torch.allclose(tokens[0]							,				EXPECTED_OUTPUT[0]    )    )
          self.assertTrue(torch.allclose(tokens[1]							,				EXPECTED_OUTPUT[1]    )    )
          self.assertTrue(torch.allclose(tokens[2]							,				EXPECTED_OUTPUT[2]    )    )