






import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch


class 							__a							(  unittest.TestCase						):





		'''simple docstring'''



		UpperCAmelCase__      :				Any				    =						JukeboxTokenizer
		UpperCAmelCase__      :				int				    =						{
		    """artist""": """Zac Brown Band""",
		    """genres""": """Country""",
		    """lyrics""": """I met a traveller from an antique land,
        Who said \"Two vast and trunkless legs of stone
        Stand in the desert. . . . Near them, on the sand,
        Half sunk a shattered visage lies, whose frown,
        And wrinkled lip, and sneer of cold command,
        Tell that its sculptor well those passions read
        Which yet survive, stamped on these lifeless things,
        The hand that mocked them, and the heart that fed;
        And on the pedestal, these words appear:
        My name is Ozymandias, King of Kings;
        Look on my Works, ye Mighty, and despair!
        Nothing beside remains. Round the decay
        Of that colossal Wreck, boundless and bare
        The lone and level sands stretch far away
        """,
		}


		@require_torch
		def   __snake_case			(						self       ):
						import torch

						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   JukeboxTokenizer.from_pretrained('openai/jukebox-1b-lyrics'       )
						SCREAMING_SNAKE_CASE_   :	List[str]													=   tokenizer(**self.metas       )['input_ids']
						# fmt: off
						SCREAMING_SNAKE_CASE_   :	List[str]													=   [
						    torch.tensor([[
						        0, 0, 0, 7169, 507, 9, 76, 39, 31, 46, 76, 27,
						        76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
						        44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
						        47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
						        76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
						        30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
						        27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
						        45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
						        41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
						        19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
						        76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
						        76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
						        64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
						        30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
						        27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
						        34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
						        27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
						        41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
						        76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
						        44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
						        76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
						        32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
						        40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
						        20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
						        45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
						        31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
						        45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
						        76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
						        34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
						        31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
						        40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
						        38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
						        76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
						        41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
						        27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
						        46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
						        76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
						        41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
						        46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
						        41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
						        40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
						        27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
						        76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
						        76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
						        41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
						        76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
						        27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
						        34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
						        44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
						        40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
						        46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
						        38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
						        40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
						        27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
						        20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
						        76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
						        76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
						        76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
						        76, 76]]       ),
						    torch.tensor([[0, 0, 0, 1069, 11]]       ),
						    torch.tensor([[0, 0, 0, 1069, 11]]       ),
						]
						# fmt: on
						self.assertTrue(torch.allclose(tokens[0] ,    EXPECTED_OUTPUT[0]       )       )
						self.assertTrue(torch.allclose(tokens[1] ,    EXPECTED_OUTPUT[1]       )       )
						self.assertTrue(torch.allclose(tokens[2] ,    EXPECTED_OUTPUT[2]       )       )






		@require_torch
		def   __snake_case			(						self       ):
						import torch

						SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   JukeboxTokenizer.from_pretrained('openai/jukebox-5b-lyrics'       )
						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   tokenizer(**self.metas       )['input_ids']
						# fmt: off
						SCREAMING_SNAKE_CASE_   :	Dict													=   [
						    torch.tensor([[
						        0, 0, 0, 1069, 11, -1, -1, -1, -1, 9, 77, 39,
						        31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
						        31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
						        40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
						        79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
						        77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
						        27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
						        37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
						        32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
						        77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
						        77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
						        77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
						        46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
						        77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
						        77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
						        77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
						        77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
						        64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
						        40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
						        40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
						        38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
						        31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
						        41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
						        77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
						        46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
						        41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
						        31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
						        31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
						        23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
						        44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
						        31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
						        38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
						        40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
						        27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
						        31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
						        34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
						        31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
						        1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
						        31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
						        45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
						        31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
						        15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
						        11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
						        45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
						        41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
						        44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
						        46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
						        27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
						        35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
						        77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
						        31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
						        41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
						        77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
						        40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
						        77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
						        77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
						        27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
						        77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
						        77, 77, 77, 77, 77, 77]]       ),
						    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]       ),
						    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]       ),
						]
						# fmt: on
						self.assertTrue(torch.allclose(tokens[0] ,    EXPECTED_OUTPUT[0]       )       )
						self.assertTrue(torch.allclose(tokens[1] ,    EXPECTED_OUTPUT[1]       )       )
						self.assertTrue(torch.allclose(tokens[2] ,    EXPECTED_OUTPUT[2]       )       )







from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import pyarrow as pa


if TYPE_CHECKING:
				from .features import FeatureType


@dataclass
class 							__a							:





		'''simple docstring'''



		UpperCAmelCase__      :				List[str]
		UpperCAmelCase__      :				Optional[str]				    =						None
		# Automatically constructed
		UpperCAmelCase__      :				ClassVar[str]				    =						"dict"
		UpperCAmelCase__      :				ClassVar[Any]				    =						None
		UpperCAmelCase__      :				str				    =						field(default="""Translation""" ,	init=__A ,	repr=__A						)


		def __call__(						self       ):
						return pa.struct({lang: pa.string() for lang in sorted(self.languages       )}       )






		def   __snake_case			(						self       ):
						from .features import Value

						return {k: Value('string'       ) for k in sorted(self.languages       )}






@dataclass
class 							__a							:





		'''simple docstring'''



		UpperCAmelCase__      :				Optional[List]				    =						None
		UpperCAmelCase__      :				Optional[int]				    =						None
		UpperCAmelCase__      :				Optional[str]				    =						None
		# Automatically constructed
		UpperCAmelCase__      :				ClassVar[str]				    =						"dict"
		UpperCAmelCase__      :				ClassVar[Any]				    =						None
		UpperCAmelCase__      :				str				    =						field(default="""TranslationVariableLanguages""" ,	init=__A ,	repr=__A						)


		def   __snake_case			(						self       ):
						SCREAMING_SNAKE_CASE_   :	Any													=   sorted(set(self.languages       )       ) if self.languages else None
						SCREAMING_SNAKE_CASE_   :	Dict													=   len(self.languages       ) if self.languages else None


		def __call__(						self       ):
						return pa.struct({'language': pa.list_(pa.string()       ), 'translation': pa.list_(pa.string()       )}       )


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   set(self.languages       )
						if self.languages and set(UpperCamelCase__       ) - lang_set:
										raise ValueError(
										    F'''Some languages in example ({', '.join(sorted(set(UpperCamelCase__       ) - lang_set       )       )}) are not in valid set ({', '.join(UpperCamelCase__       )}).'''       )

						# Convert dictionary into tuples, splitting out cases where there are
						# multiple translations for a single language.
						SCREAMING_SNAKE_CASE_   :	List[str]													=   []
						for lang, text in translation_dict.items():
										if isinstance(UpperCamelCase__ ,    UpperCamelCase__       ):
														translation_tuples.append((lang, text)       )
										else:
														translation_tuples.extend([(lang, el) for el in text]       )

        # Ensure translations are in ascending order by language code.
						SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_   :	Dict													=   zip(*sorted(UpperCamelCase__       )       )

						return {"language": languages, "translation": translations}






		def   __snake_case			(						self       ):
						from .features import Sequence, Value

						return {
						    "language": Sequence(Value('string'       )       ),
						    "translation": Sequence(Value('string'       )       ),
						}