


'''simple docstring'''



import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch


class 						_snake_case   (       unittest.TestCase		):
					lowerCAmelCase :Tuple						=   JukeboxTokenizer
					lowerCAmelCase :Optional[int]						=   {
					    '''artist''': '''Zac Brown Band''',
					    '''genres''': '''Country''',
					    '''lyrics''': '''I met a traveller from an antique land,
        Who said "Two vast and trunkless legs of stone
        Stand in the desert. . . . Near them, on the sand,
        Half sunk a shattered visage lies, whose frown,
        And wrinkled lip, and sneer of cold command,
        Tell that its sculptor well those passions read
        Which yet survive, stamped on these lifeless things,
        The hand that mocked them, and the heart that fed;
        And on the pedestal, these words appear:
        My name is Ozymandias, King of Kings;
        Look on my Works, ye Mighty, and despair!
        Nothing beside remains. Round the decay
        Of that colossal Wreck, boundless and bare
        The lone and level sands stretch far away
        ''',
					}




					@require_torch
					def 			snake_case__  ( self):
								import torch

								UpperCAmelCase__					:   str       				=       JukeboxTokenizer.from_pretrained("""openai/jukebox-1b-lyrics""")
								UpperCAmelCase__					:   Optional[int]       				=       tokenizer(**self.metas)["""input_ids"""]
								# fmt: off
								UpperCAmelCase__					:   Optional[Any]       				=       [
								    torch.tensor([[
								        0, 0, 0, 7169, 507, 9, 76, 39, 31, 46, 76, 27,
								        76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
								        44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
								        47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
								        76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
								        30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
								        27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
								        45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
								        41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
								        19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
								        76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
								        76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
								        64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
								        30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
								        27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
								        34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
								        27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
								        41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
								        76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
								        44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
								        76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
								        32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
								        40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
								        20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
								        45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
								        31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
								        45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
								        76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
								        34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
								        31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
								        40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
								        38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
								        76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
								        41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
								        27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
								        46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
								        76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
								        41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
								        46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
								        41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
								        40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
								        27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
								        76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
								        76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
								        41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
								        76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
								        27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
								        34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
								        44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
								        40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
								        46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
								        38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
								        40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
								        27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
								        20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
								        76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
								        76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
								        76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
								        76, 76]]),
								    torch.tensor([[0, 0, 0, 1069, 11]]),
								    torch.tensor([[0, 0, 0, 1069, 11]]),
								]
								# fmt: on
								self.assertTrue(torch.allclose(tokens[0]							,							EXPECTED_OUTPUT[0]))
								self.assertTrue(torch.allclose(tokens[1]							,							EXPECTED_OUTPUT[1]))
								self.assertTrue(torch.allclose(tokens[2]							,							EXPECTED_OUTPUT[2]))




					@require_torch
					def 			snake_case__  ( self):
								import torch

								UpperCAmelCase__					:   List[str]       				=       JukeboxTokenizer.from_pretrained("""openai/jukebox-5b-lyrics""")
								UpperCAmelCase__					:   Union[str, Any]       				=       tokenizer(**self.metas)["""input_ids"""]
								# fmt: off
								UpperCAmelCase__					:   List[Any]       				=       [
								    torch.tensor([[
								        0, 0, 0, 1069, 11, -1, -1, -1, -1, 9, 77, 39,
								        31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
								        31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
								        40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
								        79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
								        77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
								        27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
								        37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
								        32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
								        77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
								        77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
								        77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
								        46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
								        77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
								        77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
								        77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
								        77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
								        64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
								        40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
								        40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
								        38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
								        31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
								        41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
								        77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
								        46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
								        41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
								        31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
								        31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
								        23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
								        44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
								        31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
								        38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
								        40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
								        27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
								        31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
								        34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
								        31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
								        1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
								        31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
								        45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
								        31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
								        15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
								        11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
								        45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
								        41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
								        44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
								        46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
								        27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
								        35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
								        77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
								        31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
								        41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
								        77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
								        40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
								        77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
								        77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
								        27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
								        77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
								        77, 77, 77, 77, 77, 77]]),
								    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]),
								    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]),
								]
								# fmt: on
								self.assertTrue(torch.allclose(tokens[0]							,							EXPECTED_OUTPUT[0]))
								self.assertTrue(torch.allclose(tokens[1]							,							EXPECTED_OUTPUT[1]))
								self.assertTrue(torch.allclose(tokens[2]							,							EXPECTED_OUTPUT[2]))



'''simple docstring'''



import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE


class 						_snake_case   (       a__		):
					lowerCAmelCase :Optional[int]						=   ''''''
					lowerCAmelCase :str						=   (
					    None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
					)
					lowerCAmelCase :str						=   None  # compression type in fsspec. ex: "gzip"
					lowerCAmelCase :str						=   None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz




					def __init__( self							,							_lowerCamelCase = ""							,							_lowerCamelCase = None							,							_lowerCamelCase = None							,							**_lowerCamelCase):
								super().__init__(self							,							**_lowerCamelCase)
								# always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
								UpperCAmelCase__					:   Optional[Any]       				=       fsspec.open(
								    _lowerCamelCase							,							mode="""rb"""							,							protocol=_lowerCamelCase							,							compression=self.compression							,							client_kwargs={
								        """requote_redirect_url""": False,  # see https://github.com/huggingface/datasets/pull/5459
								        """trust_env""": True,  # Enable reading proxy env variables.
								        **(target_options or {}).pop("""client_kwargs"""							,							{}),  # To avoid issues if it was already passed.
								    }							,							**(target_options or {})							,							)
								UpperCAmelCase__					:   List[Any]       				=       os.path.basename(self.file.path.split("""::""")[0])
								UpperCAmelCase__					:   Dict       				=       (
								    self.compressed_name[: self.compressed_name.rindex(""".""")]
								    if """.""" in self.compressed_name
								    else self.compressed_name
								)
								UpperCAmelCase__					:   Tuple       				=       None




					@classmethod
					def 			snake_case__  ( cls							,							_lowerCamelCase):
								# compressed file paths are always relative to the archive root
								return super()._strip_protocol(_lowerCamelCase).lstrip("""/""")




					def 			snake_case__  ( self):
								if self.dir_cache is None:
											UpperCAmelCase__					:   Optional[Any]       				=       {**self.file.fs.info(self.file.path), """name""": self.uncompressed_name}
											UpperCAmelCase__					:   Union[str, Any]       				=       {f["""name"""]: f}




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.file.open().read()




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = "rb"							,							_lowerCamelCase=None							,							_lowerCamelCase=True							,							_lowerCamelCase=None							,							**_lowerCamelCase							,							):
								UpperCAmelCase__					:   List[str]       				=       self._strip_protocol(_lowerCamelCase)
								if mode != "rb":
											raise ValueError(f'''Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\'''')
								return self.file.open()


class 						_snake_case   (       a__		):
					lowerCAmelCase :Dict						=   '''bz2'''
					lowerCAmelCase :List[str]						=   '''bz2'''
					lowerCAmelCase :Dict						=   '''.bz2'''


class 						_snake_case   (       a__		):
					lowerCAmelCase :int						=   '''gzip'''
					lowerCAmelCase :Tuple						=   '''gzip'''
					lowerCAmelCase :str						=   '''.gz'''


class 						_snake_case   (       a__		):
					lowerCAmelCase :List[str]						=   '''lz4'''
					lowerCAmelCase :Any						=   '''lz4'''
					lowerCAmelCase :int						=   '''.lz4'''


class 						_snake_case   (       a__		):
					lowerCAmelCase :Union[str, Any]						=   '''xz'''
					lowerCAmelCase :int						=   '''xz'''
					lowerCAmelCase :List[Any]						=   '''.xz'''


class 						_snake_case   (       a__		):
					lowerCAmelCase :Tuple						=   '''zstd'''
					lowerCAmelCase :List[str]						=   '''zstd'''
					lowerCAmelCase :Union[str, Any]						=   '''.zst'''




					def __init__( self							,							_lowerCamelCase							,							_lowerCamelCase = "rb"							,							_lowerCamelCase = None							,							_lowerCamelCase = None							,							_lowerCamelCase = DEFAULT_BLOCK_SIZE							,							**_lowerCamelCase							,							):
								super().__init__(
								    fo=_lowerCamelCase							,							mode=_lowerCamelCase							,							target_protocol=_lowerCamelCase							,							target_options=_lowerCamelCase							,							block_size=_lowerCamelCase							,							**_lowerCamelCase							,							)
								# We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
								#
								# File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
								#     out.close = close
								# AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
								#
								# see https://github.com/intake/filesystem_spec/issues/725
								UpperCAmelCase__					:   Dict       				=       self.file.__enter__


								class 						_snake_case   :




													def __init__( self							,							_lowerCamelCase):
																UpperCAmelCase__					:   Optional[int]       				=       file_




													def __enter__( self):
																self._file.__enter__()
																return self




													def __exit__( self							,							*_lowerCamelCase							,							**_lowerCamelCase):
																self._file.__exit__(*_lowerCamelCase							,							**_lowerCamelCase)




													def __iter__( self):
																return iter(self._file)




													def 			snake_case__  ( self):
																return next(self._file)




													def __getattr__( self							,							_lowerCamelCase):
																return getattr(self._file							,							_lowerCamelCase)

								def fixed_enter(*_lowerCamelCase							,							**_lowerCamelCase):
											return WrappedFile(_enter(*_lowerCamelCase							,							**_lowerCamelCase))

								UpperCAmelCase__					:   List[Any]       				=       fixed_enter