

import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch



class        _a		(unittest.TestCase							):


					'''simple docstring'''






					UpperCAmelCase__:  Union[str, Any]    				=			JukeboxTokenizer
					UpperCAmelCase__:  Tuple    				=			{
					    '''artist''': '''Zac Brown Band''',
					    '''genres''': '''Country''',
					    '''lyrics''': '''I met a traveller from an antique land,
        Who said "Two vast and trunkless legs of stone
        Stand in the desert. . . . Near them, on the sand,
        Half sunk a shattered visage lies, whose frown,
        And wrinkled lip, and sneer of cold command,
        Tell that its sculptor well those passions read
        Which yet survive, stamped on these lifeless things,
        The hand that mocked them, and the heart that fed;
        And on the pedestal, these words appear:
        My name is Ozymandias, King of Kings;
        Look on my Works, ye Mighty, and despair!
        Nothing beside remains. Round the decay
        Of that colossal Wreck, boundless and bare
        The lone and level sands stretch far away
        ''',
					}






					@require_torch
					def     __A     (					self     ):
						import torch

						A__				:       str	  =						JukeboxTokenizer.from_pretrained("""openai/jukebox-1b-lyrics"""     )
						A__				:       Any	  =						tokenizer(**self.metas     )["""input_ids"""]
						# fmt: off
						A__				:       Optional[int]	  =						[
						    torch.tensor([[
						        0, 0, 0, 7169, 507, 9, 76, 39, 31, 46, 76, 27,
						        76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
						        44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
						        47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
						        76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
						        30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
						        27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
						        45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
						        41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
						        19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
						        76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
						        76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
						        64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
						        30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
						        27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
						        34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
						        27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
						        41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
						        76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
						        44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
						        76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
						        32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
						        40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
						        20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
						        45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
						        31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
						        45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
						        76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
						        34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
						        31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
						        40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
						        38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
						        76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
						        41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
						        27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
						        46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
						        76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
						        41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
						        46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
						        41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
						        40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
						        27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
						        76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
						        76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
						        41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
						        76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
						        27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
						        34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
						        44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
						        40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
						        78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
						        46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
						        38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
						        40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
						        27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
						        20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
						        76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
						        76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
						        76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
						        76, 76]]     ),
						    torch.tensor([[0, 0, 0, 1069, 11]]     ),
						    torch.tensor([[0, 0, 0, 1069, 11]]     ),
						]
						# fmt: on
						self.assertTrue(torch.allclose(tokens[0]       ,			EXPECTED_OUTPUT[0]     )     )
						self.assertTrue(torch.allclose(tokens[1]       ,			EXPECTED_OUTPUT[1]     )     )
						self.assertTrue(torch.allclose(tokens[2]       ,			EXPECTED_OUTPUT[2]     )     )






					@require_torch
					def     __A     (					self     ):
						import torch

						A__				:       Tuple	  =						JukeboxTokenizer.from_pretrained("""openai/jukebox-5b-lyrics"""     )
						A__				:       Dict	  =						tokenizer(**self.metas     )["""input_ids"""]
						# fmt: off
						A__				:       List[str]	  =						[
						    torch.tensor([[
						        0, 0, 0, 1069, 11, -1, -1, -1, -1, 9, 77, 39,
						        31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
						        31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
						        40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
						        79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
						        77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
						        27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
						        37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
						        32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
						        77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
						        77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
						        77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
						        46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
						        77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
						        77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
						        77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
						        77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
						        64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
						        40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
						        40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
						        38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
						        31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
						        41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
						        77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
						        46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
						        41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
						        31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
						        31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
						        23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
						        44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
						        31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
						        38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
						        40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
						        27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
						        31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
						        34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
						        31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
						        1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
						        31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
						        45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
						        31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
						        15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
						        11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
						        45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
						        41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
						        44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
						        46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
						        27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
						        35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
						        77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
						        31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
						        77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
						        41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
						        77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
						        40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
						        77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
						        77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
						        27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
						        77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
						        77, 77, 77, 77, 77, 77]]     ),
						    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]     ),
						    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]     ),
						]
						# fmt: on
						self.assertTrue(torch.allclose(tokens[0]       ,			EXPECTED_OUTPUT[0]     )     )
						self.assertTrue(torch.allclose(tokens[1]       ,			EXPECTED_OUTPUT[1]     )     )
						self.assertTrue(torch.allclose(tokens[2]       ,			EXPECTED_OUTPUT[2]     )     )






import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A_			:					Tuple    			=     logging.get_logger(__name__)

A_			:					Tuple    			=     {'vocab_file': 'spiece.model'}

A_			:					Optional[Any]    			=     {
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

A_			:					Tuple    			=     {
    'google/bigbird-roberta-base': 4096,
    'google/bigbird-roberta-large': 4096,
    'google/bigbird-base-trivia-itc': 4096,
}



class        _a		(__magic_name__							):


					'''simple docstring'''






					UpperCAmelCase__:  Optional[int]    				=			VOCAB_FILES_NAMES
					UpperCAmelCase__:  str    				=			PRETRAINED_VOCAB_FILES_MAP
					UpperCAmelCase__:  List[str]    				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCAmelCase__:  str    				=			['''input_ids''', '''attention_mask''']
					UpperCAmelCase__:  List[int]    				=			[]






					def __init__(					self       ,			A__       ,			A__="<unk>"       ,			A__="<s>"       ,			A__="</s>"       ,			A__="<pad>"       ,			A__="[SEP]"       ,			A__="[MASK]"       ,			A__="[CLS]"       ,			A__ = None       ,			**A__       ,			):
						A__				:       Tuple	  =						AddedToken(A__       ,			lstrip=A__       ,			rstrip=A__     ) if isinstance(A__       ,			A__     ) else bos_token
						A__				:       int	  =						AddedToken(A__       ,			lstrip=A__       ,			rstrip=A__     ) if isinstance(A__       ,			A__     ) else eos_token
						A__				:       Tuple	  =						AddedToken(A__       ,			lstrip=A__       ,			rstrip=A__     ) if isinstance(A__       ,			A__     ) else unk_token
						A__				:       str	  =						AddedToken(A__       ,			lstrip=A__       ,			rstrip=A__     ) if isinstance(A__       ,			A__     ) else pad_token
						A__				:       Dict	  =						AddedToken(A__       ,			lstrip=A__       ,			rstrip=A__     ) if isinstance(A__       ,			A__     ) else cls_token
						A__				:       Any	  =						AddedToken(A__       ,			lstrip=A__       ,			rstrip=A__     ) if isinstance(A__       ,			A__     ) else sep_token

						# Mask token behave like a normal word, i.e. include the space before it
						A__				:       Optional[Any]	  =						AddedToken(A__       ,			lstrip=A__       ,			rstrip=A__     ) if isinstance(A__       ,			A__     ) else mask_token

						A__				:       Optional[Any]	  =						{} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    bos_token=A__       ,			eos_token=A__       ,			unk_token=A__       ,			pad_token=A__       ,			sep_token=A__       ,			mask_token=A__       ,			cls_token=A__       ,			sp_model_kwargs=self.sp_model_kwargs       ,			**A__       ,			)

						A__				:       Any	  =						vocab_file

						A__				:       Optional[int]	  =						spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.Load(A__     )






					@property
					def     __A     (					self     ):
						return self.sp_model.get_piece_size()






					def     __A     (					self     ):
						A__				:       List[str]	  =						{self.convert_ids_to_tokens(A__     ): i for i in range(self.vocab_size     )}
						vocab.update(self.added_tokens_encoder     )
						return vocab






					def __getstate__(					self     ):
						A__				:       Optional[Any]	  =						self.__dict__.copy()
						A__				:       Optional[Any]	  =						None
						return state






					def __setstate__(					self       ,			A__     ):
						A__				:       Dict	  =						d

						# for backward compatibility
						if not hasattr(self       ,			"""sp_model_kwargs"""     ):
							A__				:       str	  =						{}

						A__				:       Tuple	  =						spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.Load(self.vocab_file     )






					def     __A     (					self       ,			A__     ):
						return self.sp_model.encode(A__       ,			out_type=A__     )






					def     __A     (					self       ,			A__     ):
						return self.sp_model.piece_to_id(A__     )






					def     __A     (					self       ,			A__     ):
						A__				:       Dict	  =						self.sp_model.IdToPiece(A__     )
						return token






					def     __A     (					self       ,			A__     ):
						A__				:       str	  =						[]
						A__				:       Optional[int]	  =						""""""
						A__				:       Any	  =						False
						for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
								if not prev_is_special:
									out_string += " "
								out_string += self.sp_model.decode(A__     ) + token
								A__				:       Union[str, Any]	  =						True
								A__				:       Union[str, Any]	  =						[]
							else:
								current_sub_tokens.append(A__     )
								A__				:       Any	  =						False
						out_string += self.sp_model.decode(A__     )
						return out_string.strip()






					def     __A     (					self       ,			A__       ,			A__ = False       ,			A__ = None       ,			A__ = True       ,			**A__       ,			):
						A__				:       List[str]	  =						kwargs.pop("""use_source_tokenizer"""       ,			A__     )

						A__				:       Union[str, Any]	  =						self.convert_ids_to_tokens(A__       ,			skip_special_tokens=A__     )

						# To avoid mixing byte-level and unicode for byte-level BPT
						# we need to build string separately for added tokens and byte-level tokens
						# cf. https://github.com/huggingface/transformers/issues/1133
						A__				:       Dict	  =						[]
						A__				:       List[Any]	  =						[]
						for token in filtered_tokens:
							if skip_special_tokens and token in self.all_special_ids:
								continue
							if token in self.added_tokens_encoder:
								if current_sub_text:
									sub_texts.append(self.convert_tokens_to_string(A__     )     )
									A__				:       Optional[Any]	  =						[]
								sub_texts.append(A__     )
							else:
								current_sub_text.append(A__     )
						if current_sub_text:
							sub_texts.append(self.convert_tokens_to_string(A__     )     )

						# Mimic the behavior of the Rust tokenizer:
						# No space before [MASK] and [SEP]
						if spaces_between_special_tokens:
							A__				:       List[Any]	  =						re.sub(r""" (\[(MASK|SEP)\])"""       ,			r"""\1"""       ,			""" """.join(A__     )     )
						else:
							A__				:       Optional[Any]	  =						"""""".join(A__     )

						A__				:       Union[str, Any]	  =						(
						    clean_up_tokenization_spaces
						    if clean_up_tokenization_spaces is not None
						    else self.clean_up_tokenization_spaces
						)
						if clean_up_tokenization_spaces:
							A__				:       Union[str, Any]	  =						self.clean_up_tokenization(A__     )
							return clean_text
						else:
							return text






					def     __A     (					self       ,			A__       ,			A__ = None     ):
						if not os.path.isdir(A__     ):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
							return
						A__				:       str	  =						os.path.join(
						    A__       ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

						if os.path.abspath(self.vocab_file     ) != os.path.abspath(A__     ) and os.path.isfile(self.vocab_file     ):
							copyfile(self.vocab_file       ,			A__     )
						elif not os.path.isfile(self.vocab_file     ):
							with open(A__       ,			"""wb"""     ) as fi:
								A__				:       Dict	  =						self.sp_model.serialized_model_proto()
								fi.write(A__     )

						return (out_vocab_file,)






					def     __A     (					self       ,			A__       ,			A__ = None     ):
						if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						A__				:       List[Any]	  =						[self.cls_token_id]
						A__				:       List[Any]	  =						[self.sep_token_id]
						return cls + token_ids_a + sep + token_ids_a + sep






					def     __A     (					self       ,			A__       ,			A__ = None       ,			A__ = False     ):
						if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=A__       ,			token_ids_a=A__       ,			already_has_special_tokens=A__     )

						if token_ids_a is None:
							return [1] + ([0] * len(A__     )) + [1]
						return [1] + ([0] * len(A__     )) + [1] + ([0] * len(A__     )) + [1]






					def     __A     (					self       ,			A__       ,			A__ = None     ):
						A__				:       Any	  =						[self.sep_token_id]
						A__				:       List[Any]	  =						[self.cls_token_id]
						if token_ids_a is None:
							return len(cls + token_ids_a + sep     ) * [0]
						return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]



