





import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch

class  __A       (  unittest.TestCase      ):






				"""simple docstring"""




				__A								=			JukeboxTokenizer
				__A								=			{
				    """artist""": """Zac Brown Band""",
				    """genres""": """Country""",
				    """lyrics""": """I met a traveller from an antique land,
        Who said \"Two vast and trunkless legs of stone
        Stand in the desert. . . . Near them, on the sand,
        Half sunk a shattered visage lies, whose frown,
        And wrinkled lip, and sneer of cold command,
        Tell that its sculptor well those passions read
        Which yet survive, stamped on these lifeless things,
        The hand that mocked them, and the heart that fed;
        And on the pedestal, these words appear:
        My name is Ozymandias, King of Kings;
        Look on my Works, ye Mighty, and despair!
        Nothing beside remains. Round the decay
        Of that colossal Wreck, boundless and bare
        The lone and level sands stretch far away
        """,
				}


				@require_torch
				def _snake_case    (						self							):
								import torch

								lowerCamelCase			=JukeboxTokenizer.from_pretrained("""openai/jukebox-1b-lyrics"""							)
								lowerCamelCase			=tokenizer(**self.metas							)["""input_ids"""]
								# fmt: off
								lowerCamelCase			=[
								    torch.tensor([[
								        0, 0, 0, 7169, 507, 9, 76, 39, 31, 46, 76, 27,
								        76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
								        44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
								        47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
								        76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
								        30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
								        27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
								        45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
								        41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
								        19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
								        76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
								        76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
								        64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
								        30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
								        27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
								        34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
								        27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
								        41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
								        76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
								        44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
								        76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
								        32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
								        40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
								        20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
								        45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
								        31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
								        45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
								        76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
								        34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
								        31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
								        40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
								        38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
								        76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
								        41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
								        27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
								        46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
								        76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
								        41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
								        46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
								        41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
								        40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
								        27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
								        76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
								        76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
								        41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
								        76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
								        27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
								        34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
								        44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
								        40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
								        46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
								        38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
								        40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
								        27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
								        20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
								        76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
								        76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
								        76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
								        76, 76]]							),
								    torch.tensor([[0, 0, 0, 1069, 11]]							),
								    torch.tensor([[0, 0, 0, 1069, 11]]							),
								]
								# fmt: on
								self.assertTrue(torch.allclose(tokens[0]				,	EXPECTED_OUTPUT[0]							)							)
								self.assertTrue(torch.allclose(tokens[1]				,	EXPECTED_OUTPUT[1]							)							)
								self.assertTrue(torch.allclose(tokens[2]				,	EXPECTED_OUTPUT[2]							)							)


				@require_torch
				def _snake_case    (						self							):
								import torch

								lowerCamelCase			=JukeboxTokenizer.from_pretrained("""openai/jukebox-5b-lyrics"""							)
								lowerCamelCase			=tokenizer(**self.metas							)["""input_ids"""]
								# fmt: off
								lowerCamelCase			=[
								    torch.tensor([[
								        0, 0, 0, 1069, 11, -1, -1, -1, -1, 9, 77, 39,
								        31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
								        31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
								        40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
								        79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
								        77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
								        27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
								        37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
								        32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
								        77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
								        77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
								        77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
								        46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
								        77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
								        77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
								        77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
								        77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
								        64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
								        40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
								        40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
								        38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
								        31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
								        41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
								        77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
								        46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
								        41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
								        31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
								        31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
								        23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
								        44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
								        31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
								        38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
								        40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
								        27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
								        31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
								        34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
								        31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
								        1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
								        31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
								        45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
								        31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
								        15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
								        11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
								        45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
								        41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
								        44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
								        46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
								        27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
								        35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
								        77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
								        31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
								        41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
								        77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
								        40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
								        77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
								        77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
								        27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
								        77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
								        77, 77, 77, 77, 77, 77]]							),
								    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]							),
								    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]							),
								]
								# fmt: on
								self.assertTrue(torch.allclose(tokens[0]				,	EXPECTED_OUTPUT[0]							)							)
								self.assertTrue(torch.allclose(tokens[1]				,	EXPECTED_OUTPUT[1]							)							)
								self.assertTrue(torch.allclose(tokens[2]				,	EXPECTED_OUTPUT[2]							)							)












import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCAmelCase__  :     List[Any]   			=logging.get_logger(__name__)

UpperCAmelCase__  :     Dict   			={'''vocab_file''': '''spiece.model'''}

UpperCAmelCase__  :     Dict   			={
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

UpperCAmelCase__  :     List[str]   			={
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
UpperCAmelCase__  :     Any   			=0
UpperCAmelCase__  :     List[Any]   			=1
UpperCAmelCase__  :     Union[str, Any]   			=2
UpperCAmelCase__  :     Tuple   			=3
UpperCAmelCase__  :     int   			=4

class  __A       (  a      ):
				__A								=			VOCAB_FILES_NAMES
				__A								=			PRETRAINED_VOCAB_FILES_MAP
				__A								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A								=			"""left"""


				def __init__(						self				,	UpperCAmelCase_				,	UpperCAmelCase_=False				,	UpperCAmelCase_=True				,	UpperCAmelCase_=False				,	UpperCAmelCase_="<s>"				,	UpperCAmelCase_="</s>"				,	UpperCAmelCase_="<unk>"				,	UpperCAmelCase_="<sep>"				,	UpperCAmelCase_="<pad>"				,	UpperCAmelCase_="<cls>"				,	UpperCAmelCase_="<mask>"				,	UpperCAmelCase_=["<eop>", "<eod>"]				,	UpperCAmelCase_ = None				,	**UpperCAmelCase_				,	):
								# Mask token behave like a normal word, i.e. include the space before it
								lowerCamelCase			=AddedToken(UpperCAmelCase_				,	lstrip=UpperCAmelCase_				,	rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_				,	UpperCAmelCase_							) else mask_token

								lowerCamelCase			={} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=UpperCAmelCase_				,	remove_space=UpperCAmelCase_				,	keep_accents=UpperCAmelCase_				,	bos_token=UpperCAmelCase_				,	eos_token=UpperCAmelCase_				,	unk_token=UpperCAmelCase_				,	sep_token=UpperCAmelCase_				,	pad_token=UpperCAmelCase_				,	cls_token=UpperCAmelCase_				,	mask_token=UpperCAmelCase_				,	additional_special_tokens=UpperCAmelCase_				,	sp_model_kwargs=self.sp_model_kwargs				,	**UpperCAmelCase_				,	)

								lowerCamelCase			=3

								lowerCamelCase			=do_lower_case
								lowerCamelCase			=remove_space
								lowerCamelCase			=keep_accents
								lowerCamelCase			=vocab_file

								lowerCamelCase			=spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.Load(UpperCAmelCase_							)


				@property
				def _snake_case    (						self							):
								return len(self.sp_model							)


				def _snake_case    (						self							):
								lowerCamelCase			={self.convert_ids_to_tokens(UpperCAmelCase_							): i for i in range(self.vocab_size							)}
								vocab.update(self.added_tokens_encoder							)
								return vocab


				def __getstate__(						self							):
								lowerCamelCase			=self.__dict__.copy()
								lowerCamelCase			=None
								return state


				def __setstate__(						self				,	UpperCAmelCase_							):
								lowerCamelCase			=d

								# for backward compatibility
								if not hasattr(self				,	"""sp_model_kwargs"""							):
												lowerCamelCase			={}

								lowerCamelCase			=spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.Load(self.vocab_file							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								if self.remove_space:
												lowerCamelCase			=""" """.join(inputs.strip().split()							)
								else:
												lowerCamelCase			=inputs
								lowerCamelCase			=outputs.replace("""``"""				,	"""\""""							).replace("""''"""				,	"""\""""							)

								if not self.keep_accents:
												lowerCamelCase			=unicodedata.normalize("""NFKD"""				,	UpperCAmelCase_							)
												lowerCamelCase			="""""".join([c for c in outputs if not unicodedata.combining(UpperCAmelCase_							)]							)
								if self.do_lower_case:
												lowerCamelCase			=outputs.lower()

								return outputs


				def _snake_case    (						self				,	UpperCAmelCase_							):
								lowerCamelCase			=self.preprocess_text(UpperCAmelCase_							)
								lowerCamelCase			=self.sp_model.encode(UpperCAmelCase_				,	out_type=UpperCAmelCase_							)
								lowerCamelCase			=[]
								for piece in pieces:
												if len(UpperCAmelCase_							) > 1 and piece[-1] == str(""","""							) and piece[-2].isdigit():
																lowerCamelCase			=self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase_				,	""""""							)							)
																if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																				if len(cur_pieces[0]							) == 1:
																								lowerCamelCase			=cur_pieces[1:]
																				else:
																								lowerCamelCase			=cur_pieces[0][1:]
																cur_pieces.append(piece[-1]							)
																new_pieces.extend(UpperCAmelCase_							)
												else:
																new_pieces.append(UpperCAmelCase_							)

								return new_pieces


				def _snake_case    (						self				,	UpperCAmelCase_							):
								return self.sp_model.PieceToId(UpperCAmelCase_							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								return self.sp_model.IdToPiece(UpperCAmelCase_							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								lowerCamelCase			="""""".join(UpperCAmelCase_							).replace(UpperCAmelCase_				,	""" """							).strip()
								return out_string


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = False				,	UpperCAmelCase_ = None				,	UpperCAmelCase_ = True				,	**UpperCAmelCase_				,	):
								lowerCamelCase			=kwargs.pop("""use_source_tokenizer"""				,	UpperCAmelCase_							)

								lowerCamelCase			=self.convert_ids_to_tokens(UpperCAmelCase_				,	skip_special_tokens=UpperCAmelCase_							)

								# To avoid mixing byte-level and unicode for byte-level BPT
								# we need to build string separately for added tokens and byte-level tokens
								# cf. https://github.com/huggingface/transformers/issues/1133
								lowerCamelCase			=[]
								lowerCamelCase			=[]
								for token in filtered_tokens:
												if skip_special_tokens and token in self.all_special_ids:
																continue
												if token in self.added_tokens_encoder:
																if current_sub_text:
																				sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase_							)							)
																				lowerCamelCase			=[]
																sub_texts.append(UpperCAmelCase_							)
												else:
																current_sub_text.append(UpperCAmelCase_							)
								if current_sub_text:
												sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase_							)							)

								# Mimic the behavior of the Rust tokenizer:
								# By default, there are no spaces between special tokens
								lowerCamelCase			="""""".join(UpperCAmelCase_							)

								lowerCamelCase			=(
								    clean_up_tokenization_spaces
								    if clean_up_tokenization_spaces is not None
								    else self.clean_up_tokenization_spaces
								)
								if clean_up_tokenization_spaces:
												lowerCamelCase			=self.clean_up_tokenization(UpperCAmelCase_							)
												return clean_text
								else:
												return text


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								lowerCamelCase			=[self.sep_token_id]
								lowerCamelCase			=[self.cls_token_id]
								if token_ids_a is None:
												return token_ids_a + sep + cls
								return token_ids_a + sep + token_ids_a + sep + cls


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None				,	UpperCAmelCase_ = False							):

								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCAmelCase_				,	token_ids_a=UpperCAmelCase_				,	already_has_special_tokens=UpperCAmelCase_							)

								if token_ids_a is not None:
												return ([0] * len(UpperCAmelCase_							)) + [1] + ([0] * len(UpperCAmelCase_							)) + [1, 1]
								return ([0] * len(UpperCAmelCase_							)) + [1, 1]


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								lowerCamelCase			=[self.sep_token_id]
								lowerCamelCase			=[2]

								if token_ids_a is None:
												return len(token_ids_a + sep							) * [0] + cls_segment_id
								return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								if not os.path.isdir(UpperCAmelCase_							):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
												return
								lowerCamelCase			=os.path.join(
								    UpperCAmelCase_				,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

								if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCAmelCase_							) and os.path.isfile(self.vocab_file							):
												copyfile(self.vocab_file				,	UpperCAmelCase_							)
								elif not os.path.isfile(self.vocab_file							):
												with open(UpperCAmelCase_				,	"""wb"""							) as fi:
																lowerCamelCase			=self.sp_model.serialized_model_proto()
																fi.write(UpperCAmelCase_							)

								return (out_vocab_file,)





