






import argparse
import math
import os
from copy import deepcopy

import torch
from audio_diffusion.models import DiffusionAttnUnetaD
from diffusion import sampling
from torch import nn

from diffusers import DanceDiffusionPipeline, IPNDMScheduler, UNetaDModel


SCREAMING_SNAKE_CASE__   :      Tuple				=		{
    """gwf-440k""": {
        """url""": """https://model-server.zqevans2.workers.dev/gwf-440k.ckpt""",
        """sample_rate""": 4_80_00,
        """sample_size""": 6_55_36,
    },
    """jmann-small-190k""": {
        """url""": """https://model-server.zqevans2.workers.dev/jmann-small-190k.ckpt""",
        """sample_rate""": 4_80_00,
        """sample_size""": 6_55_36,
    },
    """jmann-large-580k""": {
        """url""": """https://model-server.zqevans2.workers.dev/jmann-large-580k.ckpt""",
        """sample_rate""": 4_80_00,
        """sample_size""": 13_10_72,
    },
    """maestro-uncond-150k""": {
        """url""": """https://model-server.zqevans2.workers.dev/maestro-uncond-150k.ckpt""",
        """sample_rate""": 1_60_00,
        """sample_size""": 6_55_36,
    },
    """unlocked-uncond-250k""": {
        """url""": """https://model-server.zqevans2.workers.dev/unlocked-uncond-250k.ckpt""",
        """sample_rate""": 1_60_00,
        """sample_size""": 6_55_36,
    },
    """honk-140k""": {
        """url""": """https://model-server.zqevans2.workers.dev/honk-140k.ckpt""",
        """sample_rate""": 1_60_00,
        """sample_size""": 6_55_36,
    },
}

def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				return torch.atana(snake_case,	snake_case			) / math.pi * 2

def   __lowercase       (			snake_case			):







				"""simple docstring"""






				__magic_name__		:int   =   torch.sin(t * math.pi / 2			) ** 2
				__magic_name__		:Optional[int]   =   (1 - sigma**2) ** 0.5
				return alpha_sigma_to_t(snake_case,	snake_case			)
class     lowerCamelCase_					(   lowerCamelCase			):
				pass







class     lowerCamelCase_					(   nn.Module			):







				def __init__(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								super().__init__()

								__magic_name__		:List[str]   =   DiffusionAttnUnetaD(__lowerCAmelCase							,       n_attn_layers=4  )
								__magic_name__		:Any   =   deepcopy(self.diffusion  )
								__magic_name__		:Tuple   =   torch.quasirandom.SobolEngine(1							,       scramble=__lowerCAmelCase  )







def   __lowercase       (			snake_case			):







				"""simple docstring"""






				__magic_name__		:List[Any]   =   MODELS_MAP[model_name]['''url''']
				os.system(f'''wget {url} ./'''			)

				return f'''./{model_name}.ckpt'''


SCREAMING_SNAKE_CASE__   :      Optional[int]				=		{
    """1""": """resnets.0""",
    """2""": """attentions.0""",
    """3""": """resnets.1""",
    """4""": """attentions.1""",
    """5""": """resnets.2""",
    """6""": """attentions.2""",
}
SCREAMING_SNAKE_CASE__   :      List[str]				=		{
    """8""": """resnets.0""",
    """9""": """attentions.0""",
    """10""": """resnets.1""",
    """11""": """attentions.1""",
    """12""": """resnets.2""",
    """13""": """attentions.2""",
}
SCREAMING_SNAKE_CASE__   :      int				=		{
    """1""": """resnets.0""",
    """2""": """attentions.0""",
    """3""": """resnets.1""",
    """4""": """attentions.1""",
    """5""": """resnets.2""",
    """6""": """attentions.2""",
    """8""": """resnets.3""",
    """9""": """attentions.3""",
    """10""": """resnets.4""",
    """11""": """attentions.4""",
    """12""": """resnets.5""",
    """13""": """attentions.5""",
}
SCREAMING_SNAKE_CASE__   :      List[str]				=		{
    """0""": """resnets.0""",
    """1""": """resnets.1""",
    """2""": """resnets.2""",
    """4""": """resnets.0""",
    """5""": """resnets.1""",
    """6""": """resnets.2""",
}

SCREAMING_SNAKE_CASE__   :      str				=		{
    """skip""": """conv_skip""",
    """main.0""": """conv_1""",
    """main.1""": """group_norm_1""",
    """main.3""": """conv_2""",
    """main.4""": """group_norm_2""",
}

SCREAMING_SNAKE_CASE__   :      Optional[Any]				=		{
    """norm""": """group_norm""",
    """qkv_proj""": ["""query""", """key""", """value"""],
    """out_proj""": ["""proj_attn"""],
}

def   __lowercase       (			snake_case			):







				"""simple docstring"""






				if name.startswith('''skip'''			):
								return name.replace('''skip''',	RES_CONV_MAP['''skip''']			)

				# name has to be of format main.{digit}
				if not name.startswith('''main.'''			):
								raise ValueError(f'''ResConvBlock error with {name}'''			)

				return name.replace(name[:6],	RES_CONV_MAP[name[:6]]			)

def   __lowercase       (			snake_case			):







				"""simple docstring"""






				for key, value in ATTN_MAP.items():
								if name.startswith(snake_case			) and not isinstance(snake_case,	snake_case			):
												return name.replace(snake_case,	snake_case			)
								elif name.startswith(snake_case			):
												return [name.replace(snake_case,	snake_case			) for v in value]
				raise ValueError(f'''Attn error with {name}'''			)

def   __lowercase       (			snake_case,	snake_case=1_3			):







				"""simple docstring"""






				__magic_name__		:List[Any]   =   input_string

				if string.split('''.'''			)[0] == "timestep_embed":
								return string.replace('''timestep_embed''',	'''time_proj'''			)

				__magic_name__		:Dict   =   0
				if string.startswith('''net.3.'''			):
								depth += 1
								__magic_name__		:int   =   string[6:]
				elif string.startswith('''net.'''			):
								__magic_name__		:str   =   string[4:]

				while string.startswith('''main.7.'''			):
								depth += 1
								__magic_name__		:List[Any]   =   string[7:]

				if string.startswith('''main.'''			):
								__magic_name__		:Optional[int]   =   string[5:]

				# mid block
				if string[:2].isdigit():
								__magic_name__		:Optional[Any]   =   string[:2]
								__magic_name__		:int   =   string[2:]
				else:
								__magic_name__		:Dict   =   string[0]
								__magic_name__		:Dict   =   string[1:]

				if depth == max_depth:
								__magic_name__		:Optional[int]   =   MID_NUM_TO_LAYER[layer_num]
								__magic_name__		:Tuple   =   '''mid_block'''
				elif depth > 0 and int(snake_case			) < 7:
								__magic_name__		:List[Any]   =   DOWN_NUM_TO_LAYER[layer_num]
								__magic_name__		:Optional[Any]   =   f'''down_blocks.{depth}'''
				elif depth > 0 and int(snake_case			) > 7:
								__magic_name__		:int   =   UP_NUM_TO_LAYER[layer_num]
								__magic_name__		:Union[str, Any]   =   f'''up_blocks.{max_depth - depth - 1}'''
				elif depth == 0:
								__magic_name__		:Dict   =   DEPTH_0_TO_LAYER[layer_num]
								__magic_name__		:Any   =   f'''up_blocks.{max_depth - 1}''' if int(snake_case			) > 3 else '''down_blocks.0'''

				if not string_left.startswith('''.'''			):
								raise ValueError(f'''Naming error with {input_string} and string_left: {string_left}.'''			)

				__magic_name__		:str   =   string_left[1:]

				if "resnets" in new_layer:
								__magic_name__		:Optional[Any]   =   convert_resconv_naming(snake_case			)
				elif "attentions" in new_layer:
								__magic_name__		:List[Any]   =   convert_attn_naming(snake_case			)
								__magic_name__		:Optional[Any]   =   new_string_left

				if not isinstance(snake_case,	snake_case			):
								__magic_name__		:List[Any]   =   prefix + '''.''' + new_layer + '''.''' + string_left
				else:
								__magic_name__		:str   =   [prefix + '''.''' + new_layer + '''.''' + s for s in string_left]
				return new_string

def   __lowercase       (			snake_case			):







				"""simple docstring"""






				__magic_name__		:int   =   {}
				for k, v in state_dict.items():
								if k.endswith('''kernel'''			):
												# up- and downsample layers, don't have trainable weights
												continue

								__magic_name__		:Tuple   =   rename(snake_case			)

								# check if we need to transform from Conv => Linear for attention
								if isinstance(snake_case,	snake_case			):
												__magic_name__		:Any   =   transform_conv_attns(snake_case,	snake_case,	snake_case			)
								else:
												__magic_name__		:List[Any]   =   v

				return new_state_dict

def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				if len(snake_case			) == 1:
								if len(v.shape			) == 3:
												# weight
												__magic_name__		:int   =   v[:, :, 0]
								else:
												# bias
												__magic_name__		:Union[str, Any]   =   v
				else:
								# qkv matrices
								__magic_name__		:Tuple   =   v.shape[0]
								__magic_name__		:Optional[Any]   =   trippled_shape // 3
								for i in range(3			):
												if len(v.shape			) == 3:
																__magic_name__		:str   =   v[i * single_shape : (i + 1) * single_shape, :, 0]
												else:
																__magic_name__		:Any   =   v[i * single_shape : (i + 1) * single_shape]
				return new_state_dict

def   __lowercase       (			snake_case			):







				"""simple docstring"""






				__magic_name__		:int   =   torch.device('''cuda''' if torch.cuda.is_available() else '''cpu'''			)

				__magic_name__		:Optional[int]   =   args.model_path.split('''/'''			)[-1].split('''.'''			)[0]
				if not os.path.isfile(args.model_path			):
								assert (
								    model_name == args.model_path
								), f'''Make sure to provide one of the official model names {MODELS_MAP.keys()}'''
								__magic_name__		:Optional[Any]   =   download(snake_case			)

				__magic_name__		:int   =   MODELS_MAP[model_name]['''sample_rate''']
				__magic_name__		:Optional[int]   =   MODELS_MAP[model_name]['''sample_size''']

				__magic_name__		:List[Any]   =   Object()
				__magic_name__		:Any   =   sample_size
				__magic_name__		:str   =   sample_rate
				__magic_name__		:str   =   0

				__magic_name__		:Tuple   =   UNetaDModel(sample_size=snake_case,	sample_rate=snake_case			)
				__magic_name__		:Any   =   diffusers_model.state_dict()

				__magic_name__		:Dict   =   DiffusionUncond(snake_case			)
				orig_model.load_state_dict(torch.load(args.model_path,	map_location=snake_case			)['''state_dict''']			)
				__magic_name__		:List[Any]   =   orig_model.diffusion_ema.eval()
				__magic_name__		:Union[str, Any]   =   orig_model.state_dict()
				__magic_name__		:Tuple   =   rename_orig_weights(snake_case			)

				__magic_name__		:int   =   set(renamed_state_dict.keys()			) - set(diffusers_state_dict.keys()			)
				__magic_name__		:List[str]   =   set(diffusers_state_dict.keys()			) - set(renamed_state_dict.keys()			)

				assert len(snake_case			) == 0, f'''Problem with {renamed_minus_diffusers}'''
				assert all(k.endswith('''kernel'''			) for k in list(snake_case			)			), f'''Problem with {diffusers_minus_renamed}'''

				for key, value in renamed_state_dict.items():
								assert (
								    diffusers_state_dict[key].squeeze().shape == value.squeeze().shape
								), f'''Shape for {key} doesn\'t match. Diffusers: {diffusers_state_dict[key].shape} vs. {value.shape}'''
								if key == "time_proj.weight":
												__magic_name__		:Dict   =   value.squeeze()

								__magic_name__		:Optional[int]   =   value

				diffusers_model.load_state_dict(snake_case			)

				__magic_name__		:Tuple   =   1_0_0
				__magic_name__		:List[Any]   =   3_3

				__magic_name__		:Union[str, Any]   =   IPNDMScheduler(num_train_timesteps=snake_case			)

				__magic_name__		:Any   =   torch.manual_seed(snake_case			)
				__magic_name__		:List[str]   =   torch.randn([1, 2, config.sample_size],	generator=snake_case			).to(snake_case			)

				__magic_name__		:Any   =   torch.linspace(1,	0,	steps + 1,	device=snake_case			)[:-1]
				__magic_name__		:Optional[Any]   =   get_crash_schedule(snake_case			)

				__magic_name__		:int   =   DanceDiffusionPipeline(unet=snake_case,	scheduler=snake_case			)

				__magic_name__		:Any   =   torch.manual_seed(3_3			)
				__magic_name__		:int   =   pipe(num_inference_steps=snake_case,	generator=snake_case			).audios

				__magic_name__		:Optional[Any]   =   sampling.iplms_sample(snake_case,	snake_case,	snake_case,	{}			)
				__magic_name__		:Tuple   =   generated.clamp(-1,	1			)

				__magic_name__		:Tuple   =   (generated - audio).abs().sum()
				__magic_name__		:Optional[Any]   =   (generated - audio).abs().max()

				if args.save:
								pipe.save_pretrained(args.checkpoint_path			)

				print('''Diff sum''',	snake_case			)
				print('''Diff max''',	snake_case			)

				assert diff_max < 1E-3, f'''Diff max: {diff_max} is too much :-/'''

				print(f'''Conversion for {model_name} successful!'''			)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE__   :      List[Any]				=		argparse.ArgumentParser()

					parser.add_argument("""--model_path""", default=None, type=str, required=True, help="""Path to the model to convert.""")
					parser.add_argument(
					    """--save""", default=True, type=bool, required=False, help="""Whether to save the converted model or not."""
					)
					parser.add_argument("""--checkpoint_path""", default=None, type=str, required=True, help="""Path to the output model.""")
					SCREAMING_SNAKE_CASE__   :      str				=		parser.parse_args()

					main(args)












import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast

class 						__lowerCAmelCase      (      unittest.TestCase					):
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				0
							@slow
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)
									self.assertIsNotNone(__magic_name__							)
									self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
									self.assertGreater(len(__magic_name__							)						, 0							)

								for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)
									self.assertIsNotNone(__magic_name__							)
									self.assertIsInstance(__magic_name__						, (GPTaTokenizer, GPTaTokenizerFast)							)
									self.assertGreater(len(__magic_name__							)						, 0							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (RobertaTokenizer, RobertaTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 20							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoConfig.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, __magic_name__							)
								# Check that tokenizer_type ≠ model_type
								a									=				AutoTokenizer.from_pretrained(__magic_name__						, config=__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.txt"""						, os.path.join(__magic_name__						, """vocab.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""bert"""						, use_fast=__magic_name__							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)

								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.json"""						, os.path.join(__magic_name__						, """vocab.json"""							)							)
									shutil.copy("""./tests/fixtures/merges.txt"""						, os.path.join(__magic_name__						, """merges.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""gpt2"""						, use_fast=__magic_name__							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.txt"""						, os.path.join(__magic_name__						, """vocab.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""bert"""							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)

								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.json"""						, os.path.join(__magic_name__						, """vocab.json"""							)							)
									shutil.copy("""./tests/fixtures/merges.txt"""						, os.path.join(__magic_name__						, """merges.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""gpt2"""							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								with pytest.raises(__magic_name__							):
									AutoTokenizer.from_pretrained("""./"""						, tokenizer_type="""xxx"""							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									a									=				tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""							)
									self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)

									if isinstance(__magic_name__						, __magic_name__							):
										self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						, __magic_name__							)
									else:
										self.assertEqual(tokenizer.do_lower_case						, __magic_name__							)

									self.assertEqual(tokenizer.model_max_length						, 512							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									with self.assertRaisesRegex(
									    __magic_name__						, """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""						, ):
										a									=				tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				TOKENIZER_MAPPING.values()
								a									=				[]

								for slow_tok, fast_tok in tokenizers:
									if slow_tok is not None:
										tokenizer_names.append(slow_tok.__name__							)

									if fast_tok is not None:
										tokenizer_names.append(fast_tok.__name__							)

								for tokenizer_name in tokenizer_names:
									# must find the right class
									tokenizer_class_from_name(__magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""						, use_fast=__magic_name__							)						, __magic_name__							)
								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""							)						, __magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""distilbert-base-uncased"""						, do_lower_case=__magic_name__							)
								a									=				"""Hello, world. How are you?"""
								a									=				tokenizer.tokenize(__magic_name__							)
								self.assertEqual("""[UNK]"""						, tokens[0]							)

								a									=				AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""						, do_lower_case=__magic_name__							)
								a									=				tokenizer.tokenize(__magic_name__							)
								self.assertEqual("""[UNK]"""						, tokens[0]							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""							)
								self.assertEqual(type(__magic_name__							)						, __magic_name__							)
								self.assertEqual(tokenizer.model_max_length						, 512							)
								self.assertEqual(tokenizer.vocab_size						, 3_0000							)
								self.assertEqual(tokenizer.unk_token						, """[UNK]"""							)
								self.assertEqual(tokenizer.padding_side						, """right"""							)
								self.assertEqual(tokenizer.truncation_side						, """right"""							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)

								self.assertIsInstance(__magic_name__						, tokenizer.__class__							)
								self.assertEqual(tokenizera.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""ctrl"""							)
								# There is no fast CTRL so this always gives us a slow tokenizer.
								self.assertIsInstance(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				get_tokenizer_config("""bert-base-cased"""							)
								a									=				config.pop("""_commit_hash"""						, __magic_name__							)
								# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
								self.assertEqual(__magic_name__						, {"""do_lower_case""": False}							)

								# This model does not have a tokenizer_config so we get back an empty dict.
								a									=				get_tokenizer_config(__magic_name__							)
								self.assertDictEqual(__magic_name__						, {}							)

								# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				get_tokenizer_config(__magic_name__							)

								# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
								self.assertEqual(config["""tokenizer_class"""]						, """BertTokenizer"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								try:
									AutoConfig.register("""custom"""						, __magic_name__							)

									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(__magic_name__							):
										AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)

									a									=				CustomTokenizer.from_pretrained(__magic_name__							)
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								try:
									AutoConfig.register("""custom"""						, __magic_name__							)

									# Can register in two steps
									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, None)							)
									AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, CustomTokenizerFast)							)

									del TOKENIZER_MAPPING._extra_content[CustomConfig]
									# Can register in one step
									AutoTokenizer.register(
									    __magic_name__						, slow_tokenizer_class=__magic_name__						, fast_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, CustomTokenizerFast)							)

									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(__magic_name__							):
										AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)

									# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
									# and that model does not have a tokenizer.json
									with tempfile.TemporaryDirectory() as tmp_dir:
										a									=				BertTokenizerFast.from_pretrained(__magic_name__							)
										bert_tokenizer.save_pretrained(__magic_name__							)
										a									=				CustomTokenizerFast.from_pretrained(__magic_name__							)

									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__						, use_fast=__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								with self.assertRaises(__magic_name__							):
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(__magic_name__							):
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)

								a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
								self.assertTrue(tokenizer.special_attribute_present							)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				AutoTokenizer.from_pretrained(__magic_name__						, trust_remote_code=__magic_name__							)
								self.assertTrue(reloaded_tokenizer.special_attribute_present							)

								if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizerFast"""							)

									# Test we can also load the slow version
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertTrue(tokenizer.special_attribute_present							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									# Test tokenizer can be reloaded.
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)
										a									=				AutoTokenizer.from_pretrained(__magic_name__						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertTrue(reloaded_tokenizer.special_attribute_present							)
								else:
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizer"""							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								class 						__lowerCAmelCase      (      __magic_name__					):
															UpperCamelCase__   	=    False







								class 						__lowerCAmelCase      (      __magic_name__					):
															UpperCamelCase__   	=    NewTokenizer
															UpperCamelCase__   	=    False

								try:
									AutoConfig.register("""custom"""						, __magic_name__							)
									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)
									# If remote code is not set, the default is to use local
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertFalse(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertFalse(tokenizer.special_attribute_present							)

									# If remote code is disabled, we load the local one.
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertFalse(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertFalse(tokenizer.special_attribute_present							)

									# If remote is enabled, we load from the Hub
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertTrue(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertTrue(tokenizer.special_attribute_present							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(
								    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						, trust_remote_code=__magic_name__							)
								self.assertTrue(tokenizer.special_attribute_present							)
								if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)

									# Test we can also load the slow version
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertTrue(tokenizer.special_attribute_present							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
								else:
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								with self.assertRaisesRegex(
								    __magic_name__						, """bert-base is not a local folder and is not a valid model identifier"""							):
									a									=				AutoTokenizer.from_pretrained("""bert-base"""							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								with self.assertRaisesRegex(
								    __magic_name__						, r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
									a									=				AutoTokenizer.from_pretrained(__magic_name__						, revision="""aaaaaa"""							)





							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
								with RequestCounter() as counter:
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
									self.assertEqual(counter.get_request_count						, 0							)
									self.assertEqual(counter.head_request_count						, 1							)
									self.assertEqual(counter.other_request_count						, 0							)

