




import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)





def 						__A						(      __lowerCamelCase       ) ->			List[Any]:
	a									=				torch.exp(__lowerCamelCase       )
	a									=				torch.sum(__lowerCamelCase				,     dim=1       )  # sum of exp(x_i)
	a									=				torch.sum(x * exp_x				,     dim=1       )  # sum of x_i * exp(x_i)
	return torch.log(__lowerCamelCase       ) - B / A

class 						__lowerCAmelCase      (      nn.Module					):
							def __init__(		self		:Any						, __magic_name__		:Any							):







								'''simple docstring'''







								super().__init__()
								a									=				config.output_attentions
								a									=				config.output_hidden_states
								a									=				nn.ModuleList([BertLayer(__magic_name__							) for _ in range(config.num_hidden_layers							)]							)
								a									=				nn.ModuleList([BertHighway(__magic_name__							) for _ in range(config.num_hidden_layers							)]							)

								a									=				[-1 for _ in range(config.num_hidden_layers							)]
							def  lowerCamelCase__				(		self		:Optional[int]						, __magic_name__		:Union[str, Any]							):







								'''simple docstring'''







								if (type(__magic_name__							) is float) or (type(__magic_name__							) is int):
									for i in range(len(self.early_exit_entropy							)							):
										a									=				x
								else:
									a									=				x
							def  lowerCamelCase__				(		self		:str						, __magic_name__		:List[Any]							):







								'''simple docstring'''







								a									=				pooler.state_dict()
								for highway in self.highway:
									for name, param in highway.pooler.state_dict().items():
										param.copy_(loaded_model[name]							)





							def  lowerCamelCase__				(		self		:Optional[Any]						, __magic_name__		:Tuple						, __magic_name__		:Union[str, Any]=None						, __magic_name__		:List[str]=None						, __magic_name__		:Optional[int]=None						, __magic_name__		:Any=None						, ):







								'''simple docstring'''







								a									=				()
								a									=				()
								a									=				()
								for i, layer_module in enumerate(self.layer							):
									if self.output_hidden_states:
										a									=				all_hidden_states + (hidden_states,)

									a									=				layer_module(
									    __magic_name__						, __magic_name__						, head_mask[i]						, __magic_name__						, __magic_name__							)
									a									=				layer_outputs[0]

									if self.output_attentions:
										a									=				all_attentions + (layer_outputs[1],)

									a									=				(hidden_states,)
									if self.output_hidden_states:
										a									=				current_outputs + (all_hidden_states,)
									if self.output_attentions:
										a									=				current_outputs + (all_attentions,)

									a									=				self.highway[i](__magic_name__							)
									# logits, pooled_output

									if not self.training:
										a									=				highway_exit[0]
										a									=				entropy(__magic_name__							)
										a									=				highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
										a									=				all_highway_exits + (highway_exit,)

										if highway_entropy < self.early_exit_entropy[i]:
											a									=				(highway_logits,) + current_outputs[1:] + (all_highway_exits,)
											raise HighwayException(__magic_name__						, i + 1							)
									else:
										a									=				all_highway_exits + (highway_exit,)

        # Add last layer
								if self.output_hidden_states:
									a									=				all_hidden_states + (hidden_states,)

								a									=				(hidden_states,)
								if self.output_hidden_states:
									a									=				outputs + (all_hidden_states,)
								if self.output_attentions:
									a									=				outputs + (all_attentions,)

								a									=				outputs + (all_highway_exits,)
								return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits







@add_start_docstrings(
    '''The Bert Model transformer with early exiting (DeeBERT). ''' ,			__magic_name__ ,			)
class 						__lowerCAmelCase      (      __magic_name__					):
							def __init__(		self		:Union[str, Any]						, __magic_name__		:List[str]							):







								'''simple docstring'''







								super().__init__(__magic_name__							)
								a									=				config

								a									=				BertEmbeddings(__magic_name__							)
								a									=				DeeBertEncoder(__magic_name__							)
								a									=				BertPooler(__magic_name__							)

								self.init_weights()
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								self.encoder.init_highway_pooler(self.pooler							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								return self.embeddings.word_embeddings
							def  lowerCamelCase__				(		self		:int						, __magic_name__		:str							):







								'''simple docstring'''







								a									=				value
							def  lowerCamelCase__				(		self		:List[Any]						, __magic_name__		:str							):







								'''simple docstring'''







								for layer, heads in heads_to_prune.items():
									self.encoder.layer[layer].attention.prune_heads(__magic_name__							)





							@add_start_docstrings_to_model_forward(__magic_name__							)
							def  lowerCamelCase__				(		self		:str						, __magic_name__		:str=None						, __magic_name__		:Optional[Any]=None						, __magic_name__		:Any=None						, __magic_name__		:Union[str, Any]=None						, __magic_name__		:List[Any]=None						, __magic_name__		:Tuple=None						, __magic_name__		:Optional[int]=None						, __magic_name__		:Union[str, Any]=None						, ):







								'''simple docstring'''







								if input_ids is not None and inputs_embeds is not None:
									raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""							)
								elif input_ids is not None:
									a									=				input_ids.size()
								elif inputs_embeds is not None:
									a									=				inputs_embeds.size()[:-1]
								else:
									raise ValueError("""You have to specify either input_ids or inputs_embeds"""							)

								a									=				input_ids.device if input_ids is not None else inputs_embeds.device

								if attention_mask is None:
									a									=				torch.ones(__magic_name__						, device=__magic_name__							)
								if encoder_attention_mask is None:
									a									=				torch.ones(__magic_name__						, device=__magic_name__							)
								if token_type_ids is None:
									a									=				torch.zeros(__magic_name__						, dtype=torch.long						, device=__magic_name__							)

								# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
								# ourselves in which case we just need to make it broadcastable to all heads.
								a									=				self.get_extended_attention_mask(__magic_name__						, __magic_name__						, __magic_name__							)

								# If a 2D ou 3D attention mask is provided for the cross-attention
								# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
								if encoder_attention_mask.dim() == 3:
									a									=				encoder_attention_mask[:, None, :, :]
								if encoder_attention_mask.dim() == 2:
									a									=				encoder_attention_mask[:, None, None, :]

								a									=				encoder_extended_attention_mask.to(
								    dtype=next(self.parameters()							).dtype							)  # fp16 compatibility
								a									=				(1.0 - encoder_extended_attention_mask) * -10000.0

								# Prepare head mask if needed
								# 1.0 in head_mask indicate we keep the head
								# attention_probs has shape bsz x n_heads x N x N
								# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
								# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
								a									=				self.get_head_mask(__magic_name__						, self.config.num_hidden_layers							)

								a									=				self.embeddings(
								    input_ids=__magic_name__						, position_ids=__magic_name__						, token_type_ids=__magic_name__						, inputs_embeds=__magic_name__							)
								a									=				self.encoder(
								    __magic_name__						, attention_mask=__magic_name__						, head_mask=__magic_name__						, encoder_hidden_states=__magic_name__						, encoder_attention_mask=__magic_name__						, )
								a									=				encoder_outputs[0]
								a									=				self.pooler(__magic_name__							)

								a									=				(
								    sequence_output,
								    pooled_output,
								) + encoder_outputs[
								    1:
								]  # add hidden_states and attentions if they are here
								return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits







class 						__lowerCAmelCase      (      __magic_name__					):
							def __init__(		self		:Optional[Any]						, __magic_name__		:Optional[Any]						, __magic_name__		:Any							):







								'''simple docstring'''







								a									=				message
								a									=				exit_layer  # start from 1!







class 						__lowerCAmelCase      (      nn.Module					):
							def __init__(		self		:int						, __magic_name__		:Dict							):







								'''simple docstring'''







								super().__init__()
								a									=				BertPooler(__magic_name__							)
								a									=				nn.Dropout(config.hidden_dropout_prob							)
								a									=				nn.Linear(config.hidden_size						, config.num_labels							)





							def  lowerCamelCase__				(		self		:int						, __magic_name__		:Dict							):







								'''simple docstring'''







								a									=				encoder_outputs[0]
								a									=				self.pooler(__magic_name__							)
								# "return" pooler_output

								# BertModel
								a									=				(pooler_input, pooler_output) + encoder_outputs[1:]
								# "return" bmodel_output

								# Dropout and classification
								a									=				bmodel_output[1]

								a									=				self.dropout(__magic_name__							)
								a									=				self.classifier(__magic_name__							)

								return logits, pooled_output







@add_start_docstrings(
    '''Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. ''' ,			__magic_name__ ,			)
class 						__lowerCAmelCase      (      __magic_name__					):
							def __init__(		self		:int						, __magic_name__		:List[str]							):







								'''simple docstring'''







								super().__init__(__magic_name__							)
								a									=				config.num_labels
								a									=				config.num_hidden_layers

								a									=				DeeBertModel(__magic_name__							)
								a									=				nn.Dropout(config.hidden_dropout_prob							)
								a									=				nn.Linear(config.hidden_size						, self.config.num_labels							)

								self.init_weights()





							@add_start_docstrings_to_model_forward(__magic_name__							)
							def  lowerCamelCase__				(		self		:List[Any]						, __magic_name__		:int=None						, __magic_name__		:Dict=None						, __magic_name__		:str=None						, __magic_name__		:List[str]=None						, __magic_name__		:Union[str, Any]=None						, __magic_name__		:Any=None						, __magic_name__		:Tuple=None						, __magic_name__		:Any=-1						, __magic_name__		:Tuple=False						, ):







								'''simple docstring'''







								a									=				self.num_layers
								try:
									a									=				self.bert(
									    __magic_name__						, attention_mask=__magic_name__						, token_type_ids=__magic_name__						, position_ids=__magic_name__						, head_mask=__magic_name__						, inputs_embeds=__magic_name__						, )
									# sequence_output, pooled_output, (hidden_states), (attentions), highway exits

									a									=				outputs[1]

									a									=				self.dropout(__magic_name__							)
									a									=				self.classifier(__magic_name__							)
									a									=				(logits,) + outputs[2:]  # add hidden states and attention if they are here
								except HighwayException as e:
									a									=				e.message
									a									=				e.exit_layer
									a									=				outputs[0]

								if not self.training:
									a									=				entropy(__magic_name__							)
									a									=				[]
									a									=				[]
								if labels is not None:
									if self.num_labels == 1:
										#  We are doing regression
										a									=				MSELoss()
										a									=				loss_fct(logits.view(-1							)						, labels.view(-1							)							)
									else:
										a									=				CrossEntropyLoss()
										a									=				loss_fct(logits.view(-1						, self.num_labels							)						, labels.view(-1							)							)

									# work with highway exits
									a									=				[]
									for highway_exit in outputs[-1]:
										a									=				highway_exit[0]
										if not self.training:
											highway_logits_all.append(__magic_name__							)
											highway_entropy.append(highway_exit[2]							)
										if self.num_labels == 1:
											#  We are doing regression
											a									=				MSELoss()
											a									=				loss_fct(highway_logits.view(-1							)						, labels.view(-1							)							)
										else:
											a									=				CrossEntropyLoss()
											a									=				loss_fct(highway_logits.view(-1						, self.num_labels							)						, labels.view(-1							)							)
										highway_losses.append(__magic_name__							)

									if train_highway:
										a									=				(sum(highway_losses[:-1]							),) + outputs
										# exclude the final highway, of course
									else:
										a									=				(loss,) + outputs
								if not self.training:
									a									=				outputs + ((original_entropy, highway_entropy), exit_layer)
									if output_layer >= 0:
										a									=				(
										    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
										)  # use the highway of the last layer

								return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)







import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast

class 						__lowerCAmelCase      (      unittest.TestCase					):
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				0
							@slow
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)
									self.assertIsNotNone(__magic_name__							)
									self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
									self.assertGreater(len(__magic_name__							)						, 0							)

								for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)
									self.assertIsNotNone(__magic_name__							)
									self.assertIsInstance(__magic_name__						, (GPTaTokenizer, GPTaTokenizerFast)							)
									self.assertGreater(len(__magic_name__							)						, 0							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (RobertaTokenizer, RobertaTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 20							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoConfig.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, __magic_name__							)
								# Check that tokenizer_type ≠ model_type
								a									=				AutoTokenizer.from_pretrained(__magic_name__						, config=__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.txt"""						, os.path.join(__magic_name__						, """vocab.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""bert"""						, use_fast=__magic_name__							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)

								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.json"""						, os.path.join(__magic_name__						, """vocab.json"""							)							)
									shutil.copy("""./tests/fixtures/merges.txt"""						, os.path.join(__magic_name__						, """merges.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""gpt2"""						, use_fast=__magic_name__							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.txt"""						, os.path.join(__magic_name__						, """vocab.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""bert"""							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)

								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.json"""						, os.path.join(__magic_name__						, """vocab.json"""							)							)
									shutil.copy("""./tests/fixtures/merges.txt"""						, os.path.join(__magic_name__						, """merges.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""gpt2"""							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								with pytest.raises(__magic_name__							):
									AutoTokenizer.from_pretrained("""./"""						, tokenizer_type="""xxx"""							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									a									=				tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""							)
									self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)

									if isinstance(__magic_name__						, __magic_name__							):
										self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						, __magic_name__							)
									else:
										self.assertEqual(tokenizer.do_lower_case						, __magic_name__							)

									self.assertEqual(tokenizer.model_max_length						, 512							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									with self.assertRaisesRegex(
									    __magic_name__						, """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""						, ):
										a									=				tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				TOKENIZER_MAPPING.values()
								a									=				[]

								for slow_tok, fast_tok in tokenizers:
									if slow_tok is not None:
										tokenizer_names.append(slow_tok.__name__							)

									if fast_tok is not None:
										tokenizer_names.append(fast_tok.__name__							)

								for tokenizer_name in tokenizer_names:
									# must find the right class
									tokenizer_class_from_name(__magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""						, use_fast=__magic_name__							)						, __magic_name__							)
								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""							)						, __magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""distilbert-base-uncased"""						, do_lower_case=__magic_name__							)
								a									=				"""Hello, world. How are you?"""
								a									=				tokenizer.tokenize(__magic_name__							)
								self.assertEqual("""[UNK]"""						, tokens[0]							)

								a									=				AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""						, do_lower_case=__magic_name__							)
								a									=				tokenizer.tokenize(__magic_name__							)
								self.assertEqual("""[UNK]"""						, tokens[0]							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""							)
								self.assertEqual(type(__magic_name__							)						, __magic_name__							)
								self.assertEqual(tokenizer.model_max_length						, 512							)
								self.assertEqual(tokenizer.vocab_size						, 3_0000							)
								self.assertEqual(tokenizer.unk_token						, """[UNK]"""							)
								self.assertEqual(tokenizer.padding_side						, """right"""							)
								self.assertEqual(tokenizer.truncation_side						, """right"""							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)

								self.assertIsInstance(__magic_name__						, tokenizer.__class__							)
								self.assertEqual(tokenizera.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""ctrl"""							)
								# There is no fast CTRL so this always gives us a slow tokenizer.
								self.assertIsInstance(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				get_tokenizer_config("""bert-base-cased"""							)
								a									=				config.pop("""_commit_hash"""						, __magic_name__							)
								# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
								self.assertEqual(__magic_name__						, {"""do_lower_case""": False}							)

								# This model does not have a tokenizer_config so we get back an empty dict.
								a									=				get_tokenizer_config(__magic_name__							)
								self.assertDictEqual(__magic_name__						, {}							)

								# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				get_tokenizer_config(__magic_name__							)

								# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
								self.assertEqual(config["""tokenizer_class"""]						, """BertTokenizer"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								try:
									AutoConfig.register("""custom"""						, __magic_name__							)

									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(__magic_name__							):
										AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)

									a									=				CustomTokenizer.from_pretrained(__magic_name__							)
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								try:
									AutoConfig.register("""custom"""						, __magic_name__							)

									# Can register in two steps
									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, None)							)
									AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, CustomTokenizerFast)							)

									del TOKENIZER_MAPPING._extra_content[CustomConfig]
									# Can register in one step
									AutoTokenizer.register(
									    __magic_name__						, slow_tokenizer_class=__magic_name__						, fast_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, CustomTokenizerFast)							)

									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(__magic_name__							):
										AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)

									# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
									# and that model does not have a tokenizer.json
									with tempfile.TemporaryDirectory() as tmp_dir:
										a									=				BertTokenizerFast.from_pretrained(__magic_name__							)
										bert_tokenizer.save_pretrained(__magic_name__							)
										a									=				CustomTokenizerFast.from_pretrained(__magic_name__							)

									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__						, use_fast=__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								with self.assertRaises(__magic_name__							):
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(__magic_name__							):
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)

								a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
								self.assertTrue(tokenizer.special_attribute_present							)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				AutoTokenizer.from_pretrained(__magic_name__						, trust_remote_code=__magic_name__							)
								self.assertTrue(reloaded_tokenizer.special_attribute_present							)

								if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizerFast"""							)

									# Test we can also load the slow version
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertTrue(tokenizer.special_attribute_present							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									# Test tokenizer can be reloaded.
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)
										a									=				AutoTokenizer.from_pretrained(__magic_name__						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertTrue(reloaded_tokenizer.special_attribute_present							)
								else:
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizer"""							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								class 						__lowerCAmelCase      (      __magic_name__					):
															UpperCamelCase__   	=    False







								class 						__lowerCAmelCase      (      __magic_name__					):
															UpperCamelCase__   	=    NewTokenizer
															UpperCamelCase__   	=    False

								try:
									AutoConfig.register("""custom"""						, __magic_name__							)
									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)
									# If remote code is not set, the default is to use local
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertFalse(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertFalse(tokenizer.special_attribute_present							)

									# If remote code is disabled, we load the local one.
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertFalse(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertFalse(tokenizer.special_attribute_present							)

									# If remote is enabled, we load from the Hub
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertTrue(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertTrue(tokenizer.special_attribute_present							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(
								    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						, trust_remote_code=__magic_name__							)
								self.assertTrue(tokenizer.special_attribute_present							)
								if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)

									# Test we can also load the slow version
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertTrue(tokenizer.special_attribute_present							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
								else:
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								with self.assertRaisesRegex(
								    __magic_name__						, """bert-base is not a local folder and is not a valid model identifier"""							):
									a									=				AutoTokenizer.from_pretrained("""bert-base"""							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								with self.assertRaisesRegex(
								    __magic_name__						, r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
									a									=				AutoTokenizer.from_pretrained(__magic_name__						, revision="""aaaaaa"""							)





							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
								with RequestCounter() as counter:
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
									self.assertEqual(counter.get_request_count						, 0							)
									self.assertEqual(counter.head_request_count						, 1							)
									self.assertEqual(counter.other_request_count						, 0							)

