






import collections
import inspect
import unittest

from transformers import FocalNetConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch
							from torch import nn

							from transformers import (
							    FocalNetBackbone,
							    FocalNetForImageClassification,
							    FocalNetForMaskedImageModeling,
							    FocalNetModel,
							)
							from transformers.models.focalnet.modeling_focalnet import FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
							from PIL import Image

							from transformers import AutoImageProcessor

class 				lowerCAmelCase_	:


				def __init__(					self ,						_lowerCAmelCase ,						_lowerCAmelCase=1_3 ,						_lowerCAmelCase=3_2 ,						_lowerCAmelCase=2 ,						_lowerCAmelCase=3 ,						_lowerCAmelCase=1_6 ,						_lowerCAmelCase=[3_2, 6_4, 1_2_8] ,						_lowerCAmelCase=[1, 2, 1] ,						_lowerCAmelCase=[2, 2, 4] ,						_lowerCAmelCase=2 ,						_lowerCAmelCase=2.0 ,						_lowerCAmelCase=True ,						_lowerCAmelCase=0.0 ,						_lowerCAmelCase=0.0 ,						_lowerCAmelCase=0.1 ,						_lowerCAmelCase="gelu" ,						_lowerCAmelCase=False ,						_lowerCAmelCase=True ,						_lowerCAmelCase=0.02 ,						_lowerCAmelCase=1E-5 ,						_lowerCAmelCase=True ,						_lowerCAmelCase=None ,						_lowerCAmelCase=True ,						_lowerCAmelCase=1_0 ,						_lowerCAmelCase=8 ,						_lowerCAmelCase=["stage1", "stage2"] ,						_lowerCAmelCase=[1, 2] ,						):
											_lowercase							:   int 							=   parent
											_lowercase							:   str 							=   batch_size
											_lowercase							:   Optional[Any] 							=   image_size
											_lowercase							:   Union[str, Any] 							=   patch_size
											_lowercase							:   Optional[Any] 							=   num_channels
											_lowercase							:   List[str] 							=   embed_dim
											_lowercase							:   Optional[Any] 							=   hidden_sizes
											_lowercase							:   Dict 							=   depths
											_lowercase							:   int 							=   num_heads
											_lowercase							:   Union[str, Any] 							=   window_size
											_lowercase							:   Optional[int] 							=   mlp_ratio
											_lowercase							:   Dict 							=   qkv_bias
											_lowercase							:   Tuple 							=   hidden_dropout_prob
											_lowercase							:   Optional[int] 							=   attention_probs_dropout_prob
											_lowercase							:   Any 							=   drop_path_rate
											_lowercase							:   Optional[Any] 							=   hidden_act
											_lowercase							:   List[Any] 							=   use_absolute_embeddings
											_lowercase							:   List[Any] 							=   patch_norm
											_lowercase							:   Optional[Any] 							=   layer_norm_eps
											_lowercase							:   Optional[Any] 							=   initializer_range
											_lowercase							:   int 							=   is_training
											_lowercase							:   List[str] 							=   scope
											_lowercase							:   List[str] 							=   use_labels
											_lowercase							:   Tuple 							=   type_sequence_label_size
											_lowercase							:   str 							=   encoder_stride
											_lowercase							:   Dict 							=   out_features
											_lowercase							:   Optional[int] 							=   out_indices


				def 						__a					(					self      ):
											_lowercase							:   Tuple 							=   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]      )

											_lowercase							:   Dict 							=   None
											if self.use_labels:
																		_lowercase							:   Optional[Any] 							=   ids_tensor([self.batch_size] ,						self.type_sequence_label_size      )

											_lowercase							:   List[Any] 							=   self.get_config()

											return config, pixel_values, labels


				def 						__a					(					self      ):
											return FocalNetConfig(
											    image_size=self.image_size ,						patch_size=self.patch_size ,						num_channels=self.num_channels ,						embed_dim=self.embed_dim ,						hidden_sizes=self.hidden_sizes ,						depths=self.depths ,						num_heads=self.num_heads ,						window_size=self.window_size ,						mlp_ratio=self.mlp_ratio ,						qkv_bias=self.qkv_bias ,						hidden_dropout_prob=self.hidden_dropout_prob ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob ,						drop_path_rate=self.drop_path_rate ,						hidden_act=self.hidden_act ,						use_absolute_embeddings=self.use_absolute_embeddings ,						path_norm=self.patch_norm ,						layer_norm_eps=self.layer_norm_eps ,						initializer_range=self.initializer_range ,						encoder_stride=self.encoder_stride ,						out_features=self.out_features ,						out_indices=self.out_indices ,						)


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase      ):
											_lowercase							:   Optional[Any] 							=   FocalNetModel(config=_lowerCAmelCase      )
											model.to(_lowerCAmelCase      )
											model.eval()
											_lowercase							:   int 							=   model(_lowerCAmelCase      )

											_lowercase							:   List[str] 							=   ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths      ) - 1))
											_lowercase							:   Any 							=   int(config.embed_dim * 2 ** (len(config.depths      ) - 1)      )

											self.parent.assertEqual(result.last_hidden_state.shape ,						(self.batch_size, expected_seq_len, expected_dim)      )


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase      ):
											_lowercase							:   Optional[int] 							=   FocalNetBackbone(config=_lowerCAmelCase      )
											model.to(_lowerCAmelCase      )
											model.eval()
											_lowercase							:   Dict 							=   model(_lowerCAmelCase      )

											# verify feature maps
											self.parent.assertEqual(len(result.feature_maps      ) ,						len(config.out_features      )      )
											self.parent.assertListEqual(list(result.feature_maps[0].shape      ) ,						[self.batch_size, self.image_size, 8, 8]      )

											# verify channels
											self.parent.assertEqual(len(model.channels      ) ,						len(config.out_features      )      )
											self.parent.assertListEqual(model.channels ,						config.hidden_sizes[:-1]      )

											# verify backbone works with out_features=None
											_lowercase							:   List[str] 							=   None
											_lowercase							:   Optional[Any] 							=   FocalNetBackbone(config=_lowerCAmelCase      )
											model.to(_lowerCAmelCase      )
											model.eval()
											_lowercase							:   int 							=   model(_lowerCAmelCase      )

											# verify feature maps
											self.parent.assertEqual(len(result.feature_maps      ) ,						1      )
											self.parent.assertListEqual(list(result.feature_maps[0].shape      ) ,						[self.batch_size, self.image_size * 2, 4, 4]      )

											# verify channels
											self.parent.assertEqual(len(model.channels      ) ,						1      )
											self.parent.assertListEqual(model.channels ,						[config.hidden_sizes[-1]]      )


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase      ):
											_lowercase							:   Any 							=   FocalNetForMaskedImageModeling(config=_lowerCAmelCase      )
											model.to(_lowerCAmelCase      )
											model.eval()
											_lowercase							:   Union[str, Any] 							=   model(_lowerCAmelCase      )
											self.parent.assertEqual(
											    result.reconstruction.shape ,						(self.batch_size, self.num_channels, self.image_size, self.image_size)      )

											# test greyscale images
											_lowercase							:   Optional[int] 							=   1
											_lowercase							:   List[str] 							=   FocalNetForMaskedImageModeling(_lowerCAmelCase      )
											model.to(_lowerCAmelCase      )
											model.eval()

											_lowercase							:   Optional[int] 							=   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]      )
											_lowercase							:   Optional[int] 							=   model(_lowerCAmelCase      )
											self.parent.assertEqual(result.reconstruction.shape ,						(self.batch_size, 1, self.image_size, self.image_size)      )


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase      ):
											_lowercase							:   Optional[int] 							=   self.type_sequence_label_size
											_lowercase							:   str 							=   FocalNetForImageClassification(_lowerCAmelCase      )
											model.to(_lowerCAmelCase      )
											model.eval()
											_lowercase							:   Any 							=   model(_lowerCAmelCase ,						labels=_lowerCAmelCase      )
											self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.type_sequence_label_size)      )

											# test greyscale images
											_lowercase							:   Dict 							=   1
											_lowercase							:   Tuple 							=   FocalNetForImageClassification(_lowerCAmelCase      )
											model.to(_lowerCAmelCase      )
											model.eval()

											_lowercase							:   List[Any] 							=   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]      )
											_lowercase							:   int 							=   model(_lowerCAmelCase      )
											self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.type_sequence_label_size)      )


				def 						__a					(					self      ):
											_lowercase							:   int 							=   self.prepare_config_and_inputs()

											_lowercase							,							_lowercase							,							_lowercase							:   Tuple 							=   config_and_inputs
											_lowercase							:   Dict 							=   {'pixel_values': pixel_values}
											return config, inputs_dict




@require_torch
class 				lowerCAmelCase_	(		__snake_case   ,		__snake_case   ,		unittest.TestCase			):
				_UpperCamelCase					:       int							       = (
				    (
				        FocalNetModel,
				        FocalNetForImageClassification,
				        FocalNetForMaskedImageModeling,
				        FocalNetBackbone,
				    )
				    if is_torch_available()
				    else ()
				)
				_UpperCamelCase					:       List[str]							       = (
				    {"feature-extraction": FocalNetModel, "image-classification": FocalNetForImageClassification}
				    if is_torch_available()
				    else {}
				)
				_UpperCamelCase					:       int							       = False

				_UpperCamelCase					:       Any							       = False
				_UpperCamelCase					:       List[str]							       = False
				_UpperCamelCase					:       Union[str, Any]							       = False
				_UpperCamelCase					:       Optional[int]							       = False


				def 						__a					(					self      ):
											_lowercase							:   int 							=   FocalNetModelTester(self      )
											_lowercase							:   int 							=   ConfigTester(self ,						config_class=_lowerCAmelCase ,						embed_dim=3_7 ,						has_text_modality=_lowerCAmelCase      )


				def 						__a					(					self      ):
											self.create_and_test_config_common_properties()
											self.config_tester.create_and_test_config_to_json_string()
											self.config_tester.create_and_test_config_to_json_file()
											self.config_tester.create_and_test_config_from_and_save_pretrained()
											self.config_tester.create_and_test_config_with_num_labels()
											self.config_tester.check_config_can_be_init_without_params()
											self.config_tester.check_config_arguments_init()


				def 						__a					(					self      ):
											return


				def 						__a					(					self      ):
											_lowercase							:   Dict 							=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*_lowerCAmelCase      )


				def 						__a					(					self      ):
											_lowercase							:   Any 							=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_backbone(*_lowerCAmelCase      )


				def 						__a					(					self      ):
											_lowercase							:   Optional[Any] 							=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_image_modeling(*_lowerCAmelCase      )


				def 						__a					(					self      ):
											_lowercase							:   int 							=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_image_classification(*_lowerCAmelCase      )


				@unittest.skip(reason='FocalNet does not use inputs_embeds'      )
				def 						__a					(					self      ):
											pass


				@unittest.skip(reason='FocalNet does not use feedforward chunking'      )
				def 						__a					(					self      ):
											pass


				def 						__a					(					self      ):
											_lowercase							,							_lowercase							:   int 							=   self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes[:-1]:
																		_lowercase							:   Optional[Any] 							=   model_class(_lowerCAmelCase      )
																		self.assertIsInstance(model.get_input_embeddings() ,						(nn.Module)      )
																		_lowercase							:   Any 							=   model.get_output_embeddings()
																		self.assertTrue(x is None or isinstance(_lowerCAmelCase ,						nn.Linear      )      )


				def 						__a					(					self      ):
											_lowercase							,							_lowercase							:   Optional[int] 							=   self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes[:-1]:
																		_lowercase							:   Union[str, Any] 							=   model_class(_lowerCAmelCase      )
																		_lowercase							:   Any 							=   inspect.signature(model.forward      )
																		# signature.parameters is an OrderedDict => so arg_names order is deterministic
																		_lowercase							:   str 							=   [*signature.parameters.keys()]

																		_lowercase							:   int 							=   ['pixel_values']
																		self.assertListEqual(arg_names[:1] ,						_lowerCAmelCase      )


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase      ):
											_lowercase							:   List[Any] 							=   model_class(_lowerCAmelCase      )
											model.to(_lowerCAmelCase      )
											model.eval()

											with torch.no_grad():
																		_lowercase							:   Any 							=   model(**self._prepare_for_class(_lowerCAmelCase ,						_lowerCAmelCase      )      )

											_lowercase							:   Any 							=   outputs.hidden_states

											_lowercase							:   List[Any] 							=   getattr(
											    self.model_tester ,						'expected_num_hidden_layers' ,						len(self.model_tester.depths      ) + 1      )
											self.assertEqual(len(_lowerCAmelCase      ) ,						_lowerCAmelCase      )

											# FocalNet has a different seq_length
											_lowercase							:   Tuple 							=   (
											    config.patch_size
											    if isinstance(config.patch_size ,						collections.abc.Iterable      )
											    else (config.patch_size, config.patch_size)
											)

											_lowercase							:   Union[str, Any] 							=   (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

											self.assertListEqual(
											    list(hidden_states[0].shape[-2:]      ) ,						[num_patches, self.model_tester.embed_dim] ,						)

											_lowercase							:   Union[str, Any] 							=   outputs.reshaped_hidden_states
											self.assertEqual(len(_lowerCAmelCase      ) ,						_lowerCAmelCase      )

											_lowercase							,							_lowercase							,							_lowercase							,							_lowercase							:   Optional[int] 							=   reshaped_hidden_states[0].shape
											_lowercase							:   Optional[int] 							=   (
											    reshaped_hidden_states[0].view(_lowerCAmelCase ,						_lowerCAmelCase ,						height * width      ).permute(0 ,						2 ,						1      )
											)
											self.assertListEqual(
											    list(reshaped_hidden_states.shape[-2:]      ) ,						[num_patches, self.model_tester.embed_dim] ,						)


				def 						__a					(					self      ):
											_lowercase							,							_lowercase							:   Tuple 							=   self.model_tester.prepare_config_and_inputs_for_common()

											_lowercase							:   Optional[int] 							=   (
											    self.model_tester.image_size
											    if isinstance(self.model_tester.image_size ,						collections.abc.Iterable      )
											    else (self.model_tester.image_size, self.model_tester.image_size)
											)

											for model_class in self.all_model_classes[:-1]:
																		_lowercase							:   Tuple 							=   True
																		self.check_hidden_states_output(_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase      )

																		# check that output_hidden_states also work using config
																		del inputs_dict["output_hidden_states"]
																		_lowercase							:   Dict 							=   True

																		self.check_hidden_states_output(_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase      )


				def 						__a					(					self      ):
											_lowercase							,							_lowercase							:   Any 							=   self.model_tester.prepare_config_and_inputs_for_common()
											_lowercase							:   Optional[Any] 							=   3

											_lowercase							:   Tuple 							=   (
											    self.model_tester.image_size
											    if isinstance(self.model_tester.image_size ,						collections.abc.Iterable      )
											    else (self.model_tester.image_size, self.model_tester.image_size)
											)
											_lowercase							:   Tuple 							=   (
											    config.patch_size
											    if isinstance(config.patch_size ,						collections.abc.Iterable      )
											    else (config.patch_size, config.patch_size)
											)

											_lowercase							:   Any 							=   image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
											_lowercase							:   Union[str, Any] 							=   image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

											for model_class in self.all_model_classes[:-1]:
																		_lowercase							:   List[Any] 							=   True
																		self.check_hidden_states_output(_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase ,						(padded_height, padded_width)      )

																		# check that output_hidden_states also work using config
																		del inputs_dict["output_hidden_states"]
																		_lowercase							:   Optional[Any] 							=   True
																		self.check_hidden_states_output(_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase ,						(padded_height, padded_width)      )


				@slow
				def 						__a					(					self      ):
											for model_name in FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		_lowercase							:   List[str] 							=   FocalNetModel.from_pretrained(_lowerCAmelCase      )
																		self.assertIsNotNone(_lowerCAmelCase      )


				def 						__a					(					self      ):
											_lowercase							,							_lowercase							:   List[Any] 							=   self.model_tester.prepare_config_and_inputs_for_common()

											_lowercase							:   Tuple 							=   _config_zero_init(_lowerCAmelCase      )
											for model_class in self.all_model_classes:
																		_lowercase							:   Optional[Any] 							=   model_class(config=_lowerCAmelCase      )
																		for name, param in model.named_parameters():
																									if "embeddings" not in name and param.requires_grad:
																																self.assertIn(
																																    ((param.data.mean() * 1E9).round() / 1E9).item() ,						[0.0, 1.0] ,						msg=F"""Parameter {name} of model {model_class} seems not properly initialized""" ,						)




@require_vision
@require_torch
class 				lowerCAmelCase_	(		unittest.TestCase			):


				@cached_property
				def 						__a					(					self      ):
											# TODO update organization
											return AutoImageProcessor.from_pretrained('microsoft/focalnet-tiny'      ) if is_vision_available() else None


				@slow
				def 						__a					(					self      ):
											_lowercase							:   str 							=   FocalNetForImageClassification.from_pretrained('microsoft/focalnet-tiny'      ).to(_lowerCAmelCase      )
											_lowercase							:   Optional[int] 							=   self.default_image_processor

											_lowercase							:   Tuple 							=   Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'      )
											_lowercase							:   List[Any] 							=   image_processor(images=_lowerCAmelCase ,						return_tensors='pt'      ).to(_lowerCAmelCase      )

											# forward pass
											with torch.no_grad():
																		_lowercase							:   int 							=   model(**_lowerCAmelCase      )

											# verify the logits
											_lowercase							:   Dict 							=   torch.Size((1, 1_0_0_0)      )
											self.assertEqual(outputs.logits.shape ,						_lowerCAmelCase      )
											_lowercase							:   Any 							=   torch.tensor([0.21_66, -0.43_68, 0.21_91]      ).to(_lowerCAmelCase      )
											self.assertTrue(torch.allclose(outputs.logits[0, :3] ,						_lowerCAmelCase ,						atol=1E-4      )      )
											self.assertTrue(outputs.logits.argmax(dim=-1      ).item() ,						2_8_1      )




@require_torch
class 				lowerCAmelCase_	(		__snake_case   ,		unittest.TestCase			):
				_UpperCamelCase					:       List[str]							       = (FocalNetBackbone,) if is_torch_available() else ()
				_UpperCamelCase					:       Dict							       = FocalNetConfig

				_UpperCamelCase					:       Optional[Any]							       = False


				def 						__a					(					self      ):
											_lowercase							:   str 							=   FocalNetModelTester(self      )










import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast

class 						__lowerCAmelCase      (      unittest.TestCase					):
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				0
							@slow
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)
									self.assertIsNotNone(__magic_name__							)
									self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
									self.assertGreater(len(__magic_name__							)						, 0							)

								for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)
									self.assertIsNotNone(__magic_name__							)
									self.assertIsInstance(__magic_name__						, (GPTaTokenizer, GPTaTokenizerFast)							)
									self.assertGreater(len(__magic_name__							)						, 0							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (RobertaTokenizer, RobertaTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 20							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoConfig.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, __magic_name__							)
								# Check that tokenizer_type ≠ model_type
								a									=				AutoTokenizer.from_pretrained(__magic_name__						, config=__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.txt"""						, os.path.join(__magic_name__						, """vocab.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""bert"""						, use_fast=__magic_name__							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)

								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.json"""						, os.path.join(__magic_name__						, """vocab.json"""							)							)
									shutil.copy("""./tests/fixtures/merges.txt"""						, os.path.join(__magic_name__						, """merges.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""gpt2"""						, use_fast=__magic_name__							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.txt"""						, os.path.join(__magic_name__						, """vocab.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""bert"""							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)

								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.json"""						, os.path.join(__magic_name__						, """vocab.json"""							)							)
									shutil.copy("""./tests/fixtures/merges.txt"""						, os.path.join(__magic_name__						, """merges.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""gpt2"""							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								with pytest.raises(__magic_name__							):
									AutoTokenizer.from_pretrained("""./"""						, tokenizer_type="""xxx"""							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									a									=				tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""							)
									self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)

									if isinstance(__magic_name__						, __magic_name__							):
										self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						, __magic_name__							)
									else:
										self.assertEqual(tokenizer.do_lower_case						, __magic_name__							)

									self.assertEqual(tokenizer.model_max_length						, 512							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									with self.assertRaisesRegex(
									    __magic_name__						, """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""						, ):
										a									=				tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				TOKENIZER_MAPPING.values()
								a									=				[]

								for slow_tok, fast_tok in tokenizers:
									if slow_tok is not None:
										tokenizer_names.append(slow_tok.__name__							)

									if fast_tok is not None:
										tokenizer_names.append(fast_tok.__name__							)

								for tokenizer_name in tokenizer_names:
									# must find the right class
									tokenizer_class_from_name(__magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""						, use_fast=__magic_name__							)						, __magic_name__							)
								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""							)						, __magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""distilbert-base-uncased"""						, do_lower_case=__magic_name__							)
								a									=				"""Hello, world. How are you?"""
								a									=				tokenizer.tokenize(__magic_name__							)
								self.assertEqual("""[UNK]"""						, tokens[0]							)

								a									=				AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""						, do_lower_case=__magic_name__							)
								a									=				tokenizer.tokenize(__magic_name__							)
								self.assertEqual("""[UNK]"""						, tokens[0]							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""							)
								self.assertEqual(type(__magic_name__							)						, __magic_name__							)
								self.assertEqual(tokenizer.model_max_length						, 512							)
								self.assertEqual(tokenizer.vocab_size						, 3_0000							)
								self.assertEqual(tokenizer.unk_token						, """[UNK]"""							)
								self.assertEqual(tokenizer.padding_side						, """right"""							)
								self.assertEqual(tokenizer.truncation_side						, """right"""							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)

								self.assertIsInstance(__magic_name__						, tokenizer.__class__							)
								self.assertEqual(tokenizera.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""ctrl"""							)
								# There is no fast CTRL so this always gives us a slow tokenizer.
								self.assertIsInstance(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				get_tokenizer_config("""bert-base-cased"""							)
								a									=				config.pop("""_commit_hash"""						, __magic_name__							)
								# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
								self.assertEqual(__magic_name__						, {"""do_lower_case""": False}							)

								# This model does not have a tokenizer_config so we get back an empty dict.
								a									=				get_tokenizer_config(__magic_name__							)
								self.assertDictEqual(__magic_name__						, {}							)

								# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				get_tokenizer_config(__magic_name__							)

								# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
								self.assertEqual(config["""tokenizer_class"""]						, """BertTokenizer"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								try:
									AutoConfig.register("""custom"""						, __magic_name__							)

									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(__magic_name__							):
										AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)

									a									=				CustomTokenizer.from_pretrained(__magic_name__							)
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								try:
									AutoConfig.register("""custom"""						, __magic_name__							)

									# Can register in two steps
									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, None)							)
									AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, CustomTokenizerFast)							)

									del TOKENIZER_MAPPING._extra_content[CustomConfig]
									# Can register in one step
									AutoTokenizer.register(
									    __magic_name__						, slow_tokenizer_class=__magic_name__						, fast_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, CustomTokenizerFast)							)

									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(__magic_name__							):
										AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)

									# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
									# and that model does not have a tokenizer.json
									with tempfile.TemporaryDirectory() as tmp_dir:
										a									=				BertTokenizerFast.from_pretrained(__magic_name__							)
										bert_tokenizer.save_pretrained(__magic_name__							)
										a									=				CustomTokenizerFast.from_pretrained(__magic_name__							)

									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__						, use_fast=__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								with self.assertRaises(__magic_name__							):
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(__magic_name__							):
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)

								a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
								self.assertTrue(tokenizer.special_attribute_present							)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				AutoTokenizer.from_pretrained(__magic_name__						, trust_remote_code=__magic_name__							)
								self.assertTrue(reloaded_tokenizer.special_attribute_present							)

								if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizerFast"""							)

									# Test we can also load the slow version
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertTrue(tokenizer.special_attribute_present							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									# Test tokenizer can be reloaded.
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)
										a									=				AutoTokenizer.from_pretrained(__magic_name__						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertTrue(reloaded_tokenizer.special_attribute_present							)
								else:
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizer"""							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								class 						__lowerCAmelCase      (      __magic_name__					):
															UpperCamelCase__   	=    False







								class 						__lowerCAmelCase      (      __magic_name__					):
															UpperCamelCase__   	=    NewTokenizer
															UpperCamelCase__   	=    False

								try:
									AutoConfig.register("""custom"""						, __magic_name__							)
									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)
									# If remote code is not set, the default is to use local
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertFalse(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertFalse(tokenizer.special_attribute_present							)

									# If remote code is disabled, we load the local one.
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertFalse(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertFalse(tokenizer.special_attribute_present							)

									# If remote is enabled, we load from the Hub
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertTrue(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertTrue(tokenizer.special_attribute_present							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(
								    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						, trust_remote_code=__magic_name__							)
								self.assertTrue(tokenizer.special_attribute_present							)
								if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)

									# Test we can also load the slow version
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertTrue(tokenizer.special_attribute_present							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
								else:
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								with self.assertRaisesRegex(
								    __magic_name__						, """bert-base is not a local folder and is not a valid model identifier"""							):
									a									=				AutoTokenizer.from_pretrained("""bert-base"""							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								with self.assertRaisesRegex(
								    __magic_name__						, r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
									a									=				AutoTokenizer.from_pretrained(__magic_name__						, revision="""aaaaaa"""							)





							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
								with RequestCounter() as counter:
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
									self.assertEqual(counter.get_request_count						, 0							)
									self.assertEqual(counter.head_request_count						, 1							)
									self.assertEqual(counter.other_request_count						, 0							)

