





import argparse
import json
import os

import numpy as np
import PIL
import requests
import tensorflow.keras.applications.efficientnet as efficientnet
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from tensorflow.keras.preprocessing import image

from transformers import (
    EfficientNetConfig,
    EfficientNetForImageClassification,
    EfficientNetImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
_a     :    Optional[Any]         =						logging.get_logger(__name__)

_a     :    Dict         =						{
    'b0': efficientnet.EfficientNetBa,
    'b1': efficientnet.EfficientNetBa,
    'b2': efficientnet.EfficientNetBa,
    'b3': efficientnet.EfficientNetBa,
    'b4': efficientnet.EfficientNetBa,
    'b5': efficientnet.EfficientNetBa,
    'b6': efficientnet.EfficientNetBa,
    'b7': efficientnet.EfficientNetBa,
}

_a     :    List[Any]         =						{
    'b0': {
        'hidden_dim': 1_280,
        'width_coef': 1.0,
        'depth_coef': 1.0,
        'image_size': 224,
        'dropout_rate': 0.2,
        'dw_padding': [],
    },
    'b1': {
        'hidden_dim': 1_280,
        'width_coef': 1.0,
        'depth_coef': 1.1,
        'image_size': 240,
        'dropout_rate': 0.2,
        'dw_padding': [16],
    },
    'b2': {
        'hidden_dim': 1_408,
        'width_coef': 1.1,
        'depth_coef': 1.2,
        'image_size': 260,
        'dropout_rate': 0.3,
        'dw_padding': [5, 8, 16],
    },
    'b3': {
        'hidden_dim': 1_536,
        'width_coef': 1.2,
        'depth_coef': 1.4,
        'image_size': 300,
        'dropout_rate': 0.3,
        'dw_padding': [5, 18],
    },
    'b4': {
        'hidden_dim': 1_792,
        'width_coef': 1.4,
        'depth_coef': 1.8,
        'image_size': 380,
        'dropout_rate': 0.4,
        'dw_padding': [6],
    },
    'b5': {
        'hidden_dim': 2_048,
        'width_coef': 1.6,
        'depth_coef': 2.2,
        'image_size': 456,
        'dropout_rate': 0.4,
        'dw_padding': [13, 27],
    },
    'b6': {
        'hidden_dim': 2_304,
        'width_coef': 1.8,
        'depth_coef': 2.6,
        'image_size': 528,
        'dropout_rate': 0.5,
        'dw_padding': [31],
    },
    'b7': {
        'hidden_dim': 2_560,
        'width_coef': 2.0,
        'depth_coef': 3.1,
        'image_size': 600,
        'dropout_rate': 0.5,
        'dw_padding': [18],
    },
}
def 							a_	(    __magic_name__   )				->			Dict:




				"""simple docstring"""






				snake_case   :					Optional[Any]    						=				EfficientNetConfig()
				snake_case   :					List[str]    						=				CONFIG_MAP[model_name]['''hidden_dim''']
				snake_case   :					Any    						=				CONFIG_MAP[model_name]['''width_coef''']
				snake_case   :					Any    						=				CONFIG_MAP[model_name]['''depth_coef''']
				snake_case   :					List[Any]    						=				CONFIG_MAP[model_name]['''image_size''']
				snake_case   :					Union[str, Any]    						=				CONFIG_MAP[model_name]['''dropout_rate''']
				snake_case   :					int    						=				CONFIG_MAP[model_name]['''dw_padding''']

				snake_case   :					Optional[int]    						=				'''huggingface/label-files'''
				snake_case   :					int    						=				'''imagenet-1k-id2label.json'''
				snake_case   :					Optional[int]    						=				1_000
				snake_case   :					List[str]    						=				json.load(open(hf_hub_download(__magic_name__		,     __magic_name__		,     repo_type='''dataset'''   )		,     '''r'''   )   )
				snake_case   :					int    						=				{int(__magic_name__   ): v for k, v in idalabel.items()}

				snake_case   :					Any    						=				idalabel
				snake_case   :					str    						=				{v: k for k, v in idalabel.items()}
				return config
def 							a_	(    )				->			List[Any]:




				"""simple docstring"""






				snake_case   :					List[str]    						=				'''http://images.cocodataset.org/val2017/000000039769.jpg'''
				snake_case   :					Union[str, Any]    						=				Image.open(requests.get(__magic_name__		,     stream=__magic_name__   ).raw   )
				return im
def 							a_	(    __magic_name__   )				->			str:




				"""simple docstring"""






				snake_case   :					Tuple    						=				CONFIG_MAP[model_name]['''image_size''']
				snake_case   :					List[Any]    						=				EfficientNetImageProcessor(
				    size={'''height''': size, '''width''': size}		,     image_mean=[0.485, 0.456, 0.406]		,     image_std=[0.4785_3944, 0.473_2864, 0.4743_4163]		,     do_center_crop=__magic_name__		,     )
				return preprocessor
def 							a_	(    __magic_name__   )				->			Tuple:




				"""simple docstring"""






				snake_case   :					str    						=				[v.split('''_'''   )[0].split('''block'''   )[1] for v in original_param_names if v.startswith('''block'''   )]
				snake_case   :					Dict    						=				sorted(set(__magic_name__   )   )
				snake_case   :					List[str]    						=				len(__magic_name__   )
				snake_case   :					List[str]    						=				{b: str(__magic_name__   ) for b, i in zip(__magic_name__		,     range(__magic_name__   )   )}

				snake_case   :					Any    						=				[]
				rename_keys.append(('''stem_conv/kernel:0''', '''embeddings.convolution.weight''')   )
				rename_keys.append(('''stem_bn/gamma:0''', '''embeddings.batchnorm.weight''')   )
				rename_keys.append(('''stem_bn/beta:0''', '''embeddings.batchnorm.bias''')   )
				rename_keys.append(('''stem_bn/moving_mean:0''', '''embeddings.batchnorm.running_mean''')   )
				rename_keys.append(('''stem_bn/moving_variance:0''', '''embeddings.batchnorm.running_var''')   )

				for b in block_names:
								snake_case   :					Any    						=				block_name_mapping[b]
								rename_keys.append((F"block{b}_expand_conv/kernel:0", F"encoder.blocks.{hf_b}.expansion.expand_conv.weight")   )
								rename_keys.append((F"block{b}_expand_bn/gamma:0", F"encoder.blocks.{hf_b}.expansion.expand_bn.weight")   )
								rename_keys.append((F"block{b}_expand_bn/beta:0", F"encoder.blocks.{hf_b}.expansion.expand_bn.bias")   )
								rename_keys.append(
								    (F"block{b}_expand_bn/moving_mean:0", F"encoder.blocks.{hf_b}.expansion.expand_bn.running_mean")   )
								rename_keys.append(
								    (F"block{b}_expand_bn/moving_variance:0", F"encoder.blocks.{hf_b}.expansion.expand_bn.running_var")   )
								rename_keys.append(
								    (F"block{b}_dwconv/depthwise_kernel:0", F"encoder.blocks.{hf_b}.depthwise_conv.depthwise_conv.weight")   )
								rename_keys.append((F"block{b}_bn/gamma:0", F"encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.weight")   )
								rename_keys.append((F"block{b}_bn/beta:0", F"encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.bias")   )
								rename_keys.append(
								    (F"block{b}_bn/moving_mean:0", F"encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_mean")   )
								rename_keys.append(
								    (F"block{b}_bn/moving_variance:0", F"encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_var")   )

								rename_keys.append((F"block{b}_se_reduce/kernel:0", F"encoder.blocks.{hf_b}.squeeze_excite.reduce.weight")   )
								rename_keys.append((F"block{b}_se_reduce/bias:0", F"encoder.blocks.{hf_b}.squeeze_excite.reduce.bias")   )
								rename_keys.append((F"block{b}_se_expand/kernel:0", F"encoder.blocks.{hf_b}.squeeze_excite.expand.weight")   )
								rename_keys.append((F"block{b}_se_expand/bias:0", F"encoder.blocks.{hf_b}.squeeze_excite.expand.bias")   )
								rename_keys.append(
								    (F"block{b}_project_conv/kernel:0", F"encoder.blocks.{hf_b}.projection.project_conv.weight")   )
								rename_keys.append((F"block{b}_project_bn/gamma:0", F"encoder.blocks.{hf_b}.projection.project_bn.weight")   )
								rename_keys.append((F"block{b}_project_bn/beta:0", F"encoder.blocks.{hf_b}.projection.project_bn.bias")   )
								rename_keys.append(
								    (F"block{b}_project_bn/moving_mean:0", F"encoder.blocks.{hf_b}.projection.project_bn.running_mean")   )
								rename_keys.append(
								    (F"block{b}_project_bn/moving_variance:0", F"encoder.blocks.{hf_b}.projection.project_bn.running_var")   )

				rename_keys.append(('''top_conv/kernel:0''', '''encoder.top_conv.weight''')   )
				rename_keys.append(('''top_bn/gamma:0''', '''encoder.top_bn.weight''')   )
				rename_keys.append(('''top_bn/beta:0''', '''encoder.top_bn.bias''')   )
				rename_keys.append(('''top_bn/moving_mean:0''', '''encoder.top_bn.running_mean''')   )
				rename_keys.append(('''top_bn/moving_variance:0''', '''encoder.top_bn.running_var''')   )

				snake_case   :					Dict    						=				{}
				for item in rename_keys:
								if item[0] in original_param_names:
												snake_case   :					List[Any]    						=				'''efficientnet.''' + item[1]

				snake_case   :					Any    						=				'''classifier.weight'''
				snake_case   :					Optional[Any]    						=				'''classifier.bias'''
				return key_mapping
def 							a_	(    __magic_name__		,     __magic_name__		,     __magic_name__   )				->			str:




				"""simple docstring"""






				for key, value in tf_params.items():
								if "normalization" in key:
												continue

								snake_case   :					Dict    						=				key_mapping[key]
								if "_conv" in key and "kernel" in key:
												snake_case   :					Union[str, Any]    						=				torch.from_numpy(__magic_name__   ).permute(3		,     2		,     0		,     1   )
								elif "depthwise_kernel" in key:
												snake_case   :					int    						=				torch.from_numpy(__magic_name__   ).permute(2		,     3		,     0		,     1   )
								elif "kernel" in key:
												snake_case   :					str    						=				torch.from_numpy(np.transpose(__magic_name__   )   )
								else:
												snake_case   :					int    						=				torch.from_numpy(__magic_name__   )

								# Replace HF parameters with original TF model parameters
								assert hf_params[hf_key].shape == new_hf_value.shape
								hf_params[hf_key].copy_(__magic_name__   )
@torch.no_grad()
def 							a_	(    __magic_name__		,     __magic_name__		,     __magic_name__		,     __magic_name__   )				->			List[str]:




				"""simple docstring"""






				snake_case   :					Any    						=				model_classes[model_name](
				    include_top=__magic_name__		,     weights='''imagenet'''		,     input_tensor=__magic_name__		,     input_shape=__magic_name__		,     pooling=__magic_name__		,     classes=1_000		,     classifier_activation='''softmax'''		,     )

				snake_case   :					List[Any]    						=				original_model.trainable_variables
				snake_case   :					Any    						=				original_model.non_trainable_variables
				snake_case   :					Optional[int]    						=				{param.name: param.numpy() for param in tf_params}
				for param in tf_non_train_params:
								snake_case   :					Optional[int]    						=				param.numpy()
				snake_case   :					int    						=				list(tf_params.keys()   )

				# Load HuggingFace model
				snake_case   :					int    						=				get_efficientnet_config(__magic_name__   )
				snake_case   :					Dict    						=				EfficientNetForImageClassification(__magic_name__   ).eval()
				snake_case   :					List[Any]    						=				hf_model.state_dict()

				# Create src-to-dst parameter name mapping dictionary
				print('''Converting parameters...'''   )
				snake_case   :					Any    						=				rename_keys(__magic_name__   )
				replace_params(__magic_name__		,     __magic_name__		,     __magic_name__   )

				# Initialize preprocessor and preprocess input image
				snake_case   :					List[str]    						=				convert_image_processor(__magic_name__   )
				snake_case   :					Union[str, Any]    						=				preprocessor(images=prepare_img()		,     return_tensors='''pt'''   )

				# HF model inference
				hf_model.eval()
				with torch.no_grad():
								snake_case   :					Optional[Any]    						=				hf_model(**__magic_name__   )
				snake_case   :					Dict    						=				outputs.logits.detach().numpy()

				# Original model inference
				snake_case   :					Tuple    						=				False
				snake_case   :					Optional[Any]    						=				CONFIG_MAP[model_name]['''image_size''']
				snake_case   :					Optional[Any]    						=				prepare_img().resize((image_size, image_size)		,     resample=PIL.Image.NEAREST   )
				snake_case   :					Any    						=				image.img_to_array(__magic_name__   )
				snake_case   :					Dict    						=				np.expand_dims(__magic_name__		,     axis=0   )
				snake_case   :					int    						=				original_model.predict(__magic_name__   )

				# Check whether original and HF model outputs match  -> np.allclose
				assert np.allclose(__magic_name__		,     __magic_name__		,     atol=1e-3   ), "The predicted logits are not the same."
				print('''Model outputs match!'''   )

				if save_model:
								# Create folder to save model
								if not os.path.isdir(__magic_name__   ):
												os.mkdir(__magic_name__   )
								# Save converted model and image processor
								hf_model.save_pretrained(__magic_name__   )
								preprocessor.save_pretrained(__magic_name__   )

				if push_to_hub:
								# Push model and image processor to hub
								print(F"Pushing converted {model_name} to the hub..."   )
								snake_case   :					Tuple    						=				F"efficientnet-{model_name}"
								preprocessor.push_to_hub(__magic_name__   )
								hf_model.push_to_hub(__magic_name__   )


if __name__ == "__main__":
			_a     :    List[Any]         =						argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    '--model_name',
			    default='b0',
			    type=str,
			    help='Version name of the EfficientNet model you want to convert, select from [b0, b1, b2, b3, b4, b5, b6, b7].',
			)
			parser.add_argument(
			    '--pytorch_dump_folder_path',
			    default='hf_model',
			    type=str,
			    help='Path to the output PyTorch model directory.',
			)
			parser.add_argument('--save_model', action='store_true', help='Save model to local')
			parser.add_argument('--push_to_hub', action='store_true', help='Push model and image processor to the hub')

			_a     :    Tuple         =						parser.parse_args()
			convert_efficientnet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub)







"""simple docstring"""





import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin

class 						lowercase   (   _UpperCAmelCase	,    unittest.TestCase   ):
      _SCREAMING_SNAKE_CASE							      = CanineTokenizer
      _SCREAMING_SNAKE_CASE							      = False





      def        _snake_case     (			self					)						->	Any:
            super().setUp()
            lowerCAmelCase    		=			CanineTokenizer()
            tokenizer.save_pretrained(self.tmpdirname					)





      @cached_property
      def        _snake_case     (			self					)						->	Union[str, Any]:
            return CanineTokenizer.from_pretrained("""google/canine-s"""					)





      def        _snake_case     (			self ,		**lowercase					)						->	CanineTokenizer:
            lowerCAmelCase    		=			self.tokenizer_class.from_pretrained(self.tmpdirname ,		**lowercase					)
            lowerCAmelCase    		=			1_024
            return tokenizer





      @require_torch
      def        _snake_case     (			self					)						->	str:
            lowerCAmelCase    		=			self.canine_tokenizer
            lowerCAmelCase    		=			["""Life is like a box of chocolates.""", """You never know what you're gonna get."""]
            # fmt: off
            lowerCAmelCase    		=			[57_344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57_345, 0, 0, 0, 0]
            # fmt: on
            lowerCAmelCase    		=			tokenizer(lowercase ,		padding=lowercase ,		return_tensors="""pt"""					)
            self.assertIsInstance(lowercase ,		lowercase					)

            lowerCAmelCase    		=			list(batch.input_ids.numpy()[0]					)

            self.assertListEqual(lowercase ,		lowercase					)

            self.assertEqual((2, 39) ,		batch.input_ids.shape					)
            self.assertEqual((2, 39) ,		batch.attention_mask.shape					)





      @require_torch
      def        _snake_case     (			self					)						->	Optional[Any]:
            lowerCAmelCase    		=			self.canine_tokenizer
            lowerCAmelCase    		=			["""Once there was a man.""", """He wrote a test in HuggingFace Tranformers."""]
            lowerCAmelCase    		=			tokenizer(lowercase ,		padding=lowercase ,		return_tensors="""pt"""					)
            # check if input_ids, attention_mask and token_type_ids are returned
            self.assertIn("""input_ids""" ,		lowercase					)
            self.assertIn("""attention_mask""" ,		lowercase					)
            self.assertIn("""token_type_ids""" ,		lowercase					)





      @require_torch
      def        _snake_case     (			self					)						->	Optional[int]:
            lowerCAmelCase    		=			self.canine_tokenizer
            lowerCAmelCase    		=			[
                """What's the weater?""",
                """It's about 25 degrees.""",
            ]
            lowerCAmelCase    		=			tokenizer(
                text_target=lowercase ,		max_length=32 ,		padding="""max_length""" ,		truncation=lowercase ,		return_tensors="""pt"""					)
            self.assertEqual(32 ,		targets["""input_ids"""].shape[1]					)





      def        _snake_case     (			self					)						->	Tuple:
            # safety check on max_len default value so we are sure the test works
            lowerCAmelCase    		=			self.get_tokenizers()
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'					):
                        self.assertNotEqual(tokenizer.model_max_length ,		42					)

        # Now let's start the test
            lowerCAmelCase    		=			self.get_tokenizers()
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'					):
                        # Isolate this from the other tests because we save additional tokens/etc
                        lowerCAmelCase    		=			tempfile.mkdtemp()

                        lowerCAmelCase    		=			""" He is very happy, UNwant\u00E9d,running"""
                        lowerCAmelCase    		=			tokenizer.encode(lowercase ,		add_special_tokens=lowercase					)
                        tokenizer.save_pretrained(lowercase					)

                        lowerCAmelCase    		=			tokenizer.__class__.from_pretrained(lowercase					)
                        lowerCAmelCase    		=			after_tokenizer.encode(lowercase ,		add_special_tokens=lowercase					)
                        self.assertListEqual(lowercase ,		lowercase					)

                        shutil.rmtree(lowercase					)

            lowerCAmelCase    		=			self.get_tokenizers(model_max_length=42					)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'					):
                        # Isolate this from the other tests because we save additional tokens/etc
                        lowerCAmelCase    		=			tempfile.mkdtemp()

                        lowerCAmelCase    		=			""" He is very happy, UNwant\u00E9d,running"""

                        lowerCAmelCase    		=			tokenizer.additional_special_tokens

                        # We can add a new special token for Canine as follows:
                        lowerCAmelCase    		=			chr(0XE_007					)
                        additional_special_tokens.append(lowercase					)
                        tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}					)
                        lowerCAmelCase    		=			tokenizer.encode(lowercase ,		add_special_tokens=lowercase					)
                        tokenizer.save_pretrained(lowercase					)

                        lowerCAmelCase    		=			tokenizer.__class__.from_pretrained(lowercase					)
                        lowerCAmelCase    		=			after_tokenizer.encode(lowercase ,		add_special_tokens=lowercase					)
                        self.assertListEqual(lowercase ,		lowercase					)
                        self.assertIn(lowercase ,		after_tokenizer.additional_special_tokens					)
                        self.assertEqual(after_tokenizer.model_max_length ,		42					)

                        lowerCAmelCase    		=			tokenizer.__class__.from_pretrained(lowercase ,		model_max_length=43					)
                        self.assertEqual(tokenizer.model_max_length ,		43					)

                        shutil.rmtree(lowercase					)





      def        _snake_case     (			self					)						->	Tuple:
            lowerCAmelCase    		=			self.get_tokenizers(do_lower_case=lowercase					)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'					):
                        lowerCAmelCase		,						lowerCAmelCase    		=			self.get_clean_sequence(lowercase					)

                        # a special token for Canine can be defined as follows:
                        lowerCAmelCase    		=			0XE_005
                        lowerCAmelCase    		=			chr(lowercase					)

                        tokenizer.add_special_tokens({"""cls_token""": special_token}					)
                        lowerCAmelCase    		=			tokenizer.encode(lowercase ,		add_special_tokens=lowercase					)
                        self.assertEqual(len(lowercase					) ,		1					)

                        lowerCAmelCase    		=			tokenizer.decode(ids + encoded_special_token ,		clean_up_tokenization_spaces=lowercase					)
                        lowerCAmelCase    		=			tokenizer.encode(lowercase ,		add_special_tokens=lowercase					)

                        lowerCAmelCase    		=			tokenizer.encode(lowercase ,		add_special_tokens=lowercase					)
                        lowerCAmelCase    		=			tokenizer.encode(lowercase ,		add_special_tokens=lowercase					)
                        self.assertEqual(lowercase ,		input_encoded + special_token_id					)

                        lowerCAmelCase    		=			tokenizer.decode(lowercase ,		skip_special_tokens=lowercase					)
                        self.assertTrue(special_token not in decoded					)





      def        _snake_case     (			self					)						->	Union[str, Any]:
            lowerCAmelCase    		=			self.get_tokenizers(do_lower_case=lowercase					)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'					):
                        lowerCAmelCase    		=			chr(0XE_005					)
                        lowerCAmelCase    		=			chr(0XE_006					)

                        # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
                        tokenizer.add_tokens([SPECIAL_TOKEN_1] ,		special_tokens=lowercase					)
                        # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
                        # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
                        tokenizer.add_special_tokens({"""additional_special_tokens""": [SPECIAL_TOKEN_2]}					)

                        lowerCAmelCase    		=			tokenizer.tokenize(lowercase					)
                        lowerCAmelCase    		=			tokenizer.tokenize(lowercase					)

                        self.assertEqual(len(lowercase					) ,		1					)
                        self.assertEqual(len(lowercase					) ,		1					)
                        self.assertEqual(token_a[0] ,		lowercase					)
                        self.assertEqual(token_a[0] ,		lowercase					)





      @require_tokenizers
      def        _snake_case     (			self					)						->	Union[str, Any]:
            lowerCAmelCase    		=			self.get_tokenizers(do_lower_case=lowercase					)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'					):
                        # a special token for Canine can be defined as follows:
                        lowerCAmelCase    		=			0XE_006
                        lowerCAmelCase    		=			chr(lowercase					)

                        lowerCAmelCase    		=			AddedToken(lowercase ,		lstrip=lowercase					)
                        tokenizer.add_special_tokens({"""additional_special_tokens""": [new_token]}					)

                        with tempfile.TemporaryDirectory() as tmp_dir_name:
                              tokenizer.save_pretrained(lowercase					)
                              tokenizer.from_pretrained(lowercase					)





      def        _snake_case     (			self					)						->	Dict:
            lowerCAmelCase    		=			[]
            if self.test_slow_tokenizer:
                  tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

            if self.test_rust_tokenizer:
                  tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

            for tokenizer_class, tokenizer_utils in tokenizer_list:
                  with tempfile.TemporaryDirectory() as tmp_dir:
                        tokenizer_utils.save_pretrained(lowercase					)

                        with open(os.path.join(lowercase ,		"""special_tokens_map.json"""					) ,		encoding="""utf-8"""					) as json_file:
                              lowerCAmelCase    		=			json.load(lowercase					)

                        with open(os.path.join(lowercase ,		"""tokenizer_config.json"""					) ,		encoding="""utf-8"""					) as json_file:
                              lowerCAmelCase    		=			json.load(lowercase					)

                        # a special token for Canine can be defined as follows:
                        lowerCAmelCase    		=			0XE_006
                        lowerCAmelCase    		=			chr(lowercase					)

                        lowerCAmelCase    		=			[new_token_a]
                        lowerCAmelCase    		=			[new_token_a]

                        with open(os.path.join(lowercase ,		"""special_tokens_map.json"""					) ,		"""w""" ,		encoding="""utf-8"""					) as outfile:
                              json.dump(lowercase ,		lowercase					)
                        with open(os.path.join(lowercase ,		"""tokenizer_config.json"""					) ,		"""w""" ,		encoding="""utf-8"""					) as outfile:
                              json.dump(lowercase ,		lowercase					)

                        # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                        # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                        # "special_tokens_map.json" files
                        lowerCAmelCase    		=			tokenizer_class.from_pretrained(lowercase ,		extra_ids=0					)
                        self.assertIn(lowercase ,		tokenizer_without_change_in_init.additional_special_tokens					)
                        # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                        self.assertEqual(
                            [new_token_a] ,		tokenizer_without_change_in_init.convert_ids_to_tokens(
                                tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]					)					) ,		)

                        lowerCAmelCase    		=			0XE_007
                        lowerCAmelCase    		=			chr(lowercase					)
                        # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                        lowerCAmelCase    		=			[AddedToken(lowercase ,		lstrip=lowercase					)]
                        lowerCAmelCase    		=			tokenizer_class.from_pretrained(
                            lowercase ,		additional_special_tokens=lowercase ,		extra_ids=0					)

                        self.assertIn(lowercase ,		tokenizer.additional_special_tokens					)
                        # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
                        self.assertEqual(
                            [new_token_a] ,		tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]					)					)					)





      @require_tokenizers
      def        _snake_case     (			self					)						->	Tuple:
            lowerCAmelCase    		=			self.get_tokenizers(do_lower_case=lowercase					)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'					):
                        lowerCAmelCase    		=			"""hello world"""
                        if self.space_between_special_tokens:
                              lowerCAmelCase    		=			"""[CLS] hello world [SEP]"""
                        else:
                              lowerCAmelCase    		=			input
                        lowerCAmelCase    		=			tokenizer.encode(lowercase ,		add_special_tokens=lowercase					)
                        lowerCAmelCase    		=			tokenizer.decode(lowercase ,		spaces_between_special_tokens=self.space_between_special_tokens					)
                        self.assertIn(lowercase ,		[output, output.lower()]					)





      def        _snake_case     (			self					)						->	List[str]:
            lowerCAmelCase    		=			self.get_tokenizers()
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'					):
                        lowerCAmelCase    		=			[
                            """bos_token""",
                            """eos_token""",
                            """unk_token""",
                            """sep_token""",
                            """pad_token""",
                            """cls_token""",
                            """mask_token""",
                        ]

            lowerCAmelCase    		=			"""a"""
            lowerCAmelCase    		=			ord(lowercase					)

            for attr in attributes_list:
                  setattr(lowercase ,		attr + """_id""" ,		lowercase					)
                  self.assertEqual(getattr(lowercase ,		lowercase					) ,		lowercase					)
                  self.assertEqual(getattr(lowercase ,		attr + """_id"""					) ,		lowercase					)

                  setattr(lowercase ,		attr + """_id""" ,		lowercase					)
                  self.assertEqual(getattr(lowercase ,		lowercase					) ,		lowercase					)
                  self.assertEqual(getattr(lowercase ,		attr + """_id"""					) ,		lowercase					)

            setattr(lowercase ,		"""additional_special_tokens_ids""" ,		[]					)
            self.assertListEqual(getattr(lowercase ,		"""additional_special_tokens"""					) ,		[]					)
            self.assertListEqual(getattr(lowercase ,		"""additional_special_tokens_ids"""					) ,		[]					)

            lowerCAmelCase    		=			0XE_006
            lowerCAmelCase    		=			chr(lowercase					)
            setattr(lowercase ,		"""additional_special_tokens_ids""" ,		[additional_special_token_id]					)
            self.assertListEqual(getattr(lowercase ,		"""additional_special_tokens"""					) ,		[additional_special_token]					)
            self.assertListEqual(getattr(lowercase ,		"""additional_special_tokens_ids"""					) ,		[additional_special_token_id]					)





      def        _snake_case     (			self					)						->	Tuple:
            pass





      def        _snake_case     (			self					)						->	Optional[Any]:
            pass





      def        _snake_case     (			self					)						->	Dict:
            pass





      def        _snake_case     (			self					)						->	str:
            pass





      def        _snake_case     (			self					)						->	Optional[int]:
            pass





      def        _snake_case     (			self					)						->	Optional[int]:
            pass





      def        _snake_case     (			self					)						->	Union[str, Any]:
            pass





      def        _snake_case     (			self					)						->	Optional[Any]:
            pass






