



"""simple docstring"""







import argparse
import json
import os

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ConvNextImageProcessor, ConvNextVaConfig, ConvNextVaForImageClassification
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase:   Optional[int]       			=							logging.get_logger(__name__)


def       __SCREAMING_SNAKE_CASE      (   __UpperCAmelCase	):
		_lowercase							:      Optional[int]			=					ConvNextVaConfig()

		if "atto" in checkpoint_url:
				_lowercase							:      Any			=					[2, 2, 6, 2]
				_lowercase							:      int			=					[40, 80, 160, 320]
		if "femto" in checkpoint_url:
				_lowercase							:      Union[str, Any]			=					[2, 2, 6, 2]
				_lowercase							:      Dict			=					[48, 96, 192, 384]
		if "pico" in checkpoint_url:
				_lowercase							:      Optional[Any]			=					[2, 2, 6, 2]
				_lowercase							:      List[Any]			=					[64, 128, 256, 512]
		if "nano" in checkpoint_url:
				_lowercase							:      Union[str, Any]			=					[2, 2, 8, 2]
				_lowercase							:      Dict			=					[80, 160, 320, 640]
		if "tiny" in checkpoint_url:
				_lowercase							:      Any			=					[3, 3, 9, 3]
				_lowercase							:      Dict			=					[96, 192, 384, 768]
		if "base" in checkpoint_url:
				_lowercase							:      List[str]			=					[3, 3, 27, 3]
				_lowercase							:      Optional[int]			=					[128, 256, 512, 1024]
		if "large" in checkpoint_url:
				_lowercase							:      Any			=					[3, 3, 27, 3]
				_lowercase							:      Union[str, Any]			=					[192, 384, 768, 1536]
		if "huge" in checkpoint_url:
				_lowercase							:      List[Any]			=					[3, 3, 27, 3]
				_lowercase							:      Tuple			=					[352, 704, 1408, 2816]

		_lowercase							:      Union[str, Any]			=					1000
		_lowercase							:      int			=					"""imagenet-1k-id2label.json"""
		_lowercase							:      str			=					(1, 1000)

		_lowercase							:      Optional[int]			=					"""huggingface/label-files"""
		_lowercase							:      Dict			=					num_labels
		_lowercase							:      List[str]			=					json.load(open(hf_hub_download(__UpperCAmelCase					,							__UpperCAmelCase					,							repo_type="""dataset"""	)					,							"""r"""	)	)
		_lowercase							:      Union[str, Any]			=					{int(__UpperCAmelCase	): v for k, v in idalabel.items()}

		_lowercase							:      Dict			=					idalabel
		_lowercase							:      str			=					{v: k for k, v in idalabel.items()}
		_lowercase							:      Optional[Any]			=					hidden_sizes
		_lowercase							:      Union[str, Any]			=					depths

		return config, expected_shape


def       __SCREAMING_SNAKE_CASE      (   __UpperCAmelCase	):
		if "downsample_layers.0.0" in name:
				_lowercase							:      str			=					name.replace("""downsample_layers.0.0"""					,							"""embeddings.patch_embeddings"""	)
		if "downsample_layers.0.1" in name:
				_lowercase							:      Optional[int]			=					name.replace("""downsample_layers.0.1"""					,							"""embeddings.norm"""	)  # we rename to layernorm later on
		if "downsample_layers.1.0" in name:
				_lowercase							:      List[str]			=					name.replace("""downsample_layers.1.0"""					,							"""stages.1.downsampling_layer.0"""	)
		if "downsample_layers.1.1" in name:
				_lowercase							:      Any			=					name.replace("""downsample_layers.1.1"""					,							"""stages.1.downsampling_layer.1"""	)
		if "downsample_layers.2.0" in name:
				_lowercase							:      Any			=					name.replace("""downsample_layers.2.0"""					,							"""stages.2.downsampling_layer.0"""	)
		if "downsample_layers.2.1" in name:
				_lowercase							:      Union[str, Any]			=					name.replace("""downsample_layers.2.1"""					,							"""stages.2.downsampling_layer.1"""	)
		if "downsample_layers.3.0" in name:
				_lowercase							:      List[Any]			=					name.replace("""downsample_layers.3.0"""					,							"""stages.3.downsampling_layer.0"""	)
		if "downsample_layers.3.1" in name:
				_lowercase							:      str			=					name.replace("""downsample_layers.3.1"""					,							"""stages.3.downsampling_layer.1"""	)
		if "stages" in name and "downsampling_layer" not in name:
				# stages.0.0. for instance should be renamed to stages.0.layers.0.
				_lowercase							:      List[Any]			=					name[: len("""stages.0"""	)] + """.layers""" + name[len("""stages.0"""	) :]
		if "gamma" in name:
				_lowercase							:      Union[str, Any]			=					name.replace("""gamma"""					,							"""weight"""	)
		if "beta" in name:
				_lowercase							:      List[str]			=					name.replace("""beta"""					,							"""bias"""	)
		if "stages" in name:
				_lowercase							:      int			=					name.replace("""stages"""					,							"""encoder.stages"""	)
		if "norm" in name:
				_lowercase							:      Any			=					name.replace("""norm"""					,							"""layernorm"""	)
		if "head" in name:
				_lowercase							:      str			=					name.replace("""head"""					,							"""classifier"""	)

		return name


def       __SCREAMING_SNAKE_CASE      (   ):
		_lowercase							:      str			=					"""http://images.cocodataset.org/val2017/000000039769.jpg"""
		_lowercase							:      List[str]			=					Image.open(requests.get(__UpperCAmelCase					,							stream=__UpperCAmelCase	).raw	)
		return im


def       __SCREAMING_SNAKE_CASE      (   __UpperCAmelCase	):
		if "224" in checkpoint_url:
				_lowercase							:      Optional[int]			=					224
				_lowercase							:      Optional[Any]			=					224 / 256
		elif "384" in checkpoint_url:
				_lowercase							:      Any			=					384
				_lowercase							:      Tuple			=					None
		else:
				_lowercase							:      Dict			=					512
				_lowercase							:      str			=					None

		return ConvNextImageProcessor(
		    size=__UpperCAmelCase					,							crop_pct=__UpperCAmelCase					,							image_mean=[0.4_8_5, 0.4_5_6, 0.4_0_6]					,							image_std=[0.2_2_9, 0.2_2_4, 0.2_2_5]					,							resample=PILImageResampling.BICUBIC					,							)


@torch.no_grad()
def       __SCREAMING_SNAKE_CASE      (   __UpperCAmelCase					,							__UpperCAmelCase					,							__UpperCAmelCase					,							__UpperCAmelCase	):
		print("""Downloading original model from checkpoint..."""	)
		# define ConvNeXTV2 configuration based on URL
		_lowercase			,   _lowercase							:      Tuple			=					get_convnextva_config(__UpperCAmelCase	)
		# load original state_dict from URL
		_lowercase							:      List[Any]			=					torch.hub.load_state_dict_from_url(__UpperCAmelCase	)["""model"""]

		print("""Converting model parameters..."""	)
		# rename keys
		for key in state_dict.copy().keys():
				_lowercase							:      List[Any]			=					state_dict.pop(__UpperCAmelCase	)
				_lowercase							:      Optional[int]			=					val
		# add prefix to all keys expect classifier head
		for key in state_dict.copy().keys():
				_lowercase							:      str			=					state_dict.pop(__UpperCAmelCase	)
				if not key.startswith("""classifier"""	):
						_lowercase							:      int			=					"""convnextv2.""" + key
				_lowercase							:      Union[str, Any]			=					val

		# load HuggingFace model
		_lowercase							:      Optional[Any]			=					ConvNextVaForImageClassification(__UpperCAmelCase	)
		model.load_state_dict(__UpperCAmelCase	)
		model.eval()

		# Check outputs on an image, prepared by ConvNextImageProcessor
		_lowercase							:      Any			=					convert_preprocessor(__UpperCAmelCase	)
		_lowercase							:      Optional[int]			=					preprocessor(images=prepare_img()					,							return_tensors="""pt"""	)
		_lowercase							:      int			=					model(**__UpperCAmelCase	).logits

		# note: the logits below were obtained without center cropping
		if checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt":
				_lowercase							:      Tuple			=					torch.tensor([-0.3_9_3_0, 0.1_7_4_7, -0.5_2_4_6, 0.4_1_7_7, 0.4_2_9_5]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_femto_1k_224_ema.pt":
				_lowercase							:      Dict			=					torch.tensor([-0.1_7_2_7, -0.5_3_4_1, -0.7_8_1_8, -0.4_7_4_5, -0.6_5_6_6]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_pico_1k_224_ema.pt":
				_lowercase							:      Dict			=					torch.tensor([-0.0_3_3_3, 0.1_5_6_3, -0.9_1_3_7, 0.1_0_5_4, 0.0_3_8_1]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_nano_1k_224_ema.pt":
				_lowercase							:      Union[str, Any]			=					torch.tensor([-0.1_7_4_4, -0.1_5_5_5, -0.0_7_1_3, 0.0_9_5_0, -0.1_4_3_1]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_tiny_1k_224_ema.pt":
				_lowercase							:      Union[str, Any]			=					torch.tensor([0.9_9_9_6, 0.1_9_6_6, -0.4_3_8_6, -0.3_4_7_2, 0.6_6_6_1]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_base_1k_224_ema.pt":
				_lowercase							:      str			=					torch.tensor([-0.2_5_5_3, -0.6_7_0_8, -0.1_3_5_9, 0.2_5_1_8, -0.2_4_8_8]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_large_1k_224_ema.pt":
				_lowercase							:      Dict			=					torch.tensor([-0.0_6_7_3, -0.5_6_2_7, -0.3_7_5_3, -0.2_7_2_2, 0.0_1_7_8]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_huge_1k_224_ema.pt":
				_lowercase							:      Any			=					torch.tensor([-0.6_3_7_7, -0.7_4_5_8, -0.2_1_5_0, 0.1_1_8_4, -0.0_5_9_7]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_224_ema.pt":
				_lowercase							:      Any			=					torch.tensor([1.0_7_9_9, 0.2_3_2_2, -0.8_8_6_0, 1.0_2_1_9, 0.6_2_3_1]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_384_ema.pt":
				_lowercase							:      Tuple			=					torch.tensor([0.3_7_6_6, 0.4_9_1_7, -1.1_4_2_6, 0.9_9_4_2, 0.6_0_2_4]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_224_ema.pt":
				_lowercase							:      str			=					torch.tensor([0.4_2_2_0, -0.6_9_1_9, -0.4_3_1_7, -0.2_8_8_1, -0.6_6_0_9]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_384_ema.pt":
				_lowercase							:      List[str]			=					torch.tensor([0.1_0_8_2, -0.8_2_8_6, -0.5_0_9_5, 0.4_6_8_1, -0.8_0_8_5]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_224_ema.pt":
				_lowercase							:      int			=					torch.tensor([-0.2_4_1_9, -0.6_2_2_1, 0.2_1_7_6, -0.0_9_8_0, -0.7_5_2_7]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_384_ema.pt":
				_lowercase							:      List[str]			=					torch.tensor([0.0_3_9_1, -0.4_3_7_1, 0.3_7_8_6, 0.1_2_5_1, -0.2_7_8_4]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_224_ema.pt":
				_lowercase							:      Any			=					torch.tensor([-0.0_5_0_4, 0.5_6_3_6, -0.1_7_2_9, -0.6_5_0_7, -0.3_9_4_9]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_384_ema.pt":
				_lowercase							:      Dict			=					torch.tensor([0.3_5_6_0, 0.9_4_8_6, 0.3_1_4_9, -0.2_6_6_7, -0.5_1_3_8]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_384_ema.pt":
				_lowercase							:      Dict			=					torch.tensor([-0.2_4_6_9, -0.4_5_5_0, -0.5_8_5_3, -0.0_8_1_0, 0.0_3_0_9]	)
		elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_512_ema.pt":
				_lowercase							:      List[str]			=					torch.tensor([-0.3_0_9_0, 0.0_8_0_2, -0.0_6_8_2, -0.1_9_7_9, -0.2_8_2_6]	)
		else:
				raise ValueError(F"""Unknown URL: {checkpoint_url}"""	)

		assert torch.allclose(logits[0, :5]					,							__UpperCAmelCase					,							atol=1E-3	)
		assert logits.shape == expected_shape
		print("""Model outputs match the original results!"""	)

		if save_model:
				print("""Saving model to local..."""	)
				# Create folder to save model
				if not os.path.isdir(__UpperCAmelCase	):
						os.mkdir(__UpperCAmelCase	)

				model.save_pretrained(__UpperCAmelCase	)
				preprocessor.save_pretrained(__UpperCAmelCase	)

		_lowercase							:      List[str]			=					"""convnextv2"""
		if "atto" in checkpoint_url:
				model_name += "-atto"
		if "femto" in checkpoint_url:
				model_name += "-femto"
		if "pico" in checkpoint_url:
				model_name += "-pico"
		if "nano" in checkpoint_url:
				model_name += "-nano"
		elif "tiny" in checkpoint_url:
				model_name += "-tiny"
		elif "base" in checkpoint_url:
				model_name += "-base"
		elif "large" in checkpoint_url:
				model_name += "-large"
		elif "huge" in checkpoint_url:
				model_name += "-huge"
		if "22k" in checkpoint_url and "1k" not in checkpoint_url:
				model_name += "-22k"
		elif "22k" in checkpoint_url and "1k" in checkpoint_url:
				model_name += "-22k-1k"
		elif "1k" in checkpoint_url:
				model_name += "-1k"
		if "224" in checkpoint_url:
				model_name += "-224"
		elif "384" in checkpoint_url:
				model_name += "-384"
		elif "512" in checkpoint_url:
				model_name += "-512"

		if push_to_hub:
				print(F"""Pushing {model_name} to the hub..."""	)
				model.push_to_hub(__UpperCAmelCase	)
				preprocessor.push_to_hub(__UpperCAmelCase	)


if __name__ == "__main__":
		UpperCAmelCase:   int       			=							argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    """--checkpoint_url""",
		    default="""https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt""",
		    type=str,
		    help="""URL of the original ConvNeXTV2 checkpoint you'd like to convert.""",
		)
		parser.add_argument(
		    """--pytorch_dump_folder_path""",
		    default="""model""",
		    type=str,
		    help="""Path to the output PyTorch model directory.""",
		)
		parser.add_argument("""--save_model""", action="""store_true""", help="""Save model to local""")
		parser.add_argument("""--push_to_hub""", action="""store_true""", help="""Push model and image preprocessor to the hub""")

		UpperCAmelCase:   Optional[Any]       			=							parser.parse_args()
		convert_convnextva_checkpoint(
		    args.checkpoint_url, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub
		)








"""simple docstring"""







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class    UpperCamelCase     (			snake_case ,					unittest.TestCase    ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_				:    str		    =   PreTrainedTokenizerFast
	SCREAMING_SNAKE_CASE_				:    Optional[Any]		    =   False
	SCREAMING_SNAKE_CASE_				:    Optional[int]		    =   True
	SCREAMING_SNAKE_CASE_				:    int		    =   "tokenizer_file"






	def 					lowerCamelCase__ (			self					):
			_lowercase							:      Optional[int]			=					False  # because we don't have pretrained_vocab_files_map
			super().setUp()
			_lowercase							:      Tuple			=					True

			_lowercase							:      Optional[Any]			=					["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
			_lowercase							:      Dict			=					"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

			# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
			_lowercase							:      Dict			=					[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

			_lowercase							:      Any			=					PreTrainedTokenizerFast.from_pretrained(model_paths[0]					)
			tokenizer.save_pretrained(self.tmpdirname					)






	def 					lowerCamelCase__ (			self					):
			# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
			# model
			pass






	def 					lowerCamelCase__ (			self					):
			# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
			# model
			pass






	def 					lowerCamelCase__ (			self					):
			# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
			# model
			pass






	def 					lowerCamelCase__ (			self					):
			# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
			pass






	def 					lowerCamelCase__ (			self					):
			_lowercase							:      List[str]			=					self.tmpdirname
			# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
							try:
									_lowercase							:      Optional[Any]			=					tempfile.mkdtemp()
									_lowercase							:      Any			=					self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_							,**UpperCAmelCase_					)

									tokenizer.save_pretrained(self.tmpdirname					)
									super().test_training_new_tokenizer()
							finally:
									# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
									# is restored
									shutil.rmtree(self.tmpdirname					)
									_lowercase							:      List[Any]			=					tmpdirname_orig






	def 					lowerCamelCase__ (			self					):
			_lowercase							:      List[str]			=					self.tmpdirname
			# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
							try:
									_lowercase							:      Optional[int]			=					tempfile.mkdtemp()
									_lowercase							:      Tuple			=					self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_							,**UpperCAmelCase_					)

									tokenizer.save_pretrained(self.tmpdirname					)
									super().test_training_new_tokenizer_with_special_tokens_change()
							finally:
									# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
									# is restored
									shutil.rmtree(self.tmpdirname					)
									_lowercase							:      Optional[int]			=					tmpdirname_orig






	def 					lowerCamelCase__ (			self					):
			_lowercase							:      List[Any]			=					self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name					)

			_lowercase							:      List[str]			=					("""a""" for _ in range(10_00					))
			_lowercase							:      Dict			=					tokenizer.train_new_from_iterator(text_iterator=UpperCAmelCase_							,length=10_00							,vocab_size=50					)

			_lowercase							:      List[str]			=					new_tokenizer.encode("""a🤗"""					)
			self.assertEqual(UpperCAmelCase_							,[64, 1_72, 2_53, 97, 2_45]					)







@require_tokenizers
class    UpperCamelCase     (			unittest.TestCase    ):




	"""simple docstring"""






	def 					lowerCamelCase__ (			self					):
			_lowercase							:      List[Any]			=					AutoTokenizer.from_pretrained("""bert-base-cased"""					)
			_lowercase							:      Optional[Any]			=					json.loads(tokenizer._tokenizer.to_str()					)
			_lowercase							:      Dict			=					len(UpperCAmelCase_					)

			with tempfile.TemporaryDirectory() as tmp_dir:
					# Hack to save this in the tokenizer_config.json
					_lowercase							:      str			=					["""tokenizer.4.0.0.json"""]
					tokenizer.save_pretrained(UpperCAmelCase_					)
					json.dump(UpperCAmelCase_							,open(os.path.join(UpperCAmelCase_							,"""tokenizer.4.0.0.json"""					)							,"""w"""					)					)

					# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
					_lowercase							:      int			=					AutoTokenizer.from_pretrained(UpperCAmelCase_					)
					self.assertEqual(len(UpperCAmelCase_					)							,len(UpperCAmelCase_					) + 1					)
					_lowercase							:      List[str]			=					json.loads(new_tokenizer._tokenizer.to_str()					)
					self.assertIn("""huggingface"""							,json_tokenizer["""model"""]["""vocab"""]					)

					# Will need to be adjusted if we reach v42 and this test is still here.
					# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
					shutil.move(os.path.join(UpperCAmelCase_							,"""tokenizer.4.0.0.json"""					)							,os.path.join(UpperCAmelCase_							,"""tokenizer.42.0.0.json"""					)					)
					_lowercase							:      Optional[int]			=					["""tokenizer.42.0.0.json"""]
					tokenizer.save_pretrained(UpperCAmelCase_					)
					_lowercase							:      List[Any]			=					AutoTokenizer.from_pretrained(UpperCAmelCase_					)
					self.assertEqual(len(UpperCAmelCase_					)							,len(UpperCAmelCase_					)					)
					_lowercase							:      List[Any]			=					json.loads(new_tokenizer._tokenizer.to_str()					)
					self.assertNotIn("""huggingface"""							,json_tokenizer["""model"""]["""vocab"""]					)






	def 					lowerCamelCase__ (			self					):
			# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
			_lowercase							:      Tuple			=					"""hf-internal-testing/test-two-tokenizers"""

			# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
			_lowercase							:      List[Any]			=					AutoTokenizer.from_pretrained(UpperCAmelCase_					)
			self.assertEqual(len(UpperCAmelCase_					)							,2_89_97					)
			_lowercase							:      List[Any]			=					json.loads(tokenizer._tokenizer.to_str()					)
			self.assertIn("""huggingface"""							,json_tokenizer["""model"""]["""vocab"""]					)

			# Testing an older version by monkey-patching the version in the module it's used.
			import transformers as old_transformers

			_lowercase							:      Optional[Any]			=					"""3.0.0"""
			_lowercase							:      int			=					old_transformers.models.auto.AutoTokenizer.from_pretrained(UpperCAmelCase_					)
			self.assertEqual(len(UpperCAmelCase_					)							,2_89_96					)
			_lowercase							:      str			=					json.loads(old_tokenizer._tokenizer.to_str()					)
			self.assertNotIn("""huggingface"""							,json_tokenizer["""model"""]["""vocab"""]					)







@require_tokenizers
class    UpperCamelCase     (			unittest.TestCase    ):




	"""simple docstring"""






	def 					lowerCamelCase__ (			self					):
			# See https://github.com/huggingface/transformers/pull/12550
			# and https://github.com/huggingface/tokenizers/issues/537
			_lowercase							:      Optional[int]			=					PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""					)
			_lowercase							:      List[str]			=					"""The Matrix is a 1999 science fiction action film."""

			with concurrent.futures.ThreadPoolExecutor() as executor:
					_lowercase							:      Tuple			=					[executor.submit(self.fetch							,UpperCAmelCase_							,UpperCAmelCase_					) for i in range(10					)]
					_lowercase							:      str			=					[future.result() for future in futures]
					self.assertEqual(UpperCAmelCase_							,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10					)]					)






	def 					lowerCamelCase__ (			self							,UpperCAmelCase_							,UpperCAmelCase_					):
			return tokenizer.encode(UpperCAmelCase_							,truncation="""longest_first"""							,padding="""longest"""					)



