



"""simple docstring"""
import argparse
import json
from dataclasses import dataclass, field
from functools import partial
from pathlib import Path
from typing import Callable, Dict, List, Tuple

import timm
import torch
import torch.nn as nn
from classy_vision.models.regnet import RegNet, RegNetParams, RegNetYaagf, RegNetYaagf, RegNetYaaagf
from huggingface_hub import cached_download, hf_hub_url
from torch import Tensor
from vissl.models.model_helpers import get_trunk_forward_outputs

from transformers import AutoImageProcessor, RegNetConfig, RegNetForImageClassification, RegNetModel
from transformers.utils import logging


logging.set_verbosity_info()
SCREAMING_SNAKE_CASE_				= logging.get_logger()




@dataclass
class 			snake_case_						:
							__lowerCAmelCase    					=					42
							__lowerCAmelCase    					=					field(default_factory=a_				)
							__lowerCAmelCase    					=					field(default_factory=a_				)






							def snake_case_  (   self			,     a_			,     a_			,     a_      ):
									a_     :      str 						=     len(list(m.modules()      )      ) == 1 or isinstance(a_			,     nn.Convad      ) or isinstance(a_			,     nn.BatchNormad      )
									if has_not_submodules:
											self.traced.append(a_      )






							def __call__(   self			,     a_      ):
									for m in self.module.modules():
											self.handles.append(m.register_forward_hook(self._forward_hook      )      )
									self.module(a_      )
									[x.remove() for x in self.handles]
									return self






							@property
							def snake_case_  (   self      ):
									# check the len of the state_dict keys to see if we have learnable params
									return list(filter(lambda a_      : len(list(x.state_dict().keys()      )      ) > 0			,     self.traced      )      )




@dataclass
class 			snake_case_						:
							__lowerCAmelCase    					=					42
							__lowerCAmelCase    					=					42
							__lowerCAmelCase    					=					1
							__lowerCAmelCase    					=					field(default_factory=a_				)
							__lowerCAmelCase    					=					field(default_factory=a_				)
							__lowerCAmelCase    					=					True






							def __call__(   self			,     a_      ):
									a_     :      List[str] 						=     Tracker(self.dest      )(a_      ).parametrized
									a_     :      Union[str, Any] 						=     Tracker(self.src      )(a_      ).parametrized

									a_     :      Tuple 						=     list(filter(lambda a_      : type(a_      ) not in self.src_skip			,     a_      )      )
									a_     :      Dict 						=     list(filter(lambda a_      : type(a_      ) not in self.dest_skip			,     a_      )      )

									if len(a_      ) != len(a_      ) and self.raise_if_mismatch:
											raise Exception(
											    F"""Numbers of operations are different. Source module has {len(a_      )} operations while"""
											    F""" destination module has {len(a_      )}."""      )

									for dest_m, src_m in zip(a_			,     a_      ):
											dest_m.load_state_dict(src_m.state_dict()      )
											if self.verbose == 1:
													print(F"""Transfered from={src_m} to={dest_m}"""      )




class 			snake_case_						(						nn.Module				):






							def __init__(   self			,     a_      ):
									super().__init__()

									a_     :      List[Tuple[str, nn.Module]] 						=     []
									# - get the stem
									feature_blocks.append(("conv1", model.stem)      )
									# - get all the feature blocks
									for k, v in model.trunk_output.named_children():
											assert k.startswith("block"      ), F"""Unexpected layer name {k}"""
											a_     :      Tuple 						=     len(a_      ) + 1
											feature_blocks.append((F"""res{block_index}""", v)      )

									a_     :      List[Any] 						=     nn.ModuleDict(a_      )






							def snake_case_  (   self			,     a_      ):
									return get_trunk_forward_outputs(
									    a_			,     out_feat_keys=a_			,     feature_blocks=self._feature_blocks			,     )




class 			snake_case_						(						a_				):






							def snake_case_  (   self			,     a_      ):
									a_     :      Union[str, Any] 						=     x.split("-"      )
									return x_split[0] + x_split[1] + "_" + "".join(x_split[2:]      )






							def __getitem__(   self			,     a_      ):
									# default to timm!
									if x not in self:
											a_     :      Tuple 						=     self.convert_name_to_timm(a_      )
											a_     :      Dict 						=     partial(lambda: (timm.create_model(a_			,     pretrained=a_      ).eval(), None)      )

									else:
											a_     :      Optional[int] 						=     super().__getitem__(a_      )

									return val




class 			snake_case_						(						a_				):






							def __getitem__(   self			,     a_      ):
									if "seer" in x and "in1k" not in x:
											a_     :      Tuple 						=     RegNetModel
									else:
											a_     :      List[Any] 						=     RegNetForImageClassification
									return val



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      List[str]:
		for from_key, to_key in keys:
				a_     :      Optional[int] 						=     from_state_dict[from_key].clone()
				print(F"""Copied key={from_key} to={to_key}"""	)
		return to_state_dict



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__ = True, )			->      Optional[int]:
		print(F"""Converting {name}..."""	)
		with torch.no_grad():
				a_						,				a_     :      List[str] 						=     from_model_func()
				a_     :      Tuple 						=     our_model_func(SCREAMING_SNAKE_CASE__	).eval()
				a_     :      Tuple 						=     ModuleTransfer(src=SCREAMING_SNAKE_CASE__, dest=SCREAMING_SNAKE_CASE__, raise_if_mismatch=SCREAMING_SNAKE_CASE__	)
				a_     :      int 						=     torch.randn((1, 3, 224, 224)	)
				module_transfer(SCREAMING_SNAKE_CASE__	)

		if from_state_dict is not None:
				a_     :      Optional[Any] 						=     []
				# for seer - in1k finetuned we have to manually copy the head
				if "seer" in name and "in1k" in name:
						a_     :      str 						=     [("0.clf.0.weight", "classifier.1.weight"), ("0.clf.0.bias", "classifier.1.bias")]
				a_     :      Tuple 						=     manually_copy_vissl_head(SCREAMING_SNAKE_CASE__, our_model.state_dict(), SCREAMING_SNAKE_CASE__	)
				our_model.load_state_dict(SCREAMING_SNAKE_CASE__	)

		a_     :      List[str] 						=     our_model(SCREAMING_SNAKE_CASE__, output_hidden_states=SCREAMING_SNAKE_CASE__	)
		a_     :      Any 						=     (
		    our_outputs.logits if isinstance(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	) else our_outputs.last_hidden_state
		)

		a_     :      int 						=     from_model(SCREAMING_SNAKE_CASE__	)
		a_     :      Optional[Any] 						=     from_output[-1] if type(SCREAMING_SNAKE_CASE__	) is list else from_output

		# now since I don't want to use any config files, vissl seer model doesn't actually have an head, so let's just check the last hidden state
		if "seer" in name and "in1k" in name:
				a_     :      Optional[int] 						=     our_outputs.hidden_states[-1]

		assert torch.allclose(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	), "The model logits don't match the original one."

		if push_to_hub:
				our_model.push_to_hub(
				    repo_path_or_name=save_directory / name, commit_message="Add model", use_temp_dir=SCREAMING_SNAKE_CASE__, )

				a_     :      Dict 						=     224 if "seer" not in name else 384
				# we can use the convnext one
				a_     :      List[str] 						=     AutoImageProcessor.from_pretrained("facebook/convnext-base-224-22k-1k", size=SCREAMING_SNAKE_CASE__	)
				image_processor.push_to_hub(
				    repo_path_or_name=save_directory / name, commit_message="Add image processor", use_temp_dir=SCREAMING_SNAKE_CASE__, )

				print(F"""Pushed {name}"""	)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__ = None, SCREAMING_SNAKE_CASE__ = True	)			->      Dict:
		a_     :      Optional[Any] 						=     "imagenet-1k-id2label.json"
		a_     :      List[str] 						=     1_000
		a_     :      int 						=     (1, num_labels)

		a_     :      List[Any] 						=     "huggingface/label-files"
		a_     :      Optional[int] 						=     num_labels
		a_     :      Dict 						=     json.load(open(cached_download(hf_hub_url(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, repo_type="dataset"	)	), "r"	)	)
		a_     :      Optional[Any] 						=     {int(SCREAMING_SNAKE_CASE__	): v for k, v in idalabel.items()}

		a_     :      Optional[int] 						=     idalabel
		a_     :      int 						=     {v: k for k, v in idalabel.items()}

		a_     :      List[str] 						=     partial(SCREAMING_SNAKE_CASE__, num_labels=SCREAMING_SNAKE_CASE__, idalabel=SCREAMING_SNAKE_CASE__, labelaid=SCREAMING_SNAKE_CASE__	)

		a_     :      Optional[int] 						=     {
		    "regnet-x-002": ImageNetPreTrainedConfig(
		        depths=[1, 1, 4, 7], hidden_sizes=[24, 56, 152, 368], groups_width=8, layer_type="x"	),
		    "regnet-x-004": ImageNetPreTrainedConfig(
		        depths=[1, 2, 7, 12], hidden_sizes=[32, 64, 160, 384], groups_width=16, layer_type="x"	),
		    "regnet-x-006": ImageNetPreTrainedConfig(
		        depths=[1, 3, 5, 7], hidden_sizes=[48, 96, 240, 528], groups_width=24, layer_type="x"	),
		    "regnet-x-008": ImageNetPreTrainedConfig(
		        depths=[1, 3, 7, 5], hidden_sizes=[64, 128, 288, 672], groups_width=16, layer_type="x"	),
		    "regnet-x-016": ImageNetPreTrainedConfig(
		        depths=[2, 4, 10, 2], hidden_sizes=[72, 168, 408, 912], groups_width=24, layer_type="x"	),
		    "regnet-x-032": ImageNetPreTrainedConfig(
		        depths=[2, 6, 15, 2], hidden_sizes=[96, 192, 432, 1_008], groups_width=48, layer_type="x"	),
		    "regnet-x-040": ImageNetPreTrainedConfig(
		        depths=[2, 5, 14, 2], hidden_sizes=[80, 240, 560, 1_360], groups_width=40, layer_type="x"	),
		    "regnet-x-064": ImageNetPreTrainedConfig(
		        depths=[2, 4, 10, 1], hidden_sizes=[168, 392, 784, 1_624], groups_width=56, layer_type="x"	),
		    "regnet-x-080": ImageNetPreTrainedConfig(
		        depths=[2, 5, 15, 1], hidden_sizes=[80, 240, 720, 1_920], groups_width=120, layer_type="x"	),
		    "regnet-x-120": ImageNetPreTrainedConfig(
		        depths=[2, 5, 11, 1], hidden_sizes=[224, 448, 896, 2_240], groups_width=112, layer_type="x"	),
		    "regnet-x-160": ImageNetPreTrainedConfig(
		        depths=[2, 6, 13, 1], hidden_sizes=[256, 512, 896, 2_048], groups_width=128, layer_type="x"	),
		    "regnet-x-320": ImageNetPreTrainedConfig(
		        depths=[2, 7, 13, 1], hidden_sizes=[336, 672, 1_344, 2_520], groups_width=168, layer_type="x"	),
		    # y variant
		    "regnet-y-002": ImageNetPreTrainedConfig(depths=[1, 1, 4, 7], hidden_sizes=[24, 56, 152, 368], groups_width=8	),
		    "regnet-y-004": ImageNetPreTrainedConfig(
		        depths=[1, 3, 6, 6], hidden_sizes=[48, 104, 208, 440], groups_width=8	),
		    "regnet-y-006": ImageNetPreTrainedConfig(
		        depths=[1, 3, 7, 4], hidden_sizes=[48, 112, 256, 608], groups_width=16	),
		    "regnet-y-008": ImageNetPreTrainedConfig(
		        depths=[1, 3, 8, 2], hidden_sizes=[64, 128, 320, 768], groups_width=16	),
		    "regnet-y-016": ImageNetPreTrainedConfig(
		        depths=[2, 6, 17, 2], hidden_sizes=[48, 120, 336, 888], groups_width=24	),
		    "regnet-y-032": ImageNetPreTrainedConfig(
		        depths=[2, 5, 13, 1], hidden_sizes=[72, 216, 576, 1_512], groups_width=24	),
		    "regnet-y-040": ImageNetPreTrainedConfig(
		        depths=[2, 6, 12, 2], hidden_sizes=[128, 192, 512, 1_088], groups_width=64	),
		    "regnet-y-064": ImageNetPreTrainedConfig(
		        depths=[2, 7, 14, 2], hidden_sizes=[144, 288, 576, 1_296], groups_width=72	),
		    "regnet-y-080": ImageNetPreTrainedConfig(
		        depths=[2, 4, 10, 1], hidden_sizes=[168, 448, 896, 2_016], groups_width=56	),
		    "regnet-y-120": ImageNetPreTrainedConfig(
		        depths=[2, 5, 11, 1], hidden_sizes=[224, 448, 896, 2_240], groups_width=112	),
		    "regnet-y-160": ImageNetPreTrainedConfig(
		        depths=[2, 4, 11, 1], hidden_sizes=[224, 448, 1_232, 3_024], groups_width=112	),
		    "regnet-y-320": ImageNetPreTrainedConfig(
		        depths=[2, 5, 12, 1], hidden_sizes=[232, 696, 1_392, 3_712], groups_width=232	),
		    # models created by SEER -> https://arxiv.org/abs/2202.08360
		    "regnet-y-320-seer": RegNetConfig(depths=[2, 5, 12, 1], hidden_sizes=[232, 696, 1_392, 3_712], groups_width=232	),
		    "regnet-y-640-seer": RegNetConfig(depths=[2, 5, 12, 1], hidden_sizes=[328, 984, 1_968, 4_920], groups_width=328	),
		    "regnet-y-1280-seer": RegNetConfig(
		        depths=[2, 7, 17, 1], hidden_sizes=[528, 1_056, 2_904, 7_392], groups_width=264	),
		    "regnet-y-2560-seer": RegNetConfig(
		        depths=[3, 7, 16, 1], hidden_sizes=[640, 1_696, 2_544, 5_088], groups_width=640	),
		    "regnet-y-10b-seer": ImageNetPreTrainedConfig(
		        depths=[2, 7, 17, 1], hidden_sizes=[2_020, 4_040, 11_110, 28_280], groups_width=1_010	),
		    # finetuned on imagenet
		    "regnet-y-320-seer-in1k": ImageNetPreTrainedConfig(
		        depths=[2, 5, 12, 1], hidden_sizes=[232, 696, 1_392, 3_712], groups_width=232	),
		    "regnet-y-640-seer-in1k": ImageNetPreTrainedConfig(
		        depths=[2, 5, 12, 1], hidden_sizes=[328, 984, 1_968, 4_920], groups_width=328	),
		    "regnet-y-1280-seer-in1k": ImageNetPreTrainedConfig(
		        depths=[2, 7, 17, 1], hidden_sizes=[528, 1_056, 2_904, 7_392], groups_width=264	),
		    "regnet-y-2560-seer-in1k": ImageNetPreTrainedConfig(
		        depths=[3, 7, 16, 1], hidden_sizes=[640, 1_696, 2_544, 5_088], groups_width=640	),
		    "regnet-y-10b-seer-in1k": ImageNetPreTrainedConfig(
		        depths=[2, 7, 17, 1], hidden_sizes=[2_020, 4_040, 11_110, 28_280], groups_width=1_010	),
		}

		a_     :      List[Any] 						=     NameToOurModelFuncMap()
		a_     :      Tuple 						=     NameToFromModelFuncMap()
		# add seer weights logic

		def load_using_classy_vision(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	) -> Tuple[nn.Module, Dict]:
				a_     :      Tuple 						=     torch.hub.load_state_dict_from_url(SCREAMING_SNAKE_CASE__, model_dir=str(SCREAMING_SNAKE_CASE__	), map_location="cpu"	)
				a_     :      str 						=     model_func()
				# check if we have a head, if yes add it
				a_     :      Union[str, Any] 						=     files["classy_state_dict"]["base_model"]["model"]
				a_     :      Union[str, Any] 						=     model_state_dict["trunk"]
				model.load_state_dict(SCREAMING_SNAKE_CASE__	)
				return model.eval(), model_state_dict["heads"]

		# pretrained
		a_     :      List[Any] 						=     partial(
		    SCREAMING_SNAKE_CASE__, "https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_regnet32d/seer_regnet32gf_model_iteration244000.torch", lambda: FakeRegNetVisslWrapper(RegNetYaagf()	), )

		a_     :      int 						=     partial(
		    SCREAMING_SNAKE_CASE__, "https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_regnet64/seer_regnet64gf_model_final_checkpoint_phase0.torch", lambda: FakeRegNetVisslWrapper(RegNetYaagf()	), )

		a_     :      List[str] 						=     partial(
		    SCREAMING_SNAKE_CASE__, "https://dl.fbaipublicfiles.com/vissl/model_zoo/swav_ig1b_regnet128Gf_cnstant_bs32_node16_sinkhorn10_proto16k_syncBN64_warmup8k/model_final_checkpoint_phase0.torch", lambda: FakeRegNetVisslWrapper(RegNetYaaagf()	), )

		a_     :      List[Any] 						=     partial(
		    SCREAMING_SNAKE_CASE__, "https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_regnet10B/model_iteration124500_conso.torch", lambda: FakeRegNetVisslWrapper(
		        RegNet(RegNetParams(depth=27, group_width=1_010, w_a=1_744, w_a=6_20.83, w_m=2.52	)	)	), )

		# IN1K finetuned
		a_     :      Any 						=     partial(
		    SCREAMING_SNAKE_CASE__, "https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_finetuned/seer_regnet32_finetuned_in1k_model_final_checkpoint_phase78.torch", lambda: FakeRegNetVisslWrapper(RegNetYaagf()	), )

		a_     :      Optional[int] 						=     partial(
		    SCREAMING_SNAKE_CASE__, "https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_finetuned/seer_regnet64_finetuned_in1k_model_final_checkpoint_phase78.torch", lambda: FakeRegNetVisslWrapper(RegNetYaagf()	), )

		a_     :      Optional[Any] 						=     partial(
		    SCREAMING_SNAKE_CASE__, "https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_finetuned/seer_regnet128_finetuned_in1k_model_final_checkpoint_phase78.torch", lambda: FakeRegNetVisslWrapper(RegNetYaaagf()	), )

		a_     :      str 						=     partial(
		    SCREAMING_SNAKE_CASE__, "https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_finetuned/seer_10b_finetuned_in1k_model_phase28_conso.torch", lambda: FakeRegNetVisslWrapper(
		        RegNet(RegNetParams(depth=27, group_width=1_010, w_a=1_744, w_a=6_20.83, w_m=2.52	)	)	), )

		if model_name:
				convert_weight_and_push(
				    SCREAMING_SNAKE_CASE__, names_to_from_model_map[model_name], names_to_ours_model_map[model_name], names_to_config[model_name], SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, )
		else:
				for model_name, config in names_to_config.items():
						convert_weight_and_push(
						    SCREAMING_SNAKE_CASE__, names_to_from_model_map[model_name], names_to_ours_model_map[model_name], SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, )
		return config, expected_shape


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE_				= argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--model_name""",
					    default=None,
					    type=str,
					    help=(
					        """The name of the model you wish to convert, it must be one of the supported regnet* architecture,"""
					        """ currently: regnetx-*, regnety-*. If `None`, all of them will the converted."""
					    ),
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""",
					    default=None,
					    type=Path,
					    required=True,
					    help="""Path to the output PyTorch model directory.""",
					)
					parser.add_argument(
					    """--push_to_hub""",
					    default=True,
					    type=bool,
					    required=False,
					    help="""If True, push model and image processor to the hub.""",
					)

					SCREAMING_SNAKE_CASE_				= parser.parse_args()

					SCREAMING_SNAKE_CASE_				= args.pytorch_dump_folder_path
					pytorch_dump_folder_path.mkdir(exist_ok=True, parents=True)
					convert_weights_and_push(pytorch_dump_folder_path, args.model_name, args.push_to_hub)




"""simple docstring"""
import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_				= get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")




@require_sentencepiece
@require_tokenizers
class 			snake_case_						(						a_     ,unittest.TestCase				):
							__lowerCAmelCase    					=					SpeechTaTokenizer
							__lowerCAmelCase    					=					False
							__lowerCAmelCase    					=					True






							def snake_case_  (   self      ):
									super().setUp()

									# We have a SentencePiece fixture for testing
									a_     :      Any 						=     SpeechTaTokenizer(a_      )

									a_     :      Optional[int] 						=     AddedToken("<mask>"			,     lstrip=a_			,     rstrip=a_      )
									a_     :      Any 						=     mask_token
									tokenizer.add_special_tokens({"mask_token": mask_token}      )
									tokenizer.add_tokens(["<ctc_blank>"]      )

									tokenizer.save_pretrained(self.tmpdirname      )






							def snake_case_  (   self			,     a_      ):
									a_     :      Tuple 						=     "this is a test"
									a_     :      Any 						=     "this is a test"
									return input_text, output_text






							def snake_case_  (   self			,     a_			,     a_=False			,     a_=2_0			,     a_=5      ):
									a_						,				a_     :      Optional[Any] 						=     self.get_input_output_texts(a_      )
									a_     :      Optional[Any] 						=     tokenizer.encode(a_			,     add_special_tokens=a_      )
									a_     :      Dict 						=     tokenizer.decode(a_			,     clean_up_tokenization_spaces=a_      )
									return text, ids






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     "<pad>"
									a_     :      Optional[int] 						=     1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(a_      )			,     a_      )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(a_      )			,     a_      )






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     list(self.get_tokenizer().get_vocab().keys()      )

									self.assertEqual(vocab_keys[0]			,     "<s>"      )
									self.assertEqual(vocab_keys[1]			,     "<pad>"      )
									self.assertEqual(vocab_keys[-4]			,     "œ"      )
									self.assertEqual(vocab_keys[-2]			,     "<mask>"      )
									self.assertEqual(vocab_keys[-1]			,     "<ctc_blank>"      )
									self.assertEqual(len(a_      )			,     8_1      )






							def snake_case_  (   self      ):
									self.assertEqual(self.get_tokenizer().vocab_size			,     7_9      )






							def snake_case_  (   self      ):
									a_     :      Any 						=     self.get_tokenizers(do_lower_case=a_      )
									for tokenizer in tokenizers:
											with self.subTest(F"""{tokenizer.__class__.__name__}"""      ):
													a_     :      Dict 						=     tokenizer.vocab_size
													a_     :      List[str] 						=     len(a_      )

													self.assertNotEqual(a_			,     0      )

													# We usually have added tokens from the start in tests because our vocab fixtures are
													# smaller than the original vocabs - let's not assert this
													# self.assertEqual(vocab_size, all_size)

													a_     :      Optional[int] 						=     ["aaaaa bbbbbb", "cccccccccdddddddd"]
													a_     :      int 						=     tokenizer.add_tokens(a_      )
													a_     :      List[Any] 						=     tokenizer.vocab_size
													a_     :      Tuple 						=     len(a_      )

													self.assertNotEqual(a_			,     0      )
													self.assertEqual(a_			,     a_      )
													self.assertEqual(a_			,     len(a_      )      )
													self.assertEqual(a_			,     all_size + len(a_      )      )

													a_     :      str 						=     tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l"			,     add_special_tokens=a_      )

													self.assertGreaterEqual(len(a_      )			,     4      )
													self.assertGreater(tokens[0]			,     tokenizer.vocab_size - 1      )
													self.assertGreater(tokens[-3]			,     tokenizer.vocab_size - 1      )

													a_     :      Tuple 						=     {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"}
													a_     :      Dict 						=     tokenizer.add_special_tokens(a_      )
													a_     :      Optional[Any] 						=     tokenizer.vocab_size
													a_     :      Any 						=     len(a_      )

													self.assertNotEqual(a_			,     0      )
													self.assertEqual(a_			,     a_      )
													self.assertEqual(a_			,     len(a_      )      )
													self.assertEqual(a_			,     all_size_a + len(a_      )      )

													a_     :      Any 						=     tokenizer.encode(
													    ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"			,     add_special_tokens=a_      )

													self.assertGreaterEqual(len(a_      )			,     6      )
													self.assertGreater(tokens[0]			,     tokenizer.vocab_size - 1      )
													self.assertGreater(tokens[0]			,     tokens[1]      )
													self.assertGreater(tokens[-3]			,     tokenizer.vocab_size - 1      )
													self.assertGreater(tokens[-3]			,     tokens[-4]      )
													self.assertEqual(tokens[0]			,     tokenizer.eos_token_id      )
													self.assertEqual(tokens[-3]			,     tokenizer.pad_token_id      )






							def snake_case_  (   self      ):
									pass






							def snake_case_  (   self      ):
									pass






							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     self.get_tokenizer()

									a_     :      Any 						=     tokenizer.tokenize("This is a test"      )
									# fmt: off
									self.assertListEqual(a_			,     [SPIECE_UNDERLINE, "T", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "a", SPIECE_UNDERLINE, "t", "e", "s", "t"]      )
									# fmt: on

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(a_      )			,     [4, 3_2, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 7, 4, 6, 5, 1_2, 6]			,     )

									a_     :      Optional[int] 						=     tokenizer.tokenize("I was born in 92000, and this is falsé."      )
									self.assertListEqual(
									    a_			,     [SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "92000", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]      )

									a_     :      Tuple 						=     tokenizer.convert_tokens_to_ids(a_      )
									# fmt: off
									self.assertListEqual(a_			,     [4, 3_0, 4, 2_0, 7, 1_2, 4, 2_5, 8, 1_3, 9, 4, 1_0, 9, 4, 3, 2_3, 4, 7, 9, 1_4, 4, 6, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 1_9, 7, 1_5, 1_2, 7_3, 2_6]      )
									# fmt: on

									a_     :      Tuple 						=     tokenizer.convert_ids_to_tokens(a_      )
									self.assertListEqual(
									    a_			,     [SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "<unk>", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]      )






							@slow
							def snake_case_  (   self      ):
									# Use custom sequence because this tokenizer does not handle numbers.
									a_     :      List[Any] 						=     [
									    "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "
									    "general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural "
									    "Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained "
									    "models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.",
									    "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "
									    "conditioning on both left and right context in all layers.",
									    "The quick brown fox jumps over the lazy dog.",
									]

									# fmt: off
									a_     :      Tuple 						=     {
									    "input_ids": [
									        [4, 3_2, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 6_4, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_5, 2_2, 4, 2_8, 9, 8, 2_0, 9, 4, 7, 1_2, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 6, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 7, 9, 1_4, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 3_9, 2_5, 5, 1_3, 6, 6_3, 4, 2_4, 1_3, 8, 2_7, 1_0, 1_4, 5, 1_2, 4, 2_1, 5, 9, 5, 1_3, 7, 1_5, 3_9, 2_4, 1_6, 1_3, 2_4, 8, 1_2, 5, 4, 7, 1_3, 1_7, 1_1, 1_0, 6, 5, 1_7, 6, 1_6, 1_3, 5, 1_2, 4, 6_4, 4_0, 4_7, 5_4, 3_2, 2_3, 4, 5_3, 4_9, 3_2, 2_3, 4, 5_4, 8, 4_0, 4_7, 5_4, 3_2, 7, 2_3, 4, 6_9, 5_2, 4_3, 2_3, 4, 5_1, 1_0, 1_2, 6, 1_0, 1_5, 4_0, 5, 1_3, 6, 2_3, 4, 6_9, 5_2, 4_8, 5, 6, 2_6, 2_6, 2_6, 6_3, 4, 1_9, 8, 1_3, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 6_1, 9, 1_4, 5, 1_3, 1_2, 6, 7, 9, 1_4, 1_0, 9, 2_1, 4, 6_4, 4_8, 5_2, 6_1, 6_3, 4, 7, 9, 1_4, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 5_3, 5, 9, 5, 1_3, 7, 6, 1_0, 8, 9, 4, 6_4, 4_8, 5_2, 5_3, 6_3, 4, 2_0, 1_0, 6, 1_1, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 1_0, 1_3, 6, 2_2, 3_9, 6, 2_0, 8, 4, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 4, 1_8, 8, 1_4, 5, 1_5, 1_2, 4, 1_0, 9, 4, 8, 9, 5, 4, 1_1, 1_6, 9, 1_4, 1_3, 5, 1_4, 4, 2_4, 1_5, 1_6, 1_2, 4, 1_5, 7, 9, 2_1, 1_6, 7, 2_1, 5, 1_2, 4, 7, 9, 1_4, 4, 1_4, 5, 5, 2_4, 4, 1_0, 9, 6, 5, 1_3, 8, 2_4, 5, 1_3, 7, 2_5, 1_0, 1_5, 1_0, 6, 2_2, 4, 2_5, 5, 6, 2_0, 5, 5, 9, 4, 5_8, 7, 3_7, 2_3, 4, 4_9, 2_2, 3_2, 8, 1_3, 1_7, 1_1, 4, 7, 9, 1_4, 4, 3_2, 5, 9, 1_2, 8, 1_3, 5_5, 1_5, 8, 2_0, 2_6, 2],
									        [4, 4_0, 4_7, 5_4, 3_2, 4, 1_0, 1_2, 4, 1_4, 5, 1_2, 1_0, 2_1, 9, 5, 1_4, 4, 6, 8, 4, 2_4, 1_3, 5, 3_9, 6, 1_3, 7, 1_0, 9, 4, 1_4, 5, 5, 2_4, 4, 2_5, 1_0, 1_4, 1_0, 1_3, 5, 1_7, 6, 1_0, 8, 9, 7, 1_5, 4, 1_3, 5, 2_4, 1_3, 5, 1_2, 5, 9, 6, 7, 6, 1_0, 8, 9, 1_2, 4, 1_9, 1_3, 8, 1_8, 4, 1_6, 9, 1_5, 7, 2_5, 5, 1_5, 5, 1_4, 4, 6, 5, 3_7, 6, 4, 2_5, 2_2, 4, 4_6, 8, 1_0, 9, 6, 1_5, 2_2, 4, 1_7, 8, 9, 1_4, 1_0, 6, 1_0, 8, 9, 1_0, 9, 2_1, 4, 8, 9, 4, 2_5, 8, 6, 1_1, 4, 1_5, 5, 1_9, 6, 4, 7, 9, 1_4, 4, 1_3, 1_0, 2_1, 1_1, 6, 4, 1_7, 8, 9, 6, 5, 3_7, 6, 4, 1_0, 9, 4, 7, 1_5, 1_5, 4, 1_5, 7, 2_2, 5, 1_3, 1_2, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
									        [4, 3_2, 1_1, 5, 4, 4_5, 1_6, 1_0, 1_7, 2_8, 4, 2_5, 1_3, 8, 2_0, 9, 4, 1_9, 8, 3_7, 4, 4_6, 1_6, 1_8, 2_4, 1_2, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 5, 4, 1_5, 7, 5_7, 2_2, 4, 1_4, 8, 2_1, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
									    ],
									    "attention_mask": [
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									    ]
									}
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=a_			,     model_name="microsoft/speecht5_asr"			,     revision="c5ef64c71905caeccde0e4462ef3f9077224c524"			,     sequences=a_			,     )