






import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    MobileViTConfig,
    MobileViTForImageClassification,
    MobileViTForSemanticSegmentation,
    MobileViTImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
lowercase  :							Optional[Any]			  =    logging.get_logger(__name__)

def      lowerCAmelCase__	(    _a       :			List[str]    ):
		snake_case_				:							str						 =       MobileViTConfig()

		# size of the architecture
		if "mobilevit_s" in mobilevit_name:
				snake_case_				:							int						 =       [1_44, 1_92, 2_40]
				snake_case_				:							str						 =       [16, 32, 64, 96, 1_28, 1_60, 6_40]
		elif "mobilevit_xs" in mobilevit_name:
				snake_case_				:							Tuple						 =       [96, 1_20, 1_44]
				snake_case_				:							Optional[int]						 =       [16, 32, 48, 64, 80, 96, 3_84]
		elif "mobilevit_xxs" in mobilevit_name:
				snake_case_				:							Tuple						 =       [64, 80, 96]
				snake_case_				:							Optional[Any]						 =       [16, 16, 24, 48, 64, 80, 3_20]
				snake_case_				:							Optional[int]						 =       0.05
				snake_case_				:							Any						 =       2.0

		if mobilevit_name.startswith("deeplabv3_"    ):
				snake_case_				:							Any						 =       5_12
				snake_case_				:							List[str]						 =       16
				snake_case_				:							List[str]						 =       21
				snake_case_				:							Dict						 =       "pascal-voc-id2label.json"
		else:
				snake_case_				:							int						 =       10_00
				snake_case_				:							Union[str, Any]						 =       "imagenet-1k-id2label.json"

		snake_case_				:							List[Any]						 =       "huggingface/label-files"
		snake_case_				:							Union[str, Any]						 =       json.load(open(hf_hub_download(_a  ,      _a  ,      repo_type="dataset"    )  ,      "r"    )    )
		snake_case_				:							Any						 =       {int(_a    ): v for k, v in idalabel.items()}
		snake_case_				:							str						 =       idalabel
		snake_case_				:							List[Any]						 =       {v: k for k, v in idalabel.items()}

		return config

def      lowerCAmelCase__	(    _a       :			Any  ,      _a       :			Union[str, Any]=False    ):
		for i in range(1  ,      6    ):
				if F'''layer_{i}.''' in name:
						snake_case_				:							Any						 =       name.replace(F'''layer_{i}.'''  ,      F'''encoder.layer.{i - 1}.'''    )

		if "conv_1." in name:
				snake_case_				:							List[str]						 =       name.replace("conv_1."  ,      "conv_stem."    )
		if ".block." in name:
				snake_case_				:							str						 =       name.replace(".block."  ,      "."    )

		if "exp_1x1" in name:
				snake_case_				:							Any						 =       name.replace("exp_1x1"  ,      "expand_1x1"    )
		if "red_1x1" in name:
				snake_case_				:							str						 =       name.replace("red_1x1"  ,      "reduce_1x1"    )
		if ".local_rep.conv_3x3." in name:
				snake_case_				:							List[Any]						 =       name.replace(".local_rep.conv_3x3."  ,      ".conv_kxk."    )
		if ".local_rep.conv_1x1." in name:
				snake_case_				:							Optional[Any]						 =       name.replace(".local_rep.conv_1x1."  ,      ".conv_1x1."    )
		if ".norm." in name:
				snake_case_				:							Union[str, Any]						 =       name.replace(".norm."  ,      ".normalization."    )
		if ".conv." in name:
				snake_case_				:							List[str]						 =       name.replace(".conv."  ,      ".convolution."    )
		if ".conv_proj." in name:
				snake_case_				:							Optional[int]						 =       name.replace(".conv_proj."  ,      ".conv_projection."    )

		for i in range(0  ,      2    ):
				for j in range(0  ,      4    ):
						if F'''.{i}.{j}.''' in name:
								snake_case_				:							Tuple						 =       name.replace(F'''.{i}.{j}.'''  ,      F'''.{i}.layer.{j}.'''    )

		for i in range(2  ,      6    ):
				for j in range(0  ,      4    ):
						if F'''.{i}.{j}.''' in name:
								snake_case_				:							Tuple						 =       name.replace(F'''.{i}.{j}.'''  ,      F'''.{i}.'''    )
								if "expand_1x1" in name:
										snake_case_				:							Dict						 =       name.replace("expand_1x1"  ,      "downsampling_layer.expand_1x1"    )
								if "conv_3x3" in name:
										snake_case_				:							Optional[Any]						 =       name.replace("conv_3x3"  ,      "downsampling_layer.conv_3x3"    )
								if "reduce_1x1" in name:
										snake_case_				:							Optional[Any]						 =       name.replace("reduce_1x1"  ,      "downsampling_layer.reduce_1x1"    )

		for i in range(2  ,      5    ):
				if F'''.global_rep.{i}.weight''' in name:
						snake_case_				:							List[Any]						 =       name.replace(F'''.global_rep.{i}.weight'''  ,      ".layernorm.weight"    )
				if F'''.global_rep.{i}.bias''' in name:
						snake_case_				:							Optional[Any]						 =       name.replace(F'''.global_rep.{i}.bias'''  ,      ".layernorm.bias"    )

		if ".global_rep." in name:
				snake_case_				:							int						 =       name.replace(".global_rep."  ,      ".transformer."    )
		if ".pre_norm_mha.0." in name:
				snake_case_				:							Any						 =       name.replace(".pre_norm_mha.0."  ,      ".layernorm_before."    )
		if ".pre_norm_mha.1.out_proj." in name:
				snake_case_				:							str						 =       name.replace(".pre_norm_mha.1.out_proj."  ,      ".attention.output.dense."    )
		if ".pre_norm_ffn.0." in name:
				snake_case_				:							Optional[int]						 =       name.replace(".pre_norm_ffn.0."  ,      ".layernorm_after."    )
		if ".pre_norm_ffn.1." in name:
				snake_case_				:							Tuple						 =       name.replace(".pre_norm_ffn.1."  ,      ".intermediate.dense."    )
		if ".pre_norm_ffn.4." in name:
				snake_case_				:							Optional[int]						 =       name.replace(".pre_norm_ffn.4."  ,      ".output.dense."    )
		if ".transformer." in name:
				snake_case_				:							Optional[int]						 =       name.replace(".transformer."  ,      ".transformer.layer."    )

		if ".aspp_layer." in name:
				snake_case_				:							Any						 =       name.replace(".aspp_layer."  ,      "."    )
		if ".aspp_pool." in name:
				snake_case_				:							Optional[int]						 =       name.replace(".aspp_pool."  ,      "."    )
		if "seg_head." in name:
				snake_case_				:							Tuple						 =       name.replace("seg_head."  ,      "segmentation_head."    )
		if "segmentation_head.classifier.classifier." in name:
				snake_case_				:							Tuple						 =       name.replace("segmentation_head.classifier.classifier."  ,      "segmentation_head.classifier."    )

		if "classifier.fc." in name:
				snake_case_				:							Dict						 =       name.replace("classifier.fc."  ,      "classifier."    )
		elif (not base_model) and ("segmentation_head." not in name):
				snake_case_				:							List[str]						 =       "mobilevit." + name

		return name

def      lowerCAmelCase__	(    _a       :			int  ,      _a       :			Optional[int]  ,      _a       :			Optional[Any]=False    ):
		if base_model:
				snake_case_				:							Optional[Any]						 =       ""
		else:
				snake_case_				:							Tuple						 =       "mobilevit."

		for key in orig_state_dict.copy().keys():
				snake_case_				:							int						 =       orig_state_dict.pop(_a    )

				if key[:8] == "encoder.":
						snake_case_				:							Union[str, Any]						 =       key[8:]

				if "qkv" in key:
						snake_case_				:							Optional[int]						 =       key.split("."    )
						snake_case_				:							Any						 =       int(key_split[0][6:]    ) - 1
						snake_case_				:							Dict						 =       int(key_split[3]    )
						snake_case_				:							Optional[int]						 =       model.get_submodule(F'''{model_prefix}encoder.layer.{layer_num}'''    )
						snake_case_				:							int						 =       layer.transformer.layer[transformer_num].attention.attention.all_head_size
						snake_case_				:							Any						 =       (
						    F'''{model_prefix}encoder.layer.{layer_num}.transformer.layer.{transformer_num}.attention.attention.'''
						)
						if "weight" in key:
								snake_case_				:							str						 =       val[:dim, :]
								snake_case_				:							List[str]						 =       val[dim : dim * 2, :]
								snake_case_				:							Union[str, Any]						 =       val[-dim:, :]
						else:
								snake_case_				:							Optional[int]						 =       val[:dim]
								snake_case_				:							str						 =       val[dim : dim * 2]
								snake_case_				:							Union[str, Any]						 =       val[-dim:]
				else:
						snake_case_				:							Dict						 =       val

		return orig_state_dict

def      lowerCAmelCase__	(    ):
		snake_case_				:							List[str]						 =       "http://images.cocodataset.org/val2017/000000039769.jpg"
		snake_case_				:							Optional[int]						 =       Image.open(requests.get(_a  ,      stream=_a    ).raw    )
		return im

@torch.no_grad()
def      lowerCAmelCase__	(    _a       :			List[Any]  ,      _a       :			List[Any]  ,      _a       :			Union[str, Any]  ,      _a       :			Any=False    ):
		snake_case_				:							Dict						 =       get_mobilevit_config(_a    )

		# load original state_dict
		snake_case_				:							Any						 =       torch.load(_a  ,      map_location="cpu"    )

		# load 🤗 model
		if mobilevit_name.startswith("deeplabv3_"    ):
				snake_case_				:							List[Any]						 =       MobileViTForSemanticSegmentation(_a    ).eval()
		else:
				snake_case_				:							List[str]						 =       MobileViTForImageClassification(_a    ).eval()

		snake_case_				:							int						 =       convert_state_dict(_a  ,      _a    )
		model.load_state_dict(_a    )

		# Check outputs on an image, prepared by MobileViTImageProcessor
		snake_case_				:							Optional[int]						 =       MobileViTImageProcessor(crop_size=config.image_size  ,      size=config.image_size + 32    )
		snake_case_				:							Union[str, Any]						 =       image_processor(images=prepare_img()  ,      return_tensors="pt"    )
		snake_case_				:							Tuple						 =       model(**_a    )
		snake_case_				:							List[Any]						 =       outputs.logits

		if mobilevit_name.startswith("deeplabv3_"    ):
				assert logits.shape == (1, 21, 32, 32)

				if mobilevit_name == "deeplabv3_mobilevit_s":
						snake_case_				:							Dict						 =       torch.tensor(
						    [
						        [[6.2_065, 6.1_292, 6.2_070], [6.1_079, 6.1_254, 6.1_747], [6.0_042, 6.1_071, 6.1_034]],
						        [[-6.9_253, -6.8_653, -7.0_398], [-7.3_218, -7.3_983, -7.3_670], [-7.1_961, -7.2_482, -7.1_569]],
						        [[-4.4_723, -4.4_348, -4.3_769], [-5.3_629, -5.4_632, -5.4_598], [-5.1_587, -5.3_402, -5.5_059]],
						    ]    )
				elif mobilevit_name == "deeplabv3_mobilevit_xs":
						snake_case_				:							int						 =       torch.tensor(
						    [
						        [[5.4_449, 5.5_733, 5.6_314], [5.1_815, 5.3_930, 5.5_963], [5.1_656, 5.4_333, 5.4_853]],
						        [[-9.4_423, -9.7_766, -9.6_714], [-9.1_581, -9.5_720, -9.5_519], [-9.1_006, -9.6_458, -9.5_703]],
						        [[-7.7_721, -7.3_716, -7.1_583], [-8.4_599, -8.0_624, -7.7_944], [-8.4_172, -7.8_366, -7.5_025]],
						    ]    )
				elif mobilevit_name == "deeplabv3_mobilevit_xxs":
						snake_case_				:							int						 =       torch.tensor(
						    [
						        [[6.9_811, 6.9_743, 7.3_123], [7.1_777, 7.1_931, 7.3_938], [7.5_633, 7.8_050, 7.8_901]],
						        [[-10.5_536, -10.2_332, -10.2_924], [-10.2_336, -9.8_624, -9.5_964], [-10.8_840, -10.8_158, -10.6_659]],
						        [[-3.4_938, -3.0_631, -2.8_620], [-3.4_205, -2.8_135, -2.6_875], [-3.4_179, -2.7_945, -2.8_750]],
						    ]    )
				else:
						raise ValueError(F'''Unknown mobilevit_name: {mobilevit_name}'''    )

				assert torch.allclose(logits[0, :3, :3, :3]  ,      _a  ,      atol=1E-4    )
		else:
				assert logits.shape == (1, 10_00)

				if mobilevit_name == "mobilevit_s":
						snake_case_				:							Tuple						 =       torch.tensor([-0.9_866, 0.2_392, -1.1_241]    )
				elif mobilevit_name == "mobilevit_xs":
						snake_case_				:							Optional[Any]						 =       torch.tensor([-2.4_761, -0.9_399, -1.9_587]    )
				elif mobilevit_name == "mobilevit_xxs":
						snake_case_				:							int						 =       torch.tensor([-1.9_364, -1.2_327, -0.4_653]    )
				else:
						raise ValueError(F'''Unknown mobilevit_name: {mobilevit_name}'''    )

				assert torch.allclose(logits[0, :3]  ,      _a  ,      atol=1E-4    )

		Path(_a    ).mkdir(exist_ok=_a    )
		print(F'''Saving model {mobilevit_name} to {pytorch_dump_folder_path}'''    )
		model.save_pretrained(_a    )
		print(F'''Saving image processor to {pytorch_dump_folder_path}'''    )
		image_processor.save_pretrained(_a    )

		if push_to_hub:
				snake_case_				:							Union[str, Any]						 =       {
				    "mobilevit_s": "mobilevit-small",
				    "mobilevit_xs": "mobilevit-x-small",
				    "mobilevit_xxs": "mobilevit-xx-small",
				    "deeplabv3_mobilevit_s": "deeplabv3-mobilevit-small",
				    "deeplabv3_mobilevit_xs": "deeplabv3-mobilevit-x-small",
				    "deeplabv3_mobilevit_xxs": "deeplabv3-mobilevit-xx-small",
				}

				print("Pushing to the hub..."    )
				snake_case_				:							Optional[int]						 =       model_mapping[mobilevit_name]
				image_processor.push_to_hub(_a  ,      organization="apple"    )
				model.push_to_hub(_a  ,      organization="apple"    )


if __name__ == "__main__":
							lowercase  :							Union[str, Any]			  =    argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '''--mobilevit_name''',
							    default='''mobilevit_s''',
							    type=str,
							    help=(
							        '''Name of the MobileViT model you\'d like to convert. Should be one of \'mobilevit_s\', \'mobilevit_xs\','''
							        ''' \'mobilevit_xxs\', \'deeplabv3_mobilevit_s\', \'deeplabv3_mobilevit_xs\', \'deeplabv3_mobilevit_xxs\'.'''
							    ),
							)
							parser.add_argument(
							    '''--checkpoint_path''', required=True, type=str, help='''Path to the original state dict (.pt file).'''
							)
							parser.add_argument(
							    '''--pytorch_dump_folder_path''', required=True, type=str, help='''Path to the output PyTorch model directory.'''
							)
							parser.add_argument(
							    '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
							)

							lowercase  :							Any			  =    parser.parse_args()
							convert_movilevit_checkpoint(
							    args.mobilevit_name, args.checkpoint_path, args.pytorch_dump_folder_path, args.push_to_hub
							)








import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					,  unittest.TestCase					):


						'''simple docstring'''




						A     :   str						=					BertTokenizer
						A     :   Union[str, Any]						=					BertTokenizerFast
						A     :   int						=					True
						A     :   Union[str, Any]						=					True
						A     :   Union[str, Any]						=					filter_non_english






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								super().setUp()

								snake_case_				:							Optional[int]						 =       [
								    "[UNK]",
								    "[CLS]",
								    "[SEP]",
								    "[PAD]",
								    "[MASK]",
								    "want",
								    "##want",
								    "##ed",
								    "wa",
								    "un",
								    "runn",
								    "##ing",
								    ",",
								    "low",
								    "lowest",
								]
								snake_case_				:							str						 =       os.path.join(self.tmpdirname		,				VOCAB_FILES_NAMES["vocab_file"]				)
								with open(self.vocab_file		,				"w"		,				encoding="utf-8"				) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				Tuple:
								snake_case_				:							str						 =       "UNwant\u00E9d,running"
								snake_case_				:							Optional[int]						 =       "unwanted, running"
								return input_text, output_text






						def       _lowerCAmelCase       (	self				) ->				str:
								snake_case_				:							str						 =       self.tokenizer_class(self.vocab_file				)

								snake_case_				:							Tuple						 =       tokenizer.tokenize("UNwant\u00E9d,running"				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				["un", "##want", "##ed", ",", "runn", "##ing"]				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				)		,				[9, 6, 7, 12, 10, 11]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:
								if not self.test_rust_tokenizer:
										return

								snake_case_				:							Any						 =       self.get_tokenizer()
								snake_case_				:							str						 =       self.get_rust_tokenizer()

								snake_case_				:							List[Any]						 =       "UNwant\u00E9d,running"

								snake_case_				:							Union[str, Any]						 =       tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							str						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							int						 =       self.get_rust_tokenizer()
								snake_case_				:							Any						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								# With lower casing
								snake_case_				:							Tuple						 =       self.get_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       self.get_rust_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Tuple						 =       "UNwant\u00E9d,running"

								snake_case_				:							Tuple						 =       tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							List[Any]						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[int]						 =       self.get_rust_tokenizer()
								snake_case_				:							Dict						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				List[Any]:
								snake_case_				:							Union[str, Any]						 =       BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)		,				["ah", "\u535A", "\u63A8", "zz"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							List[Any]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)		,				["hello", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							List[str]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hällo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["h\u00E9llo"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							Optional[Any]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hallo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							Dict						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hallo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							List[str]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)		,				["HeLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				int:
								snake_case_				:							str						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["HäLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							str						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["HaLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				Union[str, Any]:
								snake_case_				:							Any						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				never_split=["[UNK]"]				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)		,				["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:
								snake_case_				:							Optional[Any]						 =       BasicTokenizer()
								snake_case_				:							Optional[Any]						 =       "a\n'll !!to?'d of, can't."
								snake_case_				:							Tuple						 =       ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
								self.assertListEqual(tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							List[Any]						 =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

								snake_case_				:							Optional[int]						 =       {}
								for i, token in enumerate(_SCREAMING_SNAKE_CASE				):
										snake_case_				:							Any						 =       i
								snake_case_				:							Any						 =       WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE		,				unk_token="[UNK]"				)

								self.assertListEqual(tokenizer.tokenize(""				)		,				[]				)

								self.assertListEqual(tokenizer.tokenize("unwanted running"				)		,				["un", "##want", "##ed", "runn", "##ing"]				)

								self.assertListEqual(tokenizer.tokenize("unwantedX running"				)		,				["[UNK]", "runn", "##ing"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								self.assertTrue(_is_whitespace(" "				)				)
								self.assertTrue(_is_whitespace("\t"				)				)
								self.assertTrue(_is_whitespace("\r"				)				)
								self.assertTrue(_is_whitespace("\n"				)				)
								self.assertTrue(_is_whitespace("\u00A0"				)				)

								self.assertFalse(_is_whitespace("A"				)				)
								self.assertFalse(_is_whitespace("-"				)				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								self.assertTrue(_is_control("\u0005"				)				)

								self.assertFalse(_is_control("A"				)				)
								self.assertFalse(_is_control(" "				)				)
								self.assertFalse(_is_control("\t"				)				)
								self.assertFalse(_is_control("\r"				)				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								self.assertTrue(_is_punctuation("-"				)				)
								self.assertTrue(_is_punctuation("$"				)				)
								self.assertTrue(_is_punctuation("`"				)				)
								self.assertTrue(_is_punctuation("."				)				)

								self.assertFalse(_is_punctuation("A"				)				)
								self.assertFalse(_is_punctuation(" "				)				)






						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Dict						 =       self.get_tokenizer()
								snake_case_				:							Any						 =       self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE				) for t in ["Test", "\xad", "test"]]		,				[["[UNK]"], [], ["[UNK]"]]				)

								self.assertListEqual(
								    [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				) for t in ["Test", "\xad", "test"]]		,				[["[UNK]"], [], ["[UNK]"]]				)






						@slow
						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Union[str, Any]						 =       self.tokenizer_class.from_pretrained("bert-base-uncased"				)

								snake_case_				:							Tuple						 =       tokenizer.encode("sequence builders"		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							List[Any]						 =       tokenizer.encode("multi-sequence build"		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[Any]						 =       tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Union[str, Any]						 =       tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]






						def       _lowerCAmelCase       (	self				) ->				int:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
												snake_case_				:							Union[str, Any]						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							List[Any]						 =       f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
												snake_case_				:							List[Any]						 =       tokenizer_r.encode_plus(
												    _SCREAMING_SNAKE_CASE		,				return_attention_mask=_SCREAMING_SNAKE_CASE		,				return_token_type_ids=_SCREAMING_SNAKE_CASE		,				return_offsets_mapping=_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE		,				)

												snake_case_				:							Optional[Any]						 =       tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE		,				"do_lower_case"				) else False
												snake_case_				:							Union[str, Any]						 =       (
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), "A"),
												        ((1, 2), ","),
												        ((3, 5), "na"),
												        ((5, 6), "##ï"),
												        ((6, 8), "##ve"),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), "Allen"),
												        ((21, 23), "##NL"),
												        ((23, 24), "##P"),
												        ((25, 33), "sentence"),
												        ((33, 34), "."),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), "a"),
												        ((1, 2), ","),
												        ((3, 8), "naive"),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), "allen"),
												        ((21, 23), "##nl"),
												        ((23, 24), "##p"),
												        ((25, 33), "sentence"),
												        ((33, 34), "."),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]		,				tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
												self.assertEqual([e[0] for e in expected_results]		,				tokens["offset_mapping"]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Union[str, Any]						 =       ["的", "人", "有"]
								snake_case_				:							int						 =       "".join(_SCREAMING_SNAKE_CASE				)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
												snake_case_				:							int						 =       True
												snake_case_				:							Union[str, Any]						 =       self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)
												snake_case_				:							int						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							List[str]						 =       tokenizer_p.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Union[str, Any]						 =       tokenizer_r.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Tuple						 =       tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
												snake_case_				:							int						 =       tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Optional[int]						 =       False
												snake_case_				:							Dict						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)
												snake_case_				:							List[Any]						 =       self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							str						 =       tokenizer_r.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Any						 =       tokenizer_p.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Any						 =       tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Any						 =       tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)

												# it is expected that only the first Chinese character is not preceded by "##".
												snake_case_				:							int						 =       [
												    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_SCREAMING_SNAKE_CASE				)
												]
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
