





import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from timm.data import resolve_data_config
from timm.data.transforms_factory import create_transform

from transformers import (
    BitConfig,
    ViTHybridConfig,
    ViTHybridForImageClassification,
    ViTHybridImageProcessor,
    ViTHybridModel,
)
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase_ 							=       logging.get_logger(__name__)





def   lowerCamelCase__       (       UpperCamelCase__     :    List[str]    ,      UpperCamelCase__     :    str=False  )     ->	Optional[int]:



					'''simple docstring'''




					_snake_case													=			[]

					# fmt: off
					# stem:
					rename_keys.append(('cls_token', 'vit.embeddings.cls_token')  )
					rename_keys.append(('pos_embed', 'vit.embeddings.position_embeddings')  )

					rename_keys.append(('patch_embed.proj.weight', 'vit.embeddings.patch_embeddings.projection.weight')  )
					rename_keys.append(('patch_embed.proj.bias', 'vit.embeddings.patch_embeddings.projection.bias')  )

					# backbone
					rename_keys.append(('patch_embed.backbone.stem.conv.weight', 'vit.embeddings.patch_embeddings.backbone.bit.embedder.convolution.weight')  )
					rename_keys.append(('patch_embed.backbone.stem.norm.weight', 'vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.weight')  )
					rename_keys.append(('patch_embed.backbone.stem.norm.bias', 'vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.bias')  )

					for stage_idx in range(len(config.backbone_config.depths  )  ):
										for layer_idx in range(config.backbone_config.depths[stage_idx]  ):
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv1.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv1.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.bias''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv2.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv2.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.bias''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv3.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv3.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.bias''')  )

										rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.conv.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.conv.weight''')  )
										rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.weight''')  )
										rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.bias''')  )

					# transformer encoder
					for i in range(config.num_hidden_layers  ):
										# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
										rename_keys.append((F'''blocks.{i}.norm1.weight''', F'''vit.encoder.layer.{i}.layernorm_before.weight''')  )
										rename_keys.append((F'''blocks.{i}.norm1.bias''', F'''vit.encoder.layer.{i}.layernorm_before.bias''')  )
										rename_keys.append((F'''blocks.{i}.attn.proj.weight''', F'''vit.encoder.layer.{i}.attention.output.dense.weight''')  )
										rename_keys.append((F'''blocks.{i}.attn.proj.bias''', F'''vit.encoder.layer.{i}.attention.output.dense.bias''')  )
										rename_keys.append((F'''blocks.{i}.norm2.weight''', F'''vit.encoder.layer.{i}.layernorm_after.weight''')  )
										rename_keys.append((F'''blocks.{i}.norm2.bias''', F'''vit.encoder.layer.{i}.layernorm_after.bias''')  )
										rename_keys.append((F'''blocks.{i}.mlp.fc1.weight''', F'''vit.encoder.layer.{i}.intermediate.dense.weight''')  )
										rename_keys.append((F'''blocks.{i}.mlp.fc1.bias''', F'''vit.encoder.layer.{i}.intermediate.dense.bias''')  )
										rename_keys.append((F'''blocks.{i}.mlp.fc2.weight''', F'''vit.encoder.layer.{i}.output.dense.weight''')  )
										rename_keys.append((F'''blocks.{i}.mlp.fc2.bias''', F'''vit.encoder.layer.{i}.output.dense.bias''')  )

					if base_model:
										# layernorm + pooler
										rename_keys.extend(
										    [
										        ('norm.weight', 'layernorm.weight'),
										        ('norm.bias', 'layernorm.bias'),
										        ('pre_logits.fc.weight', 'pooler.dense.weight'),
										        ('pre_logits.fc.bias', 'pooler.dense.bias'),
										    ]  )

										# if just the base model, we should remove "vit" from all keys that start with "vit"
										_snake_case													=			[(pair[0], pair[1][4:]) if pair[1].startswith('vit'  ) else pair for pair in rename_keys]
					else:
										# layernorm + classification head
										rename_keys.extend(
										    [
										        ('norm.weight', 'vit.layernorm.weight'),
										        ('norm.bias', 'vit.layernorm.bias'),
										        ('head.weight', 'classifier.weight'),
										        ('head.bias', 'classifier.bias'),
										    ]  )
					# fmt: on

					return rename_keys





def   lowerCamelCase__       (       UpperCamelCase__     :    str    ,      UpperCamelCase__     :    int    ,      UpperCamelCase__     :    int=False  )     ->	Any:



					'''simple docstring'''




					for i in range(config.num_hidden_layers  ):
										if base_model:
															_snake_case													=			''
										else:
															_snake_case													=			'vit.'
										# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
										_snake_case													=			state_dict.pop(F'''blocks.{i}.attn.qkv.weight'''  )
										_snake_case													=			state_dict.pop(F'''blocks.{i}.attn.qkv.bias'''  )
										# next, add query, keys and values (in that order) to the state dict
										_snake_case													=			in_proj_weight[
										    : config.hidden_size, :
										]
										_snake_case													=			in_proj_bias[: config.hidden_size]
										_snake_case													=			in_proj_weight[
										    config.hidden_size : config.hidden_size * 2, :
										]
										_snake_case													=			in_proj_bias[
										    config.hidden_size : config.hidden_size * 2
										]
										_snake_case													=			in_proj_weight[
										    -config.hidden_size :, :
										]
										_snake_case													=			in_proj_bias[-config.hidden_size :]





def   lowerCamelCase__       (       UpperCamelCase__     :    Any  )     ->	Dict:



					'''simple docstring'''




					_snake_case													=			['head.weight', 'head.bias']
					for k in ignore_keys:
										state_dict.pop(UpperCamelCase__    ,      UpperCamelCase__  )





def   lowerCamelCase__       (       UpperCamelCase__     :    Dict    ,      UpperCamelCase__     :    int    ,      UpperCamelCase__     :    Optional[int]  )     ->	List[str]:



					'''simple docstring'''




					_snake_case													=			dct.pop(UpperCamelCase__  )
					_snake_case													=			val





def   lowerCamelCase__       (       )     ->	Tuple:



					'''simple docstring'''




					_snake_case													=			'http://images.cocodataset.org/val2017/000000039769.jpg'
					_snake_case													=			Image.open(requests.get(UpperCamelCase__    ,      stream=UpperCamelCase__  ).raw  )
					return im





@torch.no_grad()
def   lowerCamelCase__       (       UpperCamelCase__     :    Dict    ,      UpperCamelCase__     :    List[str]    ,      UpperCamelCase__     :    List[Any]=False  )     ->	Union[str, Any]:



					'''simple docstring'''




					_snake_case													=			BitConfig(
					    global_padding='same'    ,      layer_type='bottleneck'    ,      depths=(3, 4, 9)    ,      out_features=['stage3']    ,      embedding_dynamic_padding=UpperCamelCase__    ,      )
					_snake_case													=			ViTHybridConfig(backbone_config=UpperCamelCase__    ,      image_size=384    ,      num_labels=1_000  )
					_snake_case													=			False

					# load original model from timm
					_snake_case													=			timm.create_model(UpperCamelCase__    ,      pretrained=UpperCamelCase__  )
					timm_model.eval()

					# load state_dict of original model, remove and rename some keys
					_snake_case													=			timm_model.state_dict()
					if base_model:
										remove_classification_head_(UpperCamelCase__  )
					_snake_case													=			create_rename_keys(UpperCamelCase__    ,      UpperCamelCase__  )
					for src, dest in rename_keys:
										rename_key(UpperCamelCase__    ,      UpperCamelCase__    ,      UpperCamelCase__  )
					read_in_q_k_v(UpperCamelCase__    ,      UpperCamelCase__    ,      UpperCamelCase__  )

					_snake_case													=			'huggingface/label-files'
					_snake_case													=			'imagenet-1k-id2label.json'
					_snake_case													=			json.load(open(hf_hub_download(UpperCamelCase__    ,      UpperCamelCase__    ,      repo_type='dataset'  )    ,      'r'  )  )
					_snake_case													=			{int(UpperCamelCase__  ): v for k, v in idalabel.items()}
					_snake_case													=			idalabel
					_snake_case													=			{v: k for k, v in idalabel.items()}

					# load HuggingFace model
					if vit_name[-5:] == "in21k":
										_snake_case													=			ViTHybridModel(UpperCamelCase__  ).eval()
					else:
										_snake_case													=			ViTHybridForImageClassification(UpperCamelCase__  ).eval()
					model.load_state_dict(UpperCamelCase__  )

					# create image processor
					_snake_case													=			create_transform(**resolve_data_config({}    ,      model=UpperCamelCase__  )  )
					_snake_case													=			transform.transforms

					_snake_case													=			{
					    'bilinear': PILImageResampling.BILINEAR,
					    'bicubic': PILImageResampling.BICUBIC,
					    'nearest': PILImageResampling.NEAREST,
					}

					_snake_case													=			ViTHybridImageProcessor(
					    do_resize=UpperCamelCase__    ,      size={'shortest_edge': timm_transforms[0].size}    ,      resample=pillow_resamplings[timm_transforms[0].interpolation.value]    ,      do_center_crop=UpperCamelCase__    ,      crop_size={'height': timm_transforms[1].size[0], 'width': timm_transforms[1].size[1]}    ,      do_normalize=UpperCamelCase__    ,      image_mean=timm_transforms[-1].mean.tolist()    ,      image_std=timm_transforms[-1].std.tolist()    ,      )

					_snake_case													=			prepare_img()
					_snake_case													=			transform(UpperCamelCase__  ).unsqueeze(0  )
					_snake_case													=			processor(UpperCamelCase__    ,      return_tensors='pt'  ).pixel_values

					# verify pixel values
					assert torch.allclose(UpperCamelCase__    ,      UpperCamelCase__  )

					# verify logits
					with torch.no_grad():
										_snake_case													=			model(UpperCamelCase__  )
										_snake_case													=			outputs.logits

					print('Predicted class:'    ,      logits.argmax(-1  ).item()  )
					if base_model:
										_snake_case													=			timm_model.forward_features(UpperCamelCase__  )
										assert timm_pooled_output.shape == outputs.pooler_output.shape
										assert torch.allclose(UpperCamelCase__    ,      outputs.pooler_output    ,      atol=1e-3  )
					else:
										_snake_case													=			timm_model(UpperCamelCase__  )
										assert timm_logits.shape == outputs.logits.shape
										assert torch.allclose(UpperCamelCase__    ,      outputs.logits    ,      atol=1e-3  )
					print('Looks ok!'  )

					if pytorch_dump_folder_path is not None:
										Path(UpperCamelCase__  ).mkdir(exist_ok=UpperCamelCase__  )
										print(F'''Saving model {vit_name} to {pytorch_dump_folder_path}'''  )
										model.save_pretrained(UpperCamelCase__  )
										print(F'''Saving processor to {pytorch_dump_folder_path}'''  )
										processor.save_pretrained(UpperCamelCase__  )

					if push_to_hub:
										print(F'''Pushing model and processor to the hub {vit_name}'''  )
										model.push_to_hub(F'''ybelkada/{vit_name}'''  )
										processor.push_to_hub(F'''ybelkada/{vit_name}'''  )


if __name__ == "__main__":
					UpperCAmelCase_ 							=       argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--vit_name""",
					    default="""vit_base_r50_s16_384""",
					    type=str,
					    help="""Name of the hybrid ViT timm model you'd like to convert.""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
					)
					parser.add_argument(
					    """--push_to_hub""", action="""store_true""", help="""Whether to upload the model to the HuggingFace hub."""
					)

					UpperCAmelCase_ 							=       parser.parse_args()
					convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path, args.push_to_hub)










import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      UpperCamelCase_						(					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		BertTokenizer
					lowerCAmelCase_							=		BertTokenizerFast
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		filter_non_english




					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										super().setUp()

										_snake_case													=			[
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['vocab_file']					)
										with open(self.vocab_file			,  'w'			,  encoding='utf-8'					) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Optional[Any]:
										_snake_case													=			'UNwant\u00E9d,running'
										_snake_case													=			'unwanted, running'
										return input_text, output_text




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			self.tokenizer_class(self.vocab_file					)

										_snake_case													=			tokenizer.tokenize('UNwant\u00E9d,running'					)
										self.assertListEqual(lowerCAmelCase_			,  ['un', '##want', '##ed', ',', 'runn', '##ing']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_					)			,  [9, 6, 7, 12, 10, 11]					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										if not self.test_rust_tokenizer:
															return

										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										# With lower casing
										_snake_case													=			self.get_tokenizer(do_lower_case=lowerCAmelCase_					)
										_snake_case													=			self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_					)

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)			,  ['ah', '\u535A', '\u63A8', 'zz']					)




					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['hello', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hällo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['h\u00E9llo']					)




					def 		lowerCAmelCase     (						self					)						->							Any:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  never_split=['[UNK]']					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer()
										_snake_case													=			'a\n\'ll !!to?\'d of, can\'t.'
										_snake_case													=			['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
										self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_					)			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										_snake_case													=			{}
										for i, token in enumerate(lowerCAmelCase_					):
															_snake_case													=			i
										_snake_case													=			WordpieceTokenizer(vocab=lowerCAmelCase_			,  unk_token='[UNK]'					)

										self.assertListEqual(tokenizer.tokenize(''					)			,  []					)

										self.assertListEqual(tokenizer.tokenize('unwanted running'					)			,  ['un', '##want', '##ed', 'runn', '##ing']					)

										self.assertListEqual(tokenizer.tokenize('unwantedX running'					)			,  ['[UNK]', 'runn', '##ing']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										self.assertTrue(_is_whitespace(' '					)					)
										self.assertTrue(_is_whitespace('\t'					)					)
										self.assertTrue(_is_whitespace('\r'					)					)
										self.assertTrue(_is_whitespace('\n'					)					)
										self.assertTrue(_is_whitespace('\u00A0'					)					)

										self.assertFalse(_is_whitespace('A'					)					)
										self.assertFalse(_is_whitespace('-'					)					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										self.assertTrue(_is_control('\u0005'					)					)

										self.assertFalse(_is_control('A'					)					)
										self.assertFalse(_is_control(' '					)					)
										self.assertFalse(_is_control('\t'					)					)
										self.assertFalse(_is_control('\r'					)					)




					def 		lowerCAmelCase     (						self					)						->							int:
										self.assertTrue(_is_punctuation('-'					)					)
										self.assertTrue(_is_punctuation('$'					)					)
										self.assertTrue(_is_punctuation('`'					)					)
										self.assertTrue(_is_punctuation('.'					)					)

										self.assertFalse(_is_punctuation('A'					)					)
										self.assertFalse(_is_punctuation(' '					)					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)

										self.assertListEqual(
										    [rust_tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)




					@slow
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			self.tokenizer_class.from_pretrained('bert-base-uncased'					)

										_snake_case													=			tokenizer.encode('sequence builders'			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			tokenizer.encode('multi-sequence build'			,  add_special_tokens=lowerCAmelCase_					)

										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					)
										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_			,  lowerCAmelCase_					)

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																				_snake_case													=			tokenizer_r.encode_plus(
																				    lowerCAmelCase_			,  return_attention_mask=lowerCAmelCase_			,  return_token_type_ids=lowerCAmelCase_			,  return_offsets_mapping=lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_			,  )

																				_snake_case													=			tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_			,  'do_lower_case'					) else False
																				_snake_case													=			(
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'A'),
																				        ((1, 2), ','),
																				        ((3, 5), 'na'),
																				        ((5, 6), '##ï'),
																				        ((6, 8), '##ve'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'Allen'),
																				        ((21, 23), '##NL'),
																				        ((23, 24), '##P'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'a'),
																				        ((1, 2), ','),
																				        ((3, 8), 'naive'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'allen'),
																				        ((21, 23), '##nl'),
																				        ((23, 24), '##p'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]			,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
																				self.assertEqual([e[0] for e in expected_results]			,  tokens['offset_mapping']					)






					def 		lowerCAmelCase     (						self					)						->							str:
										_snake_case													=			['的', '人', '有']
										_snake_case													=			''.join(lowerCAmelCase_					)
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			True
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that each Chinese character is not preceded by "##"
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

																				_snake_case													=			False
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that only the first Chinese character is not preceded by "##".
																				_snake_case													=			[
																				    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_					)
																				]
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)



