

import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import DetrConfig, DetrForObjectDetection, DetrForSegmentation, DetrImageProcessor, ResNetConfig
from transformers.utils import logging


logging.set_verbosity_info()
A_			:					Tuple    			=     logging.get_logger(__name__)

def 		UpperCamelCase   (lowercase_:					Union[str, Any]   )	-> str:
	# initialize config
	if "resnet-50" in model_name:
		A__				:       Any	  =						ResNetConfig.from_pretrained("""microsoft/resnet-50"""   )
	elif "resnet-101" in model_name:
		A__				:       Union[str, Any]	  =						ResNetConfig.from_pretrained("""microsoft/resnet-101"""   )
	else:
		raise ValueError("""Model name should include either resnet50 or resnet101"""   )

	A__				:       Dict	  =						DetrConfig(use_timm_backbone=lowercase_       , backbone_config=lowercase_   )

	# set label attributes
	A__				:       List[str]	  =						"""panoptic""" in model_name
	if is_panoptic:
		A__				:       int	  =						250
	else:
		A__				:       Tuple	  =						91
		A__				:       str	  =						"""huggingface/label-files"""
		A__				:       int	  =						"""coco-detection-id2label.json"""
		A__				:       Optional[int]	  =						json.load(open(hf_hub_download(lowercase_       , lowercase_       , repo_type="""dataset"""   )       , """r"""   )   )
		A__				:       Union[str, Any]	  =						{int(lowercase_   ): v for k, v in idalabel.items()}
		A__				:       Optional[Any]	  =						idalabel
		A__				:       Union[str, Any]	  =						{v: k for k, v in idalabel.items()}

	return config, is_panoptic

def 		UpperCamelCase   (lowercase_:					List[str]   )	-> Union[str, Any]:
	# here we list all keys to be renamed (original name on the left, our name on the right)
	A__				:       Optional[int]	  =						[]

	# stem
	# fmt: off
	rename_keys.append(("""backbone.0.body.conv1.weight""", """backbone.conv_encoder.model.embedder.embedder.convolution.weight""")   )
	rename_keys.append(("""backbone.0.body.bn1.weight""", """backbone.conv_encoder.model.embedder.embedder.normalization.weight""")   )
	rename_keys.append(("""backbone.0.body.bn1.bias""", """backbone.conv_encoder.model.embedder.embedder.normalization.bias""")   )
	rename_keys.append(("""backbone.0.body.bn1.running_mean""", """backbone.conv_encoder.model.embedder.embedder.normalization.running_mean""")   )
	rename_keys.append(("""backbone.0.body.bn1.running_var""", """backbone.conv_encoder.model.embedder.embedder.normalization.running_var""")   )
	# stages
	for stage_idx in range(len(config.backbone_config.depths   )   ):
		for layer_idx in range(config.backbone_config.depths[stage_idx]   ):
			# shortcut
			if layer_idx == 0:
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.0.weight""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.convolution.weight""",
				    )   )
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.weight""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.weight""",
				    )   )
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.bias""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.bias""",
				    )   )
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_mean""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_mean""",
				    )   )
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_var""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_var""",
				    )   )
			# 3 convs
			for i in range(3   ):
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.conv{i+1}.weight""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.convolution.weight""",
				    )   )
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.weight""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.weight""",
				    )   )
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.bias""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.bias""",
				    )   )
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_mean""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_mean""",
				    )   )
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_var""",
				        f"""backbone.conv_encoder.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_var""",
				    )   )
    # fmt: on

	for i in range(config.encoder_layers   ):
		# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
		rename_keys.append(
		    (
		        f"""transformer.encoder.layers.{i}.self_attn.out_proj.weight""",
		        f"""encoder.layers.{i}.self_attn.out_proj.weight""",
		    )   )
		rename_keys.append(
		    (f"""transformer.encoder.layers.{i}.self_attn.out_proj.bias""", f"""encoder.layers.{i}.self_attn.out_proj.bias""")   )
		rename_keys.append((f"""transformer.encoder.layers.{i}.linear1.weight""", f"""encoder.layers.{i}.fc1.weight""")   )
		rename_keys.append((f"""transformer.encoder.layers.{i}.linear1.bias""", f"""encoder.layers.{i}.fc1.bias""")   )
		rename_keys.append((f"""transformer.encoder.layers.{i}.linear2.weight""", f"""encoder.layers.{i}.fc2.weight""")   )
		rename_keys.append((f"""transformer.encoder.layers.{i}.linear2.bias""", f"""encoder.layers.{i}.fc2.bias""")   )
		rename_keys.append(
		    (f"""transformer.encoder.layers.{i}.norm1.weight""", f"""encoder.layers.{i}.self_attn_layer_norm.weight""")   )
		rename_keys.append(
		    (f"""transformer.encoder.layers.{i}.norm1.bias""", f"""encoder.layers.{i}.self_attn_layer_norm.bias""")   )
		rename_keys.append(
		    (f"""transformer.encoder.layers.{i}.norm2.weight""", f"""encoder.layers.{i}.final_layer_norm.weight""")   )
		rename_keys.append((f"""transformer.encoder.layers.{i}.norm2.bias""", f"""encoder.layers.{i}.final_layer_norm.bias""")   )
		# decoder layers: 2 times output projection, 2 feedforward neural networks and 3 layernorms
		rename_keys.append(
		    (
		        f"""transformer.decoder.layers.{i}.self_attn.out_proj.weight""",
		        f"""decoder.layers.{i}.self_attn.out_proj.weight""",
		    )   )
		rename_keys.append(
		    (f"""transformer.decoder.layers.{i}.self_attn.out_proj.bias""", f"""decoder.layers.{i}.self_attn.out_proj.bias""")   )
		rename_keys.append(
		    (
		        f"""transformer.decoder.layers.{i}.multihead_attn.out_proj.weight""",
		        f"""decoder.layers.{i}.encoder_attn.out_proj.weight""",
		    )   )
		rename_keys.append(
		    (
		        f"""transformer.decoder.layers.{i}.multihead_attn.out_proj.bias""",
		        f"""decoder.layers.{i}.encoder_attn.out_proj.bias""",
		    )   )
		rename_keys.append((f"""transformer.decoder.layers.{i}.linear1.weight""", f"""decoder.layers.{i}.fc1.weight""")   )
		rename_keys.append((f"""transformer.decoder.layers.{i}.linear1.bias""", f"""decoder.layers.{i}.fc1.bias""")   )
		rename_keys.append((f"""transformer.decoder.layers.{i}.linear2.weight""", f"""decoder.layers.{i}.fc2.weight""")   )
		rename_keys.append((f"""transformer.decoder.layers.{i}.linear2.bias""", f"""decoder.layers.{i}.fc2.bias""")   )
		rename_keys.append(
		    (f"""transformer.decoder.layers.{i}.norm1.weight""", f"""decoder.layers.{i}.self_attn_layer_norm.weight""")   )
		rename_keys.append(
		    (f"""transformer.decoder.layers.{i}.norm1.bias""", f"""decoder.layers.{i}.self_attn_layer_norm.bias""")   )
		rename_keys.append(
		    (f"""transformer.decoder.layers.{i}.norm2.weight""", f"""decoder.layers.{i}.encoder_attn_layer_norm.weight""")   )
		rename_keys.append(
		    (f"""transformer.decoder.layers.{i}.norm2.bias""", f"""decoder.layers.{i}.encoder_attn_layer_norm.bias""")   )
		rename_keys.append(
		    (f"""transformer.decoder.layers.{i}.norm3.weight""", f"""decoder.layers.{i}.final_layer_norm.weight""")   )
		rename_keys.append((f"""transformer.decoder.layers.{i}.norm3.bias""", f"""decoder.layers.{i}.final_layer_norm.bias""")   )

	# convolutional projection + query embeddings + layernorm of decoder + class and bounding box heads
	rename_keys.extend(
	    [
	        ("""input_proj.weight""", """input_projection.weight"""),
	        ("""input_proj.bias""", """input_projection.bias"""),
	        ("""query_embed.weight""", """query_position_embeddings.weight"""),
	        ("""transformer.decoder.norm.weight""", """decoder.layernorm.weight"""),
	        ("""transformer.decoder.norm.bias""", """decoder.layernorm.bias"""),
	        ("""class_embed.weight""", """class_labels_classifier.weight"""),
	        ("""class_embed.bias""", """class_labels_classifier.bias"""),
	        ("""bbox_embed.layers.0.weight""", """bbox_predictor.layers.0.weight"""),
	        ("""bbox_embed.layers.0.bias""", """bbox_predictor.layers.0.bias"""),
	        ("""bbox_embed.layers.1.weight""", """bbox_predictor.layers.1.weight"""),
	        ("""bbox_embed.layers.1.bias""", """bbox_predictor.layers.1.bias"""),
	        ("""bbox_embed.layers.2.weight""", """bbox_predictor.layers.2.weight"""),
	        ("""bbox_embed.layers.2.bias""", """bbox_predictor.layers.2.bias"""),
	    ]   )

	return rename_keys

def 		UpperCamelCase   (lowercase_:					List[Any]       , lowercase_:					Optional[int]       , lowercase_:					List[str]   )	-> Union[str, Any]:
	A__				:       str	  =						state_dict.pop(lowercase_   )
	A__				:       int	  =						val

def 		UpperCamelCase   (lowercase_:					Union[str, Any]       , lowercase_:					List[Any]=False   )	-> int:
	A__				:       List[str]	  =						""""""
	if is_panoptic:
		A__				:       int	  =						"""detr."""

	# first: transformer encoder
	for i in range(6   ):
		# read in weights + bias of input projection layer (in PyTorch's MultiHeadAttention, this is a single matrix + bias)
		A__				:       Tuple	  =						state_dict.pop(f"""{prefix}transformer.encoder.layers.{i}.self_attn.in_proj_weight"""   )
		A__				:       Tuple	  =						state_dict.pop(f"""{prefix}transformer.encoder.layers.{i}.self_attn.in_proj_bias"""   )
		# next, add query, keys and values (in that order) to the state dict
		A__				:       int	  =						in_proj_weight[:256, :]
		A__				:       str	  =						in_proj_bias[:256]
		A__				:       Tuple	  =						in_proj_weight[256:512, :]
		A__				:       Dict	  =						in_proj_bias[256:512]
		A__				:       Dict	  =						in_proj_weight[-256:, :]
		A__				:       Dict	  =						in_proj_bias[-256:]
	# next: transformer decoder (which is a bit more complex because it also includes cross-attention)
	for i in range(6   ):
		# read in weights + bias of input projection layer of self-attention
		A__				:       Union[str, Any]	  =						state_dict.pop(f"""{prefix}transformer.decoder.layers.{i}.self_attn.in_proj_weight"""   )
		A__				:       Union[str, Any]	  =						state_dict.pop(f"""{prefix}transformer.decoder.layers.{i}.self_attn.in_proj_bias"""   )
		# next, add query, keys and values (in that order) to the state dict
		A__				:       Union[str, Any]	  =						in_proj_weight[:256, :]
		A__				:       Optional[int]	  =						in_proj_bias[:256]
		A__				:       str	  =						in_proj_weight[256:512, :]
		A__				:       Tuple	  =						in_proj_bias[256:512]
		A__				:       Any	  =						in_proj_weight[-256:, :]
		A__				:       str	  =						in_proj_bias[-256:]
		# read in weights + bias of input projection layer of cross-attention
		A__				:       Any	  =						state_dict.pop(
		    f"""{prefix}transformer.decoder.layers.{i}.multihead_attn.in_proj_weight"""   )
		A__				:       Optional[Any]	  =						state_dict.pop(f"""{prefix}transformer.decoder.layers.{i}.multihead_attn.in_proj_bias"""   )
		# next, add query, keys and values (in that order) of cross-attention to the state dict
		A__				:       Dict	  =						in_proj_weight_cross_attn[:256, :]
		A__				:       Any	  =						in_proj_bias_cross_attn[:256]
		A__				:       Any	  =						in_proj_weight_cross_attn[256:512, :]
		A__				:       int	  =						in_proj_bias_cross_attn[256:512]
		A__				:       Optional[Any]	  =						in_proj_weight_cross_attn[-256:, :]
		A__				:       Tuple	  =						in_proj_bias_cross_attn[-256:]

def 		UpperCamelCase   ()	-> List[Any]:
	A__				:       int	  =						"""http://images.cocodataset.org/val2017/000000039769.jpg"""
	A__				:       Optional[int]	  =						Image.open(requests.get(lowercase_       , stream=lowercase_   ).raw   )

	return im

@torch.no_grad()
def 		UpperCamelCase   (lowercase_:					Union[str, Any]       , lowercase_:					Any=None       , lowercase_:					Any=False   )	-> str:
	A__  ,  A__				:       int	  =						get_detr_config(lowercase_   )

	# load original model from torch hub
	A__				:       Tuple	  =						{
	    """detr-resnet-50""": """detr_resnet50""",
	    """detr-resnet-101""": """detr_resnet101""",
	}
	logger.info(f"""Converting model {model_name}..."""   )
	A__				:       Optional[int]	  =						torch.hub.load("""facebookresearch/detr"""       , model_name_to_original_name[model_name]       , pretrained=lowercase_   ).eval()
	A__				:       str	  =						detr.state_dict()
	# rename keys
	for src, dest in create_rename_keys(lowercase_   ):
		if is_panoptic:
			A__				:       Optional[int]	  =						"""detr.""" + src
		rename_key(lowercase_       , lowercase_       , lowercase_   )
	# query, key and value matrices need special treatment
	read_in_q_k_v(lowercase_       , is_panoptic=lowercase_   )
	# important: we need to prepend a prefix to each of the base model keys as the head models use different attributes for them
	A__				:       Union[str, Any]	  =						"""detr.model.""" if is_panoptic else """model."""
	for key in state_dict.copy().keys():
		if is_panoptic:
			if (
			    key.startswith("""detr"""   )
			    and not key.startswith("""class_labels_classifier"""   )
			    and not key.startswith("""bbox_predictor"""   )
			):
				A__				:       List[Any]	  =						state_dict.pop(lowercase_   )
				A__				:       Optional[Any]	  =						val
			elif "class_labels_classifier" in key or "bbox_predictor" in key:
				A__				:       Union[str, Any]	  =						state_dict.pop(lowercase_   )
				A__				:       Any	  =						val
			elif key.startswith("""bbox_attention"""   ) or key.startswith("""mask_head"""   ):
				continue
			else:
				A__				:       Optional[int]	  =						state_dict.pop(lowercase_   )
				A__				:       Dict	  =						val
		else:
			if not key.startswith("""class_labels_classifier"""   ) and not key.startswith("""bbox_predictor"""   ):
				A__				:       int	  =						state_dict.pop(lowercase_   )
				A__				:       List[str]	  =						val

    # finally, create HuggingFace model and load state dict
	A__				:       List[str]	  =						DetrForSegmentation(lowercase_   ) if is_panoptic else DetrForObjectDetection(lowercase_   )
	model.load_state_dict(lowercase_   )
	model.eval()

	# verify our conversion on an image
	A__				:       str	  =						"""coco_panoptic""" if is_panoptic else """coco_detection"""
	A__				:       str	  =						DetrImageProcessor(format=lowercase_   )

	A__				:       Dict	  =						processor(images=prepare_img()       , return_tensors="""pt"""   )
	A__				:       List[Any]	  =						encoding["""pixel_values"""]

	A__				:       Any	  =						detr(lowercase_   )
	A__				:       Union[str, Any]	  =						model(lowercase_   )

	assert torch.allclose(outputs.logits       , original_outputs["""pred_logits"""]       , atol=1E-3   )
	assert torch.allclose(outputs.pred_boxes       , original_outputs["""pred_boxes"""]       , atol=1E-3   )
	if is_panoptic:
		assert torch.allclose(outputs.pred_masks       , original_outputs["""pred_masks"""]       , atol=1E-4   )
	print("""Looks ok!"""   )

	if pytorch_dump_folder_path is not None:
		# Save model and image processor
		logger.info(f"""Saving PyTorch model and image processor to {pytorch_dump_folder_path}..."""   )
		Path(lowercase_   ).mkdir(exist_ok=lowercase_   )
		model.save_pretrained(lowercase_   )
		processor.save_pretrained(lowercase_   )

	if push_to_hub:
		# Upload model and image processor to the hub
		logger.info("""Uploading PyTorch model and image processor to the hub..."""   )
		model.push_to_hub(f"""nielsr/{model_name}"""   )
		processor.push_to_hub(f"""nielsr/{model_name}"""   )


if __name__ == "__main__":
	A_			:					Optional[Any]    			=     argparse.ArgumentParser()

	parser.add_argument(
	    '--model_name',
	    default='detr-resnet-50',
	    type=str,
	    choices=['detr-resnet-50', 'detr-resnet-101'],
	    help='Name of the DETR model you\'d like to convert.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the folder to output PyTorch model.'
	)
	parser.add_argument('--push_to_hub', action='store_true', help='Whether to push the model to the hub or not.')
	A_			:					Any    			=     parser.parse_args()
	convert_detr_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class        _a		(__magic_name__						,  unittest.TestCase							):


					'''simple docstring'''






					UpperCAmelCase__:  List[str]    				=			RoCBertTokenizer
					UpperCAmelCase__:  Dict    				=			None
					UpperCAmelCase__:  Optional[Any]    				=			False
					UpperCAmelCase__:  Union[str, Any]    				=			True
					UpperCAmelCase__:  Union[str, Any]    				=			filter_non_english






					def     __A     (					self     ):
						super().setUp()

						A__				:       Tuple	  =						["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
						A__				:       Union[str, Any]	  =						{}
						A__				:       Dict	  =						{}
						for i, value in enumerate(A__     ):
							A__				:       Optional[int]	  =						i
							A__				:       Optional[int]	  =						i
						A__				:       Dict	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""vocab_file"""]     )
						A__				:       Optional[Any]	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""word_shape_file"""]     )
						A__				:       int	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""word_pronunciation_file"""]     )
						with open(self.vocab_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )
						with open(self.word_shape_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as word_shape_writer:
							json.dump(A__       ,			A__       ,			ensure_ascii=A__     )
						with open(self.word_pronunciation_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as word_pronunciation_writer:
							json.dump(A__       ,			A__       ,			ensure_ascii=A__     )






					def     __A     (					self     ):
						A__				:       Dict	  =						self.tokenizer_class(self.vocab_file       ,			self.word_shape_file       ,			self.word_pronunciation_file     )

						A__				:       Optional[Any]	  =						tokenizer.tokenize("""你好[SEP]你是谁"""     )
						self.assertListEqual(A__       ,			["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]     )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""     )       ,			["""ah""", """\u535A""", """\u63A8""", """zz"""]     )






					def     __A     (					self     ):
						A__				:       List[str]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )       ,			["""hello""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       int	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hällo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""h\u00E9llo"""]     )






					def     __A     (					self     ):
						A__				:       Optional[Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )       ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Optional[Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			never_split=["""[UNK]"""]     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""     )       ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]     )






					def     __A     (					self     ):
						A__				:       List[Any]	  =						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

						A__				:       Any	  =						{}
						for i, token in enumerate(A__     ):
							A__				:       Optional[int]	  =						i
						A__				:       Optional[Any]	  =						RoCBertWordpieceTokenizer(vocab=A__       ,			unk_token="""[UNK]"""     )

						self.assertListEqual(tokenizer.tokenize(""""""     )       ,			[]     )

						self.assertListEqual(tokenizer.tokenize("""unwanted running"""     )       ,			["""un""", """##want""", """##ed""", """runn""", """##ing"""]     )

						self.assertListEqual(tokenizer.tokenize("""unwantedX running"""     )       ,			["""[UNK]""", """runn""", """##ing"""]     )






					def     __A     (					self     ):
						self.assertTrue(_is_whitespace(""" """     )     )
						self.assertTrue(_is_whitespace("""\t"""     )     )
						self.assertTrue(_is_whitespace("""\r"""     )     )
						self.assertTrue(_is_whitespace("""\n"""     )     )
						self.assertTrue(_is_whitespace("""\u00A0"""     )     )

						self.assertFalse(_is_whitespace("""A"""     )     )
						self.assertFalse(_is_whitespace("""-"""     )     )






					def     __A     (					self     ):
						self.assertTrue(_is_control("""\u0005"""     )     )

						self.assertFalse(_is_control("""A"""     )     )
						self.assertFalse(_is_control(""" """     )     )
						self.assertFalse(_is_control("""\t"""     )     )
						self.assertFalse(_is_control("""\r"""     )     )






					def     __A     (					self     ):
						self.assertTrue(_is_punctuation("""-"""     )     )
						self.assertTrue(_is_punctuation("""$"""     )     )
						self.assertTrue(_is_punctuation("""`"""     )     )
						self.assertTrue(_is_punctuation("""."""     )     )

						self.assertFalse(_is_punctuation("""A"""     )     )
						self.assertFalse(_is_punctuation(""" """     )     )






					def     __A     (					self     ):
						A__				:       Optional[int]	  =						self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(A__     ) for t in ["""Test""", """\xad""", """test"""]]       ,			[["""[UNK]"""], [], ["""[UNK]"""]]     )

						if self.test_rust_tokenizer:
							A__				:       Tuple	  =						self.get_rust_tokenizer()
							self.assertListEqual(
							    [rust_tokenizer.tokenize(A__     ) for t in ["""Test""", """\xad""", """test"""]]       ,			[["""[UNK]"""], [], ["""[UNK]"""]]     )






					def     __A     (					self     ):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
								A__				:       Dict	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       str	  =						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
								A__				:       Union[str, Any]	  =						tokenizer_r.encode_plus(
								    A__       ,			return_attention_mask=A__       ,			return_token_type_ids=A__       ,			return_offsets_mapping=A__       ,			add_special_tokens=A__       ,			)

								A__				:       Any	  =						tokenizer_r.do_lower_case if hasattr(A__       ,			"""do_lower_case"""     ) else False
								A__				:       List[str]	  =						(
								    [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """A"""),
								        ((1, 2), ""","""),
								        ((3, 5), """na"""),
								        ((5, 6), """##ï"""),
								        ((6, 8), """##ve"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """Allen"""),
								        ((21, 23), """##NL"""),
								        ((23, 24), """##P"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								    if not do_lower_case
								    else [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """a"""),
								        ((1, 2), ""","""),
								        ((3, 8), """naive"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """allen"""),
								        ((21, 23), """##nl"""),
								        ((23, 24), """##p"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								)

								self.assertEqual(
								    [e[1] for e in expected_results]       ,			tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]     )     )
								self.assertEqual([e[0] for e in expected_results]       ,			tokens["""offset_mapping"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						["""的""", """人""", """有"""]
						A__				:       List[str]	  =						"""""".join(A__     )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
								A__				:       Any	  =						True
								A__				:       int	  =						self.tokenizer_class.from_pretrained(A__       ,			**A__     )
								A__				:       Dict	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       List[str]	  =						tokenizer_p.encode(A__       ,			add_special_tokens=A__     )
								A__				:       List[Any]	  =						tokenizer_r.encode(A__       ,			add_special_tokens=A__     )

								A__				:       Tuple	  =						tokenizer_r.convert_ids_to_tokens(A__     )
								A__				:       int	  =						tokenizer_p.convert_ids_to_tokens(A__     )

								# it is expected that each Chinese character is not preceded by "##"
								self.assertListEqual(A__       ,			A__     )
								self.assertListEqual(A__       ,			A__     )

								A__				:       Optional[int]	  =						False
								A__				:       Union[str, Any]	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )
								A__				:       str	  =						self.tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       Tuple	  =						tokenizer_r.encode(A__       ,			add_special_tokens=A__     )
								A__				:       List[str]	  =						tokenizer_p.encode(A__       ,			add_special_tokens=A__     )

								A__				:       Union[str, Any]	  =						tokenizer_r.convert_ids_to_tokens(A__     )
								A__				:       Tuple	  =						tokenizer_p.convert_ids_to_tokens(A__     )

								# it is expected that only the first Chinese character is not preceded by "##".
								A__				:       str	  =						[
								    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A__     )
								]
								self.assertListEqual(A__       ,			A__     )
								self.assertListEqual(A__       ,			A__     )






					@slow
					def     __A     (					self     ):
						A__				:       str	  =						self.tokenizer_class(self.vocab_file       ,			self.word_shape_file       ,			self.word_pronunciation_file     )

						A__				:       Optional[Any]	  =						tokenizer.encode("""你好"""       ,			add_special_tokens=A__     )
						A__				:       List[Any]	  =						tokenizer.encode("""你是谁"""       ,			add_special_tokens=A__     )

						A__				:       Any	  =						tokenizer.build_inputs_with_special_tokens(A__     )
						A__				:       Any	  =						tokenizer.build_inputs_with_special_tokens(A__       ,			A__     )

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]






					def     __A     (					self     ):
						A__				:       List[str]	  =						self.get_tokenizers(do_lower_case=A__     )
						for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}"""     ):
								A__				:       Optional[int]	  =						"""你好，你是谁"""
								A__				:       List[str]	  =						tokenizer.tokenize(A__     )
								A__				:       Union[str, Any]	  =						tokenizer.convert_tokens_to_ids(A__     )
								A__				:       str	  =						tokenizer.convert_tokens_to_shape_ids(A__     )
								A__				:       Optional[int]	  =						tokenizer.convert_tokens_to_pronunciation_ids(A__     )
								A__				:       Union[str, Any]	  =						tokenizer.prepare_for_model(
								    A__       ,			A__       ,			A__       ,			add_special_tokens=A__     )

								A__				:       int	  =						tokenizer.encode_plus(A__       ,			add_special_tokens=A__     )

								self.assertEqual(A__       ,			A__     )



