



import argparse
import json
import pickle
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import MaskFormerConfig, MaskFormerForInstanceSegmentation, MaskFormerImageProcessor, SwinConfig
from transformers.utils import logging


logging.set_verbosity_info()
SCREAMING_SNAKE_CASE_				= logging.get_logger(__name__)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__	)			->      Tuple:
		a_     :      List[Any] 						=     SwinConfig.from_pretrained(
		    "microsoft/swin-tiny-patch4-window7-224", out_features=["stage1", "stage2", "stage3", "stage4"]	)
		a_     :      List[str] 						=     MaskFormerConfig(backbone_config=SCREAMING_SNAKE_CASE__	)

		a_     :      Dict 						=     "huggingface/label-files"
		if "ade20k-full" in model_name:
				# this should be ok
				a_     :      List[str] 						=     847
				a_     :      Optional[Any] 						=     "maskformer-ade20k-full-id2label.json"
		elif "ade" in model_name:
				# this should be ok
				a_     :      List[str] 						=     150
				a_     :      Tuple 						=     "ade20k-id2label.json"
		elif "coco-stuff" in model_name:
				# this should be ok
				a_     :      Union[str, Any] 						=     171
				a_     :      Union[str, Any] 						=     "maskformer-coco-stuff-id2label.json"
		elif "coco" in model_name:
				# TODO
				a_     :      Optional[Any] 						=     133
				a_     :      List[str] 						=     "coco-panoptic-id2label.json"
		elif "cityscapes" in model_name:
				# this should be ok
				a_     :      Union[str, Any] 						=     19
				a_     :      Any 						=     "cityscapes-id2label.json"
		elif "vistas" in model_name:
				# this should be ok
				a_     :      int 						=     65
				a_     :      Any 						=     "mapillary-vistas-id2label.json"

		a_     :      List[str] 						=     json.load(open(hf_hub_download(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, repo_type="dataset"	), "r"	)	)
		a_     :      Optional[Any] 						=     {int(SCREAMING_SNAKE_CASE__	): v for k, v in idalabel.items()}

		return config



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__	)			->      List[Any]:
		a_     :      str 						=     []
		# stem
		# fmt: off
		rename_keys.append(("backbone.patch_embed.proj.weight", "model.pixel_level_module.encoder.model.embeddings.patch_embeddings.projection.weight")	)
		rename_keys.append(("backbone.patch_embed.proj.bias", "model.pixel_level_module.encoder.model.embeddings.patch_embeddings.projection.bias")	)
		rename_keys.append(("backbone.patch_embed.norm.weight", "model.pixel_level_module.encoder.model.embeddings.norm.weight")	)
		rename_keys.append(("backbone.patch_embed.norm.bias", "model.pixel_level_module.encoder.model.embeddings.norm.bias")	)
		# stages
		for i in range(len(config.backbone_config.depths	)	):
				for j in range(config.backbone_config.depths[i]	):
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.norm1.weight""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_before.weight""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.norm1.bias""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_before.bias""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.attn.relative_position_bias_table""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_bias_table""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.attn.relative_position_index""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_index""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.attn.proj.weight""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.weight""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.attn.proj.bias""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.bias""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.norm2.weight""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_after.weight""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.norm2.bias""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_after.bias""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.mlp.fc1.weight""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.weight""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.mlp.fc1.bias""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.bias""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.mlp.fc2.weight""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.output.dense.weight""")	)
						rename_keys.append((F"""backbone.layers.{i}.blocks.{j}.mlp.fc2.bias""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.output.dense.bias""")	)

				if i < 3:
						rename_keys.append((F"""backbone.layers.{i}.downsample.reduction.weight""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.reduction.weight""")	)
						rename_keys.append((F"""backbone.layers.{i}.downsample.norm.weight""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.norm.weight""")	)
						rename_keys.append((F"""backbone.layers.{i}.downsample.norm.bias""", F"""model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.norm.bias""")	)
				rename_keys.append((F"""backbone.norm{i}.weight""", F"""model.pixel_level_module.encoder.hidden_states_norms.{i}.weight""")	)
				rename_keys.append((F"""backbone.norm{i}.bias""", F"""model.pixel_level_module.encoder.hidden_states_norms.{i}.bias""")	)

		# FPN
		rename_keys.append(("sem_seg_head.layer_4.weight", "model.pixel_level_module.decoder.fpn.stem.0.weight")	)
		rename_keys.append(("sem_seg_head.layer_4.norm.weight", "model.pixel_level_module.decoder.fpn.stem.1.weight")	)
		rename_keys.append(("sem_seg_head.layer_4.norm.bias", "model.pixel_level_module.decoder.fpn.stem.1.bias")	)
		for source_index, target_index in zip(range(3, 0, -1	), range(0, 3	)	):
				rename_keys.append((F"""sem_seg_head.adapter_{source_index}.weight""", F"""model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.0.weight""")	)
				rename_keys.append((F"""sem_seg_head.adapter_{source_index}.norm.weight""", F"""model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.1.weight""")	)
				rename_keys.append((F"""sem_seg_head.adapter_{source_index}.norm.bias""", F"""model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.1.bias""")	)
				rename_keys.append((F"""sem_seg_head.layer_{source_index}.weight""", F"""model.pixel_level_module.decoder.fpn.layers.{target_index}.block.0.weight""")	)
				rename_keys.append((F"""sem_seg_head.layer_{source_index}.norm.weight""", F"""model.pixel_level_module.decoder.fpn.layers.{target_index}.block.1.weight""")	)
				rename_keys.append((F"""sem_seg_head.layer_{source_index}.norm.bias""", F"""model.pixel_level_module.decoder.fpn.layers.{target_index}.block.1.bias""")	)
		rename_keys.append(("sem_seg_head.mask_features.weight", "model.pixel_level_module.decoder.mask_projection.weight")	)
		rename_keys.append(("sem_seg_head.mask_features.bias", "model.pixel_level_module.decoder.mask_projection.bias")	)

		# Transformer decoder
		for idx in range(config.decoder_config.decoder_layers	):
				# self-attention out projection
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.out_proj.weight""", F"""model.transformer_module.decoder.layers.{idx}.self_attn.out_proj.weight""")	)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.out_proj.bias""", F"""model.transformer_module.decoder.layers.{idx}.self_attn.out_proj.bias""")	)
				# cross-attention out projection
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.out_proj.weight""", F"""model.transformer_module.decoder.layers.{idx}.encoder_attn.out_proj.weight""")	)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.out_proj.bias""", F"""model.transformer_module.decoder.layers.{idx}.encoder_attn.out_proj.bias""")	)
				# MLP 1
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear1.weight""", F"""model.transformer_module.decoder.layers.{idx}.fc1.weight""")	)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear1.bias""", F"""model.transformer_module.decoder.layers.{idx}.fc1.bias""")	)
				# MLP 2
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear2.weight""", F"""model.transformer_module.decoder.layers.{idx}.fc2.weight""")	)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear2.bias""", F"""model.transformer_module.decoder.layers.{idx}.fc2.bias""")	)
				# layernorm 1 (self-attention layernorm)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm1.weight""", F"""model.transformer_module.decoder.layers.{idx}.self_attn_layer_norm.weight""")	)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm1.bias""", F"""model.transformer_module.decoder.layers.{idx}.self_attn_layer_norm.bias""")	)
				# layernorm 2 (cross-attention layernorm)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm2.weight""", F"""model.transformer_module.decoder.layers.{idx}.encoder_attn_layer_norm.weight""")	)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm2.bias""", F"""model.transformer_module.decoder.layers.{idx}.encoder_attn_layer_norm.bias""")	)
				# layernorm 3 (final layernorm)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm3.weight""", F"""model.transformer_module.decoder.layers.{idx}.final_layer_norm.weight""")	)
				rename_keys.append((F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm3.bias""", F"""model.transformer_module.decoder.layers.{idx}.final_layer_norm.bias""")	)

		rename_keys.append(("sem_seg_head.predictor.transformer.decoder.norm.weight", "model.transformer_module.decoder.layernorm.weight")	)
		rename_keys.append(("sem_seg_head.predictor.transformer.decoder.norm.bias", "model.transformer_module.decoder.layernorm.bias")	)

		# heads on top
		rename_keys.append(("sem_seg_head.predictor.query_embed.weight", "model.transformer_module.queries_embedder.weight")	)

		rename_keys.append(("sem_seg_head.predictor.input_proj.weight", "model.transformer_module.input_projection.weight")	)
		rename_keys.append(("sem_seg_head.predictor.input_proj.bias", "model.transformer_module.input_projection.bias")	)

		rename_keys.append(("sem_seg_head.predictor.class_embed.weight", "class_predictor.weight")	)
		rename_keys.append(("sem_seg_head.predictor.class_embed.bias", "class_predictor.bias")	)

		for i in range(3	):
				rename_keys.append((F"""sem_seg_head.predictor.mask_embed.layers.{i}.weight""", F"""mask_embedder.{i}.0.weight""")	)
				rename_keys.append((F"""sem_seg_head.predictor.mask_embed.layers.{i}.bias""", F"""mask_embedder.{i}.0.bias""")	)
		# fmt: on

		return rename_keys



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      int:
		a_     :      Optional[int] 						=     dct.pop(SCREAMING_SNAKE_CASE__	)
		a_     :      List[Any] 						=     val



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      List[Any]:
		a_     :      int 						=     [int(backbone_config.embed_dim * 2**i	) for i in range(len(backbone_config.depths	)	)]
		for i in range(len(backbone_config.depths	)	):
				a_     :      Optional[int] 						=     num_features[i]
				for j in range(backbone_config.depths[i]	):
						# fmt: off
						# read in weights + bias of input projection layer (in original implementation, this is a single matrix + bias)
						a_     :      List[str] 						=     state_dict.pop(F"""backbone.layers.{i}.blocks.{j}.attn.qkv.weight"""	)
						a_     :      str 						=     state_dict.pop(F"""backbone.layers.{i}.blocks.{j}.attn.qkv.bias"""	)
						# next, add query, keys and values (in that order) to the state dict
						a_     :      Tuple 						=     in_proj_weight[:dim, :]
						a_     :      Union[str, Any] 						=     in_proj_bias[: dim]
						a_     :      Dict 						=     in_proj_weight[
						    dim : dim * 2, :
						]
						a_     :      Tuple 						=     in_proj_bias[
						    dim : dim * 2
						]
						a_     :      Optional[int] 						=     in_proj_weight[
						    -dim :, :
						]
						a_     :      str 						=     in_proj_bias[-dim :]
						# fmt: on



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      Dict:
		# fmt: off
		a_     :      List[str] 						=     config.decoder_config.hidden_size
		for idx in range(config.decoder_config.decoder_layers	):
				# read in weights + bias of self-attention input projection layer (in the original implementation, this is a single matrix + bias)
				a_     :      Optional[int] 						=     state_dict.pop(F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.in_proj_weight"""	)
				a_     :      int 						=     state_dict.pop(F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.in_proj_bias"""	)
				# next, add query, keys and values (in that order) to the state dict
				a_     :      int 						=     in_proj_weight[: hidden_size, :]
				a_     :      Tuple 						=     in_proj_bias[:config.hidden_size]
				a_     :      List[Any] 						=     in_proj_weight[hidden_size : hidden_size * 2, :]
				a_     :      Dict 						=     in_proj_bias[hidden_size : hidden_size * 2]
				a_     :      Optional[int] 						=     in_proj_weight[-hidden_size :, :]
				a_     :      List[str] 						=     in_proj_bias[-hidden_size :]
				# read in weights + bias of cross-attention input projection layer (in the original implementation, this is a single matrix + bias)
				a_     :      Union[str, Any] 						=     state_dict.pop(F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.in_proj_weight"""	)
				a_     :      str 						=     state_dict.pop(F"""sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.in_proj_bias"""	)
				# next, add query, keys and values (in that order) to the state dict
				a_     :      Dict 						=     in_proj_weight[: hidden_size, :]
				a_     :      Optional[Any] 						=     in_proj_bias[:config.hidden_size]
				a_     :      List[Any] 						=     in_proj_weight[hidden_size : hidden_size * 2, :]
				a_     :      str 						=     in_proj_bias[hidden_size : hidden_size * 2]
				a_     :      List[Any] 						=     in_proj_weight[-hidden_size :, :]
				a_     :      Dict 						=     in_proj_bias[-hidden_size :]
		# fmt: on



def        lowerCAmelCase_						(							)			->      torch.Tensor:
		a_     :      Any 						=     "http://images.cocodataset.org/val2017/000000039769.jpg"
		a_     :      Optional[Any] 						=     Image.open(requests.get(SCREAMING_SNAKE_CASE__, stream=SCREAMING_SNAKE_CASE__	).raw	)
		return im



@torch.no_grad()
def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__ = False	)			->      Optional[int]:
		a_     :      List[Any] 						=     get_maskformer_config(SCREAMING_SNAKE_CASE__	)

		# load original state_dict
		with open(SCREAMING_SNAKE_CASE__, "rb"	) as f:
				a_     :      int 						=     pickle.load(SCREAMING_SNAKE_CASE__	)
		a_     :      Optional[int] 						=     data["model"]

		# for name, param in state_dict.items():
		#     print(name, param.shape)

		# rename keys
		a_     :      List[Any] 						=     create_rename_keys(SCREAMING_SNAKE_CASE__	)
		for src, dest in rename_keys:
				rename_key(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)
		read_in_swin_q_k_v(SCREAMING_SNAKE_CASE__, config.backbone_config	)
		read_in_decoder_q_k_v(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)

		# update to torch tensors
		for key, value in state_dict.items():
				a_     :      Tuple 						=     torch.from_numpy(SCREAMING_SNAKE_CASE__	)

		# load 🤗 model
		a_     :      Tuple 						=     MaskFormerForInstanceSegmentation(SCREAMING_SNAKE_CASE__	)
		model.eval()

		for name, param in model.named_parameters():
				print(SCREAMING_SNAKE_CASE__, param.shape	)

		a_     :      int 						=     model.load_state_dict(SCREAMING_SNAKE_CASE__, strict=SCREAMING_SNAKE_CASE__	)
		assert missing_keys == [
		    "model.pixel_level_module.encoder.model.layernorm.weight",
		    "model.pixel_level_module.encoder.model.layernorm.bias",
		]
		assert len(SCREAMING_SNAKE_CASE__	) == 0, F"""Unexpected keys: {unexpected_keys}"""

		# verify results
		a_     :      List[Any] 						=     prepare_img()
		if "vistas" in model_name:
				a_     :      int 						=     65
		elif "cityscapes" in model_name:
				a_     :      str 						=     65_535
		else:
				a_     :      int 						=     255
		a_     :      List[str] 						=     True if "ade" in model_name else False
		a_     :      Any 						=     MaskFormerImageProcessor(ignore_index=SCREAMING_SNAKE_CASE__, reduce_labels=SCREAMING_SNAKE_CASE__	)

		a_     :      int 						=     image_processor(SCREAMING_SNAKE_CASE__, return_tensors="pt"	)

		a_     :      Optional[int] 						=     model(**SCREAMING_SNAKE_CASE__	)

		print("Logits:", outputs.class_queries_logits[0, :3, :3]	)

		if model_name == "maskformer-swin-tiny-ade":
				a_     :      Union[str, Any] 						=     torch.tensor(
				    [[3.63_53, -4.47_70, -2.60_65], [0.50_81, -4.23_94, -3.53_43], [2.19_09, -5.03_53, -1.93_23]]	)
		assert torch.allclose(outputs.class_queries_logits[0, :3, :3], SCREAMING_SNAKE_CASE__, atol=1e-4	)
		print("Looks ok!"	)

		if pytorch_dump_folder_path is not None:
				print(F"""Saving model and image processor to {pytorch_dump_folder_path}"""	)
				Path(SCREAMING_SNAKE_CASE__	).mkdir(exist_ok=SCREAMING_SNAKE_CASE__	)
				model.save_pretrained(SCREAMING_SNAKE_CASE__	)
				image_processor.save_pretrained(SCREAMING_SNAKE_CASE__	)

		if push_to_hub:
				print("Pushing model and image processor to the hub..."	)
				model.push_to_hub(F"""nielsr/{model_name}"""	)
				image_processor.push_to_hub(F"""nielsr/{model_name}"""	)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE_				= argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--model_name""",
					    default="""maskformer-swin-tiny-ade""",
					    type=str,
					    help=("""Name of the MaskFormer model you'd like to convert""",),
					)
					parser.add_argument(
					    """--checkpoint_path""",
					    default="""/Users/nielsrogge/Documents/MaskFormer_checkpoints/MaskFormer-Swin-tiny-ADE20k/model.pkl""",
					    type=str,
					    help="""Path to the original state dict (.pth file).""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
					)
					parser.add_argument(
					    """--push_to_hub""", action="""store_true""", help="""Whether or not to push the converted model to the 🤗 hub."""
					)

					SCREAMING_SNAKE_CASE_				= parser.parse_args()
					convert_maskformer_checkpoint(
					    args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path, args.push_to_hub
					)




"""simple docstring"""
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class 			snake_case_						(						a_     ,unittest.TestCase				):
							__lowerCAmelCase    					=					BertJapaneseTokenizer
							__lowerCAmelCase    					=					False
							__lowerCAmelCase    					=					True






							def snake_case_  (   self      ):
									super().setUp()

									a_     :      str 						=     [
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "こんにちは",
									    "こん",
									    "にちは",
									    "ばんは",
									    "##こん",
									    "##にちは",
									    "##ばんは",
									    "世界",
									    "##世界",
									    "、",
									    "##、",
									    "。",
									    "##。",
									]

									a_     :      Tuple 						=     os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["vocab_file"]      )
									with open(self.vocab_file			,     "w"			,     encoding="utf-8"      ) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )






							def snake_case_  (   self			,     a_      ):
									a_     :      Dict 						=     "こんにちは、世界。 \nこんばんは、世界。"
									a_     :      int 						=     "こんにちは 、 世界 。 こんばんは 、 世界 。"
									return input_text, output_text






							def snake_case_  (   self			,     a_      ):
									a_						,				a_     :      List[str] 						=     self.get_input_output_texts(a_      )
									a_     :      Tuple 						=     tokenizer.encode(a_			,     add_special_tokens=a_      )
									a_     :      Dict 						=     tokenizer.decode(a_			,     clean_up_tokenization_spaces=a_      )
									return text, ids






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									a_     :      int 						=     self.tokenizer_class(self.vocab_file      )

									a_     :      List[Any] 						=     tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     self.tokenizer_class(self.vocab_file			,     word_tokenizer_type="mecab"      )
									self.assertIsNotNone(a_      )

									a_     :      Union[str, Any] 						=     "こんにちは、世界。\nこんばんは、世界。"
									a_     :      Union[str, Any] 						=     tokenizer.tokenize(a_      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

									a_     :      Any 						=     os.path.join(self.tmpdirname			,     "tokenizer.bin"      )
									with open(a_			,     "wb"      ) as handle:
											pickle.dump(a_			,     a_      )

									with open(a_			,     "rb"      ) as handle:
											a_     :      Optional[Any] 						=     pickle.load(a_      )

									a_     :      int 						=     tokenizer_new.tokenize(a_      )

									self.assertListEqual(a_			,     a_      )






							def snake_case_  (   self      ):
									a_     :      Tuple 						=     MecabTokenizer(mecab_dic="ipadic"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									try:
											a_     :      Dict 						=     MecabTokenizer(mecab_dic="unidic_lite"      )
									except ModuleNotFoundError:
											return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									try:
											a_     :      List[Any] 						=     MecabTokenizer(mecab_dic="unidic"      )
									except ModuleNotFoundError:
											return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     MecabTokenizer(do_lower_case=a_			,     mecab_dic="ipadic"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									try:
											a_     :      int 						=     MecabTokenizer(
											    do_lower_case=a_			,     normalize_text=a_			,     mecab_option="-d /usr/local/lib/mecab/dic/jumandic"      )
									except RuntimeError:
											# if dict doesn't exist in the system, previous code raises this error.
											return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]			,     )






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     MecabTokenizer(normalize_text=a_			,     mecab_dic="ipadic"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      int 						=     self.tokenizer_class(self.vocab_file			,     word_tokenizer_type="sudachi"      )
									self.assertIsNotNone(a_      )

									a_     :      Optional[Any] 						=     "こんにちは、世界。\nこんばんは、世界。"
									a_     :      str 						=     tokenizer.tokenize(a_      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

									a_     :      Dict 						=     os.path.join(self.tmpdirname			,     "tokenizer.bin"      )
									with open(a_			,     "wb"      ) as handle:
											pickle.dump(a_			,     a_      )

									with open(a_			,     "rb"      ) as handle:
											a_     :      Optional[Any] 						=     pickle.load(a_      )

									a_     :      int 						=     tokenizer_new.tokenize(a_      )

									self.assertListEqual(a_			,     a_      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      int 						=     SudachiTokenizer(sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      Any 						=     SudachiTokenizer(sudachi_dict_type="core"			,     sudachi_split_mode="A"      )

									self.assertListEqual(tokenizer.tokenize("外国人参政権"      )			,     ["外国", "人", "参政", "権"]      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     SudachiTokenizer(sudachi_dict_type="core"			,     sudachi_split_mode="B"      )

									self.assertListEqual(tokenizer.tokenize("外国人参政権"      )			,     ["外国人", "参政権"]      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      int 						=     SudachiTokenizer(sudachi_dict_type="core"			,     sudachi_split_mode="C"      )

									self.assertListEqual(tokenizer.tokenize("外国人参政権"      )			,     ["外国人参政権"]      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      List[str] 						=     SudachiTokenizer(do_lower_case=a_			,     sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      str 						=     SudachiTokenizer(normalize_text=a_			,     sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      Optional[int] 						=     SudachiTokenizer(trim_whitespace=a_			,     sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     self.tokenizer_class(self.vocab_file			,     word_tokenizer_type="jumanpp"      )
									self.assertIsNotNone(a_      )

									a_     :      Optional[int] 						=     "こんにちは、世界。\nこんばんは、世界。"
									a_     :      Any 						=     tokenizer.tokenize(a_      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

									a_     :      int 						=     os.path.join(self.tmpdirname			,     "tokenizer.bin"      )
									with open(a_			,     "wb"      ) as handle:
											pickle.dump(a_			,     a_      )

									with open(a_			,     "rb"      ) as handle:
											a_     :      Tuple 						=     pickle.load(a_      )

									a_     :      Union[str, Any] 						=     tokenizer_new.tokenize(a_      )

									self.assertListEqual(a_			,     a_      )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Any 						=     JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     JumanppTokenizer(do_lower_case=a_      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      int 						=     JumanppTokenizer(normalize_text=a_      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     JumanppTokenizer(trim_whitespace=a_      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Any 						=     JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"      )			,     ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]			,     )






							def snake_case_  (   self      ):
									a_     :      str 						=     ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

									a_     :      int 						=     {}
									for i, token in enumerate(a_      ):
											a_     :      Optional[int] 						=     i
									a_     :      Optional[int] 						=     WordpieceTokenizer(vocab=a_			,     unk_token="[UNK]"      )

									self.assertListEqual(tokenizer.tokenize(""      )			,     []      )

									self.assertListEqual(tokenizer.tokenize("こんにちは"      )			,     ["こんにちは"]      )

									self.assertListEqual(tokenizer.tokenize("こんばんは"      )			,     ["こん", "##ばんは"]      )

									self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"      )			,     ["こん", "##ばんは", "[UNK]", "こんにちは"]      )






							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"      )
									a_     :      Any 						=     tokenizer.subword_tokenizer

									a_     :      Union[str, Any] 						=     subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"      )
									self.assertListEqual(a_			,     ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]      )

									a_     :      int 						=     subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"      )
									self.assertListEqual(a_			,     ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]      )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"      )

									a_     :      Any 						=     tokenizer.encode("ありがとう。"			,     add_special_tokens=a_      )
									a_     :      Optional[Any] 						=     tokenizer.encode("どういたしまして。"			,     add_special_tokens=a_      )

									a_     :      Union[str, Any] 						=     tokenizer.build_inputs_with_special_tokens(a_      )
									a_     :      Tuple 						=     tokenizer.build_inputs_with_special_tokens(a_			,     a_      )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			snake_case_						(						a_     ,unittest.TestCase				):
							__lowerCAmelCase    					=					BertJapaneseTokenizer
							__lowerCAmelCase    					=					False






							def snake_case_  (   self      ):
									super().setUp()

									a_     :      Any 						=     ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

									a_     :      str 						=     os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["vocab_file"]      )
									with open(self.vocab_file			,     "w"			,     encoding="utf-8"      ) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )






							def snake_case_  (   self			,     **a_      ):
									return BertJapaneseTokenizer.from_pretrained(self.tmpdirname			,     subword_tokenizer_type="character"			,     **a_      )






							def snake_case_  (   self			,     a_      ):
									a_     :      int 						=     "こんにちは、世界。 \nこんばんは、世界。"
									a_     :      List[str] 						=     "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
									return input_text, output_text






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									a_     :      Dict 						=     self.tokenizer_class(self.vocab_file			,     subword_tokenizer_type="character"      )

									a_     :      int 						=     tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"      )
									self.assertListEqual(
									    a_			,     ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]      )
									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(a_      )			,     [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]      )






							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

									a_     :      Any 						=     {}
									for i, token in enumerate(a_      ):
											a_     :      str 						=     i
									a_     :      List[str] 						=     CharacterTokenizer(vocab=a_			,     unk_token="[UNK]"      )

									self.assertListEqual(tokenizer.tokenize(""      )			,     []      )

									self.assertListEqual(tokenizer.tokenize("こんにちは"      )			,     ["こ", "ん", "に", "ち", "は"]      )

									self.assertListEqual(tokenizer.tokenize("こんにちほ"      )			,     ["こ", "ん", "に", "ち", "[UNK]"]      )






							def snake_case_  (   self      ):
									a_     :      str 						=     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"      )

									a_     :      Optional[int] 						=     tokenizer.encode("ありがとう。"			,     add_special_tokens=a_      )
									a_     :      Tuple 						=     tokenizer.encode("どういたしまして。"			,     add_special_tokens=a_      )

									a_     :      Optional[Any] 						=     tokenizer.build_inputs_with_special_tokens(a_      )
									a_     :      Any 						=     tokenizer.build_inputs_with_special_tokens(a_			,     a_      )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			snake_case_						(						unittest.TestCase				):






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     "cl-tohoku/bert-base-japanese"
									a_     :      Dict 						=     AutoTokenizer.from_pretrained(a_      )
									self.assertIsInstance(a_			,     a_      )




class 			snake_case_						(						unittest.TestCase				):






							def snake_case_  (   self      ):
									a_     :      Any 						=     "cl-tohoku/bert-base-japanese"
									with self.assertLogs("transformers"			,     level="WARNING"      ) as cm:
											BertTokenizer.from_pretrained(a_      )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
											        " is called from."      )      )
									a_     :      Any 						=     "bert-base-cased"
									with self.assertLogs("transformers"			,     level="WARNING"      ) as cm:
											BertJapaneseTokenizer.from_pretrained(a_      )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
											        " is called from."      )      )