



"""simple docstring"""
import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import DeiTImageProcessor, ViTConfig, ViTForImageClassification, ViTImageProcessor, ViTModel
from transformers.utils import logging


logging.set_verbosity_info()
SCREAMING_SNAKE_CASE_				= logging.get_logger(__name__)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__=False	)			->      int:
		a_     :      int 						=     []
		for i in range(config.num_hidden_layers	):
				# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
				rename_keys.append((F"""blocks.{i}.norm1.weight""", F"""vit.encoder.layer.{i}.layernorm_before.weight""")	)
				rename_keys.append((F"""blocks.{i}.norm1.bias""", F"""vit.encoder.layer.{i}.layernorm_before.bias""")	)
				rename_keys.append((F"""blocks.{i}.attn.proj.weight""", F"""vit.encoder.layer.{i}.attention.output.dense.weight""")	)
				rename_keys.append((F"""blocks.{i}.attn.proj.bias""", F"""vit.encoder.layer.{i}.attention.output.dense.bias""")	)
				rename_keys.append((F"""blocks.{i}.norm2.weight""", F"""vit.encoder.layer.{i}.layernorm_after.weight""")	)
				rename_keys.append((F"""blocks.{i}.norm2.bias""", F"""vit.encoder.layer.{i}.layernorm_after.bias""")	)
				rename_keys.append((F"""blocks.{i}.mlp.fc1.weight""", F"""vit.encoder.layer.{i}.intermediate.dense.weight""")	)
				rename_keys.append((F"""blocks.{i}.mlp.fc1.bias""", F"""vit.encoder.layer.{i}.intermediate.dense.bias""")	)
				rename_keys.append((F"""blocks.{i}.mlp.fc2.weight""", F"""vit.encoder.layer.{i}.output.dense.weight""")	)
				rename_keys.append((F"""blocks.{i}.mlp.fc2.bias""", F"""vit.encoder.layer.{i}.output.dense.bias""")	)

		# projection layer + position embeddings
		rename_keys.extend(
		    [
		        ("cls_token", "vit.embeddings.cls_token"),
		        ("patch_embed.proj.weight", "vit.embeddings.patch_embeddings.projection.weight"),
		        ("patch_embed.proj.bias", "vit.embeddings.patch_embeddings.projection.bias"),
		        ("pos_embed", "vit.embeddings.position_embeddings"),
		    ]	)

		if base_model:
				# layernorm + pooler
				rename_keys.extend(
				    [
				        ("norm.weight", "layernorm.weight"),
				        ("norm.bias", "layernorm.bias"),
				        ("pre_logits.fc.weight", "pooler.dense.weight"),
				        ("pre_logits.fc.bias", "pooler.dense.bias"),
				    ]	)

				# if just the base model, we should remove "vit" from all keys that start with "vit"
				a_     :      List[Any] 						=     [(pair[0], pair[1][4:]) if pair[1].startswith("vit"	) else pair for pair in rename_keys]
		else:
				# layernorm + classification head
				rename_keys.extend(
				    [
				        ("norm.weight", "vit.layernorm.weight"),
				        ("norm.bias", "vit.layernorm.bias"),
				        ("head.weight", "classifier.weight"),
				        ("head.bias", "classifier.bias"),
				    ]	)

		return rename_keys



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__=False	)			->      Dict:
		for i in range(config.num_hidden_layers	):
				if base_model:
						a_     :      List[str] 						=     ""
				else:
						a_     :      Any 						=     "vit."
				# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
				a_     :      List[Any] 						=     state_dict.pop(F"""blocks.{i}.attn.qkv.weight"""	)
				a_     :      str 						=     state_dict.pop(F"""blocks.{i}.attn.qkv.bias"""	)
				# next, add query, keys and values (in that order) to the state dict
				a_     :      Any 						=     in_proj_weight[
				    : config.hidden_size, :
				]
				a_     :      Union[str, Any] 						=     in_proj_bias[: config.hidden_size]
				a_     :      Optional[Any] 						=     in_proj_weight[
				    config.hidden_size : config.hidden_size * 2, :
				]
				a_     :      Tuple 						=     in_proj_bias[
				    config.hidden_size : config.hidden_size * 2
				]
				a_     :      List[Any] 						=     in_proj_weight[
				    -config.hidden_size :, :
				]
				a_     :      int 						=     in_proj_bias[-config.hidden_size :]



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__	)			->      Optional[Any]:
		a_     :      str 						=     ["head.weight", "head.bias"]
		for k in ignore_keys:
				state_dict.pop(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      List[str]:
		a_     :      Tuple 						=     dct.pop(SCREAMING_SNAKE_CASE__	)
		a_     :      List[Any] 						=     val



def        lowerCAmelCase_						(							)			->      int:
		a_     :      Optional[int] 						=     "http://images.cocodataset.org/val2017/000000039769.jpg"
		a_     :      str 						=     Image.open(requests.get(SCREAMING_SNAKE_CASE__, stream=SCREAMING_SNAKE_CASE__	).raw	)
		return im



@torch.no_grad()
def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      int:
		a_     :      int 						=     ViTConfig()
		a_     :      List[Any] 						=     False
		# dataset (ImageNet-21k only or also fine-tuned on ImageNet 2012), patch_size and image_size
		if vit_name[-5:] == "in21k":
				a_     :      Tuple 						=     True
				a_     :      Any 						=     int(vit_name[-12:-10]	)
				a_     :      str 						=     int(vit_name[-9:-6]	)
		else:
				a_     :      Any 						=     1_000
				a_     :      List[str] 						=     "huggingface/label-files"
				a_     :      Optional[Any] 						=     "imagenet-1k-id2label.json"
				a_     :      List[str] 						=     json.load(open(hf_hub_download(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, repo_type="dataset"	), "r"	)	)
				a_     :      Dict 						=     {int(SCREAMING_SNAKE_CASE__	): v for k, v in idalabel.items()}
				a_     :      Optional[int] 						=     idalabel
				a_     :      str 						=     {v: k for k, v in idalabel.items()}
				a_     :      List[Any] 						=     int(vit_name[-6:-4]	)
				a_     :      Any 						=     int(vit_name[-3:]	)
		# size of the architecture
		if "deit" in vit_name:
				if vit_name[9:].startswith("tiny"	):
						a_     :      Optional[Any] 						=     192
						a_     :      int 						=     768
						a_     :      Union[str, Any] 						=     12
						a_     :      Any 						=     3
				elif vit_name[9:].startswith("small"	):
						a_     :      Optional[int] 						=     384
						a_     :      List[str] 						=     1_536
						a_     :      List[Any] 						=     12
						a_     :      Dict 						=     6
				else:
						pass
		else:
				if vit_name[4:].startswith("small"	):
						a_     :      List[str] 						=     768
						a_     :      Any 						=     2_304
						a_     :      Optional[Any] 						=     8
						a_     :      List[Any] 						=     8
				elif vit_name[4:].startswith("base"	):
						pass
				elif vit_name[4:].startswith("large"	):
						a_     :      List[Any] 						=     1_024
						a_     :      str 						=     4_096
						a_     :      List[Any] 						=     24
						a_     :      Any 						=     16
				elif vit_name[4:].startswith("huge"	):
						a_     :      Tuple 						=     1_280
						a_     :      Optional[int] 						=     5_120
						a_     :      int 						=     32
						a_     :      int 						=     16

    # load original model from timm
		a_     :      Dict 						=     timm.create_model(SCREAMING_SNAKE_CASE__, pretrained=SCREAMING_SNAKE_CASE__	)
		timm_model.eval()

		# load state_dict of original model, remove and rename some keys
		a_     :      Tuple 						=     timm_model.state_dict()
		if base_model:
				remove_classification_head_(SCREAMING_SNAKE_CASE__	)
		a_     :      List[Any] 						=     create_rename_keys(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)
		for src, dest in rename_keys:
				rename_key(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)
		read_in_q_k_v(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)

		# load HuggingFace model
		if vit_name[-5:] == "in21k":
				a_     :      List[Any] 						=     ViTModel(SCREAMING_SNAKE_CASE__	).eval()
		else:
				a_     :      Union[str, Any] 						=     ViTForImageClassification(SCREAMING_SNAKE_CASE__	).eval()
		model.load_state_dict(SCREAMING_SNAKE_CASE__	)

		# Check outputs on an image, prepared by ViTImageProcessor/DeiTImageProcessor
		if "deit" in vit_name:
				a_     :      Optional[Any] 						=     DeiTImageProcessor(size=config.image_size	)
		else:
				a_     :      Any 						=     ViTImageProcessor(size=config.image_size	)
		a_     :      int 						=     image_processor(images=prepare_img(), return_tensors="pt"	)
		a_     :      Any 						=     encoding["pixel_values"]
		a_     :      Optional[int] 						=     model(SCREAMING_SNAKE_CASE__	)

		if base_model:
				a_     :      Optional[int] 						=     timm_model.forward_features(SCREAMING_SNAKE_CASE__	)
				assert timm_pooled_output.shape == outputs.pooler_output.shape
				assert torch.allclose(SCREAMING_SNAKE_CASE__, outputs.pooler_output, atol=1e-3	)
		else:
				a_     :      Union[str, Any] 						=     timm_model(SCREAMING_SNAKE_CASE__	)
				assert timm_logits.shape == outputs.logits.shape
				assert torch.allclose(SCREAMING_SNAKE_CASE__, outputs.logits, atol=1e-3	)

		Path(SCREAMING_SNAKE_CASE__	).mkdir(exist_ok=SCREAMING_SNAKE_CASE__	)
		print(F"""Saving model {vit_name} to {pytorch_dump_folder_path}"""	)
		model.save_pretrained(SCREAMING_SNAKE_CASE__	)
		print(F"""Saving image processor to {pytorch_dump_folder_path}"""	)
		image_processor.save_pretrained(SCREAMING_SNAKE_CASE__	)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE_				= argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--vit_name""",
					    default="""vit_base_patch16_224""",
					    type=str,
					    help="""Name of the ViT timm model you'd like to convert.""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
					)

					SCREAMING_SNAKE_CASE_				= parser.parse_args()
					convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path)




"""simple docstring"""
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class 			snake_case_						(						a_     ,unittest.TestCase				):
							__lowerCAmelCase    					=					BertJapaneseTokenizer
							__lowerCAmelCase    					=					False
							__lowerCAmelCase    					=					True






							def snake_case_  (   self      ):
									super().setUp()

									a_     :      str 						=     [
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "こんにちは",
									    "こん",
									    "にちは",
									    "ばんは",
									    "##こん",
									    "##にちは",
									    "##ばんは",
									    "世界",
									    "##世界",
									    "、",
									    "##、",
									    "。",
									    "##。",
									]

									a_     :      Tuple 						=     os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["vocab_file"]      )
									with open(self.vocab_file			,     "w"			,     encoding="utf-8"      ) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )






							def snake_case_  (   self			,     a_      ):
									a_     :      Dict 						=     "こんにちは、世界。 \nこんばんは、世界。"
									a_     :      int 						=     "こんにちは 、 世界 。 こんばんは 、 世界 。"
									return input_text, output_text






							def snake_case_  (   self			,     a_      ):
									a_						,				a_     :      List[str] 						=     self.get_input_output_texts(a_      )
									a_     :      Tuple 						=     tokenizer.encode(a_			,     add_special_tokens=a_      )
									a_     :      Dict 						=     tokenizer.decode(a_			,     clean_up_tokenization_spaces=a_      )
									return text, ids






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									a_     :      int 						=     self.tokenizer_class(self.vocab_file      )

									a_     :      List[Any] 						=     tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     self.tokenizer_class(self.vocab_file			,     word_tokenizer_type="mecab"      )
									self.assertIsNotNone(a_      )

									a_     :      Union[str, Any] 						=     "こんにちは、世界。\nこんばんは、世界。"
									a_     :      Union[str, Any] 						=     tokenizer.tokenize(a_      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

									a_     :      Any 						=     os.path.join(self.tmpdirname			,     "tokenizer.bin"      )
									with open(a_			,     "wb"      ) as handle:
											pickle.dump(a_			,     a_      )

									with open(a_			,     "rb"      ) as handle:
											a_     :      Optional[Any] 						=     pickle.load(a_      )

									a_     :      int 						=     tokenizer_new.tokenize(a_      )

									self.assertListEqual(a_			,     a_      )






							def snake_case_  (   self      ):
									a_     :      Tuple 						=     MecabTokenizer(mecab_dic="ipadic"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									try:
											a_     :      Dict 						=     MecabTokenizer(mecab_dic="unidic_lite"      )
									except ModuleNotFoundError:
											return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									try:
											a_     :      List[Any] 						=     MecabTokenizer(mecab_dic="unidic"      )
									except ModuleNotFoundError:
											return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     MecabTokenizer(do_lower_case=a_			,     mecab_dic="ipadic"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									try:
											a_     :      int 						=     MecabTokenizer(
											    do_lower_case=a_			,     normalize_text=a_			,     mecab_option="-d /usr/local/lib/mecab/dic/jumandic"      )
									except RuntimeError:
											# if dict doesn't exist in the system, previous code raises this error.
											return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]			,     )






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     MecabTokenizer(normalize_text=a_			,     mecab_dic="ipadic"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      int 						=     self.tokenizer_class(self.vocab_file			,     word_tokenizer_type="sudachi"      )
									self.assertIsNotNone(a_      )

									a_     :      Optional[Any] 						=     "こんにちは、世界。\nこんばんは、世界。"
									a_     :      str 						=     tokenizer.tokenize(a_      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

									a_     :      Dict 						=     os.path.join(self.tmpdirname			,     "tokenizer.bin"      )
									with open(a_			,     "wb"      ) as handle:
											pickle.dump(a_			,     a_      )

									with open(a_			,     "rb"      ) as handle:
											a_     :      Optional[Any] 						=     pickle.load(a_      )

									a_     :      int 						=     tokenizer_new.tokenize(a_      )

									self.assertListEqual(a_			,     a_      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      int 						=     SudachiTokenizer(sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      Any 						=     SudachiTokenizer(sudachi_dict_type="core"			,     sudachi_split_mode="A"      )

									self.assertListEqual(tokenizer.tokenize("外国人参政権"      )			,     ["外国", "人", "参政", "権"]      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     SudachiTokenizer(sudachi_dict_type="core"			,     sudachi_split_mode="B"      )

									self.assertListEqual(tokenizer.tokenize("外国人参政権"      )			,     ["外国人", "参政権"]      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      int 						=     SudachiTokenizer(sudachi_dict_type="core"			,     sudachi_split_mode="C"      )

									self.assertListEqual(tokenizer.tokenize("外国人参政権"      )			,     ["外国人参政権"]      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      List[str] 						=     SudachiTokenizer(do_lower_case=a_			,     sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      str 						=     SudachiTokenizer(normalize_text=a_			,     sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      Optional[int] 						=     SudachiTokenizer(trim_whitespace=a_			,     sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     self.tokenizer_class(self.vocab_file			,     word_tokenizer_type="jumanpp"      )
									self.assertIsNotNone(a_      )

									a_     :      Optional[int] 						=     "こんにちは、世界。\nこんばんは、世界。"
									a_     :      Any 						=     tokenizer.tokenize(a_      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

									a_     :      int 						=     os.path.join(self.tmpdirname			,     "tokenizer.bin"      )
									with open(a_			,     "wb"      ) as handle:
											pickle.dump(a_			,     a_      )

									with open(a_			,     "rb"      ) as handle:
											a_     :      Tuple 						=     pickle.load(a_      )

									a_     :      Union[str, Any] 						=     tokenizer_new.tokenize(a_      )

									self.assertListEqual(a_			,     a_      )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Any 						=     JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     JumanppTokenizer(do_lower_case=a_      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      int 						=     JumanppTokenizer(normalize_text=a_      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     JumanppTokenizer(trim_whitespace=a_      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Any 						=     JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"      )			,     ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]			,     )






							def snake_case_  (   self      ):
									a_     :      str 						=     ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

									a_     :      int 						=     {}
									for i, token in enumerate(a_      ):
											a_     :      Optional[int] 						=     i
									a_     :      Optional[int] 						=     WordpieceTokenizer(vocab=a_			,     unk_token="[UNK]"      )

									self.assertListEqual(tokenizer.tokenize(""      )			,     []      )

									self.assertListEqual(tokenizer.tokenize("こんにちは"      )			,     ["こんにちは"]      )

									self.assertListEqual(tokenizer.tokenize("こんばんは"      )			,     ["こん", "##ばんは"]      )

									self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"      )			,     ["こん", "##ばんは", "[UNK]", "こんにちは"]      )






							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"      )
									a_     :      Any 						=     tokenizer.subword_tokenizer

									a_     :      Union[str, Any] 						=     subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"      )
									self.assertListEqual(a_			,     ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]      )

									a_     :      int 						=     subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"      )
									self.assertListEqual(a_			,     ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]      )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"      )

									a_     :      Any 						=     tokenizer.encode("ありがとう。"			,     add_special_tokens=a_      )
									a_     :      Optional[Any] 						=     tokenizer.encode("どういたしまして。"			,     add_special_tokens=a_      )

									a_     :      Union[str, Any] 						=     tokenizer.build_inputs_with_special_tokens(a_      )
									a_     :      Tuple 						=     tokenizer.build_inputs_with_special_tokens(a_			,     a_      )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			snake_case_						(						a_     ,unittest.TestCase				):
							__lowerCAmelCase    					=					BertJapaneseTokenizer
							__lowerCAmelCase    					=					False






							def snake_case_  (   self      ):
									super().setUp()

									a_     :      Any 						=     ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

									a_     :      str 						=     os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["vocab_file"]      )
									with open(self.vocab_file			,     "w"			,     encoding="utf-8"      ) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )






							def snake_case_  (   self			,     **a_      ):
									return BertJapaneseTokenizer.from_pretrained(self.tmpdirname			,     subword_tokenizer_type="character"			,     **a_      )






							def snake_case_  (   self			,     a_      ):
									a_     :      int 						=     "こんにちは、世界。 \nこんばんは、世界。"
									a_     :      List[str] 						=     "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
									return input_text, output_text






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									a_     :      Dict 						=     self.tokenizer_class(self.vocab_file			,     subword_tokenizer_type="character"      )

									a_     :      int 						=     tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"      )
									self.assertListEqual(
									    a_			,     ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]      )
									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(a_      )			,     [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]      )






							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

									a_     :      Any 						=     {}
									for i, token in enumerate(a_      ):
											a_     :      str 						=     i
									a_     :      List[str] 						=     CharacterTokenizer(vocab=a_			,     unk_token="[UNK]"      )

									self.assertListEqual(tokenizer.tokenize(""      )			,     []      )

									self.assertListEqual(tokenizer.tokenize("こんにちは"      )			,     ["こ", "ん", "に", "ち", "は"]      )

									self.assertListEqual(tokenizer.tokenize("こんにちほ"      )			,     ["こ", "ん", "に", "ち", "[UNK]"]      )






							def snake_case_  (   self      ):
									a_     :      str 						=     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"      )

									a_     :      Optional[int] 						=     tokenizer.encode("ありがとう。"			,     add_special_tokens=a_      )
									a_     :      Tuple 						=     tokenizer.encode("どういたしまして。"			,     add_special_tokens=a_      )

									a_     :      Optional[Any] 						=     tokenizer.build_inputs_with_special_tokens(a_      )
									a_     :      Any 						=     tokenizer.build_inputs_with_special_tokens(a_			,     a_      )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			snake_case_						(						unittest.TestCase				):






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     "cl-tohoku/bert-base-japanese"
									a_     :      Dict 						=     AutoTokenizer.from_pretrained(a_      )
									self.assertIsInstance(a_			,     a_      )




class 			snake_case_						(						unittest.TestCase				):






							def snake_case_  (   self      ):
									a_     :      Any 						=     "cl-tohoku/bert-base-japanese"
									with self.assertLogs("transformers"			,     level="WARNING"      ) as cm:
											BertTokenizer.from_pretrained(a_      )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
											        " is called from."      )      )
									a_     :      Any 						=     "bert-base-cased"
									with self.assertLogs("transformers"			,     level="WARNING"      ) as cm:
											BertJapaneseTokenizer.from_pretrained(a_      )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
											        " is called from."      )      )