



import argparse
import json
import pickle
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import MaskFormerConfig, MaskFormerForInstanceSegmentation, MaskFormerImageProcessor, SwinConfig
from transformers.utils import logging


logging.set_verbosity_info()
__snake_case     :int			       =logging.get_logger(__name__)






def  lowerCamelCase_	(      lowerCAmelCase__	:	str				)       ->							List[Any]:

							'''simple docstring'''






							A      						=    SwinConfig.from_pretrained(
							    'microsoft/swin-tiny-patch4-window7-224'			,	out_features=['stage1', 'stage2', 'stage3', 'stage4']				)
							A      						=    MaskFormerConfig(backbone_config=lowerCAmelCase__				)

							A      						=    'huggingface/label-files'
							if "ade20k-full" in model_name:
														# this should be ok
														A      						=    847
														A      						=    'maskformer-ade20k-full-id2label.json'
							elif "ade" in model_name:
														# this should be ok
														A      						=    150
														A      						=    'ade20k-id2label.json'
							elif "coco-stuff" in model_name:
														# this should be ok
														A      						=    171
														A      						=    'maskformer-coco-stuff-id2label.json'
							elif "coco" in model_name:
														# TODO
														A      						=    133
														A      						=    'coco-panoptic-id2label.json'
							elif "cityscapes" in model_name:
														# this should be ok
														A      						=    19
														A      						=    'cityscapes-id2label.json'
							elif "vistas" in model_name:
														# this should be ok
														A      						=    65
														A      						=    'mapillary-vistas-id2label.json'

							A      						=    json.load(open(hf_hub_download(lowerCAmelCase__			,	lowerCAmelCase__			,	repo_type='dataset'				)			,	'r'				)				)
							A      						=    {int(lowerCAmelCase__				): v for k, v in idalabel.items()}

							return config






def  lowerCamelCase_	(      lowerCAmelCase__	:	List[Any]				)       ->							Dict:

							'''simple docstring'''






							A      						=    []
							# stem
							# fmt: off
							rename_keys.append(('backbone.patch_embed.proj.weight', 'model.pixel_level_module.encoder.model.embeddings.patch_embeddings.projection.weight')				)
							rename_keys.append(('backbone.patch_embed.proj.bias', 'model.pixel_level_module.encoder.model.embeddings.patch_embeddings.projection.bias')				)
							rename_keys.append(('backbone.patch_embed.norm.weight', 'model.pixel_level_module.encoder.model.embeddings.norm.weight')				)
							rename_keys.append(('backbone.patch_embed.norm.bias', 'model.pixel_level_module.encoder.model.embeddings.norm.bias')				)
							# stages
							for i in range(len(config.backbone_config.depths				)				):
														for j in range(config.backbone_config.depths[i]				):
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.norm1.weight''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_before.weight''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.norm1.bias''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_before.bias''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.attn.relative_position_bias_table''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_bias_table''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.attn.relative_position_index''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_index''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.attn.proj.weight''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.weight''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.attn.proj.bias''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.bias''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.norm2.weight''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_after.weight''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.norm2.bias''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_after.bias''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.mlp.fc1.weight''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.weight''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.mlp.fc1.bias''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.bias''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.mlp.fc2.weight''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.output.dense.weight''')				)
																					rename_keys.append((F'''backbone.layers.{i}.blocks.{j}.mlp.fc2.bias''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.output.dense.bias''')				)

														if i < 3:
																					rename_keys.append((F'''backbone.layers.{i}.downsample.reduction.weight''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.reduction.weight''')				)
																					rename_keys.append((F'''backbone.layers.{i}.downsample.norm.weight''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.norm.weight''')				)
																					rename_keys.append((F'''backbone.layers.{i}.downsample.norm.bias''', F'''model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.norm.bias''')				)
														rename_keys.append((F'''backbone.norm{i}.weight''', F'''model.pixel_level_module.encoder.hidden_states_norms.{i}.weight''')				)
														rename_keys.append((F'''backbone.norm{i}.bias''', F'''model.pixel_level_module.encoder.hidden_states_norms.{i}.bias''')				)

							# FPN
							rename_keys.append(('sem_seg_head.layer_4.weight', 'model.pixel_level_module.decoder.fpn.stem.0.weight')				)
							rename_keys.append(('sem_seg_head.layer_4.norm.weight', 'model.pixel_level_module.decoder.fpn.stem.1.weight')				)
							rename_keys.append(('sem_seg_head.layer_4.norm.bias', 'model.pixel_level_module.decoder.fpn.stem.1.bias')				)
							for source_index, target_index in zip(range(3			,	0			,	-1				)			,	range(0			,	3				)				):
														rename_keys.append((F'''sem_seg_head.adapter_{source_index}.weight''', F'''model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.0.weight''')				)
														rename_keys.append((F'''sem_seg_head.adapter_{source_index}.norm.weight''', F'''model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.1.weight''')				)
														rename_keys.append((F'''sem_seg_head.adapter_{source_index}.norm.bias''', F'''model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.1.bias''')				)
														rename_keys.append((F'''sem_seg_head.layer_{source_index}.weight''', F'''model.pixel_level_module.decoder.fpn.layers.{target_index}.block.0.weight''')				)
														rename_keys.append((F'''sem_seg_head.layer_{source_index}.norm.weight''', F'''model.pixel_level_module.decoder.fpn.layers.{target_index}.block.1.weight''')				)
														rename_keys.append((F'''sem_seg_head.layer_{source_index}.norm.bias''', F'''model.pixel_level_module.decoder.fpn.layers.{target_index}.block.1.bias''')				)
							rename_keys.append(('sem_seg_head.mask_features.weight', 'model.pixel_level_module.decoder.mask_projection.weight')				)
							rename_keys.append(('sem_seg_head.mask_features.bias', 'model.pixel_level_module.decoder.mask_projection.bias')				)

							# Transformer decoder
							for idx in range(config.decoder_config.decoder_layers				):
														# self-attention out projection
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.out_proj.weight''', F'''model.transformer_module.decoder.layers.{idx}.self_attn.out_proj.weight''')				)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.out_proj.bias''', F'''model.transformer_module.decoder.layers.{idx}.self_attn.out_proj.bias''')				)
														# cross-attention out projection
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.out_proj.weight''', F'''model.transformer_module.decoder.layers.{idx}.encoder_attn.out_proj.weight''')				)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.out_proj.bias''', F'''model.transformer_module.decoder.layers.{idx}.encoder_attn.out_proj.bias''')				)
														# MLP 1
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear1.weight''', F'''model.transformer_module.decoder.layers.{idx}.fc1.weight''')				)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear1.bias''', F'''model.transformer_module.decoder.layers.{idx}.fc1.bias''')				)
														# MLP 2
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear2.weight''', F'''model.transformer_module.decoder.layers.{idx}.fc2.weight''')				)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear2.bias''', F'''model.transformer_module.decoder.layers.{idx}.fc2.bias''')				)
														# layernorm 1 (self-attention layernorm)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm1.weight''', F'''model.transformer_module.decoder.layers.{idx}.self_attn_layer_norm.weight''')				)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm1.bias''', F'''model.transformer_module.decoder.layers.{idx}.self_attn_layer_norm.bias''')				)
														# layernorm 2 (cross-attention layernorm)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm2.weight''', F'''model.transformer_module.decoder.layers.{idx}.encoder_attn_layer_norm.weight''')				)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm2.bias''', F'''model.transformer_module.decoder.layers.{idx}.encoder_attn_layer_norm.bias''')				)
														# layernorm 3 (final layernorm)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm3.weight''', F'''model.transformer_module.decoder.layers.{idx}.final_layer_norm.weight''')				)
														rename_keys.append((F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm3.bias''', F'''model.transformer_module.decoder.layers.{idx}.final_layer_norm.bias''')				)

							rename_keys.append(('sem_seg_head.predictor.transformer.decoder.norm.weight', 'model.transformer_module.decoder.layernorm.weight')				)
							rename_keys.append(('sem_seg_head.predictor.transformer.decoder.norm.bias', 'model.transformer_module.decoder.layernorm.bias')				)

							# heads on top
							rename_keys.append(('sem_seg_head.predictor.query_embed.weight', 'model.transformer_module.queries_embedder.weight')				)

							rename_keys.append(('sem_seg_head.predictor.input_proj.weight', 'model.transformer_module.input_projection.weight')				)
							rename_keys.append(('sem_seg_head.predictor.input_proj.bias', 'model.transformer_module.input_projection.bias')				)

							rename_keys.append(('sem_seg_head.predictor.class_embed.weight', 'class_predictor.weight')				)
							rename_keys.append(('sem_seg_head.predictor.class_embed.bias', 'class_predictor.bias')				)

							for i in range(3				):
														rename_keys.append((F'''sem_seg_head.predictor.mask_embed.layers.{i}.weight''', F'''mask_embedder.{i}.0.weight''')				)
														rename_keys.append((F'''sem_seg_head.predictor.mask_embed.layers.{i}.bias''', F'''mask_embedder.{i}.0.bias''')				)
							# fmt: on

							return rename_keys






def  lowerCamelCase_	(      lowerCAmelCase__	:	List[Any]			,	lowerCAmelCase__	:	Optional[int]			,	lowerCAmelCase__	:	str				)       ->							List[Any]:

							'''simple docstring'''






							A      						=    dct.pop(lowerCAmelCase__				)
							A      						=    val






def  lowerCamelCase_	(      lowerCAmelCase__	:	str			,	lowerCAmelCase__	:	Dict				)       ->							Any:

							'''simple docstring'''






							A      						=    [int(backbone_config.embed_dim * 2**i				) for i in range(len(backbone_config.depths				)				)]
							for i in range(len(backbone_config.depths				)				):
														A      						=    num_features[i]
														for j in range(backbone_config.depths[i]				):
																					# fmt: off
																					# read in weights + bias of input projection layer (in original implementation, this is a single matrix + bias)
																					A      						=    state_dict.pop(F'''backbone.layers.{i}.blocks.{j}.attn.qkv.weight'''				)
																					A      						=    state_dict.pop(F'''backbone.layers.{i}.blocks.{j}.attn.qkv.bias'''				)
																					# next, add query, keys and values (in that order) to the state dict
																					A      						=    in_proj_weight[:dim, :]
																					A      						=    in_proj_bias[: dim]
																					A      						=    in_proj_weight[
																					    dim : dim * 2, :
																					]
																					A      						=    in_proj_bias[
																					    dim : dim * 2
																					]
																					A      						=    in_proj_weight[
																					    -dim :, :
																					]
																					A      						=    in_proj_bias[-dim :]
																					# fmt: on






def  lowerCamelCase_	(      lowerCAmelCase__	:	str			,	lowerCAmelCase__	:	List[Any]				)       ->							Optional[int]:

							'''simple docstring'''






							A      						=    config.decoder_config.hidden_size
							for idx in range(config.decoder_config.decoder_layers				):
														# read in weights + bias of self-attention input projection layer (in the original implementation, this is a single matrix + bias)
														A      						=    state_dict.pop(F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.in_proj_weight'''				)
														A      						=    state_dict.pop(F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.in_proj_bias'''				)
														# next, add query, keys and values (in that order) to the state dict
														A      						=    in_proj_weight[: hidden_size, :]
														A      						=    in_proj_bias[:config.hidden_size]
														A      						=    in_proj_weight[hidden_size : hidden_size * 2, :]
														A      						=    in_proj_bias[hidden_size : hidden_size * 2]
														A      						=    in_proj_weight[-hidden_size :, :]
														A      						=    in_proj_bias[-hidden_size :]
														# read in weights + bias of cross-attention input projection layer (in the original implementation, this is a single matrix + bias)
														A      						=    state_dict.pop(F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.in_proj_weight'''				)
														A      						=    state_dict.pop(F'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.in_proj_bias'''				)
														# next, add query, keys and values (in that order) to the state dict
														A      						=    in_proj_weight[: hidden_size, :]
														A      						=    in_proj_bias[:config.hidden_size]
														A      						=    in_proj_weight[hidden_size : hidden_size * 2, :]
														A      						=    in_proj_bias[hidden_size : hidden_size * 2]
														A      						=    in_proj_weight[-hidden_size :, :]
														A      						=    in_proj_bias[-hidden_size :]
							# fmt: on






def  lowerCamelCase_	(      )       ->							torch.Tensor:

							'''simple docstring'''






							A      						=    'http://images.cocodataset.org/val2017/000000039769.jpg'
							A      						=    Image.open(requests.get(lowerCAmelCase__			,	stream=lowerCAmelCase__				).raw				)
							return im






@torch.no_grad()
def  lowerCamelCase_	(      lowerCAmelCase__	:	str			,	lowerCAmelCase__	:	str			,	lowerCAmelCase__	:	str			,	lowerCAmelCase__	:	bool = False				)       ->							Optional[Any]:

							'''simple docstring'''






							A      						=    get_maskformer_config(lowerCAmelCase__				)

							# load original state_dict
							with open(lowerCAmelCase__			,	'rb'				) as f:
														A      						=    pickle.load(lowerCAmelCase__				)
							A      						=    data['model']

							# for name, param in state_dict.items():
							#     print(name, param.shape)

							# rename keys
							A      						=    create_rename_keys(lowerCAmelCase__				)
							for src, dest in rename_keys:
														rename_key(lowerCAmelCase__			,	lowerCAmelCase__			,	lowerCAmelCase__				)
							read_in_swin_q_k_v(lowerCAmelCase__			,	config.backbone_config				)
							read_in_decoder_q_k_v(lowerCAmelCase__			,	lowerCAmelCase__				)

							# update to torch tensors
							for key, value in state_dict.items():
														A      						=    torch.from_numpy(lowerCAmelCase__				)

							# load 🤗 model
							A      						=    MaskFormerForInstanceSegmentation(lowerCAmelCase__				)
							model.eval()

							for name, param in model.named_parameters():
														print(lowerCAmelCase__			,	param.shape				)

							A						,   A      						=    model.load_state_dict(lowerCAmelCase__			,	strict=lowerCAmelCase__				)
							assert missing_keys == [
							    "model.pixel_level_module.encoder.model.layernorm.weight",
							    "model.pixel_level_module.encoder.model.layernorm.bias",
							]
							assert len(lowerCAmelCase__				) == 0, F'''Unexpected keys: {unexpected_keys}'''

							# verify results
							A      						=    prepare_img()
							if "vistas" in model_name:
														A      						=    65
							elif "cityscapes" in model_name:
														A      						=    65535
							else:
														A      						=    255
							A      						=    True if 'ade' in model_name else False
							A      						=    MaskFormerImageProcessor(ignore_index=lowerCAmelCase__			,	reduce_labels=lowerCAmelCase__				)

							A      						=    image_processor(lowerCAmelCase__			,	return_tensors='pt'				)

							A      						=    model(**lowerCAmelCase__				)

							print('Logits:'			,	outputs.class_queries_logits[0, :3, :3]				)

							if model_name == "maskformer-swin-tiny-ade":
														A      						=    torch.tensor(
														    [[3.6353, -4.4770, -2.6065], [0.5081, -4.2394, -3.5343], [2.1909, -5.0353, -1.9323]]				)
							assert torch.allclose(outputs.class_queries_logits[0, :3, :3]			,	lowerCAmelCase__			,	atol=1E-4				)
							print('Looks ok!'				)

							if pytorch_dump_folder_path is not None:
														print(F'''Saving model and image processor to {pytorch_dump_folder_path}'''				)
														Path(lowerCAmelCase__				).mkdir(exist_ok=lowerCAmelCase__				)
														model.save_pretrained(lowerCAmelCase__				)
														image_processor.save_pretrained(lowerCAmelCase__				)

							if push_to_hub:
														print('Pushing model and image processor to the hub...'				)
														model.push_to_hub(F'''nielsr/{model_name}'''				)
														image_processor.push_to_hub(F'''nielsr/{model_name}'''				)


if __name__ == "__main__":
				__snake_case     :int			       =argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '--model_name',
				    default='maskformer-swin-tiny-ade',
				    type=str,
				    help=('Name of the MaskFormer model you\'d like to convert',),
				)
				parser.add_argument(
				    '--checkpoint_path',
				    default='/Users/nielsrogge/Documents/MaskFormer_checkpoints/MaskFormer-Swin-tiny-ADE20k/model.pkl',
				    type=str,
				    help='Path to the original state dict (.pth file).',
				)
				parser.add_argument(
				    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
				)
				parser.add_argument(
				    '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
				)

				__snake_case     :List[Any]			       =parser.parse_args()
				convert_maskformer_checkpoint(
				    args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path, args.push_to_hub
				)




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        lowerCAmelCase__					(  _lowerCamelCase ,				unittest.TestCase       ):
				A_   :     Dict								=     GPTSanJapaneseTokenizer
				A_   :     Optional[int]								=     False
				A_   :     List[str]								=     {'do_clean_text': False, 'add_prefix_space': False}


				def __UpperCamelCase						(							self      :  Tuple       )		->     Any:
											super().setUp()

											# fmt: off
											A      						=    ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
											# fmt: on
											A      						=    {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
											A      						=    {'unk_token': '<unk>'}

											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['vocab_file']       )
											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['emoji_file']       )
											with open(self.vocab_file						, 'w'						, encoding='utf-8'       ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )
											with open(self.emoji_file						, 'w'       ) as emoji_writer:
																		emoji_writer.write(json.dumps(__UpperCamelCase       )       )


				def __UpperCamelCase						(							self      :  Any						, **__UpperCamelCase      :  Any       )		->     Tuple:
											kwargs.update(self.special_tokens_map       )
											return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Dict       )		->     List[str]:
											A      						=    'こんにちは、世界。 \nこんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。 \nこんばんは、世界。😀'
											return input_text, output_text


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Optional[Any]       )		->     List[str]:
											A						,   A      						=    self.get_input_output_texts(__UpperCamelCase       )
											A      						=    tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase						, clean_up_tokenization_spaces=__UpperCamelCase       )
											return text, ids


				def __UpperCamelCase						(							self      :  Any       )		->     int:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     Tuple:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     List[Any]:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     int:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、世界。　こんばんは、㔺界。'
											A      						=    ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
											A      						=    tokenizer.tokenize(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids without special tokens
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids with special tokens
											A      						=    tokens + [tokenizer.unk_token]
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[str]:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
											A      						=    'こんにちは、、、、世界。こんばんは、、、、世界。'
											A      						=    tokenizer.encode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。こんばんは、世界。😀'
											A      						=    tokenizer.encode(prefix_text + input_text       )
											A      						=    tokenizer.encode(''						, prefix_text=prefix_text + input_text       )
											A      						=    tokenizer.encode(__UpperCamelCase						, prefix_text=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'

											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2
											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2

											A      						=    [1] + [0] * (len_prefix + len_text + 1)
											A      						=    [1] * (len_prefix + len_text + 1) + [0]
											A      						=    [1] + [1] * (len_prefix) + [0] * (len_text + 1)

											A      						=    tokenizer(prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(''						, prefix_text=prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(__UpperCamelCase						, prefix_text=__UpperCamelCase       ).token_type_ids
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    tokenizer.encode('あンいワ'       )
											A      						=    tokenizer.encode(''						, prefix_text='あンいワ'       )
											A      						=    tokenizer.encode('いワ'						, prefix_text='あン'       )

											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(x_token_a[1]						, x_token_a[-1]       )  # SEG token
											self.assertEqual(x_token_a[1]						, x_token_a[3]       )  # SEG token


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
											A      						=    tokenizer(__UpperCamelCase						, padding=__UpperCamelCase       )
											A      						=    tokenizer.batch_encode_plus(__UpperCamelCase						, padding=__UpperCamelCase       )

											# fmt: off
											A      						=    [[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
											A      						=    [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
											A      						=    [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
											# fmt: on
											self.assertListEqual(x_token.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.attention_mask						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.attention_mask						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Tuple       )		->     str:
											# Intentionally convert some words to accommodate character fluctuations unique to Japanese
											pass


				def __UpperCamelCase						(							self      :  Dict       )		->     int:
											# tokenizer has no padding token
											pass