





import argparse
import os
from pathlib import Path

import torch
from bark.generation import _load_model as _bark_load_model
from huggingface_hub import hf_hub_download

from transformers import EncodecConfig, EncodecModel, set_seed
from transformers.models.bark.configuration_bark import (
    BarkCoarseConfig,
    BarkConfig,
    BarkFineConfig,
    BarkSemanticConfig,
)
from transformers.models.bark.generation_configuration_bark import (
    BarkCoarseGenerationConfig,
    BarkFineGenerationConfig,
    BarkGenerationConfig,
    BarkSemanticGenerationConfig,
)
from transformers.models.bark.modeling_bark import BarkCoarseModel, BarkFineModel, BarkModel, BarkSemanticModel
from transformers.utils import logging


logging.set_verbosity_info()
UpperCamelCase__      =logging.get_logger(__name__)

set_seed(770)


UpperCamelCase__      ={
    'c_attn': 'att_proj',
    'c_proj': 'out_proj',
    'c_fc': 'in_proj',
    'transformer.': '',
    'h.': 'layers.',
    'ln_1': 'layernorm_1',
    'ln_2': 'layernorm_2',
    'ln_f': 'layernorm_final',
    'wpe': 'position_embeds_layer',
    'wte': 'input_embeds_layer',
}


UpperCamelCase__      ={
    'text_small': {
        'repo_id': 'suno/bark',
        'file_name': 'text.pt',
    },
    'coarse_small': {
        'repo_id': 'suno/bark',
        'file_name': 'coarse.pt',
    },
    'fine_small': {
        'repo_id': 'suno/bark',
        'file_name': 'fine.pt',
    },
    'text': {
        'repo_id': 'suno/bark',
        'file_name': 'text_2.pt',
    },
    'coarse': {
        'repo_id': 'suno/bark',
        'file_name': 'coarse_2.pt',
    },
    'fine': {
        'repo_id': 'suno/bark',
        'file_name': 'fine_2.pt',
    },
}

UpperCamelCase__      =os.path.dirname(os.path.abspath(__file__))
UpperCamelCase__      =os.path.join(os.path.expanduser('~'), '.cache')
UpperCamelCase__      =os.path.join(os.getenv('XDG_CACHE_HOME', default_cache_dir), 'suno', 'bark_v0')






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase=False	):
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							model_type
     if use_small:
          key += "_small"
     return os.path.join(__lowerCamelCase,       REMOTE_MODEL_PATHS[key]["file_name"]	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     os.makedirs(__lowerCamelCase,       exist_ok=__lowerCamelCase	)
     hf_hub_download(repo_id=__lowerCamelCase,       filename=__lowerCamelCase,       local_dir=__lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase=False,       __lowerCamelCase="text"	):
     if model_type == "text":
          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							BarkSemanticModel
          _SCREAMING_SNAKE_CASE       :	Any						    =							BarkSemanticConfig
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							BarkSemanticGenerationConfig
     elif model_type == "coarse":
          _SCREAMING_SNAKE_CASE       :	List[str]						    =							BarkCoarseModel
          _SCREAMING_SNAKE_CASE       :	Any						    =							BarkCoarseConfig
          _SCREAMING_SNAKE_CASE       :	str						    =							BarkCoarseGenerationConfig
     elif model_type == "fine":
          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							BarkFineModel
          _SCREAMING_SNAKE_CASE       :	List[str]						    =							BarkFineConfig
          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							BarkFineGenerationConfig
     else:
          raise NotImplementedError()
     _SCREAMING_SNAKE_CASE       :	List[str]						    =							f"""{model_type}_small""" if use_small else model_type
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							REMOTE_MODEL_PATHS[model_key]
     if not os.path.exists(__lowerCamelCase	):
          logger.info(f"""{model_type} model not found, downloading into `{CACHE_DIR}`."""	)
          _download(model_info["repo_id"],       model_info["file_name"]	)
     _SCREAMING_SNAKE_CASE       :	str						    =							torch.load(__lowerCamelCase,       map_location=__lowerCamelCase	)
     # this is a hack
     _SCREAMING_SNAKE_CASE       :	Any						    =							checkpoint["model_args"]
     if "input_vocab_size" not in model_args:
          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							model_args["vocab_size"]
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							model_args["vocab_size"]
          del model_args["vocab_size"]

     # convert Bark model arguments to HF Bark model arguments
     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							model_args.pop("n_head"	)
     _SCREAMING_SNAKE_CASE       :	Dict						    =							model_args.pop("n_embd"	)
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							model_args.pop("n_layer"	)

     _SCREAMING_SNAKE_CASE       :	Tuple						    =							ConfigClass(**checkpoint["model_args"]	)
     _SCREAMING_SNAKE_CASE       :	int						    =							ModelClass(config=__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							GenerationConfigClass()

     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							model_generation_config
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							checkpoint["model"]
     # fixup checkpoint
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"_orig_mod."
     for k, v in list(state_dict.items()	):
          if k.startswith(__lowerCamelCase	):
               # replace part of the key with corresponding layer name in HF implementation
               _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							k[len(__lowerCamelCase	) :]
               for old_layer_name in new_layer_name_dict:
                    _SCREAMING_SNAKE_CASE       :	Tuple						    =							new_k.replace(__lowerCamelCase,       new_layer_name_dict[old_layer_name]	)

               _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							state_dict.pop(__lowerCamelCase	)

     _SCREAMING_SNAKE_CASE       :	Tuple						    =							set(state_dict.keys()	) - set(model.state_dict().keys()	)
     _SCREAMING_SNAKE_CASE       :	int						    =							{k for k in extra_keys if not k.endswith(".attn.bias"	)}
     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							set(model.state_dict().keys()	) - set(state_dict.keys()	)
     _SCREAMING_SNAKE_CASE       :	List[str]						    =							{k for k in missing_keys if not k.endswith(".attn.bias"	)}
     if len(__lowerCamelCase	) != 0:
          raise ValueError(f"""extra keys found: {extra_keys}"""	)
     if len(__lowerCamelCase	) != 0:
          raise ValueError(f"""missing keys: {missing_keys}"""	)
     model.load_state_dict(__lowerCamelCase,       strict=__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							model.num_parameters(exclude_embeddings=__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	str						    =							checkpoint["best_val_loss"].item()
     logger.info(f"""model loaded: {round(n_params/1e6,       1	)}M params, {round(__lowerCamelCase,       3	)} loss"""	)
     model.eval()
     model.to(__lowerCamelCase	)
     del checkpoint, state_dict

     return model






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase=False,       __lowerCamelCase="text"	):
     if model_type not in ("text", "coarse", "fine"):
          raise NotImplementedError()

     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							"cpu"  # do conversion on cpu

     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							_get_ckpt_path(__lowerCamelCase,       use_small=__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							_load_model(__lowerCamelCase,       __lowerCamelCase,       model_type=__lowerCamelCase,       use_small=__lowerCamelCase	)

     # load bark initial model
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							_bark_load_model(__lowerCamelCase,       "cpu",       model_type=__lowerCamelCase,       use_small=__lowerCamelCase	)

     if model_type == "text":
          _SCREAMING_SNAKE_CASE       :	str						    =							bark_model["model"]

     if model.num_parameters(exclude_embeddings=__lowerCamelCase	) != bark_model.get_num_params():
          raise ValueError("initial and new models don't have the same number of parameters"	)

     # check if same output as the bark model
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							5
     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							10

     if model_type in ["text", "coarse"]:
          _SCREAMING_SNAKE_CASE       :	Any						    =							torch.randint(256,       (batch_size, sequence_length),       dtype=torch.int	)
          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							bark_model(__lowerCamelCase	)[0]

          _SCREAMING_SNAKE_CASE       :	Any						    =							model(__lowerCamelCase	)

          # take last logits
          _SCREAMING_SNAKE_CASE       :	List[str]						    =							output_new_model_total.logits[:, [-1], :]

     else:
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							3
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							8
          _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							torch.randint(256,       (batch_size, sequence_length, n_codes_total),       dtype=torch.int	)

          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							model(__lowerCamelCase,       __lowerCamelCase	)
          _SCREAMING_SNAKE_CASE       :	Dict						    =							bark_model(__lowerCamelCase,       __lowerCamelCase	)

          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							output_new_model_total.logits

     # output difference should come from the difference of self-attention implementation design
     if output_new_model.shape != output_old_model.shape:
          raise ValueError("initial and new outputs don't have the same shape"	)
     if (output_new_model - output_old_model).abs().max().item() > 1e-3:
          raise ValueError("initial and new outputs are not equal"	)

     Path(__lowerCamelCase	).mkdir(exist_ok=__lowerCamelCase	)
     model.save_pretrained(__lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       ):
     _SCREAMING_SNAKE_CASE       :	Dict						    =							os.path.join(__lowerCamelCase,       __lowerCamelCase	)

     _SCREAMING_SNAKE_CASE       :	int						    =							BarkSemanticConfig.from_pretrained(os.path.join(__lowerCamelCase,       "config.json"	)	)
     _SCREAMING_SNAKE_CASE       :	Dict						    =							BarkCoarseConfig.from_pretrained(os.path.join(__lowerCamelCase,       "config.json"	)	)
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							BarkFineConfig.from_pretrained(os.path.join(__lowerCamelCase,       "config.json"	)	)
     _SCREAMING_SNAKE_CASE       :	Dict						    =							EncodecConfig.from_pretrained("facebook/encodec_24khz"	)

     _SCREAMING_SNAKE_CASE       :	int						    =							BarkSemanticModel.from_pretrained(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							BarkCoarseModel.from_pretrained(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							BarkFineModel.from_pretrained(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Dict						    =							EncodecModel.from_pretrained("facebook/encodec_24khz"	)

     _SCREAMING_SNAKE_CASE       :	Any						    =							BarkConfig.from_sub_model_configs(
         __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	)

     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							BarkGenerationConfig.from_sub_model_configs(
         semantic.generation_config,       coarseAcoustic.generation_config,       fineAcoustic.generation_config	)

     _SCREAMING_SNAKE_CASE       :	str						    =							BarkModel(__lowerCamelCase	)

     _SCREAMING_SNAKE_CASE       :	Tuple						    =							semantic
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							coarseAcoustic
     _SCREAMING_SNAKE_CASE       :	List[str]						    =							fineAcoustic
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							codec

     _SCREAMING_SNAKE_CASE       :	Tuple						    =							bark_generation_config

     Path(__lowerCamelCase	).mkdir(exist_ok=__lowerCamelCase	)
     bark.save_pretrained(__lowerCamelCase,       repo_id=__lowerCamelCase,       push_to_hub=__lowerCamelCase	)


if __name__ == "__main__":
       UpperCamelCase__      =argparse.ArgumentParser()
       # Required parameters

       parser.add_argument('model_type', type=str, help='text, coarse or fine.')
       parser.add_argument('pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
       parser.add_argument('--is_small', action='store_true', help='convert the small version instead of the large.')

       UpperCamelCase__      =parser.parse_args()

       load_model(args.pytorch_dump_folder_path, model_type=args.model_type, use_small=args.is_small)






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    BertJapaneseTokenizer
       __snake_case 			=    False
       __snake_case 			=    True




       def     UpperCamelCase_			(      self							)			->     Dict:
            super().setUp()

            _SCREAMING_SNAKE_CASE       :	Dict						    =							[
                "[UNK]",
                "[CLS]",
                "[SEP]",
                "こんにちは",
                "こん",
                "にちは",
                "ばんは",
                "##こん",
                "##にちは",
                "##ばんは",
                "世界",
                "##世界",
                "、",
                "##、",
                "。",
                "##。",
            ]

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"]							)
            with open(self.vocab_file    ,						"w"    ,						encoding="utf-8"							) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							"こんにちは、世界。 \nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	str						    =							"こんにちは 、 世界 。 こんばんは 、 世界 。"
            return input_text, output_text




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Dict:
            _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	int						    =							self.get_input_output_texts(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.encode(__lowerCamelCase    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.decode(__lowerCamelCase    ,						clean_up_tokenization_spaces=__lowerCamelCase							)
            return text, ids




       def     UpperCamelCase_			(      self							)			->     List[str]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     List[str]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							self.tokenizer_class(self.vocab_file							)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="mecab"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Any						    =							MecabTokenizer(mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     str:
            try:
                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							MecabTokenizer(mecab_dic="unidic_lite"							)
            except ModuleNotFoundError:
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            try:
                 _SCREAMING_SNAKE_CASE       :	Dict						    =							MecabTokenizer(mecab_dic="unidic"							)
            except ModuleNotFoundError:
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							MecabTokenizer(do_lower_case=__lowerCamelCase    ,						mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     Dict:
            try:
                 _SCREAMING_SNAKE_CASE       :	Any						    =							MecabTokenizer(
                     do_lower_case=__lowerCamelCase    ,						normalize_text=__lowerCamelCase    ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic"							)
            except RuntimeError:
                 # if dict doesn't exist in the system, previous code raises this error.
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							MecabTokenizer(normalize_text=__lowerCamelCase    ,						mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="sudachi"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="A"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国", "人", "参政", "権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     int:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="B"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国人", "参政権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="C"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国人参政権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							SudachiTokenizer(do_lower_case=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							SudachiTokenizer(normalize_text=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							SudachiTokenizer(trim_whitespace=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="jumanpp"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	Any						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Tuple:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							JumanppTokenizer(do_lower_case=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							JumanppTokenizer(normalize_text=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							JumanppTokenizer(trim_whitespace=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"							)    ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	int						    =							["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

            _SCREAMING_SNAKE_CASE       :	str						    =							{}
            for i, token in enumerate(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							i
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							WordpieceTokenizer(vocab=__lowerCamelCase    ,						unk_token="[UNK]"							)

            self.assertListEqual(tokenizer.tokenize(""							)    ,						[]							)

            self.assertListEqual(tokenizer.tokenize("こんにちは"							)    ,						["こんにちは"]							)

            self.assertListEqual(tokenizer.tokenize("こんばんは"							)    ,						["こん", "##ばんは"]							)

            self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"							)    ,						["こん", "##ばんは", "[UNK]", "こんにちは"]							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	int						    =							BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.subword_tokenizer

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]							)

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]							)







       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.encode("ありがとう。"    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.encode("どういたしまして。"    ,						add_special_tokens=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    ,						__lowerCamelCase							)

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    BertJapaneseTokenizer
       __snake_case 			=    False




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            super().setUp()

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"]							)
            with open(self.vocab_file    ,						"w"    ,						encoding="utf-8"							) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)




       def     UpperCamelCase_			(      self    ,						**__lowerCamelCase							)			->     List[Any]:
            return BertJapaneseTokenizer.from_pretrained(self.tmpdirname    ,						subword_tokenizer_type="character"    ,						**__lowerCamelCase							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							"こんにちは、世界。 \nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	Dict						    =							"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
            return input_text, output_text




       def     UpperCamelCase_			(      self							)			->     Tuple:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     int:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	int						    =							self.tokenizer_class(self.vocab_file    ,						subword_tokenizer_type="character"							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"							)
            self.assertListEqual(
                __lowerCamelCase    ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]							)
            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							{}
            for i, token in enumerate(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	int						    =							i
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							CharacterTokenizer(vocab=__lowerCamelCase    ,						unk_token="[UNK]"							)

            self.assertListEqual(tokenizer.tokenize(""							)    ,						[]							)

            self.assertListEqual(tokenizer.tokenize("こんにちは"							)    ,						["こ", "ん", "に", "ち", "は"]							)

            self.assertListEqual(tokenizer.tokenize("こんにちほ"							)    ,						["こ", "ん", "に", "ち", "[UNK]"]							)







       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"							)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.encode("ありがとう。"    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.encode("どういたしまして。"    ,						add_special_tokens=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    ,						__lowerCamelCase							)

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							"cl-tohoku/bert-base-japanese"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							AutoTokenizer.from_pretrained(__lowerCamelCase							)
            self.assertIsInstance(__lowerCamelCase    ,						__lowerCamelCase							)





class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"cl-tohoku/bert-base-japanese"
            with self.assertLogs("transformers"    ,						level="WARNING"							) as cm:
                 BertTokenizer.from_pretrained(__lowerCamelCase							)
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from."							)							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"bert-base-cased"
            with self.assertLogs("transformers"    ,						level="WARNING"							) as cm:
                 BertJapaneseTokenizer.from_pretrained(__lowerCamelCase							)
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from."							)							)