



import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
  from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
  a__    =	True
  from torch.cuda.amp import autocast


a__    =	logging.getLogger(__name__)







@dataclass
class 		snake_case						:



   '''simple docstring'''






   snake_case_					: Optional[Any]          =		field(
       metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}  )
   snake_case_					: List[Any]          =		field(
       default=lowercase__					,metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}					,)
   snake_case_					: Any          =		field(
       default=lowercase__					,metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""}  )
   snake_case_					: Any          =		field(
       default=lowercase__					,metadata={"""help""": """Whether to log verbose messages or not."""}					,)
   snake_case_					: str          =		field(
       default=2.0					,metadata={"""help""": """Maximum temperature for gumbel softmax."""}  )
   snake_case_					: Dict          =		field(
       default=0.5					,metadata={"""help""": """Minimum temperature for gumbel softmax."""}  )
   snake_case_					: Any          =		field(
       default=0.99_9995					,metadata={"""help""": """Decay of gumbel temperature during training."""}  )



def       lowercase       (   SCREAMING_SNAKE_CASE__       :	ModelArguments				,       SCREAMING_SNAKE_CASE__       :	TrainingArguments				)  ->     Optional[Any]:
 logging.basicConfig(
     format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""				,       datefmt="""%m/%d/%Y %H:%M:%S"""				,       handlers=[logging.StreamHandler(sys.stdout				)]				,       )
 _snake_case    :  str					=					logging.WARNING
 if model_args.verbose_logging:
  _snake_case    :  str					=					logging.DEBUG
 elif trainer_utils.is_main_process(training_args.local_rank				):
  _snake_case    :  List[str]					=					logging.INFO
 logger.setLevel(_lowerCamelCase				)







@dataclass
class 		snake_case						:



   '''simple docstring'''






   snake_case_					: Any          =		field(
       default=lowercase__					,metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}  )
   snake_case_					: int          =		field(
       default=lowercase__					,metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}  )
   snake_case_					: Optional[int]          =		field(
       default="""train"""					,metadata={
           """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
       }					,)
   snake_case_					: Optional[Any]          =		field(
       default="""validation"""					,metadata={
           """help""": (
               """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
           )
       }					,)
   snake_case_					: int          =		field(
       default="""file"""					,metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}					,)
   snake_case_					: List[Any]          =		field(
       default=lowercase__					,metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}  )
   snake_case_					: Any          =		field(
       default=1					,metadata={
           """help""": """The percentage of the train set used as validation set in case there's no validation split"""
       }					,)
   snake_case_					: Dict          =		field(
       default=lowercase__					,metadata={"""help""": """The number of processes to use for the preprocessing."""}					,)
   snake_case_					: List[Any]          =		field(
       default=20.0					,metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""}  )







@dataclass
class 		snake_case						:



   '''simple docstring'''






   snake_case_					: Optional[int]          =		42
   snake_case_					: Dict          =		42
   snake_case_					: Tuple          =		"""longest"""
   snake_case_					: List[str]          =		None
   snake_case_					: List[Any]          =		None

   def __call__(					self							:			Any				,		lowerCAmelCase							:			List[Dict[str, Union[List[int], torch.Tensor]]])				->				Dict[str, torch.Tensor]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					self.feature_extractor.pad(
        lowerCAmelCase				,		max_length=self.max_length				,		padding=self.padding				,		pad_to_multiple_of=self.pad_to_multiple_of				,		return_tensors="""pt"""				,		)
    _snake_case    :  Optional[Any]					=					self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1])

    _snake_case    :  int					=					batch["input_values"].shape[0]

    # make sure that no loss is computed on padded inputs
    if batch["attention_mask"] is not None:
     # compute real output lengths according to convolution formula
     _snake_case    :  List[Any]					=					self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1)).to(
         torch.long)

     _snake_case    :  int					=					torch.zeros(
         (batch_size, mask_indices_seq_length)				,		dtype=torch.long				,		device=batch["""input_values"""].device)

     # these two operations makes sure that all values
     # before the output lengths indices are attended to
     _snake_case    :  Union[str, Any]					=					1
     _snake_case    :  Optional[int]					=					attention_mask.flip([-1]).cumsum(-1).flip([-1]).bool()

    # sample randomly masked indices
    _snake_case    :  Optional[Any]					=					_compute_mask_indices(
        (batch_size, mask_indices_seq_length)				,		self.model.config.mask_time_prob				,		self.model.config.mask_time_length				,		attention_mask=lowerCAmelCase				,		min_masks=2				,		)

    return batch







class 		snake_case						(	lowercase__  ):



   '''simple docstring'''






   def __init__(					self							:			Union[str, Any]				,		*lowerCAmelCase							:			Dict				,		lowerCAmelCase							:			List[Any]=1				,		lowerCAmelCase							:			Optional[Any]=0				,		lowerCAmelCase							:			Tuple=1.0				,		**lowerCAmelCase							:			Tuple)				->				Optional[int]:

    """simple docstring"""







    super().__init__(*lowerCAmelCase				,		**lowerCAmelCase)
    _snake_case    :  List[str]					=					0
    _snake_case    :  str					=					max_gumbel_temp
    _snake_case    :  str					=					min_gumbel_temp
    _snake_case    :  Union[str, Any]					=					gumbel_temp_decay


   def      UpperCamelCase_  (					self							:			Any				,		lowerCAmelCase							:			nn.Module				,		lowerCAmelCase							:			Dict[str, Union[torch.Tensor, Any]])				->				torch.Tensor:

    """simple docstring"""







    model.train()
    _snake_case    :  int					=					self._prepare_inputs(lowerCAmelCase)

    if self.use_amp:
     with autocast():
      _snake_case    :  Optional[int]					=					self.compute_loss(lowerCAmelCase				,		lowerCAmelCase)
    else:
     _snake_case    :  List[str]					=					self.compute_loss(lowerCAmelCase				,		lowerCAmelCase)

    if self.args.n_gpu > 1 or self.deepspeed:
     if model.module.config.ctc_loss_reduction == "mean":
      _snake_case    :  Any					=					loss.mean()
     elif model.module.config.ctc_loss_reduction == "sum":
      _snake_case    :  Tuple					=					loss.sum() / (inputs["mask_time_indices"]).sum()
     else:
      raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']''')

    if self.args.gradient_accumulation_steps > 1:
     _snake_case    :  Union[str, Any]					=					loss / self.args.gradient_accumulation_steps

    if self.use_amp:
     self.scaler.scale(lowerCAmelCase).backward()
    elif self.use_apex:
     with amp.scale_loss(lowerCAmelCase				,		self.optimizer) as scaled_loss:
      scaled_loss.backward()
    elif self.deepspeed:
     self.deepspeed.backward(lowerCAmelCase)
    else:
     loss.backward()

    self.num_update_step += 1
    # make sure gumbel softmax temperature is decayed
    if self.args.n_gpu > 1 or self.deepspeed:
     model.module.set_gumbel_temperature(
         max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step				,		self.min_gumbel_temp))
    else:
     model.set_gumbel_temperature(
         max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step				,		self.min_gumbel_temp))

    return loss.detach()



def       lowercase       (   )  ->     Tuple:
 _snake_case    :  List[Any]					=					HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)				)

 _snake_case    :  int					=					parser.parse_args_into_dataclasses()
 configure_logger(_lowerCamelCase				,       _lowerCamelCase				)

 # Downloading and loading a dataset from the hub.
 _snake_case    :  Tuple					=					load_dataset(data_args.dataset_name				,       data_args.dataset_config_name				,       cache_dir=model_args.cache_dir				)

 if "validation" not in datasets.keys():
  # make sure only "validation" and "train" keys remain"
  _snake_case    :  Optional[int]					=					DatasetDict()
  _snake_case    :  Optional[int]					=					load_dataset(
      data_args.dataset_name				,       data_args.dataset_config_name				,       split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''				,       cache_dir=model_args.cache_dir				,       )
  _snake_case    :  Any					=					load_dataset(
      data_args.dataset_name				,       data_args.dataset_config_name				,       split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''				,       cache_dir=model_args.cache_dir				,       )
 else:
  # make sure only "validation" and "train" keys remain"
  _snake_case    :  str					=					DatasetDict()
  _snake_case    :  int					=					load_dataset(
      data_args.dataset_name				,       data_args.dataset_config_name				,       split="""validation"""				,       cache_dir=model_args.cache_dir				,       )
  _snake_case    :  Optional[int]					=					load_dataset(
      data_args.dataset_name				,       data_args.dataset_config_name				,       split=F'''{data_args.train_split_name}'''				,       cache_dir=model_args.cache_dir				,       )

 # only normalized-inputs-training is supported
 _snake_case    :  Optional[Any]					=					WavaVecaFeatureExtractor.from_pretrained(
     model_args.model_name_or_path				,       cache_dir=model_args.cache_dir				,       do_normalize=_lowerCamelCase				)

 def prepare_dataset(SCREAMING_SNAKE_CASE__       :	Any				):
  # check that all files have the correct sampling rate
  _snake_case    :  Dict					=					librosa.load(batch[data_args.speech_file_column]				,       sr=feature_extractor.sampling_rate				)
  return batch

 # load audio files into numpy arrays
 _snake_case    :  Optional[Any]					=					datasets.map(
     _lowerCamelCase				,       num_proc=data_args.preprocessing_num_workers				,       remove_columns=datasets["""train"""].column_names				)

 # filter audio files that are too long
 _snake_case    :  int					=					vectorized_datasets.filter(
     lambda SCREAMING_SNAKE_CASE__				: len(data["""speech"""]				) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate				)				)

 def normalize(SCREAMING_SNAKE_CASE__       :	Union[str, Any]				):
  return feature_extractor(batch["""speech"""]				,       sampling_rate=feature_extractor.sampling_rate				)

 # normalize and transform to `BatchFeatures`
 _snake_case    :  int					=					vectorized_datasets.map(
     _lowerCamelCase				,       batched=_lowerCamelCase				,       num_proc=data_args.preprocessing_num_workers				,       load_from_cache_file=not data_args.overwrite_cache				,       remove_columns=vectorized_datasets["""train"""].column_names				,       )

 # pretraining is only supported for "newer" stable layer norm architecture
 # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
 _snake_case    :  Any					=					WavaVecaConfig.from_pretrained(
     model_args.model_name_or_path				,       cache_dir=model_args.cache_dir				,       gradient_checkpointing=training_args.gradient_checkpointing				,       )

 if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
  raise ValueError(
      """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
      """ ``config.feat_extract_norm='layer'"""				)

 _snake_case    :  Dict					=					WavaVecaForPreTraining(_lowerCamelCase				)

 _snake_case    :  Optional[int]					=					DataCollatorForWavaVecaPretraining(model=_lowerCamelCase				,       feature_extractor=_lowerCamelCase				)

 _snake_case    :  Any					=					WavaVecaPreTrainer(
     model=_lowerCamelCase				,       data_collator=_lowerCamelCase				,       args=_lowerCamelCase				,       train_dataset=vectorized_datasets["""train"""]				,       eval_dataset=vectorized_datasets["""validation"""]				,       tokenizer=_lowerCamelCase				,       max_gumbel_temp=model_args.max_gumbel_temperature				,       min_gumbel_temp=model_args.min_gumbel_temperature				,       gumbel_temp_decay=model_args.gumbel_temperature_decay				,       )
 trainer.train()


if __name__ == "__main__":
  main()





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class 		snake_case						(	SCREAMING_SNAKE_CASE_					,unittest.TestCase  ):



   '''simple docstring'''






   snake_case_					: Optional[Any]          =		BertJapaneseTokenizer
   snake_case_					: Optional[int]          =		False
   snake_case_					: Optional[int]          =		True

   def      UpperCamelCase_  (					self							:			Optional[int])				->				int:

    """simple docstring"""







    super().setUp()

    _snake_case    :  List[str]					=					[
        """[UNK]""",
        """[CLS]""",
        """[SEP]""",
        """こんにちは""",
        """こん""",
        """にちは""",
        """ばんは""",
        """##こん""",
        """##にちは""",
        """##ばんは""",
        """世界""",
        """##世界""",
        """、""",
        """##、""",
        """。""",
        """##。""",
    ]

    _snake_case    :  Dict					=					os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES["""vocab_file"""])
    with open(self.vocab_file				,		"""w"""				,		encoding="""utf-8""") as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))

   def      UpperCamelCase_  (					self							:			Union[str, Any]				,		lowerCAmelCase							:			str)				->				int:

    """simple docstring"""







    _snake_case    :  List[Any]					=					"""こんにちは、世界。 \nこんばんは、世界。"""
    _snake_case    :  Optional[Any]					=					"""こんにちは 、 世界 。 こんばんは 、 世界 。"""
    return input_text, output_text

   def      UpperCamelCase_  (					self							:			Tuple				,		lowerCAmelCase							:			Any)				->				Optional[Any]:

    """simple docstring"""







    _snake_case	,    _snake_case    :  Dict					=					self.get_input_output_texts(lowerCAmelCase)
    _snake_case    :  Optional[Any]					=					tokenizer.encode(lowerCAmelCase				,		add_special_tokens=lowerCAmelCase)
    _snake_case    :  Tuple					=					tokenizer.decode(lowerCAmelCase				,		clean_up_tokenization_spaces=lowerCAmelCase)
    return text, ids

   def      UpperCamelCase_  (					self							:			List[Any])				->				int:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			List[Any])				->				Optional[Any]:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			List[Any])				->				Dict:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			Dict)				->				Any:

    """simple docstring"""







    _snake_case    :  List[str]					=					self.tokenizer_class(self.vocab_file)

    _snake_case    :  List[Any]					=					tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。""")
    self.assertListEqual(lowerCAmelCase				,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 12, 10, 14, 4, 9, 12, 10, 14])

   def      UpperCamelCase_  (					self							:			int)				->				Dict:

    """simple docstring"""







    _snake_case    :  Tuple					=					self.tokenizer_class(self.vocab_file				,		word_tokenizer_type="""mecab""")
    self.assertIsNotNone(lowerCAmelCase)

    _snake_case    :  List[str]					=					"""こんにちは、世界。\nこんばんは、世界。"""
    _snake_case    :  Dict					=					tokenizer.tokenize(lowerCAmelCase)
    self.assertListEqual(lowerCAmelCase				,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 12, 10, 14, 4, 9, 12, 10, 14])

    _snake_case    :  Tuple					=					os.path.join(self.tmpdirname				,		"""tokenizer.bin""")
    with open(lowerCAmelCase				,		"""wb""") as handle:
     pickle.dump(lowerCAmelCase				,		lowerCAmelCase)

    with open(lowerCAmelCase				,		"""rb""") as handle:
     _snake_case    :  Dict					=					pickle.load(lowerCAmelCase)

    _snake_case    :  Optional[int]					=					tokenizer_new.tokenize(lowerCAmelCase)

    self.assertListEqual(lowerCAmelCase				,		lowerCAmelCase)

   def      UpperCamelCase_  (					self							:			str)				->				int:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					MecabTokenizer(mecab_dic="""ipadic""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			Optional[int])				->				Union[str, Any]:

    """simple docstring"""







    try:
     _snake_case    :  Optional[int]					=					MecabTokenizer(mecab_dic="""unidic_lite""")
    except ModuleNotFoundError:
     return

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			List[str])				->				Union[str, Any]:

    """simple docstring"""







    try:
     _snake_case    :  List[Any]					=					MecabTokenizer(mecab_dic="""unidic""")
    except ModuleNotFoundError:
     return

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			Optional[Any])				->				List[Any]:

    """simple docstring"""







    _snake_case    :  List[str]					=					MecabTokenizer(do_lower_case=lowerCAmelCase				,		mecab_dic="""ipadic""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			Optional[int])				->				int:

    """simple docstring"""







    try:
     _snake_case    :  Dict					=					MecabTokenizer(
         do_lower_case=lowerCAmelCase				,		normalize_text=lowerCAmelCase				,		mecab_option="""-d /usr/local/lib/mecab/dic/jumandic""")
    except RuntimeError:
     # if dict doesn't exist in the system, previous code raises this error.
     return

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			Optional[Any])				->				Tuple:

    """simple docstring"""







    _snake_case    :  str					=					MecabTokenizer(normalize_text=lowerCAmelCase				,		mecab_dic="""ipadic""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]				,		)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				str:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					self.tokenizer_class(self.vocab_file				,		word_tokenizer_type="""sudachi""")
    self.assertIsNotNone(lowerCAmelCase)

    _snake_case    :  List[str]					=					"""こんにちは、世界。\nこんばんは、世界。"""
    _snake_case    :  str					=					tokenizer.tokenize(lowerCAmelCase)
    self.assertListEqual(lowerCAmelCase				,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 12, 10, 14, 4, 9, 12, 10, 14])

    _snake_case    :  Optional[Any]					=					os.path.join(self.tmpdirname				,		"""tokenizer.bin""")
    with open(lowerCAmelCase				,		"""wb""") as handle:
     pickle.dump(lowerCAmelCase				,		lowerCAmelCase)

    with open(lowerCAmelCase				,		"""rb""") as handle:
     _snake_case    :  Optional[Any]					=					pickle.load(lowerCAmelCase)

    _snake_case    :  Tuple					=					tokenizer_new.tokenize(lowerCAmelCase)

    self.assertListEqual(lowerCAmelCase				,		lowerCAmelCase)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			List[Any])				->				List[str]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					SudachiTokenizer(sudachi_dict_type="""core""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]				,		)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			str)				->				Tuple:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					SudachiTokenizer(sudachi_dict_type="""core"""				,		sudachi_split_mode="""A""")

    self.assertListEqual(tokenizer.tokenize("""外国人参政権""")				,		["""外国""", """人""", """参政""", """権"""])

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				List[str]:

    """simple docstring"""







    _snake_case    :  Dict					=					SudachiTokenizer(sudachi_dict_type="""core"""				,		sudachi_split_mode="""B""")

    self.assertListEqual(tokenizer.tokenize("""外国人参政権""")				,		["""外国人""", """参政権"""])

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Dict)				->				str:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					SudachiTokenizer(sudachi_dict_type="""core"""				,		sudachi_split_mode="""C""")

    self.assertListEqual(tokenizer.tokenize("""外国人参政権""")				,		["""外国人参政権"""])

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Tuple)				->				Tuple:

    """simple docstring"""







    _snake_case    :  List[str]					=					SudachiTokenizer(do_lower_case=lowerCAmelCase				,		sudachi_dict_type="""core""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]				,		)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Optional[int])				->				Union[str, Any]:

    """simple docstring"""







    _snake_case    :  Dict					=					SudachiTokenizer(normalize_text=lowerCAmelCase				,		sudachi_dict_type="""core""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]				,		)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Any)				->				Union[str, Any]:

    """simple docstring"""







    _snake_case    :  Tuple					=					SudachiTokenizer(trim_whitespace=lowerCAmelCase				,		sudachi_dict_type="""core""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Any)				->				Any:

    """simple docstring"""







    _snake_case    :  int					=					self.tokenizer_class(self.vocab_file				,		word_tokenizer_type="""jumanpp""")
    self.assertIsNotNone(lowerCAmelCase)

    _snake_case    :  Optional[Any]					=					"""こんにちは、世界。\nこんばんは、世界。"""
    _snake_case    :  Tuple					=					tokenizer.tokenize(lowerCAmelCase)
    self.assertListEqual(lowerCAmelCase				,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 12, 10, 14, 4, 9, 12, 10, 14])

    _snake_case    :  str					=					os.path.join(self.tmpdirname				,		"""tokenizer.bin""")
    with open(lowerCAmelCase				,		"""wb""") as handle:
     pickle.dump(lowerCAmelCase				,		lowerCAmelCase)

    with open(lowerCAmelCase				,		"""rb""") as handle:
     _snake_case    :  int					=					pickle.load(lowerCAmelCase)

    _snake_case    :  List[Any]					=					tokenizer_new.tokenize(lowerCAmelCase)

    self.assertListEqual(lowerCAmelCase				,		lowerCAmelCase)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				List[Any]:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					JumanppTokenizer()

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Dict)				->				Optional[int]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					JumanppTokenizer(do_lower_case=lowerCAmelCase)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			int)				->				List[str]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					JumanppTokenizer(normalize_text=lowerCAmelCase)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Optional[int])				->				List[str]:

    """simple docstring"""







    _snake_case    :  str					=					JumanppTokenizer(trim_whitespace=lowerCAmelCase)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Optional[int])				->				int:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					JumanppTokenizer()

    self.assertListEqual(
        tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。""")				,		["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			List[str])				->				Optional[Any]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

    _snake_case    :  str					=					{}
    for i, token in enumerate(lowerCAmelCase):
     _snake_case    :  List[Any]					=					i
    _snake_case    :  List[Any]					=					WordpieceTokenizer(vocab=lowerCAmelCase				,		unk_token="""[UNK]""")

    self.assertListEqual(tokenizer.tokenize("""""")				,		[])

    self.assertListEqual(tokenizer.tokenize("""こんにちは""")				,		["""こんにちは"""])

    self.assertListEqual(tokenizer.tokenize("""こんばんは""")				,		["""こん""", """##ばんは"""])

    self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは""")				,		["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""])

   def      UpperCamelCase_  (					self							:			Optional[Any])				->				str:

    """simple docstring"""







    _snake_case    :  Optional[int]					=					BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp""")
    _snake_case    :  Tuple					=					tokenizer.subword_tokenizer

    _snake_case    :  Tuple					=					subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。""")
    self.assertListEqual(lowerCAmelCase				,		["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""])

    _snake_case    :  Union[str, Any]					=					subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは""")
    self.assertListEqual(lowerCAmelCase				,		["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""])


   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				Optional[int]:

    """simple docstring"""







    _snake_case    :  int					=					self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese""")

    _snake_case    :  str					=					tokenizer.encode("""ありがとう。"""				,		add_special_tokens=lowerCAmelCase)
    _snake_case    :  Optional[Any]					=					tokenizer.encode("""どういたしまして。"""				,		add_special_tokens=lowerCAmelCase)

    _snake_case    :  str					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase)
    _snake_case    :  Dict					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase				,		lowerCAmelCase)

    # 2 is for "[CLS]", 3 is for "[SEP]"
    assert encoded_sentence == [2] + text + [3]
    assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 		snake_case						(	SCREAMING_SNAKE_CASE_					,unittest.TestCase  ):



   '''simple docstring'''






   snake_case_					: Tuple          =		BertJapaneseTokenizer
   snake_case_					: Dict          =		False

   def      UpperCamelCase_  (					self							:			Optional[Any])				->				List[Any]:

    """simple docstring"""







    super().setUp()

    _snake_case    :  Optional[Any]					=					["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

    _snake_case    :  Optional[Any]					=					os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES["""vocab_file"""])
    with open(self.vocab_file				,		"""w"""				,		encoding="""utf-8""") as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))

   def      UpperCamelCase_  (					self							:			str				,		**lowerCAmelCase							:			Union[str, Any])				->				Any:

    """simple docstring"""







    return BertJapaneseTokenizer.from_pretrained(self.tmpdirname				,		subword_tokenizer_type="""character"""				,		**lowerCAmelCase)

   def      UpperCamelCase_  (					self							:			str				,		lowerCAmelCase							:			Tuple)				->				Optional[int]:

    """simple docstring"""







    _snake_case    :  Any					=					"""こんにちは、世界。 \nこんばんは、世界。"""
    _snake_case    :  List[Any]					=					"""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
    return input_text, output_text

   def      UpperCamelCase_  (					self							:			str)				->				int:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			List[str])				->				Optional[Any]:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			str)				->				Any:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			Optional[int])				->				int:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					self.tokenizer_class(self.vocab_file				,		subword_tokenizer_type="""character""")

    _snake_case    :  Dict					=					tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。""")
    self.assertListEqual(
        lowerCAmelCase				,		["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""])
    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12])

   def      UpperCamelCase_  (					self							:			List[str])				->				int:

    """simple docstring"""







    _snake_case    :  str					=					["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

    _snake_case    :  int					=					{}
    for i, token in enumerate(lowerCAmelCase):
     _snake_case    :  int					=					i
    _snake_case    :  Optional[int]					=					CharacterTokenizer(vocab=lowerCAmelCase				,		unk_token="""[UNK]""")

    self.assertListEqual(tokenizer.tokenize("""""")				,		[])

    self.assertListEqual(tokenizer.tokenize("""こんにちは""")				,		["""こ""", """ん""", """に""", """ち""", """は"""])

    self.assertListEqual(tokenizer.tokenize("""こんにちほ""")				,		["""こ""", """ん""", """に""", """ち""", """[UNK]"""])


   def      UpperCamelCase_  (					self							:			int)				->				List[str]:

    """simple docstring"""







    _snake_case    :  Optional[int]					=					self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char""")

    _snake_case    :  List[str]					=					tokenizer.encode("""ありがとう。"""				,		add_special_tokens=lowerCAmelCase)
    _snake_case    :  Optional[int]					=					tokenizer.encode("""どういたしまして。"""				,		add_special_tokens=lowerCAmelCase)

    _snake_case    :  str					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase)
    _snake_case    :  Any					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase				,		lowerCAmelCase)

    # 2 is for "[CLS]", 3 is for "[SEP]"
    assert encoded_sentence == [2] + text + [3]
    assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 		snake_case						(	unittest.TestCase  ):



   '''simple docstring'''






   def      UpperCamelCase_  (					self							:			int)				->				List[Any]:

    """simple docstring"""







    _snake_case    :  List[str]					=					"""cl-tohoku/bert-base-japanese"""
    _snake_case    :  int					=					AutoTokenizer.from_pretrained(lowerCAmelCase)
    self.assertIsInstance(lowerCAmelCase				,		lowerCAmelCase)







class 		snake_case						(	unittest.TestCase  ):



   '''simple docstring'''






   def      UpperCamelCase_  (					self							:			str)				->				Any:

    """simple docstring"""







    _snake_case    :  str					=					"""cl-tohoku/bert-base-japanese"""
    with self.assertLogs("""transformers"""				,		level="""WARNING""") as cm:
     BertTokenizer.from_pretrained(lowerCAmelCase)
     self.assertTrue(
         cm.records[0].message.startswith(
             """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
             """ is called from."""))
    _snake_case    :  Any					=					"""bert-base-cased"""
    with self.assertLogs("""transformers"""				,		level="""WARNING""") as cm:
     BertJapaneseTokenizer.from_pretrained(lowerCAmelCase)
     self.assertTrue(
         cm.records[0].message.startswith(
             """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
             """ is called from."""))
