

"""simple docstring"""




from typing import List, Optional, Union

import numpy as np

from ....audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ....feature_extraction_sequence_utils import SequenceFeatureExtractor
from ....feature_extraction_utils import BatchFeature
from ....file_utils import PaddingStrategy, TensorType
from ....utils import logging


A_				:      Any           =			logging.get_logger(__name__)





class  lowerCamelCase    (A__      ):
			lowerCamelCase__ :		int       		=				['input_features', 'attention_mask']

			def __init__(       self					:    List[Any]     ,					__UpperCAmelCase					:    Union[str, Any]=8_0     ,					__UpperCAmelCase					:    Dict=1_6_0_0_0     ,					__UpperCAmelCase					:    Any=0.0     ,					__UpperCAmelCase					:    Tuple=1_0     ,					__UpperCAmelCase					:    Tuple=2_5     ,					__UpperCAmelCase					:    List[str]="hamming_window"     ,					__UpperCAmelCase					:    List[str]=32_768.0     ,					__UpperCAmelCase					:    int=0.97     ,					__UpperCAmelCase					:    Dict=1.0     ,					__UpperCAmelCase					:    Optional[Any]=True     ,					__UpperCAmelCase					:    Any=True     ,					__UpperCAmelCase					:    List[Any]=False     ,					**__UpperCAmelCase					:    Dict     ,					) ->     int:
						super().__init__(feature_size=__UpperCAmelCase     ,					sampling_rate=__UpperCAmelCase     ,					padding_value=__UpperCAmelCase     ,					**__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   feature_size
						SCREAMING_SNAKE_CASE__										=   sampling_rate
						SCREAMING_SNAKE_CASE__										=   padding_value
						SCREAMING_SNAKE_CASE__										=   hop_length
						SCREAMING_SNAKE_CASE__										=   win_length
						SCREAMING_SNAKE_CASE__										=   frame_signal_scale
						SCREAMING_SNAKE_CASE__										=   preemphasis_coeff
						SCREAMING_SNAKE_CASE__										=   mel_floor
						SCREAMING_SNAKE_CASE__										=   normalize_means
						SCREAMING_SNAKE_CASE__										=   normalize_vars
						SCREAMING_SNAKE_CASE__										=   win_function
						SCREAMING_SNAKE_CASE__										=   return_attention_mask

						SCREAMING_SNAKE_CASE__										=   win_length * sampling_rate // 1_0_0_0
						SCREAMING_SNAKE_CASE__										=   hop_length * sampling_rate // 1_0_0_0

						SCREAMING_SNAKE_CASE__										=   optimal_fft_length(self.sample_size							)
						SCREAMING_SNAKE_CASE__										=   (self.n_fft // 2) + 1

			def        SCREAMING_SNAKE_CASE					(       self					:    str     ,					__UpperCAmelCase					:    np.array							) ->     np.ndarray:
						if self.win_function == "hamming_window":
									SCREAMING_SNAKE_CASE__										=   window_function(window_length=self.sample_size     ,					name=self.win_function     ,					periodic=__UpperCAmelCase							)
						else:
									SCREAMING_SNAKE_CASE__										=   window_function(window_length=self.sample_size     ,					name=self.win_function							)

						SCREAMING_SNAKE_CASE__										=   mel_filter_bank(
						    num_frequency_bins=self.n_freqs     ,					num_mel_filters=self.feature_size     ,					min_frequency=0.0     ,					max_frequency=self.sampling_rate / 2.0     ,					sampling_rate=self.sampling_rate     ,					)

						SCREAMING_SNAKE_CASE__										=   spectrogram(
						    one_waveform * self.frame_signal_scale     ,					window=__UpperCAmelCase     ,					frame_length=self.sample_size     ,					hop_length=self.sample_stride     ,					fft_length=self.n_fft     ,					center=__UpperCAmelCase     ,					preemphasis=self.preemphasis_coeff     ,					mel_filters=__UpperCAmelCase     ,					mel_floor=self.mel_floor     ,					log_mel="""log"""     ,					)
						return msfc_features.T

			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple     ,					__UpperCAmelCase					:    List[Any]     ,					__UpperCAmelCase					:    List[Any]     ,					__UpperCAmelCase					:    Union[str, Any]							) ->     Any:
						# make sure we normalize float32 arrays
						if self.normalize_means:
									SCREAMING_SNAKE_CASE__										=   x[:input_length].mean(axis=0							)
									SCREAMING_SNAKE_CASE__										=   np.subtract(__UpperCAmelCase     ,					__UpperCAmelCase							)
						if self.normalize_vars:
									SCREAMING_SNAKE_CASE__										=   x[:input_length].std(axis=0							)
									SCREAMING_SNAKE_CASE__										=   np.divide(__UpperCAmelCase     ,					__UpperCAmelCase							)

						if input_length < x.shape[0]:
									SCREAMING_SNAKE_CASE__										=   padding_value

						# make sure array is in float32
						SCREAMING_SNAKE_CASE__										=   x.astype(np.floataa							)

						return x

			def        SCREAMING_SNAKE_CASE					(       self					:    Any     ,					__UpperCAmelCase					:    List[np.ndarray]     ,					__UpperCAmelCase					:    Optional[np.ndarray] = None							) ->     List[np.ndarray]:
						SCREAMING_SNAKE_CASE__										=   attention_mask.sum(-1							) if attention_mask is not None else [x.shape[0] for x in input_features]
						return [self._normalize_one(__UpperCAmelCase     ,					__UpperCAmelCase     ,					self.padding_value							) for x, n in zip(__UpperCAmelCase     ,					__UpperCAmelCase							)]





			def __call__(       self					:    Dict     ,					__UpperCAmelCase					:    Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]     ,					__UpperCAmelCase					:    Union[bool, str, PaddingStrategy] = False     ,					__UpperCAmelCase					:    Optional[int] = None     ,					__UpperCAmelCase					:    bool = False     ,					__UpperCAmelCase					:    Optional[int] = None     ,					__UpperCAmelCase					:    Optional[bool] = None     ,					__UpperCAmelCase					:    Optional[Union[str, TensorType]] = None     ,					__UpperCAmelCase					:    Optional[int] = None     ,					**__UpperCAmelCase					:    Dict     ,					) ->     BatchFeature:

						if sampling_rate is not None:
									if sampling_rate != self.sampling_rate:
												raise ValueError(
												    F"""The model corresponding to this feature extractor: {self} was trained using a sampling rate of"""
												    F""" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"""
												    F""" {self.sampling_rate} and not {sampling_rate}."""							)
						else:
									logger.warning(
									    """It is strongly recommended to pass the ``sampling_rate`` argument to this function. """
									    """Failing to do so can result in silent errors that might be hard to debug."""							)

						SCREAMING_SNAKE_CASE__										=   isinstance(__UpperCAmelCase     ,					np.ndarray							) and len(raw_speech.shape							) > 1
						if is_batched_numpy and len(raw_speech.shape							) > 2:
									raise ValueError(F"""Only mono-channel audio is supported for input to {self}"""							)
						SCREAMING_SNAKE_CASE__										=   is_batched_numpy or (
						    isinstance(__UpperCAmelCase     ,					(list, tuple)							) and (isinstance(raw_speech[0]     ,					(np.ndarray, tuple, list)							))
						)

						if is_batched:
									SCREAMING_SNAKE_CASE__										=   [np.asarray(__UpperCAmelCase     ,					dtype=np.floataa							) for speech in raw_speech]
						elif not is_batched and not isinstance(__UpperCAmelCase     ,					np.ndarray							):
									SCREAMING_SNAKE_CASE__										=   np.asarray(__UpperCAmelCase     ,					dtype=np.floataa							)
						elif isinstance(__UpperCAmelCase     ,					np.ndarray							) and raw_speech.dtype is np.dtype(np.floataa							):
									SCREAMING_SNAKE_CASE__										=   raw_speech.astype(np.floataa							)

						# always return batch
						if not is_batched:
									SCREAMING_SNAKE_CASE__										=   [raw_speech]

						# extract fbank features
						SCREAMING_SNAKE_CASE__										=   [self._extract_mfsc_features(__UpperCAmelCase							) for one_waveform in raw_speech]

						# convert into correct format for padding
						SCREAMING_SNAKE_CASE__										=   BatchFeature({"""input_features""": features}							)

						SCREAMING_SNAKE_CASE__										=   self.pad(
						    __UpperCAmelCase     ,					padding=__UpperCAmelCase     ,					max_length=__UpperCAmelCase     ,					truncation=__UpperCAmelCase     ,					pad_to_multiple_of=__UpperCAmelCase     ,					return_attention_mask=__UpperCAmelCase     ,					**__UpperCAmelCase     ,					)
						# make sure list is in array format
						SCREAMING_SNAKE_CASE__										=   padded_inputs.get("""input_features"""							)
						if isinstance(input_features[0]     ,					__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   [np.asarray(__UpperCAmelCase     ,					dtype=np.floataa							) for feature in input_features]

						SCREAMING_SNAKE_CASE__										=   padded_inputs.get("""attention_mask"""							)
						if attention_mask is not None:
									SCREAMING_SNAKE_CASE__										=   [np.asarray(__UpperCAmelCase     ,					dtype=np.intaa							) for array in attention_mask]

						if self.normalize_means or self.normalize_vars:
									SCREAMING_SNAKE_CASE__										=   (
									    np.array(__UpperCAmelCase     ,					dtype=np.intaa							)
									    if self._get_padding_strategies(__UpperCAmelCase     ,					max_length=__UpperCAmelCase							) is not PaddingStrategy.DO_NOT_PAD
									    and padding
									    else None
									)
									SCREAMING_SNAKE_CASE__										=   self.normalize(
									    padded_inputs["""input_features"""]     ,					attention_mask=__UpperCAmelCase							)

						if return_tensors is not None:
									SCREAMING_SNAKE_CASE__										=   padded_inputs.convert_to_tensors(__UpperCAmelCase							)

						return padded_inputs





"""simple docstring"""




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Union[str, Any]       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		str       		=				False
			lowerCamelCase__ :		Optional[int]       		=				True

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     str:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   [
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """こんにちは""",
						    """こん""",
						    """にちは""",
						    """ばんは""",
						    """##こん""",
						    """##にちは""",
						    """##ばんは""",
						    """世界""",
						    """##世界""",
						    """、""",
						    """##、""",
						    """。""",
						    """##。""",
						]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]     ,					__UpperCAmelCase					:    Optional[int]							) ->     Tuple:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こんにちは 、 世界 。 こんばんは 、 世界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]     ,					__UpperCAmelCase					:    Union[str, Any]							) ->     int:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.get_input_output_texts(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.decode(__UpperCAmelCase     ,					clean_up_tokenization_spaces=__UpperCAmelCase							)
						return text, ids

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Tuple:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""mecab"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Union[str, Any]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic_lite"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(do_lower_case=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     List[str]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(
									    do_lower_case=__UpperCAmelCase     ,					normalize_text=__UpperCAmelCase     ,					mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""							)
						except RuntimeError:
									# if dict doesn't exist in the system, previous code raises this error.
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(normalize_text=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""sudachi"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""A"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国""", """人""", """参政""", """権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""B"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人""", """参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""C"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(do_lower_case=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     str:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(normalize_text=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(trim_whitespace=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""jumanpp"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(do_lower_case=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(normalize_text=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(trim_whitespace=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""							)     ,					["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   WordpieceTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こんにちは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは"""							)     ,					["""こん""", """##ばんは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""							)     ,					["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.subword_tokenizer

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]							)

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Dict       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		Tuple       		=				False

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]     ,					**__UpperCAmelCase					:    List[Any]							) ->     List[Any]:
						return BertJapaneseTokenizer.from_pretrained(self.tmpdirname     ,					subword_tokenizer_type="""character"""     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any     ,					__UpperCAmelCase					:    Any							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Optional[Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Union[str, Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					subword_tokenizer_type="""character"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""							)
						self.assertListEqual(
						    __UpperCAmelCase     ,					["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]							)
						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   CharacterTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こ""", """ん""", """に""", """ち""", """は"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちほ"""							)     ,					["""こ""", """ん""", """に""", """ち""", """[UNK]"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     str:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						SCREAMING_SNAKE_CASE__										=   AutoTokenizer.from_pretrained(__UpperCAmelCase							)
						self.assertIsInstance(__UpperCAmelCase     ,					__UpperCAmelCase							)





class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)
						SCREAMING_SNAKE_CASE__										=   """bert-base-cased"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertJapaneseTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)


