




"""simple docstring"""

import numpy as np
import torch
from torch.utils.data import Dataset, IterableDataset

from ..utils.generic import ModelOutput






class        SCREAMING_SNAKE_CASE__   (						snake_case__      ):
   def __init__(       self			:				Union[str, Any]   ,	SCREAMING_SNAKE_CASE_			:				str   ,	SCREAMING_SNAKE_CASE_			:				int   ,	SCREAMING_SNAKE_CASE_			:				Dict							):
          lowerCamelCase__          =						dataset
          lowerCamelCase__          =						process
          lowerCamelCase__          =						params
   def __len__(       self			:				List[str]							):
          return len(self.dataset							)





   def __getitem__(       self			:				str   ,	SCREAMING_SNAKE_CASE_			:				Any							):
          lowerCamelCase__          =						self.dataset[i]
          lowerCamelCase__          =						self.process(_UpperCAmelCase   ,	**self.params							)
          return processed






class        SCREAMING_SNAKE_CASE__   (						snake_case__      ):
   def __init__(       self			:				Union[str, Any]   ,	SCREAMING_SNAKE_CASE_			:				str   ,	SCREAMING_SNAKE_CASE_			:				Any   ,	SCREAMING_SNAKE_CASE_			:				List[Any]   ,	SCREAMING_SNAKE_CASE_			:				int=None							):
          lowerCamelCase__          =						loader
          lowerCamelCase__          =						infer
          lowerCamelCase__          =						params
          if loader_batch_size == 1:
                 # Let's spare some time by deactivating altogether
                 lowerCamelCase__          =						None
          lowerCamelCase__          =						loader_batch_size

          # Internal bookkeeping
          lowerCamelCase__          =						None
          lowerCamelCase__          =						None
   def __len__(       self			:				List[str]							):
          return len(self.loader							)
   def __iter__(       self			:				List[Any]							):
          lowerCamelCase__          =						iter(self.loader							)
          return self
   def    __UpperCAmelCase       (       self			:				Any							):
          if isinstance(self._loader_batch_data   ,	torch.Tensor							):
                 # Batch data is simple tensor, just fetch the slice
                 lowerCamelCase__          =						self._loader_batch_data[self._loader_batch_index]
          else:
                 # Batch data is assumed to be BaseModelOutput (or dict)
                 lowerCamelCase__          =						{}
                 for k, element in self._loader_batch_data.items():
                        if isinstance(_UpperCAmelCase   ,	_UpperCAmelCase							):
                               # Convert ModelOutput to tuple first
                               lowerCamelCase__          =						element.to_tuple()
                               if isinstance(element[0]   ,	torch.Tensor							):
                                      lowerCamelCase__          =						tuple(el[self._loader_batch_index].unsqueeze(0							) for el in element							)
                               elif isinstance(element[0]   ,	np.ndarray							):
                                      lowerCamelCase__          =						tuple(np.expand_dims(el[self._loader_batch_index]   ,	0							) for el in element							)
                               continue
                        if k in {"hidden_states", "past_key_values", "attentions"} and isinstance(_UpperCAmelCase   ,	_UpperCAmelCase							):
                               # Those are stored as lists of tensors so need specific unbatching.
                               if isinstance(element[0]   ,	torch.Tensor							):
                                      lowerCamelCase__          =						tuple(el[self._loader_batch_index].unsqueeze(0							) for el in element							)
                               elif isinstance(element[0]   ,	np.ndarray							):
                                      lowerCamelCase__          =						tuple(np.expand_dims(el[self._loader_batch_index]   ,	0							) for el in element							)
                               continue
                        if element is None:
                               # This can happen for optional data that get passed around
                               lowerCamelCase__          =						None
                        elif isinstance(element[self._loader_batch_index]   ,	torch.Tensor							):
                               # Take correct batch data, but make it looked like batch_size=1
                               # For compatibility with other methods within transformers

                               lowerCamelCase__          =						element[self._loader_batch_index].unsqueeze(0							)
                        elif isinstance(element[self._loader_batch_index]   ,	np.ndarray							):
                               # Take correct batch data, but make it looked like batch_size=1
                               # For compatibility with other methods within transformers
                               lowerCamelCase__          =						np.expand_dims(element[self._loader_batch_index]   ,	0							)
                        else:
                               # This is typically a list, so no need to `unsqueeze`.
                               lowerCamelCase__          =						element[self._loader_batch_index]
            # Recreate the element by reusing the original class to make it look
            # batch_size=1
                 lowerCamelCase__          =						self._loader_batch_data.__class__(_UpperCAmelCase							)
          self._loader_batch_index += 1
          return result





   def    __UpperCAmelCase       (       self			:				str							):
          if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
                 # We are currently unrolling a batch so we just need to return
                 # the current item within a batch
                 return self.loader_batch_item()

          # We're out of items within a batch
          lowerCamelCase__          =						next(self.iterator							)
          lowerCamelCase__          =						self.infer(_UpperCAmelCase   ,	**self.params							)
          # We now have a batch of "inferred things".
          if self.loader_batch_size is not None:
                 # Try to infer the size of the batch
                 if isinstance(_UpperCAmelCase   ,	torch.Tensor							):
                        lowerCamelCase__          =						processed
                 else:
                        lowerCamelCase__          =						list(processed.keys()							)[0]
                        lowerCamelCase__          =						processed[key]
                 if isinstance(_UpperCAmelCase   ,	_UpperCAmelCase							):
                        lowerCamelCase__          =						len(_UpperCAmelCase							)
                 else:
                        lowerCamelCase__          =						first_tensor.shape[0]
                 if 0 < observed_batch_size < self.loader_batch_size:
                        # could be last batch so we can't unroll as many
                        # elements.
                        lowerCamelCase__          =						observed_batch_size
                 # Setting internal index to unwrap the batch
                 lowerCamelCase__          =						processed
                 lowerCamelCase__          =						0
                 return self.loader_batch_item()
          else:
                 # We're not unrolling batches
                 return processed






class        SCREAMING_SNAKE_CASE__   (						snake_case__      ):
   def __init__(       self			:				Any   ,	SCREAMING_SNAKE_CASE_			:				Union[str, Any]   ,	SCREAMING_SNAKE_CASE_			:				Optional[int]   ,	SCREAMING_SNAKE_CASE_			:				Any   ,	SCREAMING_SNAKE_CASE_			:				str=None							):
          super().__init__(_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase							)
   def __iter__(       self			:				Tuple							):
          lowerCamelCase__          =						iter(self.loader							)
          lowerCamelCase__          =						None
          return self





   def    __UpperCAmelCase       (       self			:				Union[str, Any]							):
          if self.subiterator is None:
                 lowerCamelCase__          =						self.infer(next(self.iterator							)   ,	**self.params							)
          try:
                 # Try to return next item
                 lowerCamelCase__          =						next(self.subiterator							)
          except StopIteration:
                 # When a preprocess iterator ends, we can start lookig at the next item
                 # ChunkIterator will keep feeding until ALL elements of iterator
                 # all have created their subiterator and have been iterating against.
                 #
                 # Another way to look at it, is we're basically flattening lists of lists
                 # into a single list, but with generators
                 lowerCamelCase__          =						self.infer(next(self.iterator							)   ,	**self.params							)
                 lowerCamelCase__          =						next(self.subiterator							)
          return processed






class        SCREAMING_SNAKE_CASE__   (						snake_case__      ):
   def __iter__(       self			:				List[Any]							):
          lowerCamelCase__          =						iter(self.loader							)
          return self





   def    __UpperCAmelCase       (       self			:				List[Any]							):
          # Extremely similar to PipelineIterator in its unpacking mechanism
          # BUT, we have an extra required item which is the presence of `is_last`
          # That is because everything is flattened by `PipelineChunkIterator` we
          # need to keep track of how to regroup here in the original `process`
          # boundaries so that `process` and `postprocess` see the same data.

          # This iterator accumulates items (possibly while unbatching) until it
          # its a `is_last` and then just passes it on to the caller.
          lowerCamelCase__          =						False
          lowerCamelCase__          =						[]
          if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
                 while self._loader_batch_index < self.loader_batch_size:
                        lowerCamelCase__          =						self.loader_batch_item()
                        lowerCamelCase__          =						item.pop("""is_last"""							)
                        accumulator.append(_UpperCAmelCase							)
                        if is_last:
                               return accumulator

          while not is_last:
                 lowerCamelCase__          =						self.infer(next(self.iterator							)   ,	**self.params							)
                 if self.loader_batch_size is not None:
                        if isinstance(_UpperCAmelCase   ,	torch.Tensor							):
                               lowerCamelCase__          =						processed
                        else:
                               lowerCamelCase__          =						list(processed.keys()							)[0]
                               lowerCamelCase__          =						processed[key]
                        if isinstance(_UpperCAmelCase   ,	_UpperCAmelCase							):
                               lowerCamelCase__          =						len(_UpperCAmelCase							)
                        else:
                               lowerCamelCase__          =						first_tensor.shape[0]
                        if 0 < observed_batch_size < self.loader_batch_size:
                               # could be last batch so we can't unroll as many
                               # elements.
                               lowerCamelCase__          =						observed_batch_size
                        lowerCamelCase__          =						processed
                        lowerCamelCase__          =						0
                        while self._loader_batch_index < self.loader_batch_size:
                               lowerCamelCase__          =						self.loader_batch_item()
                               lowerCamelCase__          =						item.pop("""is_last"""							)
                               accumulator.append(_UpperCAmelCase							)
                               if is_last:
                                      return accumulator
                 else:
                        lowerCamelCase__          =						processed
                        lowerCamelCase__          =						item.pop("""is_last"""							)
                        accumulator.append(_UpperCAmelCase							)
          return accumulator






class        SCREAMING_SNAKE_CASE__   (						snake_case__      ):
   def __init__(       self			:				Tuple   ,	SCREAMING_SNAKE_CASE_			:				Optional[Any]   ,	SCREAMING_SNAKE_CASE_			:				str							):
          lowerCamelCase__          =						dataset
          lowerCamelCase__          =						key
   def __len__(       self			:				Tuple							):
          return len(self.dataset							)





   def __getitem__(       self			:				List[Any]   ,	SCREAMING_SNAKE_CASE_			:				Optional[int]							):
          return self.dataset[i][self.key]






class        SCREAMING_SNAKE_CASE__   (						snake_case__      ):
   def __init__(       self			:				int   ,	SCREAMING_SNAKE_CASE_			:				Optional[int]   ,	SCREAMING_SNAKE_CASE_			:				List[Any]   ,	SCREAMING_SNAKE_CASE_			:				Dict							):
          lowerCamelCase__          =						dataset
          lowerCamelCase__          =						keya
          lowerCamelCase__          =						keya
   def __len__(       self			:				List[str]							):
          return len(self.dataset							)





   def __getitem__(       self			:				Union[str, Any]   ,	SCREAMING_SNAKE_CASE_			:				Any							):
          return {"text": self.dataset[i][self.keya], "text_pair": self.dataset[i][self.keya]}











'''simple docstring'''

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: str									=      BertJapaneseTokenizer
			a_	: List[str]									=      False
			a_	: List[Any]									=      True






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					super().setUp()

					snake_case__	     =[
					    '[UNK]',
					    '[CLS]',
					    '[SEP]',
					    'こんにちは',
					    'こん',
					    'にちは',
					    'ばんは',
					    '##こん',
					    '##にちは',
					    '##ばんは',
					    '世界',
					    '##世界',
					    '、',
					    '##、',
					    '。',
					    '##。',
					]

					snake_case__	     =os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file					,   'w'					,   encoding='utf-8'						) as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							str:
					snake_case__	     ='こんにちは、世界。 \nこんばんは、世界。'
					snake_case__	     ='こんにちは 、 世界 。 こんばんは 、 世界 。'
					return input_text, output_text






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Any:
					snake_case__     ,	snake_case__	     =self.get_input_output_texts(_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode(_UpperCAmelCase					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.decode(_UpperCAmelCase					,   clean_up_tokenization_spaces=_UpperCAmelCase						)
					return text, ids






			def 						_lowercase  (     self						)							->							Dict:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							List[str]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							str:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class(self.vocab_file						)

					snake_case__	     =tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='mecab'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =MecabTokenizer(mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							str:
					try:
							snake_case__	     =MecabTokenizer(mecab_dic='unidic_lite'						)
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							List[str]:
					try:
							snake_case__	     =MecabTokenizer(mecab_dic='unidic'						)
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =MecabTokenizer(do_lower_case=_UpperCAmelCase					,   mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							List[Any]:
					try:
							snake_case__	     =MecabTokenizer(
							    do_lower_case=_UpperCAmelCase					,   normalize_text=_UpperCAmelCase					,   mecab_option='-d /usr/local/lib/mecab/dic/jumandic'						)
					except RuntimeError:
							# if dict doesn't exist in the system, previous code raises this error.
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =MecabTokenizer(normalize_text=_UpperCAmelCase					,   mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='sudachi'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Union[str, Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='A'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国', '人', '参政', '権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='B'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国人', '参政権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='C'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国人参政権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =SudachiTokenizer(do_lower_case=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Dict:
					snake_case__	     =SudachiTokenizer(normalize_text=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =SudachiTokenizer(trim_whitespace=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='jumanpp'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			@require_jumanpp
			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =JumanppTokenizer(do_lower_case=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =JumanppTokenizer(normalize_text=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =JumanppTokenizer(trim_whitespace=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							str:
					snake_case__	     =JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'						)					,   ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']					,   )






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

					snake_case__	     ={}
					for i, token in enumerate(_UpperCAmelCase						):
							snake_case__	     =i
					snake_case__	     =WordpieceTokenizer(vocab=_UpperCAmelCase					,   unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)					,   []						)

					self.assertListEqual(tokenizer.tokenize('こんにちは'						)					,   ['こんにちは']						)

					self.assertListEqual(tokenizer.tokenize('こんばんは'						)					,   ['こん', '##ばんは']						)

					self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'						)					,   ['こん', '##ばんは', '[UNK]', 'こんにちは']						)






			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'						)
					snake_case__	     =tokenizer.subword_tokenizer

					snake_case__	     =subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']						)

					snake_case__	     =subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']						)






			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'						)

					snake_case__	     =tokenizer.encode('ありがとう。'					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode('どういたしまして。'					,   add_special_tokens=_UpperCAmelCase						)

					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase					,   _UpperCAmelCase						)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: int									=      BertJapaneseTokenizer
			a_	: Optional[Any]									=      False






			def 						_lowercase  (     self						)							->							Any:
					super().setUp()

					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

					snake_case__	     =os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file					,   'w'					,   encoding='utf-8'						) as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)






			def 						_lowercase  (     self					,   **_UpperCAmelCase						)							->							Union[str, Any]:
					return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,   subword_tokenizer_type='character'					,   **_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Optional[Any]:
					snake_case__	     ='こんにちは、世界。 \nこんばんは、世界。'
					snake_case__	     ='こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
					return input_text, output_text






			def 						_lowercase  (     self						)							->							List[Any]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Tuple:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   subword_tokenizer_type='character'						)

					snake_case__	     =tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'						)
					self.assertListEqual(
					    _UpperCAmelCase					,   ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']						)
					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

					snake_case__	     ={}
					for i, token in enumerate(_UpperCAmelCase						):
							snake_case__	     =i
					snake_case__	     =CharacterTokenizer(vocab=_UpperCAmelCase					,   unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)					,   []						)

					self.assertListEqual(tokenizer.tokenize('こんにちは'						)					,   ['こ', 'ん', 'に', 'ち', 'は']						)

					self.assertListEqual(tokenizer.tokenize('こんにちほ'						)					,   ['こ', 'ん', 'に', 'ち', '[UNK]']						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'						)

					snake_case__	     =tokenizer.encode('ありがとう。'					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode('どういたしまして。'					,   add_special_tokens=_UpperCAmelCase						)

					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase					,   _UpperCAmelCase						)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     ='cl-tohoku/bert-base-japanese'
					snake_case__	     =AutoTokenizer.from_pretrained(_UpperCAmelCase						)
					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)





class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     ='cl-tohoku/bert-base-japanese'
					with self.assertLogs('transformers'					,   level='WARNING'						) as cm:
							BertTokenizer.from_pretrained(_UpperCAmelCase						)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
							        ' is called from.'						)						)
					snake_case__	     ='bert-base-cased'
					with self.assertLogs('transformers'					,   level='WARNING'						) as cm:
							BertJapaneseTokenizer.from_pretrained(_UpperCAmelCase						)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
							        ' is called from.'						)						)




