





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class 							__lowercase			(     snake_case_   ,							unittest.TestCase       ):
      __UpperCAmelCase		      =						BertJapaneseTokenizer
      __UpperCAmelCase		      =						False
      __UpperCAmelCase		      =						True






      def 			_a						(		self)     ->       List[Any]:
       super().setUp()

       __snake_case         =			[
           """[UNK]""",
           """[CLS]""",
           """[SEP]""",
           """こんにちは""",
           """こん""",
           """にちは""",
           """ばんは""",
           """##こん""",
           """##にちは""",
           """##ばんは""",
           """世界""",
           """##世界""",
           """、""",
           """##、""",
           """。""",
           """##。""",
       ]

       __snake_case         =			os.path.join(self.tmpdirname			,							VOCAB_FILES_NAMES['vocab_file'])
       with open(self.vocab_file			,							'w'			,							encoding='utf-8') as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))






      def 			_a						(		self			,							lowercase_)     ->       str:
       __snake_case         =			"""こんにちは、世界。 \nこんばんは、世界。"""
       __snake_case         =			"""こんにちは 、 世界 。 こんばんは 、 世界 。"""
       return input_text, output_text






      def 			_a						(		self			,							lowercase_)     ->       List[str]:
       __snake_case         =			self.get_input_output_texts(lowercase__)
       __snake_case         =			tokenizer.encode(lowercase__			,							add_special_tokens=lowercase__)
       __snake_case         =			tokenizer.decode(lowercase__			,							clean_up_tokenization_spaces=lowercase__)
       return text, ids






      def 			_a						(		self)     ->       List[Any]:
       pass  # TODO add if relevant






      def 			_a						(		self)     ->       Optional[Any]:
       pass  # TODO add if relevant






      def 			_a						(		self)     ->       List[Any]:
       pass  # TODO add if relevant






      def 			_a						(		self)     ->       str:
       __snake_case         =			self.tokenizer_class(self.vocab_file)

       __snake_case         =			tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。')
       self.assertListEqual(lowercase__			,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。'])
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__)			,							[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4])






      def 			_a						(		self)     ->       int:
       __snake_case         =			self.tokenizer_class(self.vocab_file			,							word_tokenizer_type='mecab')
       self.assertIsNotNone(lowercase__)

       __snake_case         =			"""こんにちは、世界。\nこんばんは、世界。"""
       __snake_case         =			tokenizer.tokenize(lowercase__)
       self.assertListEqual(lowercase__			,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。'])
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__)			,							[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4])

       __snake_case         =			os.path.join(self.tmpdirname			,							'tokenizer.bin')
       with open(lowercase__			,							'wb') as handle:
        pickle.dump(lowercase__			,							lowercase__)

       with open(lowercase__			,							'rb') as handle:
        __snake_case         =			pickle.load(lowercase__)

       __snake_case         =			tokenizer_new.tokenize(lowercase__)

       self.assertListEqual(lowercase__			,							lowercase__)






      def 			_a						(		self)     ->       str:
       __snake_case         =			MecabTokenizer(mecab_dic='ipadic')

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,							)






      def 			_a						(		self)     ->       Tuple:

       try:
        __snake_case         =			MecabTokenizer(mecab_dic='unidic_lite')
       except ModuleNotFoundError:
        return

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,							)






      def 			_a						(		self)     ->       Union[str, Any]:

       try:
        __snake_case         =			MecabTokenizer(mecab_dic='unidic')
       except ModuleNotFoundError:
        return

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,							)






      def 			_a						(		self)     ->       Optional[Any]:
       __snake_case         =			MecabTokenizer(do_lower_case=lowercase__			,							mecab_dic='ipadic')

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,							)






      def 			_a						(		self)     ->       Optional[Any]:

       try:
        __snake_case         =			MecabTokenizer(
            do_lower_case=lowercase__			,							normalize_text=lowercase__			,							mecab_option='-d /usr/local/lib/mecab/dic/jumandic')
       except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']			,							)






      def 			_a						(		self)     ->       List[str]:
       __snake_case         =			MecabTokenizer(normalize_text=lowercase__			,							mecab_dic='ipadic')

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']			,							)






      @require_sudachi
      def 			_a						(		self)     ->       Dict:
       __snake_case         =			self.tokenizer_class(self.vocab_file			,							word_tokenizer_type='sudachi')
       self.assertIsNotNone(lowercase__)

       __snake_case         =			"""こんにちは、世界。\nこんばんは、世界。"""
       __snake_case         =			tokenizer.tokenize(lowercase__)
       self.assertListEqual(lowercase__			,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。'])
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__)			,							[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4])

       __snake_case         =			os.path.join(self.tmpdirname			,							'tokenizer.bin')
       with open(lowercase__			,							'wb') as handle:
        pickle.dump(lowercase__			,							lowercase__)

       with open(lowercase__			,							'rb') as handle:
        __snake_case         =			pickle.load(lowercase__)

       __snake_case         =			tokenizer_new.tokenize(lowercase__)

       self.assertListEqual(lowercase__			,							lowercase__)






      @require_sudachi
      def 			_a						(		self)     ->       List[Any]:
       __snake_case         =			SudachiTokenizer(sudachi_dict_type='core')

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							[' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']			,							)






      @require_sudachi
      def 			_a						(		self)     ->       Optional[Any]:
       __snake_case         =			SudachiTokenizer(sudachi_dict_type='core'			,							sudachi_split_mode='A')

       self.assertListEqual(tokenizer.tokenize('外国人参政権')			,							['外国', '人', '参政', '権'])






      @require_sudachi
      def 			_a						(		self)     ->       Tuple:
       __snake_case         =			SudachiTokenizer(sudachi_dict_type='core'			,							sudachi_split_mode='B')

       self.assertListEqual(tokenizer.tokenize('外国人参政権')			,							['外国人', '参政権'])






      @require_sudachi
      def 			_a						(		self)     ->       List[Any]:
       __snake_case         =			SudachiTokenizer(sudachi_dict_type='core'			,							sudachi_split_mode='C')

       self.assertListEqual(tokenizer.tokenize('外国人参政権')			,							['外国人参政権'])






      @require_sudachi
      def 			_a						(		self)     ->       Dict:
       __snake_case         =			SudachiTokenizer(do_lower_case=lowercase__			,							sudachi_dict_type='core')

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							[' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']			,							)






      @require_sudachi
      def 			_a						(		self)     ->       List[Any]:
       __snake_case         =			SudachiTokenizer(normalize_text=lowercase__			,							sudachi_dict_type='core')

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							[' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']			,							)






      @require_sudachi
      def 			_a						(		self)     ->       Optional[int]:
       __snake_case         =			SudachiTokenizer(trim_whitespace=lowercase__			,							sudachi_dict_type='core')

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,							)






      @require_jumanpp
      def 			_a						(		self)     ->       str:
       __snake_case         =			self.tokenizer_class(self.vocab_file			,							word_tokenizer_type='jumanpp')
       self.assertIsNotNone(lowercase__)

       __snake_case         =			"""こんにちは、世界。\nこんばんは、世界。"""
       __snake_case         =			tokenizer.tokenize(lowercase__)
       self.assertListEqual(lowercase__			,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。'])
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__)			,							[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4])

       __snake_case         =			os.path.join(self.tmpdirname			,							'tokenizer.bin')
       with open(lowercase__			,							'wb') as handle:
        pickle.dump(lowercase__			,							lowercase__)

       with open(lowercase__			,							'rb') as handle:
        __snake_case         =			pickle.load(lowercase__)

       __snake_case         =			tokenizer_new.tokenize(lowercase__)

       self.assertListEqual(lowercase__			,							lowercase__)






      @require_jumanpp
      def 			_a						(		self)     ->       int:
       __snake_case         =			JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']			,							)






      @require_jumanpp
      def 			_a						(		self)     ->       Any:
       __snake_case         =			JumanppTokenizer(do_lower_case=lowercase__)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']			,							)






      @require_jumanpp
      def 			_a						(		self)     ->       Any:
       __snake_case         =			JumanppTokenizer(normalize_text=lowercase__)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']			,							)






      @require_jumanpp
      def 			_a						(		self)     ->       str:
       __snake_case         =			JumanppTokenizer(trim_whitespace=lowercase__)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ')			,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']			,							)






      @require_jumanpp
      def 			_a						(		self)     ->       List[str]:
       __snake_case         =			JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。')			,							['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']			,							)






      def 			_a						(		self)     ->       Dict:
       __snake_case         =			["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

       __snake_case         =			{}
       for i, token in enumerate(lowercase__):
        __snake_case         =			i
       __snake_case         =			WordpieceTokenizer(vocab=lowercase__			,							unk_token='[UNK]')

       self.assertListEqual(tokenizer.tokenize('')			,							[])

       self.assertListEqual(tokenizer.tokenize('こんにちは')			,							['こんにちは'])

       self.assertListEqual(tokenizer.tokenize('こんばんは')			,							['こん', '##ばんは'])

       self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは')			,							['こん', '##ばんは', '[UNK]', 'こんにちは'])






      def 			_a						(		self)     ->       int:
       __snake_case         =			BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp')
       __snake_case         =			tokenizer.subword_tokenizer

       __snake_case         =			subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。')
       self.assertListEqual(lowercase__			,							['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。'])

       __snake_case         =			subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは')
       self.assertListEqual(lowercase__			,							['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは'])







      def 			_a						(		self)     ->       int:
       __snake_case         =			self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese')

       __snake_case         =			tokenizer.encode('ありがとう。'			,							add_special_tokens=lowercase__)
       __snake_case         =			tokenizer.encode('どういたしまして。'			,							add_special_tokens=lowercase__)

       __snake_case         =			tokenizer.build_inputs_with_special_tokens(lowercase__)
       __snake_case         =			tokenizer.build_inputs_with_special_tokens(lowercase__			,							lowercase__)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 							__lowercase			(     snake_case_   ,							unittest.TestCase       ):
      __UpperCAmelCase		      =						BertJapaneseTokenizer
      __UpperCAmelCase		      =						False






      def 			_a						(		self)     ->       str:
       super().setUp()

       __snake_case         =			["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

       __snake_case         =			os.path.join(self.tmpdirname			,							VOCAB_FILES_NAMES['vocab_file'])
       with open(self.vocab_file			,							'w'			,							encoding='utf-8') as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))






      def 			_a						(		self			,							**lowercase_)     ->       int:
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname			,							subword_tokenizer_type='character'			,							**lowercase__)






      def 			_a						(		self			,							lowercase_)     ->       int:
       __snake_case         =			"""こんにちは、世界。 \nこんばんは、世界。"""
       __snake_case         =			"""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
       return input_text, output_text






      def 			_a						(		self)     ->       Union[str, Any]:
       pass  # TODO add if relevant






      def 			_a						(		self)     ->       List[Any]:
       pass  # TODO add if relevant






      def 			_a						(		self)     ->       Optional[int]:
       pass  # TODO add if relevant






      def 			_a						(		self)     ->       Optional[Any]:
       __snake_case         =			self.tokenizer_class(self.vocab_file			,							subword_tokenizer_type='character')

       __snake_case         =			tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。')
       self.assertListEqual(
           lowercase__			,							['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。'])
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowercase__)			,							[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2])






      def 			_a						(		self)     ->       Tuple:
       __snake_case         =			["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

       __snake_case         =			{}
       for i, token in enumerate(lowercase__):
        __snake_case         =			i
       __snake_case         =			CharacterTokenizer(vocab=lowercase__			,							unk_token='[UNK]')

       self.assertListEqual(tokenizer.tokenize('')			,							[])

       self.assertListEqual(tokenizer.tokenize('こんにちは')			,							['こ', 'ん', 'に', 'ち', 'は'])

       self.assertListEqual(tokenizer.tokenize('こんにちほ')			,							['こ', 'ん', 'に', 'ち', '[UNK]'])







      def 			_a						(		self)     ->       Tuple:
       __snake_case         =			self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char')

       __snake_case         =			tokenizer.encode('ありがとう。'			,							add_special_tokens=lowercase__)
       __snake_case         =			tokenizer.encode('どういたしまして。'			,							add_special_tokens=lowercase__)

       __snake_case         =			tokenizer.build_inputs_with_special_tokens(lowercase__)
       __snake_case         =			tokenizer.build_inputs_with_special_tokens(lowercase__			,							lowercase__)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 							__lowercase			(     unittest.TestCase       ):






      def 			_a						(		self)     ->       str:
       __snake_case         =			"""cl-tohoku/bert-base-japanese"""
       __snake_case         =			AutoTokenizer.from_pretrained(lowercase__)
       self.assertIsInstance(lowercase__			,							lowercase__)






class 							__lowercase			(     unittest.TestCase       ):






      def 			_a						(		self)     ->       Any:
       __snake_case         =			"""cl-tohoku/bert-base-japanese"""
       with self.assertLogs('transformers'			,							level='WARNING') as cm:
        BertTokenizer.from_pretrained(lowercase__)
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'))
       __snake_case         =			"""bert-base-cased"""
       with self.assertLogs('transformers'			,							level='WARNING') as cm:
        BertJapaneseTokenizer.from_pretrained(lowercase__)
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'))







'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


lowerCAmelCase_      :  Optional[Any]				     =  {}

try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCAmelCase_      :  Any				     =  ['GPTSw3Tokenizer']


if TYPE_CHECKING:
 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_gpt_swa import GPTSwaTokenizer

else:
 import sys

 lowerCAmelCase_      :  int				     =  _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
