




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE			,  unittest.TestCase   ):


 '''simple docstring'''







 lowercase_							       =       BertJapaneseTokenizer
 lowercase_							       =       False
 lowercase_							       =       True



 def  lowerCAmelCase_    (				self    :						int		):
       super().setUp()

       SCREAMING_SNAKE_CASE_      		=     [
           '[UNK]',
           '[CLS]',
           '[SEP]',
           'こんにちは',
           'こん',
           'にちは',
           'ばんは',
           '##こん',
           '##にちは',
           '##ばんは',
           '世界',
           '##世界',
           '、',
           '##、',
           '。',
           '##。',
       ]

       SCREAMING_SNAKE_CASE_      		=     os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES['vocab_file']		)
       with open(self.vocab_file							,							'w'							,							encoding='utf-8'		) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)



 def  lowerCAmelCase_    (				self    :						Dict							,							_lowerCAmelCase    :						int		):
       SCREAMING_SNAKE_CASE_      		=     'こんにちは、世界。 \nこんばんは、世界。'
       SCREAMING_SNAKE_CASE_      		=     'こんにちは 、 世界 。 こんばんは 、 世界 。'
       return input_text, output_text



 def  lowerCAmelCase_    (				self    :						Any							,							_lowerCAmelCase    :						Tuple		):
       SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_      		=     self.get_input_output_texts(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     tokenizer.encode(_lowerCAmelCase							,							add_special_tokens=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     tokenizer.decode(_lowerCAmelCase							,							clean_up_tokenization_spaces=_lowerCAmelCase		)
       return text, ids



 def  lowerCAmelCase_    (				self    :						str		):
       pass  # TODO add if relevant



 def  lowerCAmelCase_    (				self    :						Dict		):
       pass  # TODO add if relevant



 def  lowerCAmelCase_    (				self    :						List[Any]		):
       pass  # TODO add if relevant



 def  lowerCAmelCase_    (				self    :						str		):
       SCREAMING_SNAKE_CASE_      		=     self.tokenizer_class(self.vocab_file		)

       SCREAMING_SNAKE_CASE_      		=     tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'		)
       self.assertListEqual(_lowerCAmelCase							,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase		)							,							[3, 12, 10, 14, 4, 9, 12, 10, 14]		)



 def  lowerCAmelCase_    (				self    :						Union[str, Any]		):
       SCREAMING_SNAKE_CASE_      		=     self.tokenizer_class(self.vocab_file							,							word_tokenizer_type='mecab'		)
       self.assertIsNotNone(_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     'こんにちは、世界。\nこんばんは、世界。'
       SCREAMING_SNAKE_CASE_      		=     tokenizer.tokenize(_lowerCAmelCase		)
       self.assertListEqual(_lowerCAmelCase							,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase		)							,							[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

       SCREAMING_SNAKE_CASE_      		=     os.path.join(self.tmpdirname							,							'tokenizer.bin'		)
       with open(_lowerCAmelCase							,							'wb'		) as handle:
             pickle.dump(_lowerCAmelCase							,							_lowerCAmelCase		)

       with open(_lowerCAmelCase							,							'rb'		) as handle:
             SCREAMING_SNAKE_CASE_      		=     pickle.load(_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     tokenizer_new.tokenize(_lowerCAmelCase		)

       self.assertListEqual(_lowerCAmelCase							,							_lowerCAmelCase		)



 def  lowerCAmelCase_    (				self    :						Any		):
       SCREAMING_SNAKE_CASE_      		=     MecabTokenizer(mecab_dic='ipadic'		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,							)



 def  lowerCAmelCase_    (				self    :						int		):
       try:
             SCREAMING_SNAKE_CASE_      		=     MecabTokenizer(mecab_dic='unidic_lite'		)
       except ModuleNotFoundError:
             return

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,							)



 def  lowerCAmelCase_    (				self    :						Any		):
       try:
             SCREAMING_SNAKE_CASE_      		=     MecabTokenizer(mecab_dic='unidic'		)
       except ModuleNotFoundError:
             return

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,							)



 def  lowerCAmelCase_    (				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     MecabTokenizer(do_lower_case=_lowerCAmelCase							,							mecab_dic='ipadic'		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,							)



 def  lowerCAmelCase_    (				self    :						List[Any]		):
       try:
             SCREAMING_SNAKE_CASE_      		=     MecabTokenizer(
                 do_lower_case=_lowerCAmelCase							,							normalize_text=_lowerCAmelCase							,							mecab_option='-d /usr/local/lib/mecab/dic/jumandic'		)
       except RuntimeError:
             # if dict doesn't exist in the system, previous code raises this error.
             return

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']							,							)



 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       SCREAMING_SNAKE_CASE_      		=     MecabTokenizer(normalize_text=_lowerCAmelCase							,							mecab_dic='ipadic'		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']							,							)



 @require_sudachi
 def  lowerCAmelCase_    (				self    :						Dict		):
       SCREAMING_SNAKE_CASE_      		=     self.tokenizer_class(self.vocab_file							,							word_tokenizer_type='sudachi'		)
       self.assertIsNotNone(_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     'こんにちは、世界。\nこんばんは、世界。'
       SCREAMING_SNAKE_CASE_      		=     tokenizer.tokenize(_lowerCAmelCase		)
       self.assertListEqual(_lowerCAmelCase							,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase		)							,							[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

       SCREAMING_SNAKE_CASE_      		=     os.path.join(self.tmpdirname							,							'tokenizer.bin'		)
       with open(_lowerCAmelCase							,							'wb'		) as handle:
             pickle.dump(_lowerCAmelCase							,							_lowerCAmelCase		)

       with open(_lowerCAmelCase							,							'rb'		) as handle:
             SCREAMING_SNAKE_CASE_      		=     pickle.load(_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     tokenizer_new.tokenize(_lowerCAmelCase		)

       self.assertListEqual(_lowerCAmelCase							,							_lowerCAmelCase		)



 @require_sudachi
 def  lowerCAmelCase_    (				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     SudachiTokenizer(sudachi_dict_type='core'		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							[' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']							,							)



 @require_sudachi
 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       SCREAMING_SNAKE_CASE_      		=     SudachiTokenizer(sudachi_dict_type='core'							,							sudachi_split_mode='A'		)

       self.assertListEqual(tokenizer.tokenize('外国人参政権'		)							,							['外国', '人', '参政', '権']		)



 @require_sudachi
 def  lowerCAmelCase_    (				self    :						int		):
       SCREAMING_SNAKE_CASE_      		=     SudachiTokenizer(sudachi_dict_type='core'							,							sudachi_split_mode='B'		)

       self.assertListEqual(tokenizer.tokenize('外国人参政権'		)							,							['外国人', '参政権']		)



 @require_sudachi
 def  lowerCAmelCase_    (				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     SudachiTokenizer(sudachi_dict_type='core'							,							sudachi_split_mode='C'		)

       self.assertListEqual(tokenizer.tokenize('外国人参政権'		)							,							['外国人参政権']		)



 @require_sudachi
 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       SCREAMING_SNAKE_CASE_      		=     SudachiTokenizer(do_lower_case=_lowerCAmelCase							,							sudachi_dict_type='core'		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							[' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']							,							)



 @require_sudachi
 def  lowerCAmelCase_    (				self    :						str		):
       SCREAMING_SNAKE_CASE_      		=     SudachiTokenizer(normalize_text=_lowerCAmelCase							,							sudachi_dict_type='core'		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							[' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']							,							)



 @require_sudachi
 def  lowerCAmelCase_    (				self    :						List[Any]		):
       SCREAMING_SNAKE_CASE_      		=     SudachiTokenizer(trim_whitespace=_lowerCAmelCase							,							sudachi_dict_type='core'		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,							)



 @require_jumanpp
 def  lowerCAmelCase_    (				self    :						Dict		):
       SCREAMING_SNAKE_CASE_      		=     self.tokenizer_class(self.vocab_file							,							word_tokenizer_type='jumanpp'		)
       self.assertIsNotNone(_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     'こんにちは、世界。\nこんばんは、世界。'
       SCREAMING_SNAKE_CASE_      		=     tokenizer.tokenize(_lowerCAmelCase		)
       self.assertListEqual(_lowerCAmelCase							,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase		)							,							[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

       SCREAMING_SNAKE_CASE_      		=     os.path.join(self.tmpdirname							,							'tokenizer.bin'		)
       with open(_lowerCAmelCase							,							'wb'		) as handle:
             pickle.dump(_lowerCAmelCase							,							_lowerCAmelCase		)

       with open(_lowerCAmelCase							,							'rb'		) as handle:
             SCREAMING_SNAKE_CASE_      		=     pickle.load(_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     tokenizer_new.tokenize(_lowerCAmelCase		)

       self.assertListEqual(_lowerCAmelCase							,							_lowerCAmelCase		)



 @require_jumanpp
 def  lowerCAmelCase_    (				self    :						List[Any]		):
       SCREAMING_SNAKE_CASE_      		=     JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']							,							)



 @require_jumanpp
 def  lowerCAmelCase_    (				self    :						Any		):
       SCREAMING_SNAKE_CASE_      		=     JumanppTokenizer(do_lower_case=_lowerCAmelCase		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']							,							)



 @require_jumanpp
 def  lowerCAmelCase_    (				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     JumanppTokenizer(normalize_text=_lowerCAmelCase		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']							,							)



 @require_jumanpp
 def  lowerCAmelCase_    (				self    :						List[Any]		):
       SCREAMING_SNAKE_CASE_      		=     JumanppTokenizer(trim_whitespace=_lowerCAmelCase		)

       self.assertListEqual(
           tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '		)							,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']							,							)



 @require_jumanpp
 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       SCREAMING_SNAKE_CASE_      		=     JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'		)							,							['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']							,							)



 def  lowerCAmelCase_    (				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     ['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

       SCREAMING_SNAKE_CASE_      		=     {}
       for i, token in enumerate(_lowerCAmelCase		):
             SCREAMING_SNAKE_CASE_      		=     i
       SCREAMING_SNAKE_CASE_      		=     WordpieceTokenizer(vocab=_lowerCAmelCase							,							unk_token='[UNK]'		)

       self.assertListEqual(tokenizer.tokenize(''		)							,							[]		)

       self.assertListEqual(tokenizer.tokenize('こんにちは'		)							,							['こんにちは']		)

       self.assertListEqual(tokenizer.tokenize('こんばんは'		)							,							['こん', '##ばんは']		)

       self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'		)							,							['こん', '##ばんは', '[UNK]', 'こんにちは']		)



 def  lowerCAmelCase_    (				self    :						Tuple		):
       SCREAMING_SNAKE_CASE_      		=     BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'		)
       SCREAMING_SNAKE_CASE_      		=     tokenizer.subword_tokenizer

       SCREAMING_SNAKE_CASE_      		=     subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'		)
       self.assertListEqual(_lowerCAmelCase							,							['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']		)

       SCREAMING_SNAKE_CASE_      		=     subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'		)
       self.assertListEqual(_lowerCAmelCase							,							['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']		)




 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       SCREAMING_SNAKE_CASE_      		=     self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'		)

       SCREAMING_SNAKE_CASE_      		=     tokenizer.encode('ありがとう。'							,							add_special_tokens=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     tokenizer.encode('どういたしまして。'							,							add_special_tokens=_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase							,							_lowerCAmelCase		)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE			,  unittest.TestCase   ):


 '''simple docstring'''







 lowercase_							       =       BertJapaneseTokenizer
 lowercase_							       =       False



 def  lowerCAmelCase_    (				self    :						int		):
       super().setUp()

       SCREAMING_SNAKE_CASE_      		=     ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

       SCREAMING_SNAKE_CASE_      		=     os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES['vocab_file']		)
       with open(self.vocab_file							,							'w'							,							encoding='utf-8'		) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)



 def  lowerCAmelCase_    (				self    :						List[Any]							,							**_lowerCAmelCase    :						Union[str, Any]		):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname							,							subword_tokenizer_type='character'							,							**_lowerCAmelCase		)



 def  lowerCAmelCase_    (				self    :						Union[str, Any]							,							_lowerCAmelCase    :						str		):
       SCREAMING_SNAKE_CASE_      		=     'こんにちは、世界。 \nこんばんは、世界。'
       SCREAMING_SNAKE_CASE_      		=     'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
       return input_text, output_text



 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       pass  # TODO add if relevant



 def  lowerCAmelCase_    (				self    :						int		):
       pass  # TODO add if relevant



 def  lowerCAmelCase_    (				self    :						List[Any]		):
       pass  # TODO add if relevant



 def  lowerCAmelCase_    (				self    :						Union[str, Any]		):
       SCREAMING_SNAKE_CASE_      		=     self.tokenizer_class(self.vocab_file							,							subword_tokenizer_type='character'		)

       SCREAMING_SNAKE_CASE_      		=     tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'		)
       self.assertListEqual(
           _lowerCAmelCase							,							['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']		)
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_lowerCAmelCase		)							,							[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]		)



 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       SCREAMING_SNAKE_CASE_      		=     ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

       SCREAMING_SNAKE_CASE_      		=     {}
       for i, token in enumerate(_lowerCAmelCase		):
             SCREAMING_SNAKE_CASE_      		=     i
       SCREAMING_SNAKE_CASE_      		=     CharacterTokenizer(vocab=_lowerCAmelCase							,							unk_token='[UNK]'		)

       self.assertListEqual(tokenizer.tokenize(''		)							,							[]		)

       self.assertListEqual(tokenizer.tokenize('こんにちは'		)							,							['こ', 'ん', 'に', 'ち', 'は']		)

       self.assertListEqual(tokenizer.tokenize('こんにちほ'		)							,							['こ', 'ん', 'に', 'ち', '[UNK]']		)




 def  lowerCAmelCase_    (				self    :						List[Any]		):
       SCREAMING_SNAKE_CASE_      		=     self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'		)

       SCREAMING_SNAKE_CASE_      		=     tokenizer.encode('ありがとう。'							,							add_special_tokens=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     tokenizer.encode('どういたしまして。'							,							add_special_tokens=_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase							,							_lowerCAmelCase		)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCamelCase_				( unittest.TestCase   ):


 '''simple docstring'''







 def  lowerCAmelCase_    (				self    :						Tuple		):
       SCREAMING_SNAKE_CASE_      		=     'cl-tohoku/bert-base-japanese'
       SCREAMING_SNAKE_CASE_      		=     AutoTokenizer.from_pretrained(_lowerCAmelCase		)
       self.assertIsInstance(_lowerCAmelCase							,							_lowerCAmelCase		)





class 			lowerCamelCase_				( unittest.TestCase   ):


 '''simple docstring'''







 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       SCREAMING_SNAKE_CASE_      		=     'cl-tohoku/bert-base-japanese'
       with self.assertLogs('transformers'							,							level='WARNING'		) as cm:
             BertTokenizer.from_pretrained(_lowerCAmelCase		)
             self.assertTrue(
                 cm.records[0].message.startswith(
                     'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                     ' is called from.'		)		)
       SCREAMING_SNAKE_CASE_      		=     'bert-base-cased'
       with self.assertLogs('transformers'							,							level='WARNING'		) as cm:
             BertJapaneseTokenizer.from_pretrained(_lowerCAmelCase		)
             self.assertTrue(
                 cm.records[0].message.startswith(
                     'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                     ' is called from.'		)		)




'''simple docstring'''






from typing import Callable, Optional

from .. import Features
from ..packaged_modules.generator.generator import Generator
from .abc import AbstractDatasetInputStream

class 				a__(   lowerCamelCase__      ):







    def __init__( self					:    int		,     __snake_case					:    Callable		,     __snake_case					:    Optional[Features] = None		,     __snake_case					:    str = None		,     __snake_case					:    bool = False		,     __snake_case					:    bool = False		,     __snake_case					:    Optional[dict] = None		,     __snake_case					:    Optional[int] = None		,     **__snake_case					:    Optional[int]		,     ):
      super().__init__(
          features=__snake_case		,     cache_dir=__snake_case		,     keep_in_memory=__snake_case		,     streaming=__snake_case		,     num_proc=__snake_case		,     **__snake_case		,     )
      a		:   List[Any]	  =	Generator(
          cache_dir=__snake_case		,     features=__snake_case		,     generator=__snake_case		,     gen_kwargs=__snake_case		,     **__snake_case		,     )







    def      lowercase_			( self					:    Any      ):
      # Build iterable dataset
      if self.streaming:
        a		:   Optional[int]	  =	self.builder.as_streaming_dataset(split='train'      )
      # Build regular (map-style) dataset
      else:
        a		:   List[Any]	  =	None
        a		:   Any	  =	None
        a		:   Optional[int]	  =	None
        a		:   List[str]	  =	None

        self.builder.download_and_prepare(
            download_config=__snake_case		,     download_mode=__snake_case		,     verification_mode=__snake_case		,     base_path=__snake_case		,     num_proc=self.num_proc		,     )
        a		:   Dict	  =	self.builder.as_dataset(
            split='train'		,     verification_mode=__snake_case		,     in_memory=self.keep_in_memory      )
      return dataset