






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin


@custom_tokenizers
class 			SCREAMING_SNAKE_CASE      (      lowerCAmelCase   ,		unittest.TestCase       ):
     '''simple docstring'''


     UpperCamelCase_				:		List[Any]								=						BertJapaneseTokenizer
     UpperCamelCase_				:		Optional[Any]								=						False
     UpperCamelCase_				:		str								=						True


     def 					_A							(					self							:						Tuple   ):
       super().setUp()

       SCREAMING_SNAKE_CASE							:   Dict						    =     [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "こんにちは",
           "こん",
           "にちは",
           "ばんは",
           "##こん",
           "##にちは",
           "##ばんは",
           "世界",
           "##世界",
           "、",
           "##、",
           "。",
           "##。",
       ]

       SCREAMING_SNAKE_CASE							:   Optional[int]						    =     os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["vocab_file"]   )
       with open(self.vocab_file       ,			"w"       ,			encoding="utf-8"   ) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )


     def 					_A							(					self							:						Tuple       ,			UpperCAmelCase_							:						int   ):
       SCREAMING_SNAKE_CASE							:   List[str]						    =     "こんにちは、世界。 \nこんばんは、世界。"
       SCREAMING_SNAKE_CASE							:   Tuple						    =     "こんにちは 、 世界 。 こんばんは 、 世界 。"
       return input_text, output_text


     def 					_A							(					self							:						List[Any]       ,			UpperCAmelCase_							:						Any   ):
       SCREAMING_SNAKE_CASE    ,	SCREAMING_SNAKE_CASE							:   List[Any]						    =     self.get_input_output_texts(UpperCAmelCase_   )
       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     tokenizer.encode(UpperCAmelCase_       ,			add_special_tokens=UpperCAmelCase_   )
       SCREAMING_SNAKE_CASE							:   str						    =     tokenizer.decode(UpperCAmelCase_       ,			clean_up_tokenization_spaces=UpperCAmelCase_   )
       return text, ids


     def 					_A							(					self							:						List[Any]   ):
       pass  # TODO add if relevant


     def 					_A							(					self							:						str   ):
       pass  # TODO add if relevant


     def 					_A							(					self							:						Optional[Any]   ):
       pass  # TODO add if relevant


     def 					_A							(					self							:						Optional[Any]   ):
       SCREAMING_SNAKE_CASE							:   Tuple						    =     self.tokenizer_class(self.vocab_file   )

       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"   )
       self.assertListEqual(UpperCAmelCase_       ,			["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]   )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_   )       ,			[3, 12, 10, 14, 4, 9, 12, 10, 14]   )


     def 					_A							(					self							:						Any   ):
       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     self.tokenizer_class(self.vocab_file       ,			word_tokenizer_type="mecab"   )
       self.assertIsNotNone(UpperCAmelCase_   )

       SCREAMING_SNAKE_CASE							:   str						    =     "こんにちは、世界。\nこんばんは、世界。"
       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     tokenizer.tokenize(UpperCAmelCase_   )
       self.assertListEqual(UpperCAmelCase_       ,			["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]   )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_   )       ,			[3, 12, 10, 14, 4, 9, 12, 10, 14]   )

       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     os.path.join(self.tmpdirname       ,			"tokenizer.bin"   )
       with open(UpperCAmelCase_       ,			"wb"   ) as handle:
         pickle.dump(UpperCAmelCase_       ,			UpperCAmelCase_   )

       with open(UpperCAmelCase_       ,			"rb"   ) as handle:
         SCREAMING_SNAKE_CASE							:   List[Any]						    =     pickle.load(UpperCAmelCase_   )

       SCREAMING_SNAKE_CASE							:   str						    =     tokenizer_new.tokenize(UpperCAmelCase_   )

       self.assertListEqual(UpperCAmelCase_       ,			UpperCAmelCase_   )


     def 					_A							(					self							:						Union[str, Any]   ):
       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     MecabTokenizer(mecab_dic="ipadic"   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,			)


     def 					_A							(					self							:						Dict   ):
       try:
         SCREAMING_SNAKE_CASE							:   List[Any]						    =     MecabTokenizer(mecab_dic="unidic_lite"   )
       except ModuleNotFoundError:
         return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,			)


     def 					_A							(					self							:						Dict   ):
       try:
         SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     MecabTokenizer(mecab_dic="unidic"   )
       except ModuleNotFoundError:
         return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,			)


     def 					_A							(					self							:						Optional[int]   ):
       SCREAMING_SNAKE_CASE							:   Any						    =     MecabTokenizer(do_lower_case=UpperCAmelCase_       ,			mecab_dic="ipadic"   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,			)


     def 					_A							(					self							:						List[str]   ):
       try:
         SCREAMING_SNAKE_CASE							:   Tuple						    =     MecabTokenizer(
             do_lower_case=UpperCAmelCase_       ,			normalize_text=UpperCAmelCase_       ,			mecab_option="-d /usr/local/lib/mecab/dic/jumandic"   )
       except RuntimeError:
         # if dict doesn't exist in the system, previous code raises this error.
         return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]       ,			)


     def 					_A							(					self							:						Dict   ):
       SCREAMING_SNAKE_CASE							:   List[Any]						    =     MecabTokenizer(normalize_text=UpperCAmelCase_       ,			mecab_dic="ipadic"   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]       ,			)


     @require_sudachi
     def 					_A							(					self							:						Dict   ):
       SCREAMING_SNAKE_CASE							:   Any						    =     self.tokenizer_class(self.vocab_file       ,			word_tokenizer_type="sudachi"   )
       self.assertIsNotNone(UpperCAmelCase_   )

       SCREAMING_SNAKE_CASE							:   List[str]						    =     "こんにちは、世界。\nこんばんは、世界。"
       SCREAMING_SNAKE_CASE							:   List[str]						    =     tokenizer.tokenize(UpperCAmelCase_   )
       self.assertListEqual(UpperCAmelCase_       ,			["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]   )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_   )       ,			[3, 12, 10, 14, 4, 9, 12, 10, 14]   )

       SCREAMING_SNAKE_CASE							:   Tuple						    =     os.path.join(self.tmpdirname       ,			"tokenizer.bin"   )
       with open(UpperCAmelCase_       ,			"wb"   ) as handle:
         pickle.dump(UpperCAmelCase_       ,			UpperCAmelCase_   )

       with open(UpperCAmelCase_       ,			"rb"   ) as handle:
         SCREAMING_SNAKE_CASE							:   List[str]						    =     pickle.load(UpperCAmelCase_   )

       SCREAMING_SNAKE_CASE							:   Optional[int]						    =     tokenizer_new.tokenize(UpperCAmelCase_   )

       self.assertListEqual(UpperCAmelCase_       ,			UpperCAmelCase_   )


     @require_sudachi
     def 					_A							(					self							:						List[str]   ):
       SCREAMING_SNAKE_CASE							:   str						    =     SudachiTokenizer(sudachi_dict_type="core"   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,			)


     @require_sudachi
     def 					_A							(					self							:						Union[str, Any]   ):
       SCREAMING_SNAKE_CASE							:   Any						    =     SudachiTokenizer(sudachi_dict_type="core"       ,			sudachi_split_mode="A"   )

       self.assertListEqual(tokenizer.tokenize("外国人参政権"   )       ,			["外国", "人", "参政", "権"]   )


     @require_sudachi
     def 					_A							(					self							:						Optional[Any]   ):
       SCREAMING_SNAKE_CASE							:   str						    =     SudachiTokenizer(sudachi_dict_type="core"       ,			sudachi_split_mode="B"   )

       self.assertListEqual(tokenizer.tokenize("外国人参政権"   )       ,			["外国人", "参政権"]   )


     @require_sudachi
     def 					_A							(					self							:						str   ):
       SCREAMING_SNAKE_CASE							:   Dict						    =     SudachiTokenizer(sudachi_dict_type="core"       ,			sudachi_split_mode="C"   )

       self.assertListEqual(tokenizer.tokenize("外国人参政権"   )       ,			["外国人参政権"]   )


     @require_sudachi
     def 					_A							(					self							:						List[Any]   ):
       SCREAMING_SNAKE_CASE							:   Dict						    =     SudachiTokenizer(do_lower_case=UpperCAmelCase_       ,			sudachi_dict_type="core"   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,			)


     @require_sudachi
     def 					_A							(					self							:						Dict   ):
       SCREAMING_SNAKE_CASE							:   int						    =     SudachiTokenizer(normalize_text=UpperCAmelCase_       ,			sudachi_dict_type="core"   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]       ,			)


     @require_sudachi
     def 					_A							(					self							:						List[Any]   ):
       SCREAMING_SNAKE_CASE							:   Union[str, Any]						    =     SudachiTokenizer(trim_whitespace=UpperCAmelCase_       ,			sudachi_dict_type="core"   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,			)


     @require_jumanpp
     def 					_A							(					self							:						Union[str, Any]   ):
       SCREAMING_SNAKE_CASE							:   Any						    =     self.tokenizer_class(self.vocab_file       ,			word_tokenizer_type="jumanpp"   )
       self.assertIsNotNone(UpperCAmelCase_   )

       SCREAMING_SNAKE_CASE							:   Dict						    =     "こんにちは、世界。\nこんばんは、世界。"
       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     tokenizer.tokenize(UpperCAmelCase_   )
       self.assertListEqual(UpperCAmelCase_       ,			["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]   )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_   )       ,			[3, 12, 10, 14, 4, 9, 12, 10, 14]   )

       SCREAMING_SNAKE_CASE							:   Optional[int]						    =     os.path.join(self.tmpdirname       ,			"tokenizer.bin"   )
       with open(UpperCAmelCase_       ,			"wb"   ) as handle:
         pickle.dump(UpperCAmelCase_       ,			UpperCAmelCase_   )

       with open(UpperCAmelCase_       ,			"rb"   ) as handle:
         SCREAMING_SNAKE_CASE							:   List[Any]						    =     pickle.load(UpperCAmelCase_   )

       SCREAMING_SNAKE_CASE							:   str						    =     tokenizer_new.tokenize(UpperCAmelCase_   )

       self.assertListEqual(UpperCAmelCase_       ,			UpperCAmelCase_   )


     @require_jumanpp
     def 					_A							(					self							:						Optional[int]   ):
       SCREAMING_SNAKE_CASE							:   str						    =     JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,			)


     @require_jumanpp
     def 					_A							(					self							:						Optional[int]   ):
       SCREAMING_SNAKE_CASE							:   List[Any]						    =     JumanppTokenizer(do_lower_case=UpperCAmelCase_   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,			)


     @require_jumanpp
     def 					_A							(					self							:						Tuple   ):
       SCREAMING_SNAKE_CASE							:   str						    =     JumanppTokenizer(normalize_text=UpperCAmelCase_   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,			)


     @require_jumanpp
     def 					_A							(					self							:						Optional[Any]   ):
       SCREAMING_SNAKE_CASE							:   str						    =     JumanppTokenizer(trim_whitespace=UpperCAmelCase_   )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "   )       ,			["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]       ,			)


     @require_jumanpp
     def 					_A							(					self							:						Union[str, Any]   ):
       SCREAMING_SNAKE_CASE							:   List[str]						    =     JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"   )       ,			["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]       ,			)


     def 					_A							(					self							:						Optional[Any]   ):
       SCREAMING_SNAKE_CASE							:   List[Any]						    =     ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

       SCREAMING_SNAKE_CASE							:   str						    =     {}
       for i, token in enumerate(UpperCAmelCase_   ):
         SCREAMING_SNAKE_CASE							:   str						    =     i
       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     WordpieceTokenizer(vocab=UpperCAmelCase_       ,			unk_token="[UNK]"   )

       self.assertListEqual(tokenizer.tokenize(""   )       ,			[]   )

       self.assertListEqual(tokenizer.tokenize("こんにちは"   )       ,			["こんにちは"]   )

       self.assertListEqual(tokenizer.tokenize("こんばんは"   )       ,			["こん", "##ばんは"]   )

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"   )       ,			["こん", "##ばんは", "[UNK]", "こんにちは"]   )


     def 					_A							(					self							:						List[str]   ):
       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"   )
       SCREAMING_SNAKE_CASE							:   Union[str, Any]						    =     tokenizer.subword_tokenizer

       SCREAMING_SNAKE_CASE							:   Any						    =     subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"   )
       self.assertListEqual(UpperCAmelCase_       ,			["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]   )

       SCREAMING_SNAKE_CASE							:   List[str]						    =     subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"   )
       self.assertListEqual(UpperCAmelCase_       ,			["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]   )






     def 					_A							(					self							:						List[Any]   ):
       SCREAMING_SNAKE_CASE							:   Optional[int]						    =     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"   )

       SCREAMING_SNAKE_CASE							:   Dict						    =     tokenizer.encode("ありがとう。"       ,			add_special_tokens=UpperCAmelCase_   )
       SCREAMING_SNAKE_CASE							:   Union[str, Any]						    =     tokenizer.encode("どういたしまして。"       ,			add_special_tokens=UpperCAmelCase_   )

       SCREAMING_SNAKE_CASE							:   List[Any]						    =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_   )
       SCREAMING_SNAKE_CASE							:   List[str]						    =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_       ,			UpperCAmelCase_   )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			SCREAMING_SNAKE_CASE      (      lowerCAmelCase   ,		unittest.TestCase       ):
     '''simple docstring'''


     UpperCamelCase_				:		Optional[Any]								=						BertJapaneseTokenizer
     UpperCamelCase_				:		Tuple								=						False


     def 					_A							(					self							:						str   ):
       super().setUp()

       SCREAMING_SNAKE_CASE							:   Tuple						    =     ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       SCREAMING_SNAKE_CASE							:   Any						    =     os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["vocab_file"]   )
       with open(self.vocab_file       ,			"w"       ,			encoding="utf-8"   ) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )


     def 					_A							(					self							:						str       ,			**UpperCAmelCase_							:						int   ):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,			subword_tokenizer_type="character"       ,			**UpperCAmelCase_   )


     def 					_A							(					self							:						List[str]       ,			UpperCAmelCase_							:						Union[str, Any]   ):
       SCREAMING_SNAKE_CASE							:   Tuple						    =     "こんにちは、世界。 \nこんばんは、世界。"
       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
       return input_text, output_text


     def 					_A							(					self							:						Any   ):
       pass  # TODO add if relevant


     def 					_A							(					self							:						Optional[int]   ):
       pass  # TODO add if relevant


     def 					_A							(					self							:						Dict   ):
       pass  # TODO add if relevant


     def 					_A							(					self							:						Dict   ):
       SCREAMING_SNAKE_CASE							:   Dict						    =     self.tokenizer_class(self.vocab_file       ,			subword_tokenizer_type="character"   )

       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"   )
       self.assertListEqual(
           UpperCAmelCase_       ,			["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]   )
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(UpperCAmelCase_   )       ,			[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]   )


     def 					_A							(					self							:						Union[str, Any]   ):
       SCREAMING_SNAKE_CASE							:   List[str]						    =     ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       SCREAMING_SNAKE_CASE							:   Optional[int]						    =     {}
       for i, token in enumerate(UpperCAmelCase_   ):
         SCREAMING_SNAKE_CASE							:   str						    =     i
       SCREAMING_SNAKE_CASE							:   List[str]						    =     CharacterTokenizer(vocab=UpperCAmelCase_       ,			unk_token="[UNK]"   )

       self.assertListEqual(tokenizer.tokenize(""   )       ,			[]   )

       self.assertListEqual(tokenizer.tokenize("こんにちは"   )       ,			["こ", "ん", "に", "ち", "は"]   )

       self.assertListEqual(tokenizer.tokenize("こんにちほ"   )       ,			["こ", "ん", "に", "ち", "[UNK]"]   )






     def 					_A							(					self							:						List[str]   ):
       SCREAMING_SNAKE_CASE							:   Any						    =     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"   )

       SCREAMING_SNAKE_CASE							:   str						    =     tokenizer.encode("ありがとう。"       ,			add_special_tokens=UpperCAmelCase_   )
       SCREAMING_SNAKE_CASE							:   List[str]						    =     tokenizer.encode("どういたしまして。"       ,			add_special_tokens=UpperCAmelCase_   )

       SCREAMING_SNAKE_CASE							:   List[Any]						    =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_   )
       SCREAMING_SNAKE_CASE							:   Tuple						    =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_       ,			UpperCAmelCase_   )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			SCREAMING_SNAKE_CASE      (      unittest.TestCase       ):
     '''simple docstring'''


     def 					_A							(					self							:						Union[str, Any]   ):
       SCREAMING_SNAKE_CASE							:   List[str]						    =     "cl-tohoku/bert-base-japanese"
       SCREAMING_SNAKE_CASE							:   Any						    =     AutoTokenizer.from_pretrained(UpperCAmelCase_   )
       self.assertIsInstance(UpperCAmelCase_       ,			UpperCAmelCase_   )





class 			SCREAMING_SNAKE_CASE      (      unittest.TestCase       ):
     '''simple docstring'''


     def 					_A							(					self							:						int   ):
       SCREAMING_SNAKE_CASE							:   Any						    =     "cl-tohoku/bert-base-japanese"
       with self.assertLogs("transformers"       ,			level="WARNING"   ) as cm:
         BertTokenizer.from_pretrained(UpperCAmelCase_   )
         self.assertTrue(
             cm.records[0].message.startswith(
                 "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                 " is called from."   )   )
       SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     "bert-base-cased"
       with self.assertLogs("transformers"       ,			level="WARNING"   ) as cm:
         BertJapaneseTokenizer.from_pretrained(UpperCAmelCase_   )
         self.assertTrue(
             cm.records[0].message.startswith(
                 "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                 " is called from."   )   )









'''simple docstring'''







# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


__lowerCamelCase									=     '''Run commands across TPU VMs for initial setup before running `accelerate launch`.'''
def    UpperCAmelCase__     (   UpperCAmelCase__=None					)      ->      List[str]:
       if subparsers is not None:
              A_						       =   subparsers.add_parser("""tpu-config""",							description=_description					)
       else:
              A_						       =   argparse.ArgumentParser("""Accelerate tpu-config command""",							description=_description					)
       # Core arguments
       A_						       =   parser.add_argument_group(
           """Config Arguments""",							"""Arguments that can be configured through `accelerate config`."""					)
       config_args.add_argument(
           """--config_file""",							type=UpperCAmelCase__,							default=UpperCAmelCase__,							help="""Path to the config file to use for accelerate.""",							)
       config_args.add_argument(
           """--tpu_name""",							default=UpperCAmelCase__,							help="""The name of the TPU to use. If not specified, will use the TPU specified in the config file.""",							)
       config_args.add_argument(
           """--tpu_zone""",							default=UpperCAmelCase__,							help="""The zone of the TPU to use. If not specified, will use the zone specified in the config file.""",							)
       A_						       =   parser.add_argument_group("""TPU Arguments""",							"""Arguments for options ran inside the TPU."""					)
       pod_args.add_argument(
           """--use_alpha""",							action="""store_true""",							help="""Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.""",							)
       pod_args.add_argument(
           """--command_file""",							default=UpperCAmelCase__,							help="""The path to the file containing the commands to run on the pod on startup.""",							)
       pod_args.add_argument(
           """--command""",							action="""append""",							nargs="""+""",							help="""A command to run on the pod. Can be passed multiple times.""",							)
       pod_args.add_argument(
           """--install_accelerate""",							action="""store_true""",							help="""Whether to install accelerate on the pod. Defaults to False.""",							)
       pod_args.add_argument(
           """--accelerate_version""",							default="""latest""",							help="""The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify 'dev' to install from GitHub.""",							)
       pod_args.add_argument(
           """--debug""",							action="""store_true""",							help="""If set, will print the command that would be run instead of running it."""					)

       if subparsers is not None:
              parser.set_defaults(func=UpperCAmelCase__					)
       return parser
def    UpperCAmelCase__     (   UpperCAmelCase__					)      ->      List[Any]:
       A_						       =   None

       # Get the default from the config file if it exists.
       if args.config_file is not None or os.path.isfile(UpperCAmelCase__					):
              A_						       =   load_config_from_file(args.config_file					)
              if not args.command_file and defaults.command_file is not None and not args.command:
                     A_						       =   defaults.command_file
              if not args.command and defaults.commands is not None:
                     A_						       =   defaults.commands
              if not args.tpu_name:
                     A_						       =   defaults.tpu_name
              if not args.tpu_zone:
                     A_						       =   defaults.tpu_zone
       if args.accelerate_version == "dev":
              A_						       =   """git+https://github.com/huggingface/accelerate.git"""
       elif args.accelerate_version == "latest":
              A_						       =   """accelerate -U"""
       elif isinstance(parse(args.accelerate_version					),							UpperCAmelCase__					):
              A_						       =   F'''accelerate=={args.accelerate_version}'''

       if not args.command_file and not args.command:
              raise ValueError("""You must specify either a command file or a command to run on the pod."""					)

       if args.command_file:
              with open(args.command_file,							"""r"""					) as f:
                     A_						       =   [f.read().splitlines()]

    # To turn list of lists into list of strings
       if isinstance(args.command[0],							UpperCAmelCase__					):
              A_						       =   [line for cmd in args.command for line in cmd]
       # Default to the shared folder and install accelerate
       A_						       =   ["""cd /usr/share"""]
       if args.install_accelerate:
              new_cmd += [F'''pip install {args.accelerate_version}''']
       new_cmd += args.command
       A_						       =   """; """.join(UpperCAmelCase__					)

       # Then send it to gcloud
       # Eventually try to use google-api-core to do this instead of subprocess
       A_						       =   ["""gcloud"""]
       if args.use_alpha:
              cmd += ["alpha"]
       cmd += [
           "compute",
           "tpus",
           "tpu-vm",
           "ssh",
           args.tpu_name,
           "--zone",
           args.tpu_zone,
           "--command",
           args.command,
           "--worker",
           "all",
       ]
       if args.debug:
              print(F'''Running {" ".join(UpperCAmelCase__					)}'''					)
              return
       subprocess.run(UpperCAmelCase__					)
       print("""Successfully setup pod."""					)







def    UpperCAmelCase__     (   )      ->      int:
       A_						       =   tpu_command_parser()
       A_						       =   parser.parse_args()

       tpu_command_launcher(UpperCAmelCase__					)
