
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class 				lowerCamelCase__							(       __lowercase  ,		unittest.TestCase):




	'''simple docstring'''




	_A			=  BertJapaneseTokenizer
	_A			=  False
	_A			=  True







	def      _lowerCamelCase       (      self	:str		)	->				Union[str, Any]:
								super().setUp()

								__UpperCamelCase			:     List[Any]         =      [
								    "[UNK]",
								    "[CLS]",
								    "[SEP]",
								    "こんにちは",
								    "こん",
								    "にちは",
								    "ばんは",
								    "##こん",
								    "##にちは",
								    "##ばんは",
								    "世界",
								    "##世界",
								    "、",
								    "##、",
								    "。",
								    "##。",
								]

								__UpperCamelCase			:     Tuple         =      os.path.join(self.tmpdirname  ,     VOCAB_FILES_NAMES["vocab_file"]		)
								with open(self.vocab_file  ,     "w"  ,     encoding="utf-8"		) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)







	def      _lowerCamelCase       (      self	:str  ,     a	:Any		)	->				Union[str, Any]:
								__UpperCamelCase			:     Optional[int]         =      "こんにちは、世界。 \nこんばんは、世界。"
								__UpperCamelCase			:     Union[str, Any]         =      "こんにちは 、 世界 。 こんばんは 、 世界 。"
								return input_text, output_text







	def      _lowerCamelCase       (      self	:Dict  ,     a	:Tuple		)	->				Dict:
								__UpperCamelCase     ,		__UpperCamelCase			:     Dict         =      self.get_input_output_texts(a		)
								__UpperCamelCase			:     str         =      tokenizer.encode(a  ,     add_special_tokens=a		)
								__UpperCamelCase			:     Any         =      tokenizer.decode(a  ,     clean_up_tokenization_spaces=a		)
								return text, ids







	def      _lowerCamelCase       (      self	:Any		)	->				List[Any]:
								pass  # TODO add if relevant







	def      _lowerCamelCase       (      self	:int		)	->				Union[str, Any]:
								pass  # TODO add if relevant







	def      _lowerCamelCase       (      self	:Union[str, Any]		)	->				Tuple:
								pass  # TODO add if relevant







	def      _lowerCamelCase       (      self	:Any		)	->				List[str]:
								__UpperCamelCase			:     List[str]         =      self.tokenizer_class(self.vocab_file		)

								__UpperCamelCase			:     Tuple         =      tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"		)
								self.assertListEqual(a  ,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]		)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a		)  ,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]		)







	def      _lowerCamelCase       (      self	:str		)	->				Dict:
								__UpperCamelCase			:     List[Any]         =      self.tokenizer_class(self.vocab_file  ,     word_tokenizer_type="mecab"		)
								self.assertIsNotNone(a		)

								__UpperCamelCase			:     Any         =      "こんにちは、世界。\nこんばんは、世界。"
								__UpperCamelCase			:     Any         =      tokenizer.tokenize(a		)
								self.assertListEqual(a  ,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]		)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a		)  ,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]		)

								__UpperCamelCase			:     Union[str, Any]         =      os.path.join(self.tmpdirname  ,     "tokenizer.bin"		)
								with open(a  ,     "wb"		) as handle:
															pickle.dump(a  ,     a		)

								with open(a  ,     "rb"		) as handle:
															__UpperCamelCase			:     Union[str, Any]         =      pickle.load(a		)

								__UpperCamelCase			:     Any         =      tokenizer_new.tokenize(a		)

								self.assertListEqual(a  ,     a		)







	def      _lowerCamelCase       (      self	:Optional[Any]		)	->				Optional[Any]:
								__UpperCamelCase			:     List[Any]         =      MecabTokenizer(mecab_dic="ipadic"		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )







	def      _lowerCamelCase       (      self	:Optional[int]		)	->				List[str]:
								try:
															__UpperCamelCase			:     int         =      MecabTokenizer(mecab_dic="unidic_lite"		)
								except ModuleNotFoundError:
															return

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )







	def      _lowerCamelCase       (      self	:Dict		)	->				Union[str, Any]:
								try:
															__UpperCamelCase			:     List[Any]         =      MecabTokenizer(mecab_dic="unidic"		)
								except ModuleNotFoundError:
															return

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )







	def      _lowerCamelCase       (      self	:Tuple		)	->				int:
								__UpperCamelCase			:     str         =      MecabTokenizer(do_lower_case=a  ,     mecab_dic="ipadic"		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )







	def      _lowerCamelCase       (      self	:List[str]		)	->				List[Any]:
								try:
															__UpperCamelCase			:     Tuple         =      MecabTokenizer(
															    do_lower_case=a  ,     normalize_text=a  ,     mecab_option="-d /usr/local/lib/mecab/dic/jumandic"		)
								except RuntimeError:
															# if dict doesn't exist in the system, previous code raises this error.
															return

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]  ,     )







	def      _lowerCamelCase       (      self	:List[str]		)	->				Optional[int]:
								__UpperCamelCase			:     Dict         =      MecabTokenizer(normalize_text=a  ,     mecab_dic="ipadic"		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]  ,     )







	@require_sudachi
	def      _lowerCamelCase       (      self	:str		)	->				Optional[Any]:
								__UpperCamelCase			:     List[Any]         =      self.tokenizer_class(self.vocab_file  ,     word_tokenizer_type="sudachi"		)
								self.assertIsNotNone(a		)

								__UpperCamelCase			:     str         =      "こんにちは、世界。\nこんばんは、世界。"
								__UpperCamelCase			:     Tuple         =      tokenizer.tokenize(a		)
								self.assertListEqual(a  ,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]		)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a		)  ,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]		)

								__UpperCamelCase			:     Optional[Any]         =      os.path.join(self.tmpdirname  ,     "tokenizer.bin"		)
								with open(a  ,     "wb"		) as handle:
															pickle.dump(a  ,     a		)

								with open(a  ,     "rb"		) as handle:
															__UpperCamelCase			:     Optional[int]         =      pickle.load(a		)

								__UpperCamelCase			:     str         =      tokenizer_new.tokenize(a		)

								self.assertListEqual(a  ,     a		)







	@require_sudachi
	def      _lowerCamelCase       (      self	:Any		)	->				Tuple:
								__UpperCamelCase			:     Any         =      SudachiTokenizer(sudachi_dict_type="core"		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]  ,     )







	@require_sudachi
	def      _lowerCamelCase       (      self	:Dict		)	->				Any:
								__UpperCamelCase			:     Any         =      SudachiTokenizer(sudachi_dict_type="core"  ,     sudachi_split_mode="A"		)

								self.assertListEqual(tokenizer.tokenize("外国人参政権"		)  ,     ["外国", "人", "参政", "権"]		)







	@require_sudachi
	def      _lowerCamelCase       (      self	:List[str]		)	->				Tuple:
								__UpperCamelCase			:     Optional[Any]         =      SudachiTokenizer(sudachi_dict_type="core"  ,     sudachi_split_mode="B"		)

								self.assertListEqual(tokenizer.tokenize("外国人参政権"		)  ,     ["外国人", "参政権"]		)







	@require_sudachi
	def      _lowerCamelCase       (      self	:Dict		)	->				List[str]:
								__UpperCamelCase			:     List[str]         =      SudachiTokenizer(sudachi_dict_type="core"  ,     sudachi_split_mode="C"		)

								self.assertListEqual(tokenizer.tokenize("外国人参政権"		)  ,     ["外国人参政権"]		)







	@require_sudachi
	def      _lowerCamelCase       (      self	:Union[str, Any]		)	->				Union[str, Any]:
								__UpperCamelCase			:     Optional[int]         =      SudachiTokenizer(do_lower_case=a  ,     sudachi_dict_type="core"		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]  ,     )







	@require_sudachi
	def      _lowerCamelCase       (      self	:List[Any]		)	->				Any:
								__UpperCamelCase			:     Dict         =      SudachiTokenizer(normalize_text=a  ,     sudachi_dict_type="core"		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]  ,     )







	@require_sudachi
	def      _lowerCamelCase       (      self	:Optional[Any]		)	->				int:
								__UpperCamelCase			:     int         =      SudachiTokenizer(trim_whitespace=a  ,     sudachi_dict_type="core"		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )







	@require_jumanpp
	def      _lowerCamelCase       (      self	:List[Any]		)	->				List[Any]:
								__UpperCamelCase			:     Tuple         =      self.tokenizer_class(self.vocab_file  ,     word_tokenizer_type="jumanpp"		)
								self.assertIsNotNone(a		)

								__UpperCamelCase			:     int         =      "こんにちは、世界。\nこんばんは、世界。"
								__UpperCamelCase			:     Dict         =      tokenizer.tokenize(a		)
								self.assertListEqual(a  ,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]		)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a		)  ,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]		)

								__UpperCamelCase			:     Optional[int]         =      os.path.join(self.tmpdirname  ,     "tokenizer.bin"		)
								with open(a  ,     "wb"		) as handle:
															pickle.dump(a  ,     a		)

								with open(a  ,     "rb"		) as handle:
															__UpperCamelCase			:     Any         =      pickle.load(a		)

								__UpperCamelCase			:     List[Any]         =      tokenizer_new.tokenize(a		)

								self.assertListEqual(a  ,     a		)







	@require_jumanpp
	def      _lowerCamelCase       (      self	:Tuple		)	->				str:
								__UpperCamelCase			:     List[Any]         =      JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]  ,     )







	@require_jumanpp
	def      _lowerCamelCase       (      self	:List[str]		)	->				List[Any]:
								__UpperCamelCase			:     Any         =      JumanppTokenizer(do_lower_case=a		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]  ,     )







	@require_jumanpp
	def      _lowerCamelCase       (      self	:Dict		)	->				str:
								__UpperCamelCase			:     Union[str, Any]         =      JumanppTokenizer(normalize_text=a		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]  ,     )







	@require_jumanpp
	def      _lowerCamelCase       (      self	:Union[str, Any]		)	->				Optional[Any]:
								__UpperCamelCase			:     List[str]         =      JumanppTokenizer(trim_whitespace=a		)

								self.assertListEqual(
								    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "		)  ,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]  ,     )







	@require_jumanpp
	def      _lowerCamelCase       (      self	:List[Any]		)	->				Optional[Any]:
								__UpperCamelCase			:     Union[str, Any]         =      JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"		)  ,     ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]  ,     )







	def      _lowerCamelCase       (      self	:Union[str, Any]		)	->				List[str]:
								__UpperCamelCase			:     str         =      ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

								__UpperCamelCase			:     str         =      {}
								for i, token in enumerate(a		):
															__UpperCamelCase			:     Tuple         =      i
								__UpperCamelCase			:     Optional[Any]         =      WordpieceTokenizer(vocab=a  ,     unk_token="[UNK]"		)

								self.assertListEqual(tokenizer.tokenize(""		)  ,     []		)

								self.assertListEqual(tokenizer.tokenize("こんにちは"		)  ,     ["こんにちは"]		)

								self.assertListEqual(tokenizer.tokenize("こんばんは"		)  ,     ["こん", "##ばんは"]		)

								self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"		)  ,     ["こん", "##ばんは", "[UNK]", "こんにちは"]		)







	def      _lowerCamelCase       (      self	:List[Any]		)	->				Tuple:
								__UpperCamelCase			:     Any         =      BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"		)
								__UpperCamelCase			:     int         =      tokenizer.subword_tokenizer

								__UpperCamelCase			:     int         =      subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"		)
								self.assertListEqual(a  ,     ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]		)

								__UpperCamelCase			:     List[Any]         =      subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"		)
								self.assertListEqual(a  ,     ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]		)







	def      _lowerCamelCase       (      self	:List[Any]		)	->				List[str]:
								__UpperCamelCase			:     Dict         =      self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"		)

								__UpperCamelCase			:     Any         =      tokenizer.encode("ありがとう。"  ,     add_special_tokens=a		)
								__UpperCamelCase			:     int         =      tokenizer.encode("どういたしまして。"  ,     add_special_tokens=a		)

								__UpperCamelCase			:     str         =      tokenizer.build_inputs_with_special_tokens(a		)
								__UpperCamelCase			:     Union[str, Any]         =      tokenizer.build_inputs_with_special_tokens(a  ,     a		)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 				lowerCamelCase__							(       __lowercase  ,		unittest.TestCase):




	'''simple docstring'''




	_A			=  BertJapaneseTokenizer
	_A			=  False







	def      _lowerCamelCase       (      self	:List[str]		)	->				List[Any]:
								super().setUp()

								__UpperCamelCase			:     int         =      ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

								__UpperCamelCase			:     Union[str, Any]         =      os.path.join(self.tmpdirname  ,     VOCAB_FILES_NAMES["vocab_file"]		)
								with open(self.vocab_file  ,     "w"  ,     encoding="utf-8"		) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)







	def      _lowerCamelCase       (      self	:Dict  ,     **a	:Any		)	->				List[Any]:
								return BertJapaneseTokenizer.from_pretrained(self.tmpdirname  ,     subword_tokenizer_type="character"  ,     **a		)







	def      _lowerCamelCase       (      self	:Optional[Any]  ,     a	:Union[str, Any]		)	->				List[Any]:
								__UpperCamelCase			:     int         =      "こんにちは、世界。 \nこんばんは、世界。"
								__UpperCamelCase			:     Tuple         =      "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
								return input_text, output_text







	def      _lowerCamelCase       (      self	:Optional[int]		)	->				Any:
								pass  # TODO add if relevant







	def      _lowerCamelCase       (      self	:Union[str, Any]		)	->				Optional[Any]:
								pass  # TODO add if relevant







	def      _lowerCamelCase       (      self	:Any		)	->				List[str]:
								pass  # TODO add if relevant







	def      _lowerCamelCase       (      self	:List[Any]		)	->				Optional[Any]:
								__UpperCamelCase			:     Tuple         =      self.tokenizer_class(self.vocab_file  ,     subword_tokenizer_type="character"		)

								__UpperCamelCase			:     Optional[int]         =      tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"		)
								self.assertListEqual(
								    a  ,     ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]		)
								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(a		)  ,     [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]		)







	def      _lowerCamelCase       (      self	:Tuple		)	->				int:
								__UpperCamelCase			:     str         =      ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

								__UpperCamelCase			:     Union[str, Any]         =      {}
								for i, token in enumerate(a		):
															__UpperCamelCase			:     Optional[int]         =      i
								__UpperCamelCase			:     List[str]         =      CharacterTokenizer(vocab=a  ,     unk_token="[UNK]"		)

								self.assertListEqual(tokenizer.tokenize(""		)  ,     []		)

								self.assertListEqual(tokenizer.tokenize("こんにちは"		)  ,     ["こ", "ん", "に", "ち", "は"]		)

								self.assertListEqual(tokenizer.tokenize("こんにちほ"		)  ,     ["こ", "ん", "に", "ち", "[UNK]"]		)







	def      _lowerCamelCase       (      self	:str		)	->				int:
								__UpperCamelCase			:     Tuple         =      self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"		)

								__UpperCamelCase			:     Union[str, Any]         =      tokenizer.encode("ありがとう。"  ,     add_special_tokens=a		)
								__UpperCamelCase			:     Tuple         =      tokenizer.encode("どういたしまして。"  ,     add_special_tokens=a		)

								__UpperCamelCase			:     Tuple         =      tokenizer.build_inputs_with_special_tokens(a		)
								__UpperCamelCase			:     Optional[Any]         =      tokenizer.build_inputs_with_special_tokens(a  ,     a		)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 				lowerCamelCase__							(       unittest.TestCase):




	'''simple docstring'''




	def      _lowerCamelCase       (      self	:Any		)	->				List[str]:
								__UpperCamelCase			:     Tuple         =      "cl-tohoku/bert-base-japanese"
								__UpperCamelCase			:     Optional[int]         =      AutoTokenizer.from_pretrained(a		)
								self.assertIsInstance(a  ,     a		)




class 				lowerCamelCase__							(       unittest.TestCase):




	'''simple docstring'''




	def      _lowerCamelCase       (      self	:List[str]		)	->				Any:
								__UpperCamelCase			:     Any         =      "cl-tohoku/bert-base-japanese"
								with self.assertLogs("transformers"  ,     level="WARNING"		) as cm:
															BertTokenizer.from_pretrained(a		)
															self.assertTrue(
															    cm.records[0].message.startswith(
															        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
															        " is called from."		)		)
								__UpperCamelCase			:     Optional[Any]         =      "bert-base-cased"
								with self.assertLogs("transformers"  ,     level="WARNING"		) as cm:
															BertJapaneseTokenizer.from_pretrained(a		)
															self.assertTrue(
															    cm.records[0].message.startswith(
															        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
															        " is called from."		)		)

import argparse

import torch

from transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert
from transformers.utils import logging


logging.set_verbosity_info()




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    Dict		,						_lowerCamelCase				:    Tuple		,						_lowerCamelCase				:    Union[str, Any])      ->					List[Any]:






							'''simple docstring'''





							__UpperCamelCase			:     Tuple         =      BertConfig.from_json_file(_lowerCamelCase)
							print(F'Building PyTorch model from configuration: {config}')
							__UpperCamelCase			:     List[Any]         =      BertForPreTraining(_lowerCamelCase)

							# Load weights from tf checkpoint
							load_tf_weights_in_bert(_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase)

							# Save pytorch-model
							print(F'Save PyTorch model to {pytorch_dump_path}')
							torch.save(model.state_dict()		,						_lowerCamelCase)


if __name__ == "__main__":
		lowercase						:      Any									= argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
		)
		parser.add_argument(
		    '--bert_config_file',
		    default=None,
		    type=str,
		    required=True,
		    help=(
		        'The config json file corresponding to the pre-trained BERT model. \n'
		        'This specifies the model architecture.'
		    ),
		)
		parser.add_argument(
		    '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
		)
		lowercase						:      Tuple									= parser.parse_args()
		convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)