





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class 					lowercase__ (			__lowerCamelCase    ,       unittest.TestCase       ):

			'''simple docstring'''


			a		:							List[str]     	=	BertJapaneseTokenizer
			a		:							List[str]     	=	False
			a		:							List[Any]     	=	True




			def    UpperCamelCase__						(					self							)				->   Optional[Any]:






							"""simple docstring"""
							super().setUp()

							UpperCamelCase__							:       List[Any]      =   [
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''こんにちは''',
							    '''こん''',
							    '''にちは''',
							    '''ばんは''',
							    '''##こん''',
							    '''##にちは''',
							    '''##ばんは''',
							    '''世界''',
							    '''##世界''',
							    '''、''',
							    '''##、''',
							    '''。''',
							    '''##。''',
							]

							UpperCamelCase__							:       Any      =   os.path.join(self.tmpdirname,	VOCAB_FILES_NAMES['''vocab_file''']							)
							with open(self.vocab_file,	'''w''',	encoding='''utf-8'''							) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)




			def    UpperCamelCase__						(					self,	__magic_name__							)				->   List[Any]:






							"""simple docstring"""
							UpperCamelCase__							:       Dict      =   '''こんにちは、世界。 \nこんばんは、世界。'''
							UpperCamelCase__							:       str      =   '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
							return input_text, output_text




			def    UpperCamelCase__						(					self,	__magic_name__							)				->   Union[str, Any]:






							"""simple docstring"""
							UpperCamelCase__							:       List[str]      =   self.get_input_output_texts(__magic_name__							)
							UpperCamelCase__							:       Tuple      =   tokenizer.encode(__magic_name__,	add_special_tokens=__magic_name__							)
							UpperCamelCase__							:       Dict      =   tokenizer.decode(__magic_name__,	clean_up_tokenization_spaces=__magic_name__							)
							return text, ids




			def    UpperCamelCase__						(					self							)				->   int:






							"""simple docstring"""
							pass  # TODO add if relevant




			def    UpperCamelCase__						(					self							)				->   List[Any]:






							"""simple docstring"""
							pass  # TODO add if relevant




			def    UpperCamelCase__						(					self							)				->   int:






							"""simple docstring"""
							pass  # TODO add if relevant




			def    UpperCamelCase__						(					self							)				->   int:






							"""simple docstring"""
							UpperCamelCase__							:       str      =   self.tokenizer_class(self.vocab_file							)

							UpperCamelCase__							:       str      =   tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''							)
							self.assertListEqual(__magic_name__,	['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__							),	[3, 12, 10, 14, 4, 9, 12, 10, 14]							)




			def    UpperCamelCase__						(					self							)				->   Any:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   self.tokenizer_class(self.vocab_file,	word_tokenizer_type='''mecab'''							)
							self.assertIsNotNone(__magic_name__							)

							UpperCamelCase__							:       List[Any]      =   '''こんにちは、世界。\nこんばんは、世界。'''
							UpperCamelCase__							:       Dict      =   tokenizer.tokenize(__magic_name__							)
							self.assertListEqual(__magic_name__,	['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__							),	[3, 12, 10, 14, 4, 9, 12, 10, 14]							)

							UpperCamelCase__							:       str      =   os.path.join(self.tmpdirname,	'''tokenizer.bin'''							)
							with open(__magic_name__,	'''wb'''							) as handle:
											pickle.dump(__magic_name__,	__magic_name__							)

							with open(__magic_name__,	'''rb'''							) as handle:
											UpperCamelCase__							:       Optional[Any]      =   pickle.load(__magic_name__							)

							UpperCamelCase__							:       int      =   tokenizer_new.tokenize(__magic_name__							)

							self.assertListEqual(__magic_name__,	__magic_name__							)




			def    UpperCamelCase__						(					self							)				->   List[str]:






							"""simple docstring"""
							UpperCamelCase__							:       List[str]      =   MecabTokenizer(mecab_dic='''ipadic'''							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],	)




			def    UpperCamelCase__						(					self							)				->   List[Any]:






							"""simple docstring"""
							try:
											UpperCamelCase__							:       List[Any]      =   MecabTokenizer(mecab_dic='''unidic_lite'''							)
							except ModuleNotFoundError:
											return

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],	)




			def    UpperCamelCase__						(					self							)				->   Optional[int]:






							"""simple docstring"""
							try:
											UpperCamelCase__							:       List[Any]      =   MecabTokenizer(mecab_dic='''unidic'''							)
							except ModuleNotFoundError:
											return

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],	)




			def    UpperCamelCase__						(					self							)				->   Tuple:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   MecabTokenizer(do_lower_case=__magic_name__,	mecab_dic='''ipadic'''							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],	)




			def    UpperCamelCase__						(					self							)				->   Dict:






							"""simple docstring"""
							try:
											UpperCamelCase__							:       Union[str, Any]      =   MecabTokenizer(
											    do_lower_case=__magic_name__,	normalize_text=__magic_name__,	mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''							)
							except RuntimeError:
											# if dict doesn't exist in the system, previous code raises this error.
											return

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''],	)




			def    UpperCamelCase__						(					self							)				->   Optional[Any]:






							"""simple docstring"""
							UpperCamelCase__							:       str      =   MecabTokenizer(normalize_text=__magic_name__,	mecab_dic='''ipadic'''							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。'''],	)




			@require_sudachi
			def    UpperCamelCase__						(					self							)				->   Optional[int]:






							"""simple docstring"""
							UpperCamelCase__							:       Tuple      =   self.tokenizer_class(self.vocab_file,	word_tokenizer_type='''sudachi'''							)
							self.assertIsNotNone(__magic_name__							)

							UpperCamelCase__							:       Union[str, Any]      =   '''こんにちは、世界。\nこんばんは、世界。'''
							UpperCamelCase__							:       Union[str, Any]      =   tokenizer.tokenize(__magic_name__							)
							self.assertListEqual(__magic_name__,	['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__							),	[3, 12, 10, 14, 4, 9, 12, 10, 14]							)

							UpperCamelCase__							:       Any      =   os.path.join(self.tmpdirname,	'''tokenizer.bin'''							)
							with open(__magic_name__,	'''wb'''							) as handle:
											pickle.dump(__magic_name__,	__magic_name__							)

							with open(__magic_name__,	'''rb'''							) as handle:
											UpperCamelCase__							:       Optional[Any]      =   pickle.load(__magic_name__							)

							UpperCamelCase__							:       Union[str, Any]      =   tokenizer_new.tokenize(__magic_name__							)

							self.assertListEqual(__magic_name__,	__magic_name__							)




			@require_sudachi
			def    UpperCamelCase__						(					self							)				->   List[str]:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   SudachiTokenizer(sudachi_dict_type='''core'''							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' '''],	)




			@require_sudachi
			def    UpperCamelCase__						(					self							)				->   Dict:






							"""simple docstring"""
							UpperCamelCase__							:       Dict      =   SudachiTokenizer(sudachi_dict_type='''core''',	sudachi_split_mode='''A'''							)

							self.assertListEqual(tokenizer.tokenize('''外国人参政権'''							),	['''外国''', '''人''', '''参政''', '''権''']							)




			@require_sudachi
			def    UpperCamelCase__						(					self							)				->   Union[str, Any]:






							"""simple docstring"""
							UpperCamelCase__							:       Any      =   SudachiTokenizer(sudachi_dict_type='''core''',	sudachi_split_mode='''B'''							)

							self.assertListEqual(tokenizer.tokenize('''外国人参政権'''							),	['''外国人''', '''参政権''']							)




			@require_sudachi
			def    UpperCamelCase__						(					self							)				->   Dict:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   SudachiTokenizer(sudachi_dict_type='''core''',	sudachi_split_mode='''C'''							)

							self.assertListEqual(tokenizer.tokenize('''外国人参政権'''							),	['''外国人参政権''']							)




			@require_sudachi
			def    UpperCamelCase__						(					self							)				->   Union[str, Any]:






							"""simple docstring"""
							UpperCamelCase__							:       Optional[int]      =   SudachiTokenizer(do_lower_case=__magic_name__,	sudachi_dict_type='''core'''							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' '''],	)




			@require_sudachi
			def    UpperCamelCase__						(					self							)				->   Tuple:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   SudachiTokenizer(normalize_text=__magic_name__,	sudachi_dict_type='''core'''							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	[''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' '''],	)




			@require_sudachi
			def    UpperCamelCase__						(					self							)				->   List[Any]:






							"""simple docstring"""
							UpperCamelCase__							:       Union[str, Any]      =   SudachiTokenizer(trim_whitespace=__magic_name__,	sudachi_dict_type='''core'''							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],	)




			@require_jumanpp
			def    UpperCamelCase__						(					self							)				->   Union[str, Any]:






							"""simple docstring"""
							UpperCamelCase__							:       Tuple      =   self.tokenizer_class(self.vocab_file,	word_tokenizer_type='''jumanpp'''							)
							self.assertIsNotNone(__magic_name__							)

							UpperCamelCase__							:       str      =   '''こんにちは、世界。\nこんばんは、世界。'''
							UpperCamelCase__							:       Tuple      =   tokenizer.tokenize(__magic_name__							)
							self.assertListEqual(__magic_name__,	['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__							),	[3, 12, 10, 14, 4, 9, 12, 10, 14]							)

							UpperCamelCase__							:       List[Any]      =   os.path.join(self.tmpdirname,	'''tokenizer.bin'''							)
							with open(__magic_name__,	'''wb'''							) as handle:
											pickle.dump(__magic_name__,	__magic_name__							)

							with open(__magic_name__,	'''rb'''							) as handle:
											UpperCamelCase__							:       Tuple      =   pickle.load(__magic_name__							)

							UpperCamelCase__							:       Dict      =   tokenizer_new.tokenize(__magic_name__							)

							self.assertListEqual(__magic_name__,	__magic_name__							)




			@require_jumanpp
			def    UpperCamelCase__						(					self							)				->   Optional[int]:






							"""simple docstring"""
							UpperCamelCase__							:       Tuple      =   JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''],	)




			@require_jumanpp
			def    UpperCamelCase__						(					self							)				->   str:






							"""simple docstring"""
							UpperCamelCase__							:       Dict      =   JumanppTokenizer(do_lower_case=__magic_name__							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''],	)




			@require_jumanpp
			def    UpperCamelCase__						(					self							)				->   Optional[Any]:






							"""simple docstring"""
							UpperCamelCase__							:       Optional[Any]      =   JumanppTokenizer(normalize_text=__magic_name__							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''],	)




			@require_jumanpp
			def    UpperCamelCase__						(					self							)				->   List[str]:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   JumanppTokenizer(trim_whitespace=__magic_name__							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							),	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。'''],	)




			@require_jumanpp
			def    UpperCamelCase__						(					self							)				->   Dict:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''							),	['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。'''],	)




			def    UpperCamelCase__						(					self							)				->   Union[str, Any]:






							"""simple docstring"""
							UpperCamelCase__							:       Optional[int]      =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

							UpperCamelCase__							:       Union[str, Any]      =   {}
							for i, token in enumerate(__magic_name__							):
											UpperCamelCase__							:       Union[str, Any]      =   i
							UpperCamelCase__							:       Optional[Any]      =   WordpieceTokenizer(vocab=__magic_name__,	unk_token='''[UNK]'''							)

							self.assertListEqual(tokenizer.tokenize(''''''							),	[]							)

							self.assertListEqual(tokenizer.tokenize('''こんにちは'''							),	['''こんにちは''']							)

							self.assertListEqual(tokenizer.tokenize('''こんばんは'''							),	['''こん''', '''##ばんは''']							)

							self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''							),	['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']							)




			def    UpperCamelCase__						(					self							)				->   str:






							"""simple docstring"""
							UpperCamelCase__							:       List[str]      =   BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''							)
							UpperCamelCase__							:       Optional[Any]      =   tokenizer.subword_tokenizer

							UpperCamelCase__							:       Tuple      =   subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''							)
							self.assertListEqual(__magic_name__,	['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']							)

							UpperCamelCase__							:       List[str]      =   subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''							)
							self.assertListEqual(__magic_name__,	['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']							)




			def    UpperCamelCase__						(					self							)				->   List[str]:






							"""simple docstring"""
							UpperCamelCase__							:       Tuple      =   self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''							)

							UpperCamelCase__							:       str      =   tokenizer.encode('''ありがとう。''',	add_special_tokens=__magic_name__							)
							UpperCamelCase__							:       Optional[int]      =   tokenizer.encode('''どういたしまして。''',	add_special_tokens=__magic_name__							)

							UpperCamelCase__							:       Any      =   tokenizer.build_inputs_with_special_tokens(__magic_name__							)
							UpperCamelCase__							:       Tuple      =   tokenizer.build_inputs_with_special_tokens(__magic_name__,	__magic_name__							)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 					lowercase__ (			__lowerCamelCase    ,       unittest.TestCase       ):

			'''simple docstring'''


			a		:							Dict     	=	BertJapaneseTokenizer
			a		:							Dict     	=	False




			def    UpperCamelCase__						(					self							)				->   Tuple:






							"""simple docstring"""
							super().setUp()

							UpperCamelCase__							:       str      =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

							UpperCamelCase__							:       Tuple      =   os.path.join(self.tmpdirname,	VOCAB_FILES_NAMES['''vocab_file''']							)
							with open(self.vocab_file,	'''w''',	encoding='''utf-8'''							) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)




			def    UpperCamelCase__						(					self,	**__magic_name__							)				->   Dict:






							"""simple docstring"""
							return BertJapaneseTokenizer.from_pretrained(self.tmpdirname,	subword_tokenizer_type='''character''',	**__magic_name__							)




			def    UpperCamelCase__						(					self,	__magic_name__							)				->   Union[str, Any]:






							"""simple docstring"""
							UpperCamelCase__							:       Tuple      =   '''こんにちは、世界。 \nこんばんは、世界。'''
							UpperCamelCase__							:       Optional[Any]      =   '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
							return input_text, output_text




			def    UpperCamelCase__						(					self							)				->   List[Any]:






							"""simple docstring"""
							pass  # TODO add if relevant




			def    UpperCamelCase__						(					self							)				->   int:






							"""simple docstring"""
							pass  # TODO add if relevant




			def    UpperCamelCase__						(					self							)				->   Optional[int]:






							"""simple docstring"""
							pass  # TODO add if relevant




			def    UpperCamelCase__						(					self							)				->   Any:






							"""simple docstring"""
							UpperCamelCase__							:       Tuple      =   self.tokenizer_class(self.vocab_file,	subword_tokenizer_type='''character'''							)

							UpperCamelCase__							:       List[str]      =   tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''							)
							self.assertListEqual(
							    __magic_name__,	['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']							)
							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(__magic_name__							),	[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]							)




			def    UpperCamelCase__						(					self							)				->   Optional[Any]:






							"""simple docstring"""
							UpperCamelCase__							:       Dict      =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

							UpperCamelCase__							:       Tuple      =   {}
							for i, token in enumerate(__magic_name__							):
											UpperCamelCase__							:       Dict      =   i
							UpperCamelCase__							:       Optional[Any]      =   CharacterTokenizer(vocab=__magic_name__,	unk_token='''[UNK]'''							)

							self.assertListEqual(tokenizer.tokenize(''''''							),	[]							)

							self.assertListEqual(tokenizer.tokenize('''こんにちは'''							),	['''こ''', '''ん''', '''に''', '''ち''', '''は''']							)

							self.assertListEqual(tokenizer.tokenize('''こんにちほ'''							),	['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']							)




			def    UpperCamelCase__						(					self							)				->   Optional[int]:






							"""simple docstring"""
							UpperCamelCase__							:       int      =   self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''							)

							UpperCamelCase__							:       List[str]      =   tokenizer.encode('''ありがとう。''',	add_special_tokens=__magic_name__							)
							UpperCamelCase__							:       Dict      =   tokenizer.encode('''どういたしまして。''',	add_special_tokens=__magic_name__							)

							UpperCamelCase__							:       Optional[int]      =   tokenizer.build_inputs_with_special_tokens(__magic_name__							)
							UpperCamelCase__							:       str      =   tokenizer.build_inputs_with_special_tokens(__magic_name__,	__magic_name__							)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 					lowercase__ (			unittest.TestCase       ):

			'''simple docstring'''


			def    UpperCamelCase__						(					self							)				->   str:






							"""simple docstring"""
							UpperCamelCase__							:       Optional[int]      =   '''cl-tohoku/bert-base-japanese'''
							UpperCamelCase__							:       Dict      =   AutoTokenizer.from_pretrained(__magic_name__							)
							self.assertIsInstance(__magic_name__,	__magic_name__							)






class 					lowercase__ (			unittest.TestCase       ):

			'''simple docstring'''


			def    UpperCamelCase__						(					self							)				->   List[str]:






							"""simple docstring"""
							UpperCamelCase__							:       Tuple      =   '''cl-tohoku/bert-base-japanese'''
							with self.assertLogs('''transformers''',	level='''WARNING'''							) as cm:
											BertTokenizer.from_pretrained(__magic_name__							)
											self.assertTrue(
											    cm.records[0].message.startswith(
											        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
											        ''' is called from.'''							)							)
							UpperCamelCase__							:       List[str]      =   '''bert-base-cased'''
							with self.assertLogs('''transformers''',	level='''WARNING'''							) as cm:
											BertJapaneseTokenizer.from_pretrained(__magic_name__							)
											self.assertTrue(
											    cm.records[0].message.startswith(
											        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
											        ''' is called from.'''							)							)








import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase_  							=				logging.get_logger(__name__)




def 	lowerCAmelCase_					(						__UpperCAmelCase:    List[Any]	,     __UpperCAmelCase:    List[str]=False	,     __UpperCAmelCase:    List[Any]=False	,     __UpperCAmelCase:    int=False   )   ->			Optional[Any]:
    UpperCamelCase__							:       Union[str, Any]      =   []
    for i in range(config.num_hidden_layers   ):
        # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
        rename_keys.append((f"transformer.blocks.{i}.norm1.weight", f"vilt.encoder.layer.{i}.layernorm_before.weight")   )
        rename_keys.append((f"transformer.blocks.{i}.norm1.bias", f"vilt.encoder.layer.{i}.layernorm_before.bias")   )
        rename_keys.append(
            (f"transformer.blocks.{i}.attn.proj.weight", f"vilt.encoder.layer.{i}.attention.output.dense.weight")   )
        rename_keys.append(
            (f"transformer.blocks.{i}.attn.proj.bias", f"vilt.encoder.layer.{i}.attention.output.dense.bias")   )
        rename_keys.append((f"transformer.blocks.{i}.norm2.weight", f"vilt.encoder.layer.{i}.layernorm_after.weight")   )
        rename_keys.append((f"transformer.blocks.{i}.norm2.bias", f"vilt.encoder.layer.{i}.layernorm_after.bias")   )
        rename_keys.append(
            (f"transformer.blocks.{i}.mlp.fc1.weight", f"vilt.encoder.layer.{i}.intermediate.dense.weight")   )
        rename_keys.append((f"transformer.blocks.{i}.mlp.fc1.bias", f"vilt.encoder.layer.{i}.intermediate.dense.bias")   )
        rename_keys.append((f"transformer.blocks.{i}.mlp.fc2.weight", f"vilt.encoder.layer.{i}.output.dense.weight")   )
        rename_keys.append((f"transformer.blocks.{i}.mlp.fc2.bias", f"vilt.encoder.layer.{i}.output.dense.bias")   )

    # embeddings
    rename_keys.extend(
        [
            # text embeddings
            ('''text_embeddings.word_embeddings.weight''', '''vilt.embeddings.text_embeddings.word_embeddings.weight'''),
            (
                '''text_embeddings.position_embeddings.weight''',
                '''vilt.embeddings.text_embeddings.position_embeddings.weight''',
            ),
            ('''text_embeddings.position_ids''', '''vilt.embeddings.text_embeddings.position_ids'''),
            (
                '''text_embeddings.token_type_embeddings.weight''',
                '''vilt.embeddings.text_embeddings.token_type_embeddings.weight''',
            ),
            ('''text_embeddings.LayerNorm.weight''', '''vilt.embeddings.text_embeddings.LayerNorm.weight'''),
            ('''text_embeddings.LayerNorm.bias''', '''vilt.embeddings.text_embeddings.LayerNorm.bias'''),
            # patch embeddings
            ('''transformer.cls_token''', '''vilt.embeddings.cls_token'''),
            ('''transformer.patch_embed.proj.weight''', '''vilt.embeddings.patch_embeddings.projection.weight'''),
            ('''transformer.patch_embed.proj.bias''', '''vilt.embeddings.patch_embeddings.projection.bias'''),
            ('''transformer.pos_embed''', '''vilt.embeddings.position_embeddings'''),
            # token type embeddings
            ('''token_type_embeddings.weight''', '''vilt.embeddings.token_type_embeddings.weight'''),
        ]   )

    # final layernorm + pooler
    rename_keys.extend(
        [
            ('''transformer.norm.weight''', '''vilt.layernorm.weight'''),
            ('''transformer.norm.bias''', '''vilt.layernorm.bias'''),
            ('''pooler.dense.weight''', '''vilt.pooler.dense.weight'''),
            ('''pooler.dense.bias''', '''vilt.pooler.dense.bias'''),
        ]   )

    # classifier head(s)
    if vqa_model:
        # classification head
        rename_keys.extend(
            [
                ('''vqa_classifier.0.weight''', '''classifier.0.weight'''),
                ('''vqa_classifier.0.bias''', '''classifier.0.bias'''),
                ('''vqa_classifier.1.weight''', '''classifier.1.weight'''),
                ('''vqa_classifier.1.bias''', '''classifier.1.bias'''),
                ('''vqa_classifier.3.weight''', '''classifier.3.weight'''),
                ('''vqa_classifier.3.bias''', '''classifier.3.bias'''),
            ]   )
    elif nlvr_model:
        # classification head
        rename_keys.extend(
            [
                ('''nlvr2_classifier.0.weight''', '''classifier.0.weight'''),
                ('''nlvr2_classifier.0.bias''', '''classifier.0.bias'''),
                ('''nlvr2_classifier.1.weight''', '''classifier.1.weight'''),
                ('''nlvr2_classifier.1.bias''', '''classifier.1.bias'''),
                ('''nlvr2_classifier.3.weight''', '''classifier.3.weight'''),
                ('''nlvr2_classifier.3.bias''', '''classifier.3.bias'''),
            ]   )
    else:
        pass

    return rename_keys




def 	lowerCAmelCase_					(						__UpperCAmelCase:    List[Any]	,     __UpperCAmelCase:    List[Any]   )   ->			Optional[int]:
    for i in range(config.num_hidden_layers   ):
        UpperCamelCase__							:       Tuple      =   '''vilt.'''
        # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
        UpperCamelCase__							:       Dict      =   state_dict.pop(f"transformer.blocks.{i}.attn.qkv.weight"   )
        UpperCamelCase__							:       Tuple      =   state_dict.pop(f"transformer.blocks.{i}.attn.qkv.bias"   )
        # next, add query, keys and values (in that order) to the state dict
        UpperCamelCase__							:       Union[str, Any]      =   in_proj_weight[
            : config.hidden_size, :
        ]
        UpperCamelCase__							:       Optional[Any]      =   in_proj_bias[: config.hidden_size]
        UpperCamelCase__							:       Tuple      =   in_proj_weight[
            config.hidden_size : config.hidden_size * 2, :
        ]
        UpperCamelCase__							:       Any      =   in_proj_bias[
            config.hidden_size : config.hidden_size * 2
        ]
        UpperCamelCase__							:       List[Any]      =   in_proj_weight[
            -config.hidden_size :, :
        ]
        UpperCamelCase__							:       List[Any]      =   in_proj_bias[-config.hidden_size :]




def 	lowerCAmelCase_					(						__UpperCAmelCase:    List[Any]   )   ->			Any:
    UpperCamelCase__							:       int      =   ['''head.weight''', '''head.bias''']
    for k in ignore_keys:
        state_dict.pop(__UpperCAmelCase	,     __UpperCAmelCase   )




def 	lowerCAmelCase_					(						__UpperCAmelCase:    Any	,     __UpperCAmelCase:    Dict	,     __UpperCAmelCase:    int   )   ->			List[str]:
    UpperCamelCase__							:       str      =   dct.pop(__UpperCAmelCase   )
    UpperCamelCase__							:       Optional[int]      =   val




@torch.no_grad()
def 	lowerCAmelCase_					(						__UpperCAmelCase:    List[Any]	,     __UpperCAmelCase:    Union[str, Any]   )   ->			int:
    UpperCamelCase__							:       Any      =   ViltConfig(image_size=384	,     patch_size=32	,     tie_word_embeddings=__UpperCAmelCase   )
    UpperCamelCase__							:       Any      =   False
    UpperCamelCase__							:       Optional[int]      =   False
    UpperCamelCase__							:       List[Any]      =   False
    UpperCamelCase__							:       int      =   False
    if "vqa" in checkpoint_url:
        UpperCamelCase__							:       Any      =   True
        UpperCamelCase__							:       Optional[int]      =   3129
        UpperCamelCase__							:       Dict      =   '''huggingface/label-files'''
        UpperCamelCase__							:       Optional[Any]      =   '''vqa2-id2label.json'''
        UpperCamelCase__							:       Optional[int]      =   json.load(open(hf_hub_download(__UpperCAmelCase	,     __UpperCAmelCase	,     repo_type='''dataset'''   )	,     '''r'''   )   )
        UpperCamelCase__							:       List[str]      =   {int(__UpperCAmelCase   ): v for k, v in idalabel.items()}
        UpperCamelCase__							:       str      =   idalabel
        UpperCamelCase__							:       List[Any]      =   {v: k for k, v in idalabel.items()}
        UpperCamelCase__							:       Tuple      =   ViltForQuestionAnswering(__UpperCAmelCase   )
    elif "nlvr" in checkpoint_url:
        UpperCamelCase__							:       Optional[Any]      =   True
        UpperCamelCase__							:       Union[str, Any]      =   2
        UpperCamelCase__							:       int      =   {0: '''False''', 1: '''True'''}
        UpperCamelCase__							:       Optional[Any]      =   {v: k for k, v in config.idalabel.items()}
        UpperCamelCase__							:       Tuple      =   3
        UpperCamelCase__							:       Optional[Any]      =   ViltForImagesAndTextClassification(__UpperCAmelCase   )
    elif "irtr" in checkpoint_url:
        UpperCamelCase__							:       List[str]      =   True
        UpperCamelCase__							:       Any      =   ViltForImageAndTextRetrieval(__UpperCAmelCase   )
    elif "mlm_itm" in checkpoint_url:
        UpperCamelCase__							:       Any      =   True
        UpperCamelCase__							:       int      =   ViltForMaskedLM(__UpperCAmelCase   )
    else:
        raise ValueError('''Unknown model type'''   )

    # load state_dict of original model, remove and rename some keys
    UpperCamelCase__							:       int      =   torch.hub.load_state_dict_from_url(__UpperCAmelCase	,     map_location='''cpu'''   )['''state_dict''']
    UpperCamelCase__							:       Dict      =   create_rename_keys(__UpperCAmelCase	,     __UpperCAmelCase	,     __UpperCAmelCase	,     __UpperCAmelCase   )
    for src, dest in rename_keys:
        rename_key(__UpperCAmelCase	,     __UpperCAmelCase	,     __UpperCAmelCase   )
    read_in_q_k_v(__UpperCAmelCase	,     __UpperCAmelCase   )
    if mlm_model or irtr_model:
        UpperCamelCase__							:       str      =   ['''itm_score.fc.weight''', '''itm_score.fc.bias''']
        for k in ignore_keys:
            state_dict.pop(__UpperCAmelCase	,     __UpperCAmelCase   )

    # load state dict into HuggingFace model
    model.eval()
    if mlm_model:
        UpperCamelCase__     ,UpperCamelCase__							:       Optional[Any]      =   model.load_state_dict(__UpperCAmelCase	,     strict=__UpperCAmelCase   )
        assert missing_keys == ["mlm_score.decoder.bias"]
    else:
        model.load_state_dict(__UpperCAmelCase   )

    # Define processor
    UpperCamelCase__							:       Union[str, Any]      =   ViltImageProcessor(size=384   )
    UpperCamelCase__							:       List[str]      =   BertTokenizer.from_pretrained('''bert-base-uncased'''   )
    UpperCamelCase__							:       Optional[int]      =   ViltProcessor(__UpperCAmelCase	,     __UpperCAmelCase   )

    # Forward pass on example inputs (image + text)
    if nlvr_model:
        UpperCamelCase__							:       Optional[int]      =   Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'''	,     stream=__UpperCAmelCase   ).raw   )
        UpperCamelCase__							:       Union[str, Any]      =   Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'''	,     stream=__UpperCAmelCase   ).raw   )
        UpperCamelCase__							:       Dict      =   (
            '''The left image contains twice the number of dogs as the right image, and at least two dogs in total are'''
            ''' standing.'''
        )
        UpperCamelCase__							:       str      =   processor(__UpperCAmelCase	,     __UpperCAmelCase	,     return_tensors='''pt'''   )
        UpperCamelCase__							:       Optional[Any]      =   processor(__UpperCAmelCase	,     __UpperCAmelCase	,     return_tensors='''pt'''   )
        UpperCamelCase__							:       Union[str, Any]      =   model(
            input_ids=encoding_a.input_ids	,     pixel_values=encoding_a.pixel_values	,     pixel_values_a=encoding_a.pixel_values	,     )
    else:
        UpperCamelCase__							:       Union[str, Any]      =   Image.open(requests.get('''http://images.cocodataset.org/val2017/000000039769.jpg'''	,     stream=__UpperCAmelCase   ).raw   )
        if mlm_model:
            UpperCamelCase__							:       int      =   '''a bunch of [MASK] laying on a [MASK].'''
        else:
            UpperCamelCase__							:       Optional[Any]      =   '''How many cats are there?'''
        UpperCamelCase__							:       List[Any]      =   processor(__UpperCAmelCase	,     __UpperCAmelCase	,     return_tensors='''pt'''   )
        UpperCamelCase__							:       Union[str, Any]      =   model(**__UpperCAmelCase   )

    # Verify outputs
    if mlm_model:
        UpperCamelCase__							:       str      =   torch.Size([1, 11, 3_0522]   )
        UpperCamelCase__							:       Optional[Any]      =   torch.tensor([-12.5061, -12.5123, -12.5174]   )
        assert outputs.logits.shape == expected_shape
        assert torch.allclose(outputs.logits[0, 0, :3]	,     __UpperCAmelCase	,     atol=1e-4   )

        # verify masked token prediction equals "cats"
        UpperCamelCase__							:       Optional[Any]      =   outputs.logits[0, 4, :].argmax(-1   ).item()
        assert tokenizer.decode([predicted_id]   ) == "cats"
    elif vqa_model:
        UpperCamelCase__							:       List[Any]      =   torch.Size([1, 3129]   )
        UpperCamelCase__							:       Tuple      =   torch.tensor([-15.9495, -18.1472, -10.3041]   )
        assert torch.allclose(outputs.logits[0, :3]	,     __UpperCAmelCase	,     atol=1e-4   )
        assert outputs.logits.shape == expected_shape
        assert torch.allclose(outputs.logits[0, 0, :3]	,     __UpperCAmelCase	,     atol=1e-4   )

        # verify vqa prediction equals "2"
        UpperCamelCase__							:       Any      =   outputs.logits.argmax(-1   ).item()
        assert model.config.idalabel[predicted_idx] == "2"
    elif nlvr_model:
        UpperCamelCase__							:       Dict      =   torch.Size([1, 2]   )
        UpperCamelCase__							:       str      =   torch.tensor([-2.8721, 2.1291]   )
        assert torch.allclose(outputs.logits[0, :3]	,     __UpperCAmelCase	,     atol=1e-4   )
        assert outputs.logits.shape == expected_shape

    Path(__UpperCAmelCase   ).mkdir(exist_ok=__UpperCAmelCase   )
    print(f"Saving model and processor to {pytorch_dump_folder_path}"   )
    model.save_pretrained(__UpperCAmelCase   )
    processor.save_pretrained(__UpperCAmelCase   )


if __name__ == "__main__":
  UpperCAmelCase_  							=				argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      '--checkpoint_url',
      default='https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt',
      type=str,
      help='URL of the checkpoint you\'d like to convert.',
  )
  parser.add_argument(
      '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
  )

  UpperCAmelCase_  							=				parser.parse_args()
  convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)

