





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 				a			(      __lowercase   ,unittest.TestCase			):
				SCREAMING_SNAKE_CASE__     :   List[Any]		      =     BertJapaneseTokenizer
				SCREAMING_SNAKE_CASE__     :   Optional[Any]		      =     False
				SCREAMING_SNAKE_CASE__     :   Optional[Any]		      =     True






				def 			snake_case_    (  self					):

										"""simple docstring"""




										super().setUp()

										__SCREAMING_SNAKE_CASE:				str           =   [
										    '''[UNK]''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''こんにちは''',
										    '''こん''',
										    '''にちは''',
										    '''ばんは''',
										    '''##こん''',
										    '''##にちは''',
										    '''##ばんは''',
										    '''世界''',
										    '''##世界''',
										    '''、''',
										    '''##、''',
										    '''。''',
										    '''##。''',
										]

										__SCREAMING_SNAKE_CASE:				List[Any]           =   os.path.join(self.tmpdirname			, VOCAB_FILES_NAMES['''vocab_file''']					)
										with open(self.vocab_file			, '''w'''			, encoding='''utf-8'''					) as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   '''こんにちは、世界。 \nこんばんは、世界。'''
										__SCREAMING_SNAKE_CASE:				Tuple           =   '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
										return input_text, output_text






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				Optional[Any]           =   self.get_input_output_texts(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				List[Any]           =   tokenizer.encode(_lowerCAmelCase			, add_special_tokens=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Tuple           =   tokenizer.decode(_lowerCAmelCase			, clean_up_tokenization_spaces=_lowerCAmelCase					)
										return text, ids






				def 			snake_case_    (  self					):

										"""simple docstring"""




										pass  # TODO add if relevant






				def 			snake_case_    (  self					):

										"""simple docstring"""




										pass  # TODO add if relevant






				def 			snake_case_    (  self					):

										"""simple docstring"""




										pass  # TODO add if relevant






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   self.tokenizer_class(self.vocab_file					)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''					)
										self.assertListEqual(_lowerCAmelCase			, ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase					)			, [3, 12, 10, 14, 4, 9, 12, 10, 14]					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   self.tokenizer_class(self.vocab_file			, word_tokenizer_type='''mecab'''					)
										self.assertIsNotNone(_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Tuple           =   '''こんにちは、世界。\nこんばんは、世界。'''
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   tokenizer.tokenize(_lowerCAmelCase					)
										self.assertListEqual(_lowerCAmelCase			, ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase					)			, [3, 12, 10, 14, 4, 9, 12, 10, 14]					)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   os.path.join(self.tmpdirname			, '''tokenizer.bin'''					)
										with open(_lowerCAmelCase			, '''wb'''					) as handle:
																pickle.dump(_lowerCAmelCase			, _lowerCAmelCase					)

										with open(_lowerCAmelCase			, '''rb'''					) as handle:
																__SCREAMING_SNAKE_CASE:				Any           =   pickle.load(_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Any           =   tokenizer_new.tokenize(_lowerCAmelCase					)

										self.assertListEqual(_lowerCAmelCase			, _lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   MecabTokenizer(mecab_dic='''ipadic'''					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']			, )






				def 			snake_case_    (  self					):

										"""simple docstring"""




										try:
																__SCREAMING_SNAKE_CASE:				str           =   MecabTokenizer(mecab_dic='''unidic_lite'''					)
										except ModuleNotFoundError:
																return

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']			, )






				def 			snake_case_    (  self					):

										"""simple docstring"""




										try:
																__SCREAMING_SNAKE_CASE:				int           =   MecabTokenizer(mecab_dic='''unidic'''					)
										except ModuleNotFoundError:
																return

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']			, )






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   MecabTokenizer(do_lower_case=_lowerCAmelCase			, mecab_dic='''ipadic'''					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']			, )






				def 			snake_case_    (  self					):

										"""simple docstring"""




										try:
																__SCREAMING_SNAKE_CASE:				int           =   MecabTokenizer(
																    do_lower_case=_lowerCAmelCase			, normalize_text=_lowerCAmelCase			, mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''					)
										except RuntimeError:
																# if dict doesn't exist in the system, previous code raises this error.
																return

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']			, )






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   MecabTokenizer(normalize_text=_lowerCAmelCase			, mecab_dic='''ipadic'''					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']			, )






				@require_sudachi
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   self.tokenizer_class(self.vocab_file			, word_tokenizer_type='''sudachi'''					)
										self.assertIsNotNone(_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				str           =   '''こんにちは、世界。\nこんばんは、世界。'''
										__SCREAMING_SNAKE_CASE:				str           =   tokenizer.tokenize(_lowerCAmelCase					)
										self.assertListEqual(_lowerCAmelCase			, ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase					)			, [3, 12, 10, 14, 4, 9, 12, 10, 14]					)

										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   os.path.join(self.tmpdirname			, '''tokenizer.bin'''					)
										with open(_lowerCAmelCase			, '''wb'''					) as handle:
																pickle.dump(_lowerCAmelCase			, _lowerCAmelCase					)

										with open(_lowerCAmelCase			, '''rb'''					) as handle:
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   pickle.load(_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Optional[int]           =   tokenizer_new.tokenize(_lowerCAmelCase					)

										self.assertListEqual(_lowerCAmelCase			, _lowerCAmelCase					)






				@require_sudachi
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   SudachiTokenizer(sudachi_dict_type='''core'''					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']			, )






				@require_sudachi
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   SudachiTokenizer(sudachi_dict_type='''core'''			, sudachi_split_mode='''A'''					)

										self.assertListEqual(tokenizer.tokenize('''外国人参政権'''					)			, ['''外国''', '''人''', '''参政''', '''権''']					)






				@require_sudachi
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   SudachiTokenizer(sudachi_dict_type='''core'''			, sudachi_split_mode='''B'''					)

										self.assertListEqual(tokenizer.tokenize('''外国人参政権'''					)			, ['''外国人''', '''参政権''']					)






				@require_sudachi
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   SudachiTokenizer(sudachi_dict_type='''core'''			, sudachi_split_mode='''C'''					)

										self.assertListEqual(tokenizer.tokenize('''外国人参政権'''					)			, ['''外国人参政権''']					)






				@require_sudachi
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   SudachiTokenizer(do_lower_case=_lowerCAmelCase			, sudachi_dict_type='''core'''					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']			, )






				@require_sudachi
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   SudachiTokenizer(normalize_text=_lowerCAmelCase			, sudachi_dict_type='''core'''					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, [''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']			, )






				@require_sudachi
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   SudachiTokenizer(trim_whitespace=_lowerCAmelCase			, sudachi_dict_type='''core'''					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']			, )






				@require_jumanpp
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   self.tokenizer_class(self.vocab_file			, word_tokenizer_type='''jumanpp'''					)
										self.assertIsNotNone(_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				List[str]           =   '''こんにちは、世界。\nこんばんは、世界。'''
										__SCREAMING_SNAKE_CASE:				Any           =   tokenizer.tokenize(_lowerCAmelCase					)
										self.assertListEqual(_lowerCAmelCase			, ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase					)			, [3, 12, 10, 14, 4, 9, 12, 10, 14]					)

										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   os.path.join(self.tmpdirname			, '''tokenizer.bin'''					)
										with open(_lowerCAmelCase			, '''wb'''					) as handle:
																pickle.dump(_lowerCAmelCase			, _lowerCAmelCase					)

										with open(_lowerCAmelCase			, '''rb'''					) as handle:
																__SCREAMING_SNAKE_CASE:				List[str]           =   pickle.load(_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				int           =   tokenizer_new.tokenize(_lowerCAmelCase					)

										self.assertListEqual(_lowerCAmelCase			, _lowerCAmelCase					)






				@require_jumanpp
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   JumanppTokenizer()

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']			, )






				@require_jumanpp
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   JumanppTokenizer(do_lower_case=_lowerCAmelCase					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']			, )






				@require_jumanpp
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   JumanppTokenizer(normalize_text=_lowerCAmelCase					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']			, )






				@require_jumanpp
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   JumanppTokenizer(trim_whitespace=_lowerCAmelCase					)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''					)			, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']			, )






				@require_jumanpp
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   JumanppTokenizer()

										self.assertListEqual(
										    tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''					)			, ['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']			, )






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   {}
										for i, token in enumerate(_lowerCAmelCase					):
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   i
										__SCREAMING_SNAKE_CASE:				List[str]           =   WordpieceTokenizer(vocab=_lowerCAmelCase			, unk_token='''[UNK]'''					)

										self.assertListEqual(tokenizer.tokenize(''''''					)			, []					)

										self.assertListEqual(tokenizer.tokenize('''こんにちは'''					)			, ['''こんにちは''']					)

										self.assertListEqual(tokenizer.tokenize('''こんばんは'''					)			, ['''こん''', '''##ばんは''']					)

										self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''					)			, ['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''					)
										__SCREAMING_SNAKE_CASE:				Tuple           =   tokenizer.subword_tokenizer

										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''					)
										self.assertListEqual(_lowerCAmelCase			, ['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']					)

										__SCREAMING_SNAKE_CASE:				Dict           =   subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''					)
										self.assertListEqual(_lowerCAmelCase			, ['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''					)

										__SCREAMING_SNAKE_CASE:				List[Any]           =   tokenizer.encode('''ありがとう。'''			, add_special_tokens=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   tokenizer.encode('''どういたしまして。'''			, add_special_tokens=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				List[Any]           =   tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				List[Any]           =   tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase			, _lowerCAmelCase					)

										# 2 is for "[CLS]", 3 is for "[SEP]"
										assert encoded_sentence == [2] + text + [3]
										assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				a			(      __lowercase   ,unittest.TestCase			):
				SCREAMING_SNAKE_CASE__     :   Any		      =     BertJapaneseTokenizer
				SCREAMING_SNAKE_CASE__     :   int		      =     False






				def 			snake_case_    (  self					):

										"""simple docstring"""




										super().setUp()

										__SCREAMING_SNAKE_CASE:				str           =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

										__SCREAMING_SNAKE_CASE:				Dict           =   os.path.join(self.tmpdirname			, VOCAB_FILES_NAMES['''vocab_file''']					)
										with open(self.vocab_file			, '''w'''			, encoding='''utf-8'''					) as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)






				def 			snake_case_    (  self			, **_lowerCAmelCase					):

										"""simple docstring"""




										return BertJapaneseTokenizer.from_pretrained(self.tmpdirname			, subword_tokenizer_type='''character'''			, **_lowerCAmelCase					)






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   '''こんにちは、世界。 \nこんばんは、世界。'''
										__SCREAMING_SNAKE_CASE:				int           =   '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
										return input_text, output_text






				def 			snake_case_    (  self					):

										"""simple docstring"""




										pass  # TODO add if relevant






				def 			snake_case_    (  self					):

										"""simple docstring"""




										pass  # TODO add if relevant






				def 			snake_case_    (  self					):

										"""simple docstring"""




										pass  # TODO add if relevant






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   self.tokenizer_class(self.vocab_file			, subword_tokenizer_type='''character'''					)

										__SCREAMING_SNAKE_CASE:				Optional[int]           =   tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''					)
										self.assertListEqual(
										    _lowerCAmelCase			, ['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']					)
										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(_lowerCAmelCase					)			, [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

										__SCREAMING_SNAKE_CASE:				Tuple           =   {}
										for i, token in enumerate(_lowerCAmelCase					):
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   i
										__SCREAMING_SNAKE_CASE:				Any           =   CharacterTokenizer(vocab=_lowerCAmelCase			, unk_token='''[UNK]'''					)

										self.assertListEqual(tokenizer.tokenize(''''''					)			, []					)

										self.assertListEqual(tokenizer.tokenize('''こんにちは'''					)			, ['''こ''', '''ん''', '''に''', '''ち''', '''は''']					)

										self.assertListEqual(tokenizer.tokenize('''こんにちほ'''					)			, ['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''					)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   tokenizer.encode('''ありがとう。'''			, add_special_tokens=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Dict           =   tokenizer.encode('''どういたしまして。'''			, add_special_tokens=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				str           =   tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase			, _lowerCAmelCase					)

										# 2 is for "[CLS]", 3 is for "[SEP]"
										assert encoded_sentence == [2] + text + [3]
										assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				a			(      unittest.TestCase			):






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   '''cl-tohoku/bert-base-japanese'''
										__SCREAMING_SNAKE_CASE:				Dict           =   AutoTokenizer.from_pretrained(_lowerCAmelCase					)
										self.assertIsInstance(_lowerCAmelCase			, _lowerCAmelCase					)





class 				a			(      unittest.TestCase			):






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   '''cl-tohoku/bert-base-japanese'''
										with self.assertLogs('''transformers'''			, level='''WARNING'''					) as cm:
																BertTokenizer.from_pretrained(_lowerCAmelCase					)
																self.assertTrue(
																    cm.records[0].message.startswith(
																        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
																        ''' is called from.'''					)					)
										__SCREAMING_SNAKE_CASE:				str           =   '''bert-base-cased'''
										with self.assertLogs('''transformers'''			, level='''WARNING'''					) as cm:
																BertJapaneseTokenizer.from_pretrained(_lowerCAmelCase					)
																self.assertTrue(
																    cm.records[0].message.startswith(
																        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
																        ''' is called from.'''					)					)










import gc
import unittest

import numpy as np
import torch
from torch.backends.cuda import sdp_kernel

from diffusers import (
    CMStochasticIterativeScheduler,
    ConsistencyModelPipeline,
    UNetaDModel,
)
from diffusers.utils import randn_tensor, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_a, require_torch_gpu

from ..pipeline_params import UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS, UNCONDITIONAL_IMAGE_GENERATION_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()





class 				a			(      __lowercase   ,unittest.TestCase			):
				SCREAMING_SNAKE_CASE__     :   List[Any]		      =     ConsistencyModelPipeline
				SCREAMING_SNAKE_CASE__     :   Any		      =     UNCONDITIONAL_IMAGE_GENERATION_PARAMS
				SCREAMING_SNAKE_CASE__     :   str		      =     UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS

				# Override required_optional_params to remove num_images_per_prompt
				SCREAMING_SNAKE_CASE__     :   Optional[int]		      =     frozenset(
				    [
				        '''num_inference_steps''',
				        '''generator''',
				        '''latents''',
				        '''output_type''',
				        '''return_dict''',
				        '''callback''',
				        '''callback_steps''',
				    ]			)






				@property
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   UNetaDModel.from_pretrained(
										    '''diffusers/consistency-models-test'''			, subfolder='''test_unet'''			, )
										return unet






				@property
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   UNetaDModel.from_pretrained(
										    '''diffusers/consistency-models-test'''			, subfolder='''test_unet_class_cond'''			, )
										return unet






				def 			snake_case_    (  self			, _lowerCAmelCase=False					):

										"""simple docstring"""




										if class_cond:
																__SCREAMING_SNAKE_CASE:				List[Any]           =   self.dummy_cond_unet
										else:
																__SCREAMING_SNAKE_CASE:				Tuple           =   self.dummy_uncond_unet

										# Default to CM multistep sampler
										__SCREAMING_SNAKE_CASE:				Tuple           =   CMStochasticIterativeScheduler(
										    num_train_timesteps=40			, sigma_min=0.002			, sigma_max=80.0			, )

										__SCREAMING_SNAKE_CASE:				List[str]           =   {
										    '''unet''': unet,
										    '''scheduler''': scheduler,
										}

										return components






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase=0					):

										"""simple docstring"""




										if str(_lowerCAmelCase					).startswith('''mps'''					):
																__SCREAMING_SNAKE_CASE:				List[str]           =   torch.manual_seed(_lowerCAmelCase					)
										else:
																__SCREAMING_SNAKE_CASE:				List[str]           =   torch.Generator(device=_lowerCAmelCase					).manual_seed(_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   {
										    '''batch_size''': 1,
										    '''num_inference_steps''': None,
										    '''timesteps''': [22, 0],
										    '''generator''': generator,
										    '''output_type''': '''np''',
										}

										return inputs






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   '''cpu'''  # ensure determinism for the device-dependent torch.Generator
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.get_dummy_components()
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   ConsistencyModelPipeline(**_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Any           =   pipe.to(_lowerCAmelCase					)
										pipe.set_progress_bar_config(disable=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.get_dummy_inputs(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Any           =   pipe(**_lowerCAmelCase					).images
										assert image.shape == (1, 32, 32, 3)

										__SCREAMING_SNAKE_CASE:				int           =   image[0, -3:, -3:, -1]
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   np.array([0.3572, 0.6273, 0.4031, 0.3961, 0.4321, 0.5730, 0.5266, 0.4780, 0.5004]					)

										assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-3






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   '''cpu'''  # ensure determinism for the device-dependent torch.Generator
										__SCREAMING_SNAKE_CASE:				str           =   self.get_dummy_components(class_cond=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Any           =   ConsistencyModelPipeline(**_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Any           =   pipe.to(_lowerCAmelCase					)
										pipe.set_progress_bar_config(disable=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Dict           =   self.get_dummy_inputs(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Dict           =   0
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   pipe(**_lowerCAmelCase					).images
										assert image.shape == (1, 32, 32, 3)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   image[0, -3:, -3:, -1]
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   np.array([0.3572, 0.6273, 0.4031, 0.3961, 0.4321, 0.5730, 0.5266, 0.4780, 0.5004]					)

										assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-3






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   '''cpu'''  # ensure determinism for the device-dependent torch.Generator
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   self.get_dummy_components()
										__SCREAMING_SNAKE_CASE:				Tuple           =   ConsistencyModelPipeline(**_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   pipe.to(_lowerCAmelCase					)
										pipe.set_progress_bar_config(disable=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				List[Any]           =   self.get_dummy_inputs(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				int           =   1
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   None
										__SCREAMING_SNAKE_CASE:				List[str]           =   pipe(**_lowerCAmelCase					).images
										assert image.shape == (1, 32, 32, 3)

										__SCREAMING_SNAKE_CASE:				Tuple           =   image[0, -3:, -3:, -1]
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   np.array([0.5004, 0.5004, 0.4994, 0.5008, 0.4976, 0.5018, 0.4990, 0.4982, 0.4987]					)

										assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-3






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   '''cpu'''  # ensure determinism for the device-dependent torch.Generator
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   self.get_dummy_components(class_cond=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				List[Any]           =   ConsistencyModelPipeline(**_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				List[Any]           =   pipe.to(_lowerCAmelCase					)
										pipe.set_progress_bar_config(disable=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Tuple           =   self.get_dummy_inputs(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Any           =   1
										__SCREAMING_SNAKE_CASE:				List[Any]           =   None
										__SCREAMING_SNAKE_CASE:				Tuple           =   0
										__SCREAMING_SNAKE_CASE:				List[str]           =   pipe(**_lowerCAmelCase					).images
										assert image.shape == (1, 32, 32, 3)

										__SCREAMING_SNAKE_CASE:				List[Any]           =   image[0, -3:, -3:, -1]
										__SCREAMING_SNAKE_CASE:				Any           =   np.array([0.5004, 0.5004, 0.4994, 0.5008, 0.4976, 0.5018, 0.4990, 0.4982, 0.4987]					)

										assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-3





@slow
@require_torch_gpu
class 				a			(      unittest.TestCase			):






				def 			snake_case_    (  self					):

										"""simple docstring"""




										super().tearDown()
										gc.collect()
										torch.cuda.empty_cache()






				def 			snake_case_    (  self			, _lowerCAmelCase=0			, _lowerCAmelCase=False			, _lowerCAmelCase="cpu"			, _lowerCAmelCase=torch.floataa			, _lowerCAmelCase=(1, 3, 64, 64)					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   torch.manual_seed(_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Dict           =   {
										    '''num_inference_steps''': None,
										    '''timesteps''': [22, 0],
										    '''class_labels''': 0,
										    '''generator''': generator,
										    '''output_type''': '''np''',
										}

										if get_fixed_latents:
																__SCREAMING_SNAKE_CASE:				Dict           =   self.get_fixed_latents(seed=_lowerCAmelCase			, device=_lowerCAmelCase			, dtype=_lowerCAmelCase			, shape=_lowerCAmelCase					)
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   latents

										return inputs






				def 			snake_case_    (  self			, _lowerCAmelCase=0			, _lowerCAmelCase="cpu"			, _lowerCAmelCase=torch.floataa			, _lowerCAmelCase=(1, 3, 64, 64)					):

										"""simple docstring"""




										if type(_lowerCAmelCase					) == str:
																__SCREAMING_SNAKE_CASE:				List[Any]           =   torch.device(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Dict           =   torch.Generator(device=_lowerCAmelCase					).manual_seed(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				int           =   randn_tensor(_lowerCAmelCase			, generator=_lowerCAmelCase			, device=_lowerCAmelCase			, dtype=_lowerCAmelCase					)
										return latents






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   UNetaDModel.from_pretrained('''diffusers/consistency_models'''			, subfolder='''diffusers_cd_imagenet64_l2'''					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   CMStochasticIterativeScheduler(
										    num_train_timesteps=40			, sigma_min=0.002			, sigma_max=80.0			, )
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   ConsistencyModelPipeline(unet=_lowerCAmelCase			, scheduler=_lowerCAmelCase					)
										pipe.to(torch_device=_lowerCAmelCase					)
										pipe.set_progress_bar_config(disable=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				int           =   self.get_inputs()
										__SCREAMING_SNAKE_CASE:				List[Any]           =   pipe(**_lowerCAmelCase					).images
										assert image.shape == (1, 64, 64, 3)

										__SCREAMING_SNAKE_CASE:				Dict           =   image[0, -3:, -3:, -1]

										__SCREAMING_SNAKE_CASE:				Any           =   np.array([0.0888, 0.0881, 0.0666, 0.0479, 0.0292, 0.0195, 0.0201, 0.0163, 0.0254]					)

										assert np.abs(image_slice.flatten() - expected_slice					).max() < 2e-2






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   UNetaDModel.from_pretrained('''diffusers/consistency_models'''			, subfolder='''diffusers_cd_imagenet64_l2'''					)
										__SCREAMING_SNAKE_CASE:				str           =   CMStochasticIterativeScheduler(
										    num_train_timesteps=40			, sigma_min=0.002			, sigma_max=80.0			, )
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   ConsistencyModelPipeline(unet=_lowerCAmelCase			, scheduler=_lowerCAmelCase					)
										pipe.to(torch_device=_lowerCAmelCase					)
										pipe.set_progress_bar_config(disable=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				List[str]           =   self.get_inputs()
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   1
										__SCREAMING_SNAKE_CASE:				List[Any]           =   None
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   pipe(**_lowerCAmelCase					).images
										assert image.shape == (1, 64, 64, 3)

										__SCREAMING_SNAKE_CASE:				Dict           =   image[0, -3:, -3:, -1]

										__SCREAMING_SNAKE_CASE:				Any           =   np.array([0.0340, 0.0152, 0.0063, 0.0267, 0.0221, 0.0107, 0.0416, 0.0186, 0.0217]					)

										assert np.abs(image_slice.flatten() - expected_slice					).max() < 2e-2






				@require_torch_a
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   UNetaDModel.from_pretrained('''diffusers/consistency_models'''			, subfolder='''diffusers_cd_imagenet64_l2'''					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   CMStochasticIterativeScheduler(
										    num_train_timesteps=40			, sigma_min=0.002			, sigma_max=80.0			, )
										__SCREAMING_SNAKE_CASE:				List[Any]           =   ConsistencyModelPipeline(unet=_lowerCAmelCase			, scheduler=_lowerCAmelCase					)
										pipe.to(torch_device=_lowerCAmelCase			, torch_dtype=torch.floataa					)
										pipe.set_progress_bar_config(disable=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				int           =   self.get_inputs(get_fixed_latents=_lowerCAmelCase			, device=_lowerCAmelCase					)
										# Ensure usage of flash attention in torch 2.0
										with sdp_kernel(enable_flash=_lowerCAmelCase			, enable_math=_lowerCAmelCase			, enable_mem_efficient=_lowerCAmelCase					):
																__SCREAMING_SNAKE_CASE:				Dict           =   pipe(**_lowerCAmelCase					).images
										assert image.shape == (1, 64, 64, 3)

										__SCREAMING_SNAKE_CASE:				List[Any]           =   image[0, -3:, -3:, -1]

										__SCREAMING_SNAKE_CASE:				Optional[int]           =   np.array([0.1875, 0.1428, 0.1289, 0.2151, 0.2092, 0.1477, 0.1877, 0.1641, 0.1353]					)

										assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-3






				@require_torch_a
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   UNetaDModel.from_pretrained('''diffusers/consistency_models'''			, subfolder='''diffusers_cd_imagenet64_l2'''					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   CMStochasticIterativeScheduler(
										    num_train_timesteps=40			, sigma_min=0.002			, sigma_max=80.0			, )
										__SCREAMING_SNAKE_CASE:				List[Any]           =   ConsistencyModelPipeline(unet=_lowerCAmelCase			, scheduler=_lowerCAmelCase					)
										pipe.to(torch_device=_lowerCAmelCase			, torch_dtype=torch.floataa					)
										pipe.set_progress_bar_config(disable=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				int           =   self.get_inputs(get_fixed_latents=_lowerCAmelCase			, device=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   1
										__SCREAMING_SNAKE_CASE:				List[Any]           =   None
										# Ensure usage of flash attention in torch 2.0
										with sdp_kernel(enable_flash=_lowerCAmelCase			, enable_math=_lowerCAmelCase			, enable_mem_efficient=_lowerCAmelCase					):
																__SCREAMING_SNAKE_CASE:				Optional[int]           =   pipe(**_lowerCAmelCase					).images
										assert image.shape == (1, 64, 64, 3)

										__SCREAMING_SNAKE_CASE:				Tuple           =   image[0, -3:, -3:, -1]

										__SCREAMING_SNAKE_CASE:				Tuple           =   np.array([0.1663, 0.1948, 0.2275, 0.1680, 0.1204, 0.1245, 0.1858, 0.1338, 0.2095]					)

										assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-3



