






'''simple docstring'''



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class UpperCAmelCase_       (     __A					,							unittest.TestCase					):




      """simple docstring"""




      UpperCamelCase_             =      BertJapaneseTokenizer
      UpperCamelCase_             =      False
      UpperCamelCase_             =      True






      def 					A__   (			self      :		Optional[int]		)	->	Any:


       '''simple docstring'''

       super().setUp()

       lowercase       : Optional[int]						=[
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''こんにちは''',
           '''こん''',
           '''にちは''',
           '''ばんは''',
           '''##こん''',
           '''##にちは''',
           '''##ばんは''',
           '''世界''',
           '''##世界''',
           '''、''',
           '''##、''',
           '''。''',
           '''##。''',
       ]

       lowercase       : List[str]						=os.path.join(self.tmpdirname  ,   VOCAB_FILES_NAMES['''vocab_file''']		)
       with open(self.vocab_file  ,   '''w'''  ,   encoding='''utf-8'''		) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)






      def 					A__   (			self      :		List[str]  ,   UpperCAmelCase      :		Any		)	->	Tuple:


       '''simple docstring'''

       lowercase       : Optional[int]						='''こんにちは、世界。 \nこんばんは、世界。'''
       lowercase       : Dict						='''こんにちは 、 世界 。 こんばんは 、 世界 。'''
       return input_text, output_text






      def 					A__   (			self      :		Tuple  ,   UpperCAmelCase      :		List[str]		)	->	int:


       '''simple docstring'''

       lowercase	,		lowercase       : List[str]						=self.get_input_output_texts(UpperCAmelCase		)
       lowercase       : Optional[Any]						=tokenizer.encode(UpperCAmelCase  ,   add_special_tokens=UpperCAmelCase		)
       lowercase       : Optional[Any]						=tokenizer.decode(UpperCAmelCase  ,   clean_up_tokenization_spaces=UpperCAmelCase		)
       return text, ids






      def 					A__   (			self      :		List[str]		)	->	List[Any]:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		Tuple		)	->	List[str]:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		Union[str, Any]		)	->	Optional[Any]:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		List[Any]		)	->	Tuple:


       '''simple docstring'''

       lowercase       : str						=self.tokenizer_class(self.vocab_file		)

       lowercase       : Dict						=tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''		)
       self.assertListEqual(UpperCAmelCase  ,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]		)






      def 					A__   (			self      :		Union[str, Any]		)	->	Any:


       '''simple docstring'''

       lowercase       : Optional[int]						=self.tokenizer_class(self.vocab_file  ,   word_tokenizer_type='''mecab'''		)
       self.assertIsNotNone(UpperCAmelCase		)

       lowercase       : Dict						='''こんにちは、世界。\nこんばんは、世界。'''
       lowercase       : Optional[Any]						=tokenizer.tokenize(UpperCAmelCase		)
       self.assertListEqual(UpperCAmelCase  ,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]		)

       lowercase       : Union[str, Any]						=os.path.join(self.tmpdirname  ,   '''tokenizer.bin'''		)
       with open(UpperCAmelCase  ,   '''wb'''		) as handle:
        pickle.dump(UpperCAmelCase  ,   UpperCAmelCase		)

       with open(UpperCAmelCase  ,   '''rb'''		) as handle:
        lowercase       : Tuple						=pickle.load(UpperCAmelCase		)

       lowercase       : Dict						=tokenizer_new.tokenize(UpperCAmelCase		)

       self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)






      def 					A__   (			self      :		List[str]		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : int						=MecabTokenizer(mecab_dic='''ipadic'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      def 					A__   (			self      :		List[str]		)	->	Any:


       '''simple docstring'''

       try:
        lowercase       : Tuple						=MecabTokenizer(mecab_dic='''unidic_lite'''		)
       except ModuleNotFoundError:
        return

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      def 					A__   (			self      :		Optional[Any]		)	->	str:


       '''simple docstring'''

       try:
        lowercase       : Optional[int]						=MecabTokenizer(mecab_dic='''unidic'''		)
       except ModuleNotFoundError:
        return

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      def 					A__   (			self      :		Dict		)	->	Tuple:


       '''simple docstring'''

       lowercase       : Tuple						=MecabTokenizer(do_lower_case=UpperCAmelCase  ,   mecab_dic='''ipadic'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      def 					A__   (			self      :		str		)	->	str:


       '''simple docstring'''

       try:
        lowercase       : Any						=MecabTokenizer(
            do_lower_case=UpperCAmelCase  ,   normalize_text=UpperCAmelCase  ,   mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''		)
       except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']  ,   )






      def 					A__   (			self      :		Union[str, Any]		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : List[str]						=MecabTokenizer(normalize_text=UpperCAmelCase  ,   mecab_dic='''ipadic'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']  ,   )






      @require_sudachi
      def 					A__   (			self      :		Optional[int]		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : int						=self.tokenizer_class(self.vocab_file  ,   word_tokenizer_type='''sudachi'''		)
       self.assertIsNotNone(UpperCAmelCase		)

       lowercase       : Optional[int]						='''こんにちは、世界。\nこんばんは、世界。'''
       lowercase       : Any						=tokenizer.tokenize(UpperCAmelCase		)
       self.assertListEqual(UpperCAmelCase  ,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]		)

       lowercase       : Union[str, Any]						=os.path.join(self.tmpdirname  ,   '''tokenizer.bin'''		)
       with open(UpperCAmelCase  ,   '''wb'''		) as handle:
        pickle.dump(UpperCAmelCase  ,   UpperCAmelCase		)

       with open(UpperCAmelCase  ,   '''rb'''		) as handle:
        lowercase       : Tuple						=pickle.load(UpperCAmelCase		)

       lowercase       : Tuple						=tokenizer_new.tokenize(UpperCAmelCase		)

       self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)






      @require_sudachi
      def 					A__   (			self      :		Optional[int]		)	->	int:


       '''simple docstring'''

       lowercase       : str						=SudachiTokenizer(sudachi_dict_type='''core'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']  ,   )






      @require_sudachi
      def 					A__   (			self      :		Dict		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : Any						=SudachiTokenizer(sudachi_dict_type='''core'''  ,   sudachi_split_mode='''A'''		)

       self.assertListEqual(tokenizer.tokenize('''外国人参政権'''		)  ,   ['''外国''', '''人''', '''参政''', '''権''']		)






      @require_sudachi
      def 					A__   (			self      :		Dict		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : int						=SudachiTokenizer(sudachi_dict_type='''core'''  ,   sudachi_split_mode='''B'''		)

       self.assertListEqual(tokenizer.tokenize('''外国人参政権'''		)  ,   ['''外国人''', '''参政権''']		)






      @require_sudachi
      def 					A__   (			self      :		Optional[int]		)	->	str:


       '''simple docstring'''

       lowercase       : Optional[Any]						=SudachiTokenizer(sudachi_dict_type='''core'''  ,   sudachi_split_mode='''C'''		)

       self.assertListEqual(tokenizer.tokenize('''外国人参政権'''		)  ,   ['''外国人参政権''']		)






      @require_sudachi
      def 					A__   (			self      :		Dict		)	->	Dict:


       '''simple docstring'''

       lowercase       : Optional[int]						=SudachiTokenizer(do_lower_case=UpperCAmelCase  ,   sudachi_dict_type='''core'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']  ,   )






      @require_sudachi
      def 					A__   (			self      :		Union[str, Any]		)	->	List[str]:


       '''simple docstring'''

       lowercase       : int						=SudachiTokenizer(normalize_text=UpperCAmelCase  ,   sudachi_dict_type='''core'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   [''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']  ,   )






      @require_sudachi
      def 					A__   (			self      :		List[str]		)	->	List[str]:


       '''simple docstring'''

       lowercase       : int						=SudachiTokenizer(trim_whitespace=UpperCAmelCase  ,   sudachi_dict_type='''core'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		Optional[int]		)	->	Any:


       '''simple docstring'''

       lowercase       : int						=self.tokenizer_class(self.vocab_file  ,   word_tokenizer_type='''jumanpp'''		)
       self.assertIsNotNone(UpperCAmelCase		)

       lowercase       : Dict						='''こんにちは、世界。\nこんばんは、世界。'''
       lowercase       : List[Any]						=tokenizer.tokenize(UpperCAmelCase		)
       self.assertListEqual(UpperCAmelCase  ,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]		)

       lowercase       : Any						=os.path.join(self.tmpdirname  ,   '''tokenizer.bin'''		)
       with open(UpperCAmelCase  ,   '''wb'''		) as handle:
        pickle.dump(UpperCAmelCase  ,   UpperCAmelCase		)

       with open(UpperCAmelCase  ,   '''rb'''		) as handle:
        lowercase       : Union[str, Any]						=pickle.load(UpperCAmelCase		)

       lowercase       : str						=tokenizer_new.tokenize(UpperCAmelCase		)

       self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)






      @require_jumanpp
      def 					A__   (			self      :		Any		)	->	Any:


       '''simple docstring'''

       lowercase       : Any						=JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		int		)	->	int:


       '''simple docstring'''

       lowercase       : Any						=JumanppTokenizer(do_lower_case=UpperCAmelCase		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		str		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : List[str]						=JumanppTokenizer(normalize_text=UpperCAmelCase		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		str		)	->	str:


       '''simple docstring'''

       lowercase       : Any						=JumanppTokenizer(trim_whitespace=UpperCAmelCase		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		List[str]		)	->	Any:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''		)  ,   ['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']  ,   )






      def 					A__   (			self      :		Union[str, Any]		)	->	int:


       '''simple docstring'''

       lowercase       : int						=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

       lowercase       : Optional[Any]						={}
       for i, token in enumerate(UpperCAmelCase		):
        lowercase       : str						=i
       lowercase       : int						=WordpieceTokenizer(vocab=UpperCAmelCase  ,   unk_token='''[UNK]'''		)

       self.assertListEqual(tokenizer.tokenize(''''''		)  ,   []		)

       self.assertListEqual(tokenizer.tokenize('''こんにちは'''		)  ,   ['''こんにちは''']		)

       self.assertListEqual(tokenizer.tokenize('''こんばんは'''		)  ,   ['''こん''', '''##ばんは''']		)

       self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''		)  ,   ['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']		)






      def 					A__   (			self      :		Any		)	->	Tuple:


       '''simple docstring'''

       lowercase       : List[Any]						=BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''		)
       lowercase       : Tuple						=tokenizer.subword_tokenizer

       lowercase       : Union[str, Any]						=subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''		)
       self.assertListEqual(UpperCAmelCase  ,   ['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']		)

       lowercase       : Optional[Any]						=subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''		)
       self.assertListEqual(UpperCAmelCase  ,   ['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']		)






      def 					A__   (			self      :		str		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : Optional[Any]						=self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''		)

       lowercase       : Tuple						=tokenizer.encode('''ありがとう。'''  ,   add_special_tokens=UpperCAmelCase		)
       lowercase       : Tuple						=tokenizer.encode('''どういたしまして。'''  ,   add_special_tokens=UpperCAmelCase		)

       lowercase       : Any						=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase		)
       lowercase       : Tuple						=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase  ,   UpperCAmelCase		)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class UpperCAmelCase_       (     __A					,							unittest.TestCase					):




      """simple docstring"""




      UpperCamelCase_             =      BertJapaneseTokenizer
      UpperCamelCase_             =      False






      def 					A__   (			self      :		str		)	->	Optional[int]:


       '''simple docstring'''

       super().setUp()

       lowercase       : Any						=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

       lowercase       : Optional[int]						=os.path.join(self.tmpdirname  ,   VOCAB_FILES_NAMES['''vocab_file''']		)
       with open(self.vocab_file  ,   '''w'''  ,   encoding='''utf-8'''		) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)






      def 					A__   (			self      :		Dict  ,   **UpperCAmelCase      :		Union[str, Any]		)	->	Union[str, Any]:


       '''simple docstring'''

       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname  ,   subword_tokenizer_type='''character'''  ,   **UpperCAmelCase		)






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		Optional[int]		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : int						='''こんにちは、世界。 \nこんばんは、世界。'''
       lowercase       : Optional[int]						='''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
       return input_text, output_text






      def 					A__   (			self      :		Dict		)	->	int:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		str		)	->	str:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		Optional[int]		)	->	Dict:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		Dict		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : Dict						=self.tokenizer_class(self.vocab_file  ,   subword_tokenizer_type='''character'''		)

       lowercase       : List[str]						=tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''		)
       self.assertListEqual(
           UpperCAmelCase  ,   ['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']		)
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]		)






      def 					A__   (			self      :		Optional[Any]		)	->	Dict:


       '''simple docstring'''

       lowercase       : List[Any]						=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

       lowercase       : List[Any]						={}
       for i, token in enumerate(UpperCAmelCase		):
        lowercase       : Tuple						=i
       lowercase       : Dict						=CharacterTokenizer(vocab=UpperCAmelCase  ,   unk_token='''[UNK]'''		)

       self.assertListEqual(tokenizer.tokenize(''''''		)  ,   []		)

       self.assertListEqual(tokenizer.tokenize('''こんにちは'''		)  ,   ['''こ''', '''ん''', '''に''', '''ち''', '''は''']		)

       self.assertListEqual(tokenizer.tokenize('''こんにちほ'''		)  ,   ['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']		)






      def 					A__   (			self      :		Optional[int]		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : str						=self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''		)

       lowercase       : Tuple						=tokenizer.encode('''ありがとう。'''  ,   add_special_tokens=UpperCAmelCase		)
       lowercase       : List[str]						=tokenizer.encode('''どういたしまして。'''  ,   add_special_tokens=UpperCAmelCase		)

       lowercase       : Any						=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase		)
       lowercase       : List[str]						=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase  ,   UpperCAmelCase		)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class UpperCAmelCase_       (     unittest.TestCase					):




      """simple docstring"""




      def 					A__   (			self      :		Tuple		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : Optional[int]						='''cl-tohoku/bert-base-japanese'''
       lowercase       : Optional[Any]						=AutoTokenizer.from_pretrained(UpperCAmelCase		)
       self.assertIsInstance(UpperCAmelCase  ,   UpperCAmelCase		)





class UpperCAmelCase_       (     unittest.TestCase					):




      """simple docstring"""




      def 					A__   (			self      :		Optional[int]		)	->	Any:


       '''simple docstring'''

       lowercase       : str						='''cl-tohoku/bert-base-japanese'''
       with self.assertLogs('''transformers'''  ,   level='''WARNING'''		) as cm:
        BertTokenizer.from_pretrained(UpperCAmelCase		)
        self.assertTrue(
            cm.records[0].message.startswith(
                '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                ''' is called from.'''		)		)
       lowercase       : str						='''bert-base-cased'''
       with self.assertLogs('''transformers'''  ,   level='''WARNING'''		) as cm:
        BertJapaneseTokenizer.from_pretrained(UpperCAmelCase		)
        self.assertTrue(
            cm.records[0].message.startswith(
                '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                ''' is called from.'''		)		)






import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402

class 							UpperCAmelCase_						(     unittest.TestCase  ):
							"""simple docstring"""







							def 							__lowercase							(     self			)       ->       List[Any]:
												_a      :       int   =						0


							def 							__lowercase							(     self			)       ->       List[str]:
												_a      :       Dict   =						AutoImageProcessor.from_pretrained('''openai/clip-vit-base-patch32'''			)
												self.assertIsInstance(_a			,				_a			)


							def 							__lowercase							(     self			)       ->       Union[str, Any]:
												with tempfile.TemporaryDirectory() as tmpdirname:
																	_a      :       Tuple   =						Path(_a			) / '''preprocessor_config.json'''
																	_a      :       Optional[Any]   =						Path(_a			) / '''config.json'''
																	json.dump(
																	    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}			,				open(_a			,				'''w'''			)			,				)
																	json.dump({'''model_type''': '''clip'''}			,				open(_a			,				'''w'''			)			)

																	_a      :       List[str]   =						AutoImageProcessor.from_pretrained(_a			)
																	self.assertIsInstance(_a			,				_a			)


							def 							__lowercase							(     self			)       ->       Optional[Any]:
												# Ensure we can load the image processor from the feature extractor config
												with tempfile.TemporaryDirectory() as tmpdirname:
																	_a      :       Optional[int]   =						Path(_a			) / '''preprocessor_config.json'''
																	_a      :       Any   =						Path(_a			) / '''config.json'''
																	json.dump(
																	    {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}			,				open(_a			,				'''w'''			)			,				)
																	json.dump({'''model_type''': '''clip'''}			,				open(_a			,				'''w'''			)			)

																	_a      :       Optional[Any]   =						AutoImageProcessor.from_pretrained(_a			)
																	self.assertIsInstance(_a			,				_a			)


							def 							__lowercase							(     self			)       ->       Optional[Any]:
												with tempfile.TemporaryDirectory() as tmpdirname:
																	_a      :       Dict   =						CLIPConfig()

																	# Create a dummy config file with image_proceesor_type
																	_a      :       Tuple   =						Path(_a			) / '''preprocessor_config.json'''
																	_a      :       List[str]   =						Path(_a			) / '''config.json'''
																	json.dump(
																	    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}			,				open(_a			,				'''w'''			)			,				)
																	json.dump({'''model_type''': '''clip'''}			,				open(_a			,				'''w'''			)			)

																	# remove image_processor_type to make sure config.json alone is enough to load image processor locally
																	_a      :       Tuple   =						AutoImageProcessor.from_pretrained(_a			).to_dict()

																	config_dict.pop('''image_processor_type'''			)
																	_a      :       Tuple   =						CLIPImageProcessor(**_a			)

																	# save in new folder
																	model_config.save_pretrained(_a			)
																	config.save_pretrained(_a			)

																	_a      :       List[str]   =						AutoImageProcessor.from_pretrained(_a			)

																	# make sure private variable is not incorrectly saved
																	_a      :       Optional[int]   =						json.loads(config.to_json_string()			)
																	self.assertTrue('''_processor_class''' not in dict_as_saved			)

												self.assertIsInstance(_a			,				_a			)


							def 							__lowercase							(     self			)       ->       Dict:
												with tempfile.TemporaryDirectory() as tmpdirname:
																	_a      :       Optional[int]   =						Path(_a			) / '''preprocessor_config.json'''
																	json.dump(
																	    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}			,				open(_a			,				'''w'''			)			,				)

																	_a      :       List[str]   =						AutoImageProcessor.from_pretrained(_a			)
																	self.assertIsInstance(_a			,				_a			)


							def 							__lowercase							(     self			)       ->       Any:
												with self.assertRaisesRegex(
												    _a			,				'''clip-base is not a local folder and is not a valid model identifier'''			):
																	_a      :       Dict   =						AutoImageProcessor.from_pretrained('''clip-base'''			)


							def 							__lowercase							(     self			)       ->       List[Any]:
												with self.assertRaisesRegex(
												    _a			,				R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''			):
																	_a      :       List[str]   =						AutoImageProcessor.from_pretrained(_a			,				revision='''aaaaaa'''			)


							def 							__lowercase							(     self			)       ->       Dict:
												with self.assertRaisesRegex(
												    _a			,				'''hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'''			,				):
																	_a      :       Optional[int]   =						AutoImageProcessor.from_pretrained('''hf-internal-testing/config-no-model'''			)


							def 							__lowercase							(     self			)       ->       Union[str, Any]:
												# If remote code is not set, we will time out when asking whether to load the model.
												with self.assertRaises(_a			):
																	_a      :       str   =						AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''			)
												# If remote code is disabled, we can't load this config.
												with self.assertRaises(_a			):
																	_a      :       Optional[Any]   =						AutoImageProcessor.from_pretrained(
																	    '''hf-internal-testing/test_dynamic_image_processor'''			,				trust_remote_code=_a			)

												_a      :       Union[str, Any]   =						AutoImageProcessor.from_pretrained(
												    '''hf-internal-testing/test_dynamic_image_processor'''			,				trust_remote_code=_a			)
												self.assertEqual(image_processor.__class__.__name__			,				'''NewImageProcessor'''			)

												# Test image processor can be reloaded.
												with tempfile.TemporaryDirectory() as tmp_dir:
																	image_processor.save_pretrained(_a			)
																	_a      :       Optional[Any]   =						AutoImageProcessor.from_pretrained(_a			,				trust_remote_code=_a			)
												self.assertEqual(reloaded_image_processor.__class__.__name__			,				'''NewImageProcessor'''			)


							def 							__lowercase							(     self			)       ->       Dict:
												try:
																	AutoConfig.register('''custom'''			,				_a			)
																	AutoImageProcessor.register(_a			,				_a			)
																	# Trying to register something existing in the Transformers library will raise an error
																	with self.assertRaises(_a			):
																						AutoImageProcessor.register(_a			,				_a			)

																	with tempfile.TemporaryDirectory() as tmpdirname:
																						_a      :       int   =						Path(_a			) / '''preprocessor_config.json'''
																						_a      :       int   =						Path(_a			) / '''config.json'''
																						json.dump(
																						    {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}			,				open(_a			,				'''w'''			)			,				)
																						json.dump({'''model_type''': '''clip'''}			,				open(_a			,				'''w'''			)			)

																						_a      :       int   =						CustomImageProcessor.from_pretrained(_a			)

																	# Now that the config is registered, it can be used as any other config with the auto-API
																	with tempfile.TemporaryDirectory() as tmp_dir:
																						image_processor.save_pretrained(_a			)
																						_a      :       Optional[Any]   =						AutoImageProcessor.from_pretrained(_a			)
																						self.assertIsInstance(_a			,				_a			)

												finally:
																	if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
																	if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																						del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]




							def 							__lowercase							(     self			)       ->       Union[str, Any]:

												class 							UpperCAmelCase_						(     __lowercase  ):
																			"""simple docstring"""







																			UpperCAmelCase__			:		Tuple             =       True

												try:
																	AutoConfig.register('''custom'''			,				_a			)
																	AutoImageProcessor.register(_a			,				_a			)
																	# If remote code is not set, the default is to use local
																	_a      :       str   =						AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''			)
																	self.assertEqual(image_processor.__class__.__name__			,				'''NewImageProcessor'''			)
																	self.assertTrue(image_processor.is_local			)

																	# If remote code is disabled, we load the local one.
																	_a      :       int   =						AutoImageProcessor.from_pretrained(
																	    '''hf-internal-testing/test_dynamic_image_processor'''			,				trust_remote_code=_a			)
																	self.assertEqual(image_processor.__class__.__name__			,				'''NewImageProcessor'''			)
																	self.assertTrue(image_processor.is_local			)

																	# If remote is enabled, we load from the Hub
																	_a      :       Dict   =						AutoImageProcessor.from_pretrained(
																	    '''hf-internal-testing/test_dynamic_image_processor'''			,				trust_remote_code=_a			)
																	self.assertEqual(image_processor.__class__.__name__			,				'''NewImageProcessor'''			)
																	self.assertTrue(not hasattr(_a			,				'''is_local'''			)			)

												finally:
																	if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
																	if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																						del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]



