






'''simple docstring'''



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class UpperCAmelCase_       (     __A					,							unittest.TestCase					):




      """simple docstring"""




      UpperCamelCase_             =      BertJapaneseTokenizer
      UpperCamelCase_             =      False
      UpperCamelCase_             =      True






      def 					A__   (			self      :		Optional[int]		)	->	Any:


       '''simple docstring'''

       super().setUp()

       lowercase       : Optional[int]						=[
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''こんにちは''',
           '''こん''',
           '''にちは''',
           '''ばんは''',
           '''##こん''',
           '''##にちは''',
           '''##ばんは''',
           '''世界''',
           '''##世界''',
           '''、''',
           '''##、''',
           '''。''',
           '''##。''',
       ]

       lowercase       : List[str]						=os.path.join(self.tmpdirname  ,   VOCAB_FILES_NAMES['''vocab_file''']		)
       with open(self.vocab_file  ,   '''w'''  ,   encoding='''utf-8'''		) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)






      def 					A__   (			self      :		List[str]  ,   UpperCAmelCase      :		Any		)	->	Tuple:


       '''simple docstring'''

       lowercase       : Optional[int]						='''こんにちは、世界。 \nこんばんは、世界。'''
       lowercase       : Dict						='''こんにちは 、 世界 。 こんばんは 、 世界 。'''
       return input_text, output_text






      def 					A__   (			self      :		Tuple  ,   UpperCAmelCase      :		List[str]		)	->	int:


       '''simple docstring'''

       lowercase	,		lowercase       : List[str]						=self.get_input_output_texts(UpperCAmelCase		)
       lowercase       : Optional[Any]						=tokenizer.encode(UpperCAmelCase  ,   add_special_tokens=UpperCAmelCase		)
       lowercase       : Optional[Any]						=tokenizer.decode(UpperCAmelCase  ,   clean_up_tokenization_spaces=UpperCAmelCase		)
       return text, ids






      def 					A__   (			self      :		List[str]		)	->	List[Any]:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		Tuple		)	->	List[str]:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		Union[str, Any]		)	->	Optional[Any]:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		List[Any]		)	->	Tuple:


       '''simple docstring'''

       lowercase       : str						=self.tokenizer_class(self.vocab_file		)

       lowercase       : Dict						=tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''		)
       self.assertListEqual(UpperCAmelCase  ,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]		)






      def 					A__   (			self      :		Union[str, Any]		)	->	Any:


       '''simple docstring'''

       lowercase       : Optional[int]						=self.tokenizer_class(self.vocab_file  ,   word_tokenizer_type='''mecab'''		)
       self.assertIsNotNone(UpperCAmelCase		)

       lowercase       : Dict						='''こんにちは、世界。\nこんばんは、世界。'''
       lowercase       : Optional[Any]						=tokenizer.tokenize(UpperCAmelCase		)
       self.assertListEqual(UpperCAmelCase  ,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]		)

       lowercase       : Union[str, Any]						=os.path.join(self.tmpdirname  ,   '''tokenizer.bin'''		)
       with open(UpperCAmelCase  ,   '''wb'''		) as handle:
        pickle.dump(UpperCAmelCase  ,   UpperCAmelCase		)

       with open(UpperCAmelCase  ,   '''rb'''		) as handle:
        lowercase       : Tuple						=pickle.load(UpperCAmelCase		)

       lowercase       : Dict						=tokenizer_new.tokenize(UpperCAmelCase		)

       self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)






      def 					A__   (			self      :		List[str]		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : int						=MecabTokenizer(mecab_dic='''ipadic'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      def 					A__   (			self      :		List[str]		)	->	Any:


       '''simple docstring'''

       try:
        lowercase       : Tuple						=MecabTokenizer(mecab_dic='''unidic_lite'''		)
       except ModuleNotFoundError:
        return

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      def 					A__   (			self      :		Optional[Any]		)	->	str:


       '''simple docstring'''

       try:
        lowercase       : Optional[int]						=MecabTokenizer(mecab_dic='''unidic'''		)
       except ModuleNotFoundError:
        return

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      def 					A__   (			self      :		Dict		)	->	Tuple:


       '''simple docstring'''

       lowercase       : Tuple						=MecabTokenizer(do_lower_case=UpperCAmelCase  ,   mecab_dic='''ipadic'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      def 					A__   (			self      :		str		)	->	str:


       '''simple docstring'''

       try:
        lowercase       : Any						=MecabTokenizer(
            do_lower_case=UpperCAmelCase  ,   normalize_text=UpperCAmelCase  ,   mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''		)
       except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']  ,   )






      def 					A__   (			self      :		Union[str, Any]		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : List[str]						=MecabTokenizer(normalize_text=UpperCAmelCase  ,   mecab_dic='''ipadic'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']  ,   )






      @require_sudachi
      def 					A__   (			self      :		Optional[int]		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : int						=self.tokenizer_class(self.vocab_file  ,   word_tokenizer_type='''sudachi'''		)
       self.assertIsNotNone(UpperCAmelCase		)

       lowercase       : Optional[int]						='''こんにちは、世界。\nこんばんは、世界。'''
       lowercase       : Any						=tokenizer.tokenize(UpperCAmelCase		)
       self.assertListEqual(UpperCAmelCase  ,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]		)

       lowercase       : Union[str, Any]						=os.path.join(self.tmpdirname  ,   '''tokenizer.bin'''		)
       with open(UpperCAmelCase  ,   '''wb'''		) as handle:
        pickle.dump(UpperCAmelCase  ,   UpperCAmelCase		)

       with open(UpperCAmelCase  ,   '''rb'''		) as handle:
        lowercase       : Tuple						=pickle.load(UpperCAmelCase		)

       lowercase       : Tuple						=tokenizer_new.tokenize(UpperCAmelCase		)

       self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)






      @require_sudachi
      def 					A__   (			self      :		Optional[int]		)	->	int:


       '''simple docstring'''

       lowercase       : str						=SudachiTokenizer(sudachi_dict_type='''core'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']  ,   )






      @require_sudachi
      def 					A__   (			self      :		Dict		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : Any						=SudachiTokenizer(sudachi_dict_type='''core'''  ,   sudachi_split_mode='''A'''		)

       self.assertListEqual(tokenizer.tokenize('''外国人参政権'''		)  ,   ['''外国''', '''人''', '''参政''', '''権''']		)






      @require_sudachi
      def 					A__   (			self      :		Dict		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : int						=SudachiTokenizer(sudachi_dict_type='''core'''  ,   sudachi_split_mode='''B'''		)

       self.assertListEqual(tokenizer.tokenize('''外国人参政権'''		)  ,   ['''外国人''', '''参政権''']		)






      @require_sudachi
      def 					A__   (			self      :		Optional[int]		)	->	str:


       '''simple docstring'''

       lowercase       : Optional[Any]						=SudachiTokenizer(sudachi_dict_type='''core'''  ,   sudachi_split_mode='''C'''		)

       self.assertListEqual(tokenizer.tokenize('''外国人参政権'''		)  ,   ['''外国人参政権''']		)






      @require_sudachi
      def 					A__   (			self      :		Dict		)	->	Dict:


       '''simple docstring'''

       lowercase       : Optional[int]						=SudachiTokenizer(do_lower_case=UpperCAmelCase  ,   sudachi_dict_type='''core'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']  ,   )






      @require_sudachi
      def 					A__   (			self      :		Union[str, Any]		)	->	List[str]:


       '''simple docstring'''

       lowercase       : int						=SudachiTokenizer(normalize_text=UpperCAmelCase  ,   sudachi_dict_type='''core'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   [''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']  ,   )






      @require_sudachi
      def 					A__   (			self      :		List[str]		)	->	List[str]:


       '''simple docstring'''

       lowercase       : int						=SudachiTokenizer(trim_whitespace=UpperCAmelCase  ,   sudachi_dict_type='''core'''		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		Optional[int]		)	->	Any:


       '''simple docstring'''

       lowercase       : int						=self.tokenizer_class(self.vocab_file  ,   word_tokenizer_type='''jumanpp'''		)
       self.assertIsNotNone(UpperCAmelCase		)

       lowercase       : Dict						='''こんにちは、世界。\nこんばんは、世界。'''
       lowercase       : List[Any]						=tokenizer.tokenize(UpperCAmelCase		)
       self.assertListEqual(UpperCAmelCase  ,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]		)

       lowercase       : Any						=os.path.join(self.tmpdirname  ,   '''tokenizer.bin'''		)
       with open(UpperCAmelCase  ,   '''wb'''		) as handle:
        pickle.dump(UpperCAmelCase  ,   UpperCAmelCase		)

       with open(UpperCAmelCase  ,   '''rb'''		) as handle:
        lowercase       : Union[str, Any]						=pickle.load(UpperCAmelCase		)

       lowercase       : str						=tokenizer_new.tokenize(UpperCAmelCase		)

       self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)






      @require_jumanpp
      def 					A__   (			self      :		Any		)	->	Any:


       '''simple docstring'''

       lowercase       : Any						=JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		int		)	->	int:


       '''simple docstring'''

       lowercase       : Any						=JumanppTokenizer(do_lower_case=UpperCAmelCase		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		str		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : List[str]						=JumanppTokenizer(normalize_text=UpperCAmelCase		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		str		)	->	str:


       '''simple docstring'''

       lowercase       : Any						=JumanppTokenizer(trim_whitespace=UpperCAmelCase		)

       self.assertListEqual(
           tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)  ,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']  ,   )






      @require_jumanpp
      def 					A__   (			self      :		List[str]		)	->	Any:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''		)  ,   ['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']  ,   )






      def 					A__   (			self      :		Union[str, Any]		)	->	int:


       '''simple docstring'''

       lowercase       : int						=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

       lowercase       : Optional[Any]						={}
       for i, token in enumerate(UpperCAmelCase		):
        lowercase       : str						=i
       lowercase       : int						=WordpieceTokenizer(vocab=UpperCAmelCase  ,   unk_token='''[UNK]'''		)

       self.assertListEqual(tokenizer.tokenize(''''''		)  ,   []		)

       self.assertListEqual(tokenizer.tokenize('''こんにちは'''		)  ,   ['''こんにちは''']		)

       self.assertListEqual(tokenizer.tokenize('''こんばんは'''		)  ,   ['''こん''', '''##ばんは''']		)

       self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''		)  ,   ['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']		)






      def 					A__   (			self      :		Any		)	->	Tuple:


       '''simple docstring'''

       lowercase       : List[Any]						=BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''		)
       lowercase       : Tuple						=tokenizer.subword_tokenizer

       lowercase       : Union[str, Any]						=subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''		)
       self.assertListEqual(UpperCAmelCase  ,   ['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']		)

       lowercase       : Optional[Any]						=subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''		)
       self.assertListEqual(UpperCAmelCase  ,   ['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']		)






      def 					A__   (			self      :		str		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : Optional[Any]						=self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''		)

       lowercase       : Tuple						=tokenizer.encode('''ありがとう。'''  ,   add_special_tokens=UpperCAmelCase		)
       lowercase       : Tuple						=tokenizer.encode('''どういたしまして。'''  ,   add_special_tokens=UpperCAmelCase		)

       lowercase       : Any						=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase		)
       lowercase       : Tuple						=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase  ,   UpperCAmelCase		)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class UpperCAmelCase_       (     __A					,							unittest.TestCase					):




      """simple docstring"""




      UpperCamelCase_             =      BertJapaneseTokenizer
      UpperCamelCase_             =      False






      def 					A__   (			self      :		str		)	->	Optional[int]:


       '''simple docstring'''

       super().setUp()

       lowercase       : Any						=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

       lowercase       : Optional[int]						=os.path.join(self.tmpdirname  ,   VOCAB_FILES_NAMES['''vocab_file''']		)
       with open(self.vocab_file  ,   '''w'''  ,   encoding='''utf-8'''		) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)






      def 					A__   (			self      :		Dict  ,   **UpperCAmelCase      :		Union[str, Any]		)	->	Union[str, Any]:


       '''simple docstring'''

       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname  ,   subword_tokenizer_type='''character'''  ,   **UpperCAmelCase		)






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		Optional[int]		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : int						='''こんにちは、世界。 \nこんばんは、世界。'''
       lowercase       : Optional[int]						='''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
       return input_text, output_text






      def 					A__   (			self      :		Dict		)	->	int:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		str		)	->	str:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		Optional[int]		)	->	Dict:


       '''simple docstring'''

       pass  # TODO add if relevant






      def 					A__   (			self      :		Dict		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : Dict						=self.tokenizer_class(self.vocab_file  ,   subword_tokenizer_type='''character'''		)

       lowercase       : List[str]						=tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''		)
       self.assertListEqual(
           UpperCAmelCase  ,   ['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']		)
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]		)






      def 					A__   (			self      :		Optional[Any]		)	->	Dict:


       '''simple docstring'''

       lowercase       : List[Any]						=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

       lowercase       : List[Any]						={}
       for i, token in enumerate(UpperCAmelCase		):
        lowercase       : Tuple						=i
       lowercase       : Dict						=CharacterTokenizer(vocab=UpperCAmelCase  ,   unk_token='''[UNK]'''		)

       self.assertListEqual(tokenizer.tokenize(''''''		)  ,   []		)

       self.assertListEqual(tokenizer.tokenize('''こんにちは'''		)  ,   ['''こ''', '''ん''', '''に''', '''ち''', '''は''']		)

       self.assertListEqual(tokenizer.tokenize('''こんにちほ'''		)  ,   ['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']		)






      def 					A__   (			self      :		Optional[int]		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : str						=self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''		)

       lowercase       : Tuple						=tokenizer.encode('''ありがとう。'''  ,   add_special_tokens=UpperCAmelCase		)
       lowercase       : List[str]						=tokenizer.encode('''どういたしまして。'''  ,   add_special_tokens=UpperCAmelCase		)

       lowercase       : Any						=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase		)
       lowercase       : List[str]						=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase  ,   UpperCAmelCase		)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class UpperCAmelCase_       (     unittest.TestCase					):




      """simple docstring"""




      def 					A__   (			self      :		Tuple		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : Optional[int]						='''cl-tohoku/bert-base-japanese'''
       lowercase       : Optional[Any]						=AutoTokenizer.from_pretrained(UpperCAmelCase		)
       self.assertIsInstance(UpperCAmelCase  ,   UpperCAmelCase		)





class UpperCAmelCase_       (     unittest.TestCase					):




      """simple docstring"""




      def 					A__   (			self      :		Optional[int]		)	->	Any:


       '''simple docstring'''

       lowercase       : str						='''cl-tohoku/bert-base-japanese'''
       with self.assertLogs('''transformers'''  ,   level='''WARNING'''		) as cm:
        BertTokenizer.from_pretrained(UpperCAmelCase		)
        self.assertTrue(
            cm.records[0].message.startswith(
                '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                ''' is called from.'''		)		)
       lowercase       : str						='''bert-base-cased'''
       with self.assertLogs('''transformers'''  ,   level='''WARNING'''		) as cm:
        BertJapaneseTokenizer.from_pretrained(UpperCAmelCase		)
        self.assertTrue(
            cm.records[0].message.startswith(
                '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                ''' is called from.'''		)		)










"""simple docstring"""



import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


a_					=     get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
a_					=     get_tests_dir("""fixtures/vocab.json""")
a_					=     get_tests_dir("""fixtures""")






class 						__snake_case   (						unittest.TestCase	):





 """simple docstring"""


 _lowerCamelCase								=	["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[Any]							=	0

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		str							=	AutoProcessor.from_pretrained('''facebook/wav2vec2-base-960h'''    )
     self.assertIsInstance(__lowerCamelCase   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     with tempfile.TemporaryDirectory() as tmpdirname:
         __A       :		str							=	WavaVecaConfig()
         __A       :		Dict							=	AutoProcessor.from_pretrained('''facebook/wav2vec2-base-960h'''    )

         # save in new folder
         model_config.save_pretrained(__lowerCamelCase    )
         processor.save_pretrained(__lowerCamelCase    )

         __A       :		List[str]							=	AutoProcessor.from_pretrained(__lowerCamelCase    )

     self.assertIsInstance(__lowerCamelCase   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     with tempfile.TemporaryDirectory() as tmpdirname:
         # copy relevant files
         copyfile(__lowerCamelCase   ,		os.path.join(__lowerCamelCase   ,		__lowerCamelCase    )    )
         copyfile(__lowerCamelCase   ,		os.path.join(__lowerCamelCase   ,		'''vocab.json'''    )    )

         __A       :		Union[str, Any]							=	AutoProcessor.from_pretrained(__lowerCamelCase    )

     self.assertIsInstance(__lowerCamelCase   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     with tempfile.TemporaryDirectory() as tmpdirname:
         __A       :		Union[str, Any]							=	WavaVecaFeatureExtractor()
         __A       :		Optional[Any]							=	AutoTokenizer.from_pretrained('''facebook/wav2vec2-base-960h'''    )

         __A       :		Union[str, Any]							=	WavaVecaProcessor(__lowerCamelCase   ,		__lowerCamelCase    )

         # save in new folder
         processor.save_pretrained(__lowerCamelCase    )

         # drop `processor_class` in tokenizer
         with open(os.path.join(__lowerCamelCase   ,		__lowerCamelCase    )   ,		'''r'''    ) as f:
             __A       :		Optional[int]							=	json.load(__lowerCamelCase    )
             config_dict.pop('''processor_class'''    )

         with open(os.path.join(__lowerCamelCase   ,		__lowerCamelCase    )   ,		'''w'''    ) as f:
             f.write(json.dumps(__lowerCamelCase    )    )

         __A       :		Tuple							=	AutoProcessor.from_pretrained(__lowerCamelCase    )

     self.assertIsInstance(__lowerCamelCase   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     with tempfile.TemporaryDirectory() as tmpdirname:
         __A       :		Optional[int]							=	WavaVecaFeatureExtractor()
         __A       :		List[str]							=	AutoTokenizer.from_pretrained('''facebook/wav2vec2-base-960h'''    )

         __A       :		List[Any]							=	WavaVecaProcessor(__lowerCamelCase   ,		__lowerCamelCase    )

         # save in new folder
         processor.save_pretrained(__lowerCamelCase    )

         # drop `processor_class` in feature extractor
         with open(os.path.join(__lowerCamelCase   ,		__lowerCamelCase    )   ,		'''r'''    ) as f:
             __A       :		int							=	json.load(__lowerCamelCase    )
             config_dict.pop('''processor_class'''    )

         with open(os.path.join(__lowerCamelCase   ,		__lowerCamelCase    )   ,		'''w'''    ) as f:
             f.write(json.dumps(__lowerCamelCase    )    )

         __A       :		Optional[int]							=	AutoProcessor.from_pretrained(__lowerCamelCase    )

     self.assertIsInstance(__lowerCamelCase   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     with tempfile.TemporaryDirectory() as tmpdirname:
         __A       :		Optional[Any]							=	WavaVecaConfig(processor_class='''Wav2Vec2Processor'''    )
         model_config.save_pretrained(__lowerCamelCase    )
         # copy relevant files
         copyfile(__lowerCamelCase   ,		os.path.join(__lowerCamelCase   ,		'''vocab.json'''    )    )
         # create emtpy sample processor
         with open(os.path.join(__lowerCamelCase   ,		__lowerCamelCase    )   ,		'''w'''    ) as f:
             f.write('''{}'''    )

         __A       :		Union[str, Any]							=	AutoProcessor.from_pretrained(__lowerCamelCase    )

     self.assertIsInstance(__lowerCamelCase   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     with self.assertRaises(__lowerCamelCase    ):
         __A       :		Union[str, Any]							=	AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''    )
     # If remote code is disabled, we can't load this config.
     with self.assertRaises(__lowerCamelCase    ):
         __A       :		Dict							=	AutoProcessor.from_pretrained(
             '''hf-internal-testing/test_dynamic_processor'''   ,		trust_remote_code=__lowerCamelCase    )

     __A       :		List[Any]							=	AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''   ,		trust_remote_code=__lowerCamelCase    )
     self.assertTrue(processor.special_attribute_present    )
     self.assertEqual(processor.__class__.__name__   ,		'''NewProcessor'''    )

     __A       :		Union[str, Any]							=	processor.feature_extractor
     self.assertTrue(feature_extractor.special_attribute_present    )
     self.assertEqual(feature_extractor.__class__.__name__   ,		'''NewFeatureExtractor'''    )

     __A       :		Optional[int]							=	processor.tokenizer
     self.assertTrue(tokenizer.special_attribute_present    )
     if is_tokenizers_available():
         self.assertEqual(tokenizer.__class__.__name__   ,		'''NewTokenizerFast'''    )

         # Test we can also load the slow version
         __A       :		int							=	AutoProcessor.from_pretrained(
             '''hf-internal-testing/test_dynamic_processor'''   ,		trust_remote_code=__lowerCamelCase   ,		use_fast=__lowerCamelCase    )
         __A       :		Optional[int]							=	new_processor.tokenizer
         self.assertTrue(new_tokenizer.special_attribute_present    )
         self.assertEqual(new_tokenizer.__class__.__name__   ,		'''NewTokenizer'''    )
     else:
         self.assertEqual(tokenizer.__class__.__name__   ,		'''NewTokenizer'''    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     try:
         AutoConfig.register('''custom'''   ,		__lowerCamelCase    )
         AutoFeatureExtractor.register(__lowerCamelCase   ,		__lowerCamelCase    )
         AutoTokenizer.register(__lowerCamelCase   ,		slow_tokenizer_class=__lowerCamelCase    )
         AutoProcessor.register(__lowerCamelCase   ,		__lowerCamelCase    )
         # Trying to register something existing in the Transformers library will raise an error
         with self.assertRaises(__lowerCamelCase    ):
             AutoProcessor.register(__lowerCamelCase   ,		__lowerCamelCase    )

         # Now that the config is registered, it can be used as any other config with the auto-API
         __A       :		str							=	CustomFeatureExtractor.from_pretrained(__lowerCamelCase    )

         with tempfile.TemporaryDirectory() as tmp_dir:
             __A       :		Optional[Any]							=	os.path.join(__lowerCamelCase   ,		'''vocab.txt'''    )
             with open(__lowerCamelCase   ,		'''w'''   ,		encoding='''utf-8'''    ) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )
             __A       :		Dict							=	CustomTokenizer(__lowerCamelCase    )

         __A       :		Optional[Any]							=	CustomProcessor(__lowerCamelCase   ,		__lowerCamelCase    )

         with tempfile.TemporaryDirectory() as tmp_dir:
             processor.save_pretrained(__lowerCamelCase    )
             __A       :		List[str]							=	AutoProcessor.from_pretrained(__lowerCamelCase    )
             self.assertIsInstance(__lowerCamelCase   ,		__lowerCamelCase    )

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
             del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
         if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]
         if CustomConfig in PROCESSOR_MAPPING._extra_content:
             del PROCESSOR_MAPPING._extra_content[CustomConfig]

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     class 						__snake_case   (						SCREAMING_SNAKE_CASE__	):





      """simple docstring"""


      _lowerCamelCase								=	False






     class 						__snake_case   (						SCREAMING_SNAKE_CASE__	):





      """simple docstring"""


      _lowerCamelCase								=	False






     class 						__snake_case   (						SCREAMING_SNAKE_CASE__	):





      """simple docstring"""


      _lowerCamelCase								=	"""AutoFeatureExtractor"""
      _lowerCamelCase								=	"""AutoTokenizer"""
      _lowerCamelCase								=	False

     try:
         AutoConfig.register('''custom'''   ,		__lowerCamelCase    )
         AutoFeatureExtractor.register(__lowerCamelCase   ,		__lowerCamelCase    )
         AutoTokenizer.register(__lowerCamelCase   ,		slow_tokenizer_class=__lowerCamelCase    )
         AutoProcessor.register(__lowerCamelCase   ,		__lowerCamelCase    )
         # If remote code is not set, the default is to use local classes.
         __A       :		Union[str, Any]							=	AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''    )
         self.assertEqual(processor.__class__.__name__   ,		'''NewProcessor'''    )
         self.assertFalse(processor.special_attribute_present    )
         self.assertFalse(processor.feature_extractor.special_attribute_present    )
         self.assertFalse(processor.tokenizer.special_attribute_present    )

         # If remote code is disabled, we load the local ones.
         __A       :		int							=	AutoProcessor.from_pretrained(
             '''hf-internal-testing/test_dynamic_processor'''   ,		trust_remote_code=__lowerCamelCase    )
         self.assertEqual(processor.__class__.__name__   ,		'''NewProcessor'''    )
         self.assertFalse(processor.special_attribute_present    )
         self.assertFalse(processor.feature_extractor.special_attribute_present    )
         self.assertFalse(processor.tokenizer.special_attribute_present    )

         # If remote is enabled, we load from the Hub.
         __A       :		str							=	AutoProcessor.from_pretrained(
             '''hf-internal-testing/test_dynamic_processor'''   ,		trust_remote_code=__lowerCamelCase    )
         self.assertEqual(processor.__class__.__name__   ,		'''NewProcessor'''    )
         self.assertTrue(processor.special_attribute_present    )
         self.assertTrue(processor.feature_extractor.special_attribute_present    )
         self.assertTrue(processor.tokenizer.special_attribute_present    )

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
             del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
         if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]
         if CustomConfig in PROCESSOR_MAPPING._extra_content:
             del PROCESSOR_MAPPING._extra_content[CustomConfig]

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		int							=	AutoProcessor.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )
     self.assertEqual(processor.__class__.__name__   ,		'''BertTokenizerFast'''    )







 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		int							=	AutoProcessor.from_pretrained('''hf-internal-testing/tiny-random-convnext'''    )
     self.assertEqual(processor.__class__.__name__   ,		'''ConvNextImageProcessor'''    )






@is_staging_test
class 						__snake_case   (						unittest.TestCase	):





 """simple docstring"""


 _lowerCamelCase								=	["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]

 @classmethod
 def    UpperCamelCase__(       cls    ):



     '''simple docstring'''




     __A       :		Optional[int]							=	TOKEN
     HfFolder.save_token(__lowerCamelCase    )

 @classmethod
 def    UpperCamelCase__(       cls    ):



     '''simple docstring'''




     try:
         delete_repo(token=cls._token   ,		repo_id='''test-processor'''    )
     except HTTPError:
         pass

     try:
         delete_repo(token=cls._token   ,		repo_id='''valid_org/test-processor-org'''    )
     except HTTPError:
         pass

     try:
         delete_repo(token=cls._token   ,		repo_id='''test-dynamic-processor'''    )
     except HTTPError:
         pass

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Tuple							=	WavaVecaProcessor.from_pretrained(__lowerCamelCase    )
     with tempfile.TemporaryDirectory() as tmp_dir:
         processor.save_pretrained(
             os.path.join(__lowerCamelCase   ,		'''test-processor'''    )   ,		push_to_hub=__lowerCamelCase   ,		use_auth_token=self._token    )

         __A       :		Tuple							=	WavaVecaProcessor.from_pretrained(F"""{USER}/test-processor"""    )
         for k, v in processor.feature_extractor.__dict__.items():
             self.assertEqual(__lowerCamelCase   ,		getattr(new_processor.feature_extractor   ,		__lowerCamelCase    )    )
         self.assertDictEqual(new_processor.tokenizer.get_vocab()   ,		processor.tokenizer.get_vocab()    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[int]							=	WavaVecaProcessor.from_pretrained(__lowerCamelCase    )

     with tempfile.TemporaryDirectory() as tmp_dir:
         processor.save_pretrained(
             os.path.join(__lowerCamelCase   ,		'''test-processor-org'''    )   ,		push_to_hub=__lowerCamelCase   ,		use_auth_token=self._token   ,		organization='''valid_org'''   ,		)

         __A       :		int							=	WavaVecaProcessor.from_pretrained('''valid_org/test-processor-org'''    )
         for k, v in processor.feature_extractor.__dict__.items():
             self.assertEqual(__lowerCamelCase   ,		getattr(new_processor.feature_extractor   ,		__lowerCamelCase    )    )
         self.assertDictEqual(new_processor.tokenizer.get_vocab()   ,		processor.tokenizer.get_vocab()    )







 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     CustomFeatureExtractor.register_for_auto_class()
     CustomTokenizer.register_for_auto_class()
     CustomProcessor.register_for_auto_class()

     __A       :		Any							=	CustomFeatureExtractor.from_pretrained(__lowerCamelCase    )

     with tempfile.TemporaryDirectory() as tmp_dir:
         __A       :		List[Any]							=	os.path.join(__lowerCamelCase   ,		'''vocab.txt'''    )
         with open(__lowerCamelCase   ,		'''w'''   ,		encoding='''utf-8'''    ) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )
         __A       :		Optional[int]							=	CustomTokenizer(__lowerCamelCase    )

     __A       :		Any							=	CustomProcessor(__lowerCamelCase   ,		__lowerCamelCase    )

     with tempfile.TemporaryDirectory() as tmp_dir:
         create_repo(F"""{USER}/test-dynamic-processor"""   ,		token=self._token    )
         __A       :		Tuple							=	Repository(__lowerCamelCase   ,		clone_from=F"""{USER}/test-dynamic-processor"""   ,		token=self._token    )
         processor.save_pretrained(__lowerCamelCase    )

         # This has added the proper auto_map field to the feature extractor config
         self.assertDictEqual(
             processor.feature_extractor.auto_map   ,		{
                 '''AutoFeatureExtractor''': '''custom_feature_extraction.CustomFeatureExtractor''',
                 '''AutoProcessor''': '''custom_processing.CustomProcessor''',
             }   ,		)

         # This has added the proper auto_map field to the tokenizer config
         with open(os.path.join(__lowerCamelCase   ,		'''tokenizer_config.json'''    )    ) as f:
             __A       :		Tuple							=	json.load(__lowerCamelCase    )
         self.assertDictEqual(
             tokenizer_config['''auto_map''']   ,		{
                 '''AutoTokenizer''': ['''custom_tokenization.CustomTokenizer''', None],
                 '''AutoProcessor''': '''custom_processing.CustomProcessor''',
             }   ,		)

         # The code has been copied from fixtures
         self.assertTrue(os.path.isfile(os.path.join(__lowerCamelCase   ,		'''custom_feature_extraction.py'''    )    )    )
         self.assertTrue(os.path.isfile(os.path.join(__lowerCamelCase   ,		'''custom_tokenization.py'''    )    )    )
         self.assertTrue(os.path.isfile(os.path.join(__lowerCamelCase   ,		'''custom_processing.py'''    )    )    )

         repo.push_to_hub()

     __A       :		Any							=	AutoProcessor.from_pretrained(F"""{USER}/test-dynamic-processor"""   ,		trust_remote_code=__lowerCamelCase    )
     # Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
     self.assertEqual(new_processor.__class__.__name__   ,		'''CustomProcessor'''    )




