import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class 		_lowerCamelCase		(   __SCREAMING_SNAKE_CASE    ,     unittest.TestCase    ):
    __a  =      BertJapaneseTokenizer
    __a  =      False
    __a  =      True
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     super().setUp()

     SCREAMING_SNAKE_CASE__:   str=      [
         '''[UNK]''',
         '''[CLS]''',
         '''[SEP]''',
         '''こんにちは''',
         '''こん''',
         '''にちは''',
         '''ばんは''',
         '''##こん''',
         '''##にちは''',
         '''##ばんは''',
         '''世界''',
         '''##世界''',
         '''、''',
         '''##、''',
         '''。''',
         '''##。''',
     ]

     SCREAMING_SNAKE_CASE__:   Tuple=      os.path.join(self.tmpdirname , VOCAB_FILES_NAMES['''vocab_file''']   )
     with open(self.vocab_file , '''w''' , encoding='''utf-8'''   ) as vocab_writer:
      vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )
    def 						UpperCamelCase_				(  self , lowerCAmelCase   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   int=      '''こんにちは、世界。 \nこんばんは、世界。'''
     SCREAMING_SNAKE_CASE__:   List[str]=      '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
     return input_text, output_text
    def 						UpperCamelCase_				(  self , lowerCAmelCase   )       ->					List[str]:
     SCREAMING_SNAKE_CASE__,	SCREAMING_SNAKE_CASE__:   str=      self.get_input_output_texts(_a   )
     SCREAMING_SNAKE_CASE__:   Optional[int]=      tokenizer.encode(_a , add_special_tokens=_a   )
     SCREAMING_SNAKE_CASE__:   int=      tokenizer.decode(_a , clean_up_tokenization_spaces=_a   )
     return text, ids
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     pass  # TODO add if relevant
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     pass  # TODO add if relevant
    def 						UpperCamelCase_				(  self   )       ->					Optional[int]:
     pass  # TODO add if relevant
    def 						UpperCamelCase_				(  self   )       ->					str:
     SCREAMING_SNAKE_CASE__:   Any=      self.tokenizer_class(self.vocab_file   )

     SCREAMING_SNAKE_CASE__:   Optional[Any]=      tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''   )
     self.assertListEqual(_a , ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']   )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_a   ) , [3, 12, 10, 14, 4, 9, 12, 10, 14]   )
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     SCREAMING_SNAKE_CASE__:   int=      self.tokenizer_class(self.vocab_file , word_tokenizer_type='''mecab'''   )
     self.assertIsNotNone(_a   )

     SCREAMING_SNAKE_CASE__:   Tuple=      '''こんにちは、世界。\nこんばんは、世界。'''
     SCREAMING_SNAKE_CASE__:   Tuple=      tokenizer.tokenize(_a   )
     self.assertListEqual(_a , ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']   )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_a   ) , [3, 12, 10, 14, 4, 9, 12, 10, 14]   )

     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      os.path.join(self.tmpdirname , '''tokenizer.bin'''   )
     with open(_a , '''wb'''   ) as handle:
      pickle.dump(_a , _a   )

     with open(_a , '''rb'''   ) as handle:
      SCREAMING_SNAKE_CASE__:   List[str]=      pickle.load(_a   )

     SCREAMING_SNAKE_CASE__:   Dict=      tokenizer_new.tokenize(_a   )

     self.assertListEqual(_a , _a   )
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      MecabTokenizer(mecab_dic='''ipadic'''   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''] , )
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     try:
      SCREAMING_SNAKE_CASE__:   List[Any]=      MecabTokenizer(mecab_dic='''unidic_lite'''   )
     except ModuleNotFoundError:
      return

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''] , )
    def 						UpperCamelCase_				(  self   )       ->					Any:
     try:
      SCREAMING_SNAKE_CASE__:   List[Any]=      MecabTokenizer(mecab_dic='''unidic'''   )
     except ModuleNotFoundError:
      return

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''] , )
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     SCREAMING_SNAKE_CASE__:   Tuple=      MecabTokenizer(do_lower_case=_a , mecab_dic='''ipadic'''   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''] , )
    def 						UpperCamelCase_				(  self   )       ->					List[str]:
     try:
      SCREAMING_SNAKE_CASE__:   Optional[Any]=      MecabTokenizer(
          do_lower_case=_a , normalize_text=_a , mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''   )
     except RuntimeError:
      # if dict doesn't exist in the system, previous code raises this error.
      return

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''] , )
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      MecabTokenizer(normalize_text=_a , mecab_dic='''ipadic'''   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。'''] , )
    @require_sudachi
    def 						UpperCamelCase_				(  self   )       ->					Any:
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      self.tokenizer_class(self.vocab_file , word_tokenizer_type='''sudachi'''   )
     self.assertIsNotNone(_a   )

     SCREAMING_SNAKE_CASE__:   Tuple=      '''こんにちは、世界。\nこんばんは、世界。'''
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      tokenizer.tokenize(_a   )
     self.assertListEqual(_a , ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']   )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_a   ) , [3, 12, 10, 14, 4, 9, 12, 10, 14]   )

     SCREAMING_SNAKE_CASE__:   Optional[Any]=      os.path.join(self.tmpdirname , '''tokenizer.bin'''   )
     with open(_a , '''wb'''   ) as handle:
      pickle.dump(_a , _a   )

     with open(_a , '''rb'''   ) as handle:
      SCREAMING_SNAKE_CASE__:   int=      pickle.load(_a   )

     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      tokenizer_new.tokenize(_a   )

     self.assertListEqual(_a , _a   )
    @require_sudachi
    def 						UpperCamelCase_				(  self   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   str=      SudachiTokenizer(sudachi_dict_type='''core'''   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' '''] , )
    @require_sudachi
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      SudachiTokenizer(sudachi_dict_type='''core''' , sudachi_split_mode='''A'''   )

     self.assertListEqual(tokenizer.tokenize('''外国人参政権'''   ) , ['''外国''', '''人''', '''参政''', '''権''']   )
    @require_sudachi
    def 						UpperCamelCase_				(  self   )       ->					str:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      SudachiTokenizer(sudachi_dict_type='''core''' , sudachi_split_mode='''B'''   )

     self.assertListEqual(tokenizer.tokenize('''外国人参政権'''   ) , ['''外国人''', '''参政権''']   )
    @require_sudachi
    def 						UpperCamelCase_				(  self   )       ->					Dict:
     SCREAMING_SNAKE_CASE__:   Any=      SudachiTokenizer(sudachi_dict_type='''core''' , sudachi_split_mode='''C'''   )

     self.assertListEqual(tokenizer.tokenize('''外国人参政権'''   ) , ['''外国人参政権''']   )
    @require_sudachi
    def 						UpperCamelCase_				(  self   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   Dict=      SudachiTokenizer(do_lower_case=_a , sudachi_dict_type='''core'''   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' '''] , )
    @require_sudachi
    def 						UpperCamelCase_				(  self   )       ->					Optional[int]:
     SCREAMING_SNAKE_CASE__:   List[Any]=      SudachiTokenizer(normalize_text=_a , sudachi_dict_type='''core'''   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , [''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' '''] , )
    @require_sudachi
    def 						UpperCamelCase_				(  self   )       ->					int:
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      SudachiTokenizer(trim_whitespace=_a , sudachi_dict_type='''core'''   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''] , )
    @require_jumanpp
    def 						UpperCamelCase_				(  self   )       ->					Any:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      self.tokenizer_class(self.vocab_file , word_tokenizer_type='''jumanpp'''   )
     self.assertIsNotNone(_a   )

     SCREAMING_SNAKE_CASE__:   int=      '''こんにちは、世界。\nこんばんは、世界。'''
     SCREAMING_SNAKE_CASE__:   int=      tokenizer.tokenize(_a   )
     self.assertListEqual(_a , ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']   )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_a   ) , [3, 12, 10, 14, 4, 9, 12, 10, 14]   )

     SCREAMING_SNAKE_CASE__:   Tuple=      os.path.join(self.tmpdirname , '''tokenizer.bin'''   )
     with open(_a , '''wb'''   ) as handle:
      pickle.dump(_a , _a   )

     with open(_a , '''rb'''   ) as handle:
      SCREAMING_SNAKE_CASE__:   Tuple=      pickle.load(_a   )

     SCREAMING_SNAKE_CASE__:   Optional[int]=      tokenizer_new.tokenize(_a   )

     self.assertListEqual(_a , _a   )
    @require_jumanpp
    def 						UpperCamelCase_				(  self   )       ->					int:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''] , )
    @require_jumanpp
    def 						UpperCamelCase_				(  self   )       ->					Any:
     SCREAMING_SNAKE_CASE__:   Dict=      JumanppTokenizer(do_lower_case=_a   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''] , )
    @require_jumanpp
    def 						UpperCamelCase_				(  self   )       ->					str:
     SCREAMING_SNAKE_CASE__:   Optional[int]=      JumanppTokenizer(normalize_text=_a   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''] , )
    @require_jumanpp
    def 						UpperCamelCase_				(  self   )       ->					str:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      JumanppTokenizer(trim_whitespace=_a   )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''   ) , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。'''] , )
    @require_jumanpp
    def 						UpperCamelCase_				(  self   )       ->					Any:
     SCREAMING_SNAKE_CASE__:   str=      JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''   ) , ['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。'''] , )
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     SCREAMING_SNAKE_CASE__:   List[str]=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

     SCREAMING_SNAKE_CASE__:   List[Any]=      {}
     for i, token in enumerate(_a   ):
      SCREAMING_SNAKE_CASE__:   Tuple=      i
     SCREAMING_SNAKE_CASE__:   Dict=      WordpieceTokenizer(vocab=_a , unk_token='''[UNK]'''   )

     self.assertListEqual(tokenizer.tokenize(''''''   ) , []   )

     self.assertListEqual(tokenizer.tokenize('''こんにちは'''   ) , ['''こんにちは''']   )

     self.assertListEqual(tokenizer.tokenize('''こんばんは'''   ) , ['''こん''', '''##ばんは''']   )

     self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''   ) , ['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']   )
    def 						UpperCamelCase_				(  self   )       ->					Any:
     SCREAMING_SNAKE_CASE__:   Tuple=      BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''   )
     SCREAMING_SNAKE_CASE__:   str=      tokenizer.subword_tokenizer

     SCREAMING_SNAKE_CASE__:   Dict=      subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''   )
     self.assertListEqual(_a , ['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']   )

     SCREAMING_SNAKE_CASE__:   int=      subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''   )
     self.assertListEqual(_a , ['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']   )



    def 						UpperCamelCase_				(  self   )       ->					Optional[int]:
     SCREAMING_SNAKE_CASE__:   Dict=      self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''   )

     SCREAMING_SNAKE_CASE__:   List[Any]=      tokenizer.encode('''ありがとう。''' , add_special_tokens=_a   )
     SCREAMING_SNAKE_CASE__:   List[str]=      tokenizer.encode('''どういたしまして。''' , add_special_tokens=_a   )

     SCREAMING_SNAKE_CASE__:   Any=      tokenizer.build_inputs_with_special_tokens(_a   )
     SCREAMING_SNAKE_CASE__:   Any=      tokenizer.build_inputs_with_special_tokens(_a , _a   )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 		_lowerCamelCase		(   __SCREAMING_SNAKE_CASE    ,     unittest.TestCase    ):
    __a  =      BertJapaneseTokenizer
    __a  =      False
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     super().setUp()

     SCREAMING_SNAKE_CASE__:   Optional[int]=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

     SCREAMING_SNAKE_CASE__:   Any=      os.path.join(self.tmpdirname , VOCAB_FILES_NAMES['''vocab_file''']   )
     with open(self.vocab_file , '''w''' , encoding='''utf-8'''   ) as vocab_writer:
      vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )
    def 						UpperCamelCase_				(  self , **lowerCAmelCase   )       ->					Any:
     return BertJapaneseTokenizer.from_pretrained(self.tmpdirname , subword_tokenizer_type='''character''' , **_a   )
    def 						UpperCamelCase_				(  self , lowerCAmelCase   )       ->					List[str]:
     SCREAMING_SNAKE_CASE__:   List[Any]=      '''こんにちは、世界。 \nこんばんは、世界。'''
     SCREAMING_SNAKE_CASE__:   Tuple=      '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
     return input_text, output_text
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     pass  # TODO add if relevant
    def 						UpperCamelCase_				(  self   )       ->					Union[str, Any]:
     pass  # TODO add if relevant
    def 						UpperCamelCase_				(  self   )       ->					str:
     pass  # TODO add if relevant
    def 						UpperCamelCase_				(  self   )       ->					int:
     SCREAMING_SNAKE_CASE__:   int=      self.tokenizer_class(self.vocab_file , subword_tokenizer_type='''character'''   )

     SCREAMING_SNAKE_CASE__:   Optional[int]=      tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''   )
     self.assertListEqual(
         _a , ['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']   )
     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(_a   ) , [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]   )
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     SCREAMING_SNAKE_CASE__:   Optional[int]=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

     SCREAMING_SNAKE_CASE__:   Optional[int]=      {}
     for i, token in enumerate(_a   ):
      SCREAMING_SNAKE_CASE__:   Dict=      i
     SCREAMING_SNAKE_CASE__:   str=      CharacterTokenizer(vocab=_a , unk_token='''[UNK]'''   )

     self.assertListEqual(tokenizer.tokenize(''''''   ) , []   )

     self.assertListEqual(tokenizer.tokenize('''こんにちは'''   ) , ['''こ''', '''ん''', '''に''', '''ち''', '''は''']   )

     self.assertListEqual(tokenizer.tokenize('''こんにちほ'''   ) , ['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']   )



    def 						UpperCamelCase_				(  self   )       ->					Optional[int]:
     SCREAMING_SNAKE_CASE__:   List[str]=      self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''   )

     SCREAMING_SNAKE_CASE__:   str=      tokenizer.encode('''ありがとう。''' , add_special_tokens=_a   )
     SCREAMING_SNAKE_CASE__:   Any=      tokenizer.encode('''どういたしまして。''' , add_special_tokens=_a   )

     SCREAMING_SNAKE_CASE__:   List[str]=      tokenizer.build_inputs_with_special_tokens(_a   )
     SCREAMING_SNAKE_CASE__:   Tuple=      tokenizer.build_inputs_with_special_tokens(_a , _a   )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 		_lowerCamelCase		(   unittest.TestCase    ):
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      '''cl-tohoku/bert-base-japanese'''
     SCREAMING_SNAKE_CASE__:   List[Any]=      AutoTokenizer.from_pretrained(_a   )
     self.assertIsInstance(_a , _a   )






class 		_lowerCamelCase		(   unittest.TestCase    ):
    def 						UpperCamelCase_				(  self   )       ->					int:
     SCREAMING_SNAKE_CASE__:   Dict=      '''cl-tohoku/bert-base-japanese'''
     with self.assertLogs('''transformers''' , level='''WARNING'''   ) as cm:
      BertTokenizer.from_pretrained(_a   )
      self.assertTrue(
          cm.records[0].message.startswith(
              '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
              ''' is called from.'''   )   )
     SCREAMING_SNAKE_CASE__:   Any=      '''bert-base-cased'''
     with self.assertLogs('''transformers''' , level='''WARNING'''   ) as cm:
      BertJapaneseTokenizer.from_pretrained(_a   )
      self.assertTrue(
          cm.records[0].message.startswith(
              '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
              ''' is called from.'''   )   )


import copy
import inspect
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import VideoMAEConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch
      from torch import nn

      from transformers import (
          MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
          VideoMAEForPreTraining,
          VideoMAEForVideoClassification,
          VideoMAEModel,
      )
      from transformers.models.videomae.modeling_videomae import VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
      from transformers import VideoMAEImageProcessor






class 		_lowerCamelCase		:
    def __init__(  self , lowerCAmelCase , lowerCAmelCase=13 , lowerCAmelCase=10 , lowerCAmelCase=3 , lowerCAmelCase=2 , lowerCAmelCase=2 , lowerCAmelCase=2 , lowerCAmelCase=True , lowerCAmelCase=True , lowerCAmelCase=32 , lowerCAmelCase=5 , lowerCAmelCase=4 , lowerCAmelCase=37 , lowerCAmelCase="gelu" , lowerCAmelCase=0.1 , lowerCAmelCase=0.1 , lowerCAmelCase=10 , lowerCAmelCase=0.02 , lowerCAmelCase=0.9 , lowerCAmelCase=None , )       ->					Optional[int]:
     SCREAMING_SNAKE_CASE__:   Tuple=      parent
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      batch_size
     SCREAMING_SNAKE_CASE__:   List[str]=      image_size
     SCREAMING_SNAKE_CASE__:   List[Any]=      num_channels
     SCREAMING_SNAKE_CASE__:   str=      patch_size
     SCREAMING_SNAKE_CASE__:   Optional[int]=      tubelet_size
     SCREAMING_SNAKE_CASE__:   Dict=      num_frames
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      is_training
     SCREAMING_SNAKE_CASE__:   Dict=      use_labels
     SCREAMING_SNAKE_CASE__:   Tuple=      hidden_size
     SCREAMING_SNAKE_CASE__:   Any=      num_hidden_layers
     SCREAMING_SNAKE_CASE__:   List[Any]=      num_attention_heads
     SCREAMING_SNAKE_CASE__:   Dict=      intermediate_size
     SCREAMING_SNAKE_CASE__:   str=      hidden_act
     SCREAMING_SNAKE_CASE__:   str=      hidden_dropout_prob
     SCREAMING_SNAKE_CASE__:   str=      attention_probs_dropout_prob
     SCREAMING_SNAKE_CASE__:   Tuple=      type_sequence_label_size
     SCREAMING_SNAKE_CASE__:   List[str]=      initializer_range
     SCREAMING_SNAKE_CASE__:   str=      mask_ratio
     SCREAMING_SNAKE_CASE__:   Any=      scope

     # in VideoMAE, the number of tokens equals num_frames/tubelet_size * num_patches per frame
     SCREAMING_SNAKE_CASE__:   Tuple=      (image_size // patch_size) ** 2
     SCREAMING_SNAKE_CASE__:   Dict=      (num_frames // tubelet_size) * self.num_patches_per_frame

     # use this variable to define bool_masked_pos
     SCREAMING_SNAKE_CASE__:   str=      int(mask_ratio * self.seq_length   )
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      floats_tensor(
         [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size]   )

     SCREAMING_SNAKE_CASE__:   List[str]=      None
     if self.use_labels:
      SCREAMING_SNAKE_CASE__:   List[Any]=      ids_tensor([self.batch_size] , self.type_sequence_label_size   )

     SCREAMING_SNAKE_CASE__:   Dict=      self.get_config()

     return config, pixel_values, labels
    def 						UpperCamelCase_				(  self   )       ->					List[str]:
     return VideoMAEConfig(
         image_size=self.image_size , patch_size=self.patch_size , num_channels=self.num_channels , num_frames=self.num_frames , tubelet_size=self.tubelet_size , hidden_size=self.hidden_size , num_hidden_layers=self.num_hidden_layers , num_attention_heads=self.num_attention_heads , intermediate_size=self.intermediate_size , hidden_act=self.hidden_act , hidden_dropout_prob=self.hidden_dropout_prob , attention_probs_dropout_prob=self.attention_probs_dropout_prob , is_decoder=lowerCAmelCase , initializer_range=self.initializer_range , )
    def 						UpperCamelCase_				(  self , lowerCAmelCase , lowerCAmelCase , lowerCAmelCase   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   Any=      VideoMAEModel(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      model(lowerCAmelCase   )
     self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, self.seq_length, self.hidden_size)   )
    def 						UpperCamelCase_				(  self , lowerCAmelCase , lowerCAmelCase , lowerCAmelCase   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   Any=      VideoMAEForPreTraining(lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     # important: each video needs to have the same number of masked patches
     # hence we define a single mask, which we then repeat for each example in the batch
     SCREAMING_SNAKE_CASE__:   str=      torch.ones((self.num_masks,)   )
     SCREAMING_SNAKE_CASE__:   str=      torch.cat([mask, torch.zeros(self.seq_length - mask.size(0   )   )]   )
     SCREAMING_SNAKE_CASE__:   List[str]=      mask.expand(self.batch_size , -1   ).bool()

     SCREAMING_SNAKE_CASE__:   List[str]=      model(lowerCAmelCase , lowerCAmelCase   )
     # model only returns predictions for masked patches
     SCREAMING_SNAKE_CASE__:   int=      mask.sum().item()
     SCREAMING_SNAKE_CASE__:   Dict=      3 * self.tubelet_size * self.patch_size**2
     self.parent.assertEqual(result.logits.shape , (self.batch_size, num_masked_patches, decoder_num_labels)   )



    def 						UpperCamelCase_				(  self   )       ->					Optional[int]:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      self.prepare_config_and_inputs()
     SCREAMING_SNAKE_CASE__,	SCREAMING_SNAKE_CASE__,	SCREAMING_SNAKE_CASE__:   Union[str, Any]=      config_and_inputs
     SCREAMING_SNAKE_CASE__:   List[Any]=      {'''pixel_values''': pixel_values}
     return config, inputs_dict






@require_torch
class 		_lowerCamelCase		(   UpperCamelCase_    ,     UpperCamelCase_    ,     unittest.TestCase    ):
    __a  =      (
        (VideoMAEModel, VideoMAEForPreTraining, VideoMAEForVideoClassification) if is_torch_available() else ()
    )
    __a  =      (
        {"feature-extraction": VideoMAEModel, "video-classification": VideoMAEForVideoClassification}
        if is_torch_available()
        else {}
    )

    __a  =      False
    __a  =      False
    __a  =      False
    __a  =      False
    def 						UpperCamelCase_				(  self   )       ->					Optional[int]:
     SCREAMING_SNAKE_CASE__:   Optional[int]=      VideoMAEModelTester(self   )
     SCREAMING_SNAKE_CASE__:   Tuple=      ConfigTester(self , config_class=lowerCAmelCase , has_text_modality=lowerCAmelCase , hidden_size=37   )
    def 						UpperCamelCase_				(  self , lowerCAmelCase , lowerCAmelCase , lowerCAmelCase=False   )       ->					List[str]:
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      copy.deepcopy(lowerCAmelCase   )

     if model_class == VideoMAEForPreTraining:
      # important: each video needs to have the same number of masked patches
      # hence we define a single mask, which we then repeat for each example in the batch
      SCREAMING_SNAKE_CASE__:   str=      torch.ones((self.model_tester.num_masks,)   )
      SCREAMING_SNAKE_CASE__:   int=      torch.cat([mask, torch.zeros(self.model_tester.seq_length - mask.size(0   )   )]   )
      SCREAMING_SNAKE_CASE__:   int=      mask.expand(self.model_tester.batch_size , -1   ).bool()
      SCREAMING_SNAKE_CASE__:   Union[str, Any]=      bool_masked_pos.to(lowerCAmelCase   )

     if return_labels:
      if model_class in [
          *get_values(lowerCAmelCase   ),
      ]:
       SCREAMING_SNAKE_CASE__:   Dict=      torch.zeros(
           self.model_tester.batch_size , dtype=torch.long , device=lowerCAmelCase   )

     return inputs_dict
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     self.config_tester.run_common_tests()
    @unittest.skip(reason='''VideoMAE does not use inputs_embeds'''   )
    def 						UpperCamelCase_				(  self   )       ->					List[str]:
     pass
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     SCREAMING_SNAKE_CASE__,	SCREAMING_SNAKE_CASE__:   Tuple=      self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
      SCREAMING_SNAKE_CASE__:   Optional[Any]=      model_class(lowerCAmelCase   )
      self.assertIsInstance(model.get_input_embeddings() , (nn.Module)   )
      SCREAMING_SNAKE_CASE__:   Dict=      model.get_output_embeddings()
      self.assertTrue(x is None or isinstance(lowerCAmelCase , nn.Linear   )   )
    def 						UpperCamelCase_				(  self   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__,	SCREAMING_SNAKE_CASE__:   Optional[Any]=      self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
      SCREAMING_SNAKE_CASE__:   List[Any]=      model_class(lowerCAmelCase   )
      SCREAMING_SNAKE_CASE__:   List[str]=      inspect.signature(model.forward   )
      # signature.parameters is an OrderedDict => so arg_names order is deterministic
      SCREAMING_SNAKE_CASE__:   Union[str, Any]=      [*signature.parameters.keys()]

      SCREAMING_SNAKE_CASE__:   Tuple=      ['''pixel_values''']
      self.assertListEqual(arg_names[:1] , lowerCAmelCase   )
    def 						UpperCamelCase_				(  self   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   Any=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*lowerCAmelCase   )
    def 						UpperCamelCase_				(  self   )       ->					str:
     SCREAMING_SNAKE_CASE__:   List[Any]=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase   )
    @slow
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     for model_name in VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
      SCREAMING_SNAKE_CASE__:   int=      VideoMAEModel.from_pretrained(lowerCAmelCase   )
      self.assertIsNotNone(lowerCAmelCase   )
    def 						UpperCamelCase_				(  self   )       ->					int:
     if not self.has_attentions:
      pass

     else:
      SCREAMING_SNAKE_CASE__,	SCREAMING_SNAKE_CASE__:   List[Any]=      self.model_tester.prepare_config_and_inputs_for_common()
      SCREAMING_SNAKE_CASE__:   int=      True

      for model_class in self.all_model_classes:
       SCREAMING_SNAKE_CASE__:   Dict=      self.model_tester.seq_length - self.model_tester.num_masks
       SCREAMING_SNAKE_CASE__:   Union[str, Any]=      (
           num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length
       )

       SCREAMING_SNAKE_CASE__:   Union[str, Any]=      True
       SCREAMING_SNAKE_CASE__:   Optional[Any]=      False
       SCREAMING_SNAKE_CASE__:   Optional[int]=      True
       SCREAMING_SNAKE_CASE__:   Union[str, Any]=      model_class(lowerCAmelCase   )
       model.to(lowerCAmelCase   )
       model.eval()
       with torch.no_grad():
        SCREAMING_SNAKE_CASE__:   int=      model(**self._prepare_for_class(lowerCAmelCase , lowerCAmelCase   )   )
       SCREAMING_SNAKE_CASE__:   Any=      outputs.attentions
       self.assertEqual(len(lowerCAmelCase   ) , self.model_tester.num_hidden_layers   )

       # check that output_attentions also work using config
       del inputs_dict["output_attentions"]
       SCREAMING_SNAKE_CASE__:   Optional[Any]=      True
       SCREAMING_SNAKE_CASE__:   List[Any]=      model_class(lowerCAmelCase   )
       model.to(lowerCAmelCase   )
       model.eval()
       with torch.no_grad():
        SCREAMING_SNAKE_CASE__:   Dict=      model(**self._prepare_for_class(lowerCAmelCase , lowerCAmelCase   )   )
       SCREAMING_SNAKE_CASE__:   Optional[Any]=      outputs.attentions
       self.assertEqual(len(lowerCAmelCase   ) , self.model_tester.num_hidden_layers   )

       self.assertListEqual(
           list(attentions[0].shape[-3:]   ) , [self.model_tester.num_attention_heads, seq_len, seq_len] , )
       SCREAMING_SNAKE_CASE__:   Union[str, Any]=      len(lowerCAmelCase   )

       # Check attention is always last and order is fine
       SCREAMING_SNAKE_CASE__:   Optional[int]=      True
       SCREAMING_SNAKE_CASE__:   Optional[Any]=      True
       SCREAMING_SNAKE_CASE__:   List[Any]=      model_class(lowerCAmelCase   )
       model.to(lowerCAmelCase   )
       model.eval()
       with torch.no_grad():
        SCREAMING_SNAKE_CASE__:   Optional[Any]=      model(**self._prepare_for_class(lowerCAmelCase , lowerCAmelCase   )   )

       self.assertEqual(out_len + 1 , len(lowerCAmelCase   )   )

       SCREAMING_SNAKE_CASE__:   Optional[int]=      outputs.attentions

       self.assertEqual(len(lowerCAmelCase   ) , self.model_tester.num_hidden_layers   )
       self.assertListEqual(
           list(self_attentions[0].shape[-3:]   ) , [self.model_tester.num_attention_heads, seq_len, seq_len] , )
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     def check_hidden_states_output(lowerCAmelCase , lowerCAmelCase , lowerCAmelCase   ):
      SCREAMING_SNAKE_CASE__:   int=      model_class(lowerCAmelCase   )
      model.to(lowerCAmelCase   )
      model.eval()

      with torch.no_grad():
       SCREAMING_SNAKE_CASE__:   Dict=      model(**self._prepare_for_class(lowerCAmelCase , lowerCAmelCase   )   )

      SCREAMING_SNAKE_CASE__:   Any=      outputs.hidden_states
      SCREAMING_SNAKE_CASE__:   int=      self.model_tester.num_hidden_layers + 1
      self.assertEqual(len(lowerCAmelCase   ) , lowerCAmelCase   )

      SCREAMING_SNAKE_CASE__:   str=      self.model_tester.seq_length - self.model_tester.num_masks
      SCREAMING_SNAKE_CASE__:   Optional[int]=      num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length

      self.assertListEqual(
          list(hidden_states[0].shape[-2:]   ) , [seq_length, self.model_tester.hidden_size] , )

     SCREAMING_SNAKE_CASE__,	SCREAMING_SNAKE_CASE__:   int=      self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
      SCREAMING_SNAKE_CASE__:   List[str]=      True
      check_hidden_states_output(lowerCAmelCase , lowerCAmelCase , lowerCAmelCase   )

      # check that output_hidden_states also work using config
      del inputs_dict["output_hidden_states"]
      SCREAMING_SNAKE_CASE__:   Optional[Any]=      True

      check_hidden_states_output(lowerCAmelCase , lowerCAmelCase , lowerCAmelCase   )



    @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''   )
    def 						UpperCamelCase_				(  self   )       ->					str:
     pass







def 			A__      (	):
 SCREAMING_SNAKE_CASE__:   Optional[int]=      hf_hub_download(
     repo_id='''hf-internal-testing/spaghetti-video'''       ,				filename='''eating_spaghetti.npy'''       ,				repo_type='''dataset'''      )
 SCREAMING_SNAKE_CASE__:   List[str]=      np.load(snake_case_      )
 return list(snake_case_      )






@require_torch
@require_vision
class 		_lowerCamelCase		(   unittest.TestCase    ):
    @cached_property
    def 						UpperCamelCase_				(  self   )       ->					List[str]:
     # logits were tested with a different mean and std, so we use the same here
     return (
         VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5] , image_std=[0.5, 0.5, 0.5]   )
         if is_vision_available()
         else None
     )
    @slow
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     SCREAMING_SNAKE_CASE__:   Dict=      VideoMAEForVideoClassification.from_pretrained('''MCG-NJU/videomae-base-finetuned-kinetics'''   ).to(
         lowerCAmelCase   )

     SCREAMING_SNAKE_CASE__:   Dict=      self.default_image_processor
     SCREAMING_SNAKE_CASE__:   Any=      prepare_video()
     SCREAMING_SNAKE_CASE__:   List[Any]=      image_processor(lowerCAmelCase , return_tensors='''pt'''   ).to(lowerCAmelCase   )

     # forward pass
     with torch.no_grad():
      SCREAMING_SNAKE_CASE__:   Union[str, Any]=      model(**lowerCAmelCase   )

     # verify the logits
     SCREAMING_SNAKE_CASE__:   Any=      torch.Size((1, 400)   )
     self.assertEqual(outputs.logits.shape , lowerCAmelCase   )

     SCREAMING_SNAKE_CASE__:   Optional[Any]=      torch.tensor([0.3669, -0.0688, -0.2421]   ).to(lowerCAmelCase   )

     self.assertTrue(torch.allclose(outputs.logits[0, :3] , lowerCAmelCase , atol=1e-4   )   )



    @slow
    def 						UpperCamelCase_				(  self   )       ->					List[str]:
     SCREAMING_SNAKE_CASE__:   int=      VideoMAEForPreTraining.from_pretrained('''MCG-NJU/videomae-base-short'''   ).to(lowerCAmelCase   )

     SCREAMING_SNAKE_CASE__:   List[Any]=      self.default_image_processor
     SCREAMING_SNAKE_CASE__:   Dict=      prepare_video()
     SCREAMING_SNAKE_CASE__:   int=      image_processor(lowerCAmelCase , return_tensors='''pt'''   ).to(lowerCAmelCase   )

     # add boolean mask, indicating which patches to mask
     SCREAMING_SNAKE_CASE__:   List[Any]=      hf_hub_download(repo_id='''hf-internal-testing/bool-masked-pos''' , filename='''bool_masked_pos.pt'''   )
     SCREAMING_SNAKE_CASE__:   List[Any]=      torch.load(lowerCAmelCase   )

     # forward pass
     with torch.no_grad():
      SCREAMING_SNAKE_CASE__:   str=      model(**lowerCAmelCase   )

     # verify the logits
     SCREAMING_SNAKE_CASE__:   Tuple=      torch.Size([1, 1408, 1536]   )
     SCREAMING_SNAKE_CASE__:   Optional[int]=      torch.tensor(
         [[0.7994, 0.9612, 0.8508], [0.7401, 0.8958, 0.8302], [0.5862, 0.7468, 0.7325]] , device=lowerCAmelCase   )
     self.assertEqual(outputs.logits.shape , lowerCAmelCase   )
     self.assertTrue(torch.allclose(outputs.logits[0, :3, :3] , lowerCAmelCase , atol=1e-4   )   )

     # verify the loss (`config.norm_pix_loss` = `True`)
     SCREAMING_SNAKE_CASE__:   str=      torch.tensor([0.5142] , device=lowerCAmelCase   )
     self.assertTrue(torch.allclose(outputs.loss , lowerCAmelCase , atol=1e-4   )   )

     # verify the loss (`config.norm_pix_loss` = `False`)
     SCREAMING_SNAKE_CASE__:   Dict=      VideoMAEForPreTraining.from_pretrained('''MCG-NJU/videomae-base-short''' , norm_pix_loss=lowerCAmelCase   ).to(
         lowerCAmelCase   )

     with torch.no_grad():
      SCREAMING_SNAKE_CASE__:   List[Any]=      model(**lowerCAmelCase   )

     SCREAMING_SNAKE_CASE__:   Optional[Any]=      torch.tensor(torch.tensor([0.6469]   ) , device=lowerCAmelCase   )
     self.assertTrue(torch.allclose(outputs.loss , lowerCAmelCase , atol=1e-4   )   )

