'''simple docstring'''





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class  __magic_name__  (		__a      ,	unittest.TestCase    ):





   """simple docstring"""




   lowerCAmelCase      :    int  			=   BertJapaneseTokenizer
   lowerCAmelCase      :    Dict  			=   False
   lowerCAmelCase      :    int  			=   True



   def   lowerCAmelCase						(	self					:  Tuple							):



    """simple docstring"""



    super().setUp()

    _UpperCamelCase:     int							= [
        '''[UNK]''',
        '''[CLS]''',
        '''[SEP]''',
        '''こんにちは''',
        '''こん''',
        '''にちは''',
        '''ばんは''',
        '''##こん''',
        '''##にちは''',
        '''##ばんは''',
        '''世界''',
        '''##世界''',
        '''、''',
        '''##、''',
        '''。''',
        '''##。''',
    ]

    _UpperCamelCase:     List[Any]							= os.path.join(self.tmpdirname		,  VOCAB_FILES_NAMES['''vocab_file''']							)
    with open(self.vocab_file		,  '''w'''		,  encoding='''utf-8'''							) as vocab_writer:
     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)



   def   lowerCAmelCase						(	self					:  int		,  _lowercase					:  int							):



    """simple docstring"""



    _UpperCamelCase:     str							= '''こんにちは、世界。 \nこんばんは、世界。'''
    _UpperCamelCase:     Dict							= '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
    return input_text, output_text



   def   lowerCAmelCase						(	self					:  Union[str, Any]		,  _lowercase					:  Optional[int]							):



    """simple docstring"""



    _UpperCamelCase	,							_UpperCamelCase:     Union[str, Any]							= self.get_input_output_texts(_lowercase							)
    _UpperCamelCase:     Union[str, Any]							= tokenizer.encode(_lowercase		,  add_special_tokens=_lowercase							)
    _UpperCamelCase:     str							= tokenizer.decode(_lowercase		,  clean_up_tokenization_spaces=_lowercase							)
    return text, ids



   def   lowerCAmelCase						(	self					:  List[str]							):



    """simple docstring"""



    pass  # TODO add if relevant



   def   lowerCAmelCase						(	self					:  Tuple							):



    """simple docstring"""



    pass  # TODO add if relevant



   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    pass  # TODO add if relevant



   def   lowerCAmelCase						(	self					:  Union[str, Any]							):



    """simple docstring"""



    _UpperCamelCase:     Dict							= self.tokenizer_class(self.vocab_file							)

    _UpperCamelCase:     Optional[Any]							= tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''							)
    self.assertListEqual(_lowercase		,  ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase							)		,  [3, 12, 10, 14, 4, 9, 12, 10, 14]							)



   def   lowerCAmelCase						(	self					:  int							):



    """simple docstring"""



    _UpperCamelCase:     Optional[Any]							= self.tokenizer_class(self.vocab_file		,  word_tokenizer_type='''mecab'''							)
    self.assertIsNotNone(_lowercase							)

    _UpperCamelCase:     Optional[Any]							= '''こんにちは、世界。\nこんばんは、世界。'''
    _UpperCamelCase:     List[Any]							= tokenizer.tokenize(_lowercase							)
    self.assertListEqual(_lowercase		,  ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase							)		,  [3, 12, 10, 14, 4, 9, 12, 10, 14]							)

    _UpperCamelCase:     str							= os.path.join(self.tmpdirname		,  '''tokenizer.bin'''							)
    with open(_lowercase		,  '''wb'''							) as handle:
     pickle.dump(_lowercase		,  _lowercase							)

    with open(_lowercase		,  '''rb'''							) as handle:
     _UpperCamelCase:     Optional[int]							= pickle.load(_lowercase							)

    _UpperCamelCase:     Any							= tokenizer_new.tokenize(_lowercase							)

    self.assertListEqual(_lowercase		,  _lowercase							)



   def   lowerCAmelCase						(	self					:  List[str]							):



    """simple docstring"""



    _UpperCamelCase:     Dict							= MecabTokenizer(mecab_dic='''ipadic'''							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']		,  )



   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    try:
     _UpperCamelCase:     Any							= MecabTokenizer(mecab_dic='''unidic_lite'''							)
    except ModuleNotFoundError:
     return

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']		,  )



   def   lowerCAmelCase						(	self					:  Union[str, Any]							):



    """simple docstring"""



    try:
     _UpperCamelCase:     List[Any]							= MecabTokenizer(mecab_dic='''unidic'''							)
    except ModuleNotFoundError:
     return

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']		,  )



   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    _UpperCamelCase:     List[Any]							= MecabTokenizer(do_lower_case=_lowercase		,  mecab_dic='''ipadic'''							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']		,  )



   def   lowerCAmelCase						(	self					:  Optional[int]							):



    """simple docstring"""



    try:
     _UpperCamelCase:     Optional[Any]							= MecabTokenizer(
         do_lower_case=_lowercase		,  normalize_text=_lowercase		,  mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''							)
    except RuntimeError:
     # if dict doesn't exist in the system, previous code raises this error.
     return

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']		,  )



   def   lowerCAmelCase						(	self					:  Optional[int]							):



    """simple docstring"""



    _UpperCamelCase:     Tuple							= MecabTokenizer(normalize_text=_lowercase		,  mecab_dic='''ipadic'''							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']		,  )



   @require_sudachi
   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    _UpperCamelCase:     int							= self.tokenizer_class(self.vocab_file		,  word_tokenizer_type='''sudachi'''							)
    self.assertIsNotNone(_lowercase							)

    _UpperCamelCase:     int							= '''こんにちは、世界。\nこんばんは、世界。'''
    _UpperCamelCase:     Optional[Any]							= tokenizer.tokenize(_lowercase							)
    self.assertListEqual(_lowercase		,  ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase							)		,  [3, 12, 10, 14, 4, 9, 12, 10, 14]							)

    _UpperCamelCase:     Optional[Any]							= os.path.join(self.tmpdirname		,  '''tokenizer.bin'''							)
    with open(_lowercase		,  '''wb'''							) as handle:
     pickle.dump(_lowercase		,  _lowercase							)

    with open(_lowercase		,  '''rb'''							) as handle:
     _UpperCamelCase:     Tuple							= pickle.load(_lowercase							)

    _UpperCamelCase:     Dict							= tokenizer_new.tokenize(_lowercase							)

    self.assertListEqual(_lowercase		,  _lowercase							)



   @require_sudachi
   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    _UpperCamelCase:     str							= SudachiTokenizer(sudachi_dict_type='''core'''							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']		,  )



   @require_sudachi
   def   lowerCAmelCase						(	self					:  int							):



    """simple docstring"""



    _UpperCamelCase:     Dict							= SudachiTokenizer(sudachi_dict_type='''core'''		,  sudachi_split_mode='''A'''							)

    self.assertListEqual(tokenizer.tokenize('''外国人参政権'''							)		,  ['''外国''', '''人''', '''参政''', '''権''']							)



   @require_sudachi
   def   lowerCAmelCase						(	self					:  Optional[int]							):



    """simple docstring"""



    _UpperCamelCase:     Dict							= SudachiTokenizer(sudachi_dict_type='''core'''		,  sudachi_split_mode='''B'''							)

    self.assertListEqual(tokenizer.tokenize('''外国人参政権'''							)		,  ['''外国人''', '''参政権''']							)



   @require_sudachi
   def   lowerCAmelCase						(	self					:  Dict							):



    """simple docstring"""



    _UpperCamelCase:     int							= SudachiTokenizer(sudachi_dict_type='''core'''		,  sudachi_split_mode='''C'''							)

    self.assertListEqual(tokenizer.tokenize('''外国人参政権'''							)		,  ['''外国人参政権''']							)



   @require_sudachi
   def   lowerCAmelCase						(	self					:  List[str]							):



    """simple docstring"""



    _UpperCamelCase:     Optional[Any]							= SudachiTokenizer(do_lower_case=_lowercase		,  sudachi_dict_type='''core'''							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']		,  )



   @require_sudachi
   def   lowerCAmelCase						(	self					:  List[Any]							):



    """simple docstring"""



    _UpperCamelCase:     int							= SudachiTokenizer(normalize_text=_lowercase		,  sudachi_dict_type='''core'''							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  [''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']		,  )



   @require_sudachi
   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    _UpperCamelCase:     Union[str, Any]							= SudachiTokenizer(trim_whitespace=_lowercase		,  sudachi_dict_type='''core'''							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']		,  )



   @require_jumanpp
   def   lowerCAmelCase						(	self					:  List[Any]							):



    """simple docstring"""



    _UpperCamelCase:     Union[str, Any]							= self.tokenizer_class(self.vocab_file		,  word_tokenizer_type='''jumanpp'''							)
    self.assertIsNotNone(_lowercase							)

    _UpperCamelCase:     List[str]							= '''こんにちは、世界。\nこんばんは、世界。'''
    _UpperCamelCase:     List[Any]							= tokenizer.tokenize(_lowercase							)
    self.assertListEqual(_lowercase		,  ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase							)		,  [3, 12, 10, 14, 4, 9, 12, 10, 14]							)

    _UpperCamelCase:     Dict							= os.path.join(self.tmpdirname		,  '''tokenizer.bin'''							)
    with open(_lowercase		,  '''wb'''							) as handle:
     pickle.dump(_lowercase		,  _lowercase							)

    with open(_lowercase		,  '''rb'''							) as handle:
     _UpperCamelCase:     Union[str, Any]							= pickle.load(_lowercase							)

    _UpperCamelCase:     Tuple							= tokenizer_new.tokenize(_lowercase							)

    self.assertListEqual(_lowercase		,  _lowercase							)



   @require_jumanpp
   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    _UpperCamelCase:     Optional[Any]							= JumanppTokenizer()

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']		,  )



   @require_jumanpp
   def   lowerCAmelCase						(	self					:  List[str]							):



    """simple docstring"""



    _UpperCamelCase:     Dict							= JumanppTokenizer(do_lower_case=_lowercase							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']		,  )



   @require_jumanpp
   def   lowerCAmelCase						(	self					:  Tuple							):



    """simple docstring"""



    _UpperCamelCase:     str							= JumanppTokenizer(normalize_text=_lowercase							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']		,  )



   @require_jumanpp
   def   lowerCAmelCase						(	self					:  Union[str, Any]							):



    """simple docstring"""



    _UpperCamelCase:     Tuple							= JumanppTokenizer(trim_whitespace=_lowercase							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)		,  ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']		,  )



   @require_jumanpp
   def   lowerCAmelCase						(	self					:  Optional[int]							):



    """simple docstring"""



    _UpperCamelCase:     List[Any]							= JumanppTokenizer()

    self.assertListEqual(
        tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''							)		,  ['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']		,  )



   def   lowerCAmelCase						(	self					:  int							):



    """simple docstring"""



    _UpperCamelCase:     Optional[int]							= ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

    _UpperCamelCase:     List[str]							= {}
    for i, token in enumerate(_lowercase							):
     _UpperCamelCase:     Optional[Any]							= i
    _UpperCamelCase:     Dict							= WordpieceTokenizer(vocab=_lowercase		,  unk_token='''[UNK]'''							)

    self.assertListEqual(tokenizer.tokenize(''''''							)		,  []							)

    self.assertListEqual(tokenizer.tokenize('''こんにちは'''							)		,  ['''こんにちは''']							)

    self.assertListEqual(tokenizer.tokenize('''こんばんは'''							)		,  ['''こん''', '''##ばんは''']							)

    self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''							)		,  ['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']							)



   def   lowerCAmelCase						(	self					:  Dict							):



    """simple docstring"""



    _UpperCamelCase:     int							= BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''							)
    _UpperCamelCase:     str							= tokenizer.subword_tokenizer

    _UpperCamelCase:     int							= subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''							)
    self.assertListEqual(_lowercase		,  ['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']							)

    _UpperCamelCase:     Union[str, Any]							= subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''							)
    self.assertListEqual(_lowercase		,  ['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']							)



   def   lowerCAmelCase						(	self					:  Dict							):



    """simple docstring"""



    _UpperCamelCase:     Union[str, Any]							= self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''							)

    _UpperCamelCase:     Tuple							= tokenizer.encode('''ありがとう。'''		,  add_special_tokens=_lowercase							)
    _UpperCamelCase:     str							= tokenizer.encode('''どういたしまして。'''		,  add_special_tokens=_lowercase							)

    _UpperCamelCase:     Dict							= tokenizer.build_inputs_with_special_tokens(_lowercase							)
    _UpperCamelCase:     Union[str, Any]							= tokenizer.build_inputs_with_special_tokens(_lowercase		,  _lowercase							)

    # 2 is for "[CLS]", 3 is for "[SEP]"
    assert encoded_sentence == [2] + text + [3]
    assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class  __magic_name__  (		__a      ,	unittest.TestCase    ):





   """simple docstring"""




   lowerCAmelCase      :    List[str]  			=   BertJapaneseTokenizer
   lowerCAmelCase      :    List[str]  			=   False



   def   lowerCAmelCase						(	self					:  int							):



    """simple docstring"""



    super().setUp()

    _UpperCamelCase:     List[Any]							= ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

    _UpperCamelCase:     Optional[Any]							= os.path.join(self.tmpdirname		,  VOCAB_FILES_NAMES['''vocab_file''']							)
    with open(self.vocab_file		,  '''w'''		,  encoding='''utf-8'''							) as vocab_writer:
     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)



   def   lowerCAmelCase						(	self					:  List[Any]		,  **_lowercase					:  Tuple							):



    """simple docstring"""



    return BertJapaneseTokenizer.from_pretrained(self.tmpdirname		,  subword_tokenizer_type='''character'''		,  **_lowercase							)



   def   lowerCAmelCase						(	self					:  Optional[Any]		,  _lowercase					:  int							):



    """simple docstring"""



    _UpperCamelCase:     Optional[Any]							= '''こんにちは、世界。 \nこんばんは、世界。'''
    _UpperCamelCase:     Tuple							= '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
    return input_text, output_text



   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    pass  # TODO add if relevant



   def   lowerCAmelCase						(	self					:  List[Any]							):



    """simple docstring"""



    pass  # TODO add if relevant



   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    pass  # TODO add if relevant



   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    _UpperCamelCase:     Any							= self.tokenizer_class(self.vocab_file		,  subword_tokenizer_type='''character'''							)

    _UpperCamelCase:     Dict							= tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''							)
    self.assertListEqual(
        _lowercase		,  ['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']							)
    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(_lowercase							)		,  [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]							)



   def   lowerCAmelCase						(	self					:  List[str]							):



    """simple docstring"""



    _UpperCamelCase:     int							= ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

    _UpperCamelCase:     int							= {}
    for i, token in enumerate(_lowercase							):
     _UpperCamelCase:     int							= i
    _UpperCamelCase:     Union[str, Any]							= CharacterTokenizer(vocab=_lowercase		,  unk_token='''[UNK]'''							)

    self.assertListEqual(tokenizer.tokenize(''''''							)		,  []							)

    self.assertListEqual(tokenizer.tokenize('''こんにちは'''							)		,  ['''こ''', '''ん''', '''に''', '''ち''', '''は''']							)

    self.assertListEqual(tokenizer.tokenize('''こんにちほ'''							)		,  ['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']							)



   def   lowerCAmelCase						(	self					:  str							):



    """simple docstring"""



    _UpperCamelCase:     Tuple							= self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''							)

    _UpperCamelCase:     int							= tokenizer.encode('''ありがとう。'''		,  add_special_tokens=_lowercase							)
    _UpperCamelCase:     Optional[int]							= tokenizer.encode('''どういたしまして。'''		,  add_special_tokens=_lowercase							)

    _UpperCamelCase:     List[Any]							= tokenizer.build_inputs_with_special_tokens(_lowercase							)
    _UpperCamelCase:     Union[str, Any]							= tokenizer.build_inputs_with_special_tokens(_lowercase		,  _lowercase							)

    # 2 is for "[CLS]", 3 is for "[SEP]"
    assert encoded_sentence == [2] + text + [3]
    assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class  __magic_name__  (		unittest.TestCase    ):





   """simple docstring"""




   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    _UpperCamelCase:     Any							= '''cl-tohoku/bert-base-japanese'''
    _UpperCamelCase:     Union[str, Any]							= AutoTokenizer.from_pretrained(_lowercase							)
    self.assertIsInstance(_lowercase		,  _lowercase							)




class  __magic_name__  (		unittest.TestCase    ):





   """simple docstring"""




   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    _UpperCamelCase:     Optional[int]							= '''cl-tohoku/bert-base-japanese'''
    with self.assertLogs('''transformers'''		,  level='''WARNING'''							) as cm:
     BertTokenizer.from_pretrained(_lowercase							)
     self.assertTrue(
         cm.records[0].message.startswith(
             '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
             ''' is called from.'''							)							)
    _UpperCamelCase:     str							= '''bert-base-cased'''
    with self.assertLogs('''transformers'''		,  level='''WARNING'''							) as cm:
     BertJapaneseTokenizer.from_pretrained(_lowercase							)
     self.assertTrue(
         cm.records[0].message.startswith(
             '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
             ''' is called from.'''							)							)
import inspect
import unittest

from huggingface_hub import hf_hub_download

from transformers import ConvNextConfig, UperNetConfig
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import UperNetForSemanticSegmentation
  from transformers.models.upernet.modeling_upernet import UPERNET_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
  from PIL import Image

  from transformers import AutoImageProcessor




class  __magic_name__  :





   """simple docstring"""




   def __init__(	self					:  List[str]		,  _lowercase					:  Union[str, Any]		,  _lowercase					:  int=13		,  _lowercase					:  Optional[int]=32		,  _lowercase					:  Optional[Any]=3		,  _lowercase					:  Union[str, Any]=4		,  _lowercase					:  Any=[10, 20, 30, 40]		,  _lowercase					:  str=[2, 2, 3, 2]		,  _lowercase					:  Any=True		,  _lowercase					:  Union[str, Any]=True		,  _lowercase					:  int=37		,  _lowercase					:  Union[str, Any]="gelu"		,  _lowercase					:  Union[str, Any]=10		,  _lowercase					:  Tuple=0.02		,  _lowercase					:  int=["stage2", "stage3", "stage4"]		,  _lowercase					:  Optional[Any]=3		,  _lowercase					:  Optional[int]=None		,  ):



    """simple docstring"""



    _UpperCamelCase:     Optional[int]							= parent
    _UpperCamelCase:     str							= batch_size
    _UpperCamelCase:     str							= image_size
    _UpperCamelCase:     Any							= num_channels
    _UpperCamelCase:     Union[str, Any]							= num_stages
    _UpperCamelCase:     Any							= hidden_sizes
    _UpperCamelCase:     int							= depths
    _UpperCamelCase:     Dict							= is_training
    _UpperCamelCase:     Optional[int]							= use_labels
    _UpperCamelCase:     Optional[int]							= intermediate_size
    _UpperCamelCase:     int							= hidden_act
    _UpperCamelCase:     Tuple							= type_sequence_label_size
    _UpperCamelCase:     List[Any]							= initializer_range
    _UpperCamelCase:     Dict							= out_features
    _UpperCamelCase:     Union[str, Any]							= num_labels
    _UpperCamelCase:     Tuple							= scope
    _UpperCamelCase:     Union[str, Any]							= num_stages



   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    _UpperCamelCase:     Any							= floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

    _UpperCamelCase:     Any							= None
    if self.use_labels:
     _UpperCamelCase:     List[Any]							= ids_tensor([self.batch_size]		,  self.type_sequence_label_size							)

    _UpperCamelCase:     Union[str, Any]							= self.get_config()

    return config, pixel_values, labels



   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    return ConvNextConfig(
        num_channels=self.num_channels		,  num_stages=self.num_stages		,  hidden_sizes=self.hidden_sizes		,  depths=self.depths		,  is_training=self.is_training		,  intermediate_size=self.intermediate_size		,  hidden_act=self.hidden_act		,  out_features=self.out_features		,  )



   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    return UperNetConfig(
        backbone_config=self.get_backbone_config()		,  hidden_size=512		,  pool_scales=[1, 2, 3, 6]		,  use_auxiliary_head=_lowercase		,  auxiliary_loss_weight=0.4		,  auxiliary_in_channels=40		,  auxiliary_channels=256		,  auxiliary_num_convs=1		,  auxiliary_concat_input=_lowercase		,  loss_ignore_index=255		,  num_labels=self.num_labels		,  )



   def   lowerCAmelCase						(	self					:  Tuple		,  _lowercase					:  Tuple		,  _lowercase					:  Optional[Any]		,  _lowercase					:  Dict							):



    """simple docstring"""



    _UpperCamelCase:     Optional[Any]							= UperNetForSemanticSegmentation(config=_lowercase							)
    model.to(_lowercase							)
    model.eval()
    _UpperCamelCase:     int							= model(_lowercase							)
    self.parent.assertEqual(
        result.logits.shape		,  (self.batch_size, self.num_labels, self.image_size, self.image_size)							)



   def   lowerCAmelCase						(	self					:  Union[str, Any]							):



    """simple docstring"""



    _UpperCamelCase:     List[str]							= self.prepare_config_and_inputs()
    (
        (
        _UpperCamelCase
    )	,							(
        _UpperCamelCase
    )	,							(
        _UpperCamelCase
    )	,							
    ):     str							= config_and_inputs
    _UpperCamelCase:     Dict							= {'''pixel_values''': pixel_values}
    return config, inputs_dict




@require_torch
class  __magic_name__  (		__a      ,	__a      ,	unittest.TestCase    ):





   """simple docstring"""




   lowerCAmelCase      :    List[Any]  			=   (UperNetForSemanticSegmentation,) if is_torch_available() else ()
   lowerCAmelCase      :    Tuple  			=   {'''image-segmentation''': UperNetForSemanticSegmentation} if is_torch_available() else {}
   lowerCAmelCase      :    Dict  			=   False
   lowerCAmelCase      :    str  			=   False
   lowerCAmelCase      :    Optional[Any]  			=   False
   lowerCAmelCase      :    Optional[Any]  			=   False
   lowerCAmelCase      :    List[Any]  			=   False
   lowerCAmelCase      :    Union[str, Any]  			=   False



   def   lowerCAmelCase						(	self					:  Dict							):



    """simple docstring"""



    _UpperCamelCase:     Optional[int]							= UperNetModelTester(self							)
    _UpperCamelCase:     List[str]							= ConfigTester(self		,  config_class=_lowercase		,  has_text_modality=_lowercase		,  hidden_size=37							)



   def   lowerCAmelCase						(	self					:  str							):



    """simple docstring"""



    self.create_and_test_config_common_properties()
    self.config_tester.create_and_test_config_to_json_string()
    self.config_tester.create_and_test_config_to_json_file()
    self.config_tester.create_and_test_config_from_and_save_pretrained()
    self.config_tester.create_and_test_config_with_num_labels()
    self.config_tester.check_config_can_be_init_without_params()
    self.config_tester.check_config_arguments_init()



   def   lowerCAmelCase						(	self					:  List[Any]							):



    """simple docstring"""



    return



   def   lowerCAmelCase						(	self					:  List[str]							):



    """simple docstring"""



    _UpperCamelCase	,							_UpperCamelCase:     Dict							= self.model_tester.prepare_config_and_inputs_for_common()

    for model_class in self.all_model_classes:
     _UpperCamelCase:     List[Any]							= model_class(_lowercase							)
     _UpperCamelCase:     Tuple							= inspect.signature(model.forward							)
     # signature.parameters is an OrderedDict => so arg_names order is deterministic
     _UpperCamelCase:     Any							= [*signature.parameters.keys()]

     _UpperCamelCase:     List[Any]							= ['''pixel_values''']
     self.assertListEqual(arg_names[:1]		,  _lowercase							)



   def   lowerCAmelCase						(	self					:  Tuple							):



    """simple docstring"""



    _UpperCamelCase:     Optional[int]							= self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_semantic_segmentation(*_lowercase							)



   @unittest.skip(reason='''UperNet does not use inputs_embeds'''							)
   def   lowerCAmelCase						(	self					:  Tuple							):



    """simple docstring"""



    pass



   @unittest.skip(reason='''UperNet does not support input and output embeddings'''							)
   def   lowerCAmelCase						(	self					:  List[str]							):



    """simple docstring"""



    pass



   @unittest.skip(reason='''UperNet does not have a base model'''							)
   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    pass



   @unittest.skip(reason='''UperNet does not have a base model'''							)
   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    pass



   @require_torch_multi_gpu
   @unittest.skip(reason='''UperNet has some layers using `add_module` which doesn\'t work well with `nn.DataParallel`'''							)
   def   lowerCAmelCase						(	self					:  Dict							):



    """simple docstring"""



    pass



   @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''							)
   def   lowerCAmelCase						(	self					:  Tuple							):



    """simple docstring"""



    pass



   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    def check_hidden_states_output(_lowercase					:  List[str]		,  _lowercase					:  List[str]		,  _lowercase					:  List[str]							):
     _UpperCamelCase:     Any							= model_class(_lowercase							)
     model.to(_lowercase							)
     model.eval()

     with torch.no_grad():
      _UpperCamelCase:     Union[str, Any]							= model(**self._prepare_for_class(_lowercase		,  _lowercase							)							)

     _UpperCamelCase:     Optional[Any]							= outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

     _UpperCamelCase:     List[str]							= self.model_tester.num_stages
     self.assertEqual(len(_lowercase							)		,  expected_num_stages + 1							)

     # ConvNext's feature maps are of shape (batch_size, num_channels, height, width)
     self.assertListEqual(
         list(hidden_states[0].shape[-2:]							)		,  [self.model_tester.image_size // 4, self.model_tester.image_size // 4]		,  )

    _UpperCamelCase	,							_UpperCamelCase:     Any							= self.model_tester.prepare_config_and_inputs_for_common()

    for model_class in self.all_model_classes:
     _UpperCamelCase:     Optional[int]							= True
     check_hidden_states_output(_lowercase		,  _lowercase		,  _lowercase							)

     # check that output_hidden_states also work using config
     del inputs_dict["output_hidden_states"]
     _UpperCamelCase:     Union[str, Any]							= True

     check_hidden_states_output(_lowercase		,  _lowercase		,  _lowercase							)



   def   lowerCAmelCase						(	self					:  Tuple							):



    """simple docstring"""



    _UpperCamelCase	,							_UpperCamelCase:     List[str]							= self.model_tester.prepare_config_and_inputs_for_common()

    _UpperCamelCase:     int							= _config_zero_init(_lowercase							)
    _UpperCamelCase:     List[Any]							= _config_zero_init(configs_no_init.backbone_config							)
    for model_class in self.all_model_classes:
     _UpperCamelCase:     int							= model_class(config=_lowercase							)
     for name, param in model.named_parameters():
      if param.requires_grad:
       self.assertIn(
           ((param.data.mean() * 1E9).round() / 1E9).item()		,  [0.0, 1.0]		,  msg=f"""Parameter {name} of model {model_class} seems not properly initialized"""		,  )



   @unittest.skip(reason='''UperNet does not have tied weights'''							)
   def   lowerCAmelCase						(	self					:  Tuple							):



    """simple docstring"""



    pass



   @slow
   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    for model_name in UPERNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     _UpperCamelCase:     Union[str, Any]							= UperNetForSemanticSegmentation.from_pretrained(_lowercase							)
     self.assertIsNotNone(_lowercase							)




def       lowerCAmelCase_     (		)    ->				Dict:







 '''simple docstring'''
 _UpperCamelCase:     Any							= hf_hub_download(
     repo_id='''hf-internal-testing/fixtures_ade20k'''  ,					repo_type='''dataset'''  ,					filename='''ADE_val_00000001.jpg'''		)
 _UpperCamelCase:     Any							= Image.open(lowercase		).convert('''RGB'''		)
 return image





@require_torch
@require_vision
@slow
class  __magic_name__  (		unittest.TestCase    ):





   """simple docstring"""




   def   lowerCAmelCase						(	self					:  Optional[Any]							):



    """simple docstring"""



    _UpperCamelCase:     Any							= AutoImageProcessor.from_pretrained('''openmmlab/upernet-swin-tiny'''							)
    _UpperCamelCase:     Optional[Any]							= UperNetForSemanticSegmentation.from_pretrained('''openmmlab/upernet-swin-tiny'''							).to(_lowercase							)

    _UpperCamelCase:     List[Any]							= prepare_img()
    _UpperCamelCase:     int							= processor(images=_lowercase		,  return_tensors='''pt'''							).to(_lowercase							)

    with torch.no_grad():
     _UpperCamelCase:     Any							= model(**_lowercase							)

    _UpperCamelCase:     Any							= torch.Size((1, model.config.num_labels, 512, 512)							)
    self.assertEqual(outputs.logits.shape		,  _lowercase							)

    _UpperCamelCase:     Tuple							= torch.tensor(
        [[-7.5958, -7.5958, -7.4302], [-7.5958, -7.5958, -7.4302], [-7.4797, -7.4797, -7.3068]]							).to(_lowercase							)
    self.assertTrue(torch.allclose(outputs.logits[0, 0, :3, :3]		,  _lowercase		,  atol=1E-4							)							)



   def   lowerCAmelCase						(	self					:  int							):



    """simple docstring"""



    _UpperCamelCase:     Tuple							= AutoImageProcessor.from_pretrained('''openmmlab/upernet-convnext-tiny'''							)
    _UpperCamelCase:     int							= UperNetForSemanticSegmentation.from_pretrained('''openmmlab/upernet-convnext-tiny'''							).to(_lowercase							)

    _UpperCamelCase:     int							= prepare_img()
    _UpperCamelCase:     Union[str, Any]							= processor(images=_lowercase		,  return_tensors='''pt'''							).to(_lowercase							)

    with torch.no_grad():
     _UpperCamelCase:     List[Any]							= model(**_lowercase							)

    _UpperCamelCase:     Tuple							= torch.Size((1, model.config.num_labels, 512, 512)							)
    self.assertEqual(outputs.logits.shape		,  _lowercase							)

    _UpperCamelCase:     Optional[Any]							= torch.tensor(
        [[-8.8110, -8.8110, -8.6521], [-8.8110, -8.8110, -8.6521], [-8.7746, -8.7746, -8.6130]]							).to(_lowercase							)
    self.assertTrue(torch.allclose(outputs.logits[0, 0, :3, :3]		,  _lowercase		,  atol=1E-4							)							)