

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 			_SCREAMING_SNAKE_CASE (   __snake_case					,       unittest.TestCase		):







   '''simple docstring'''




   lowerCamelCase__				=      BertJapaneseTokenizer
   lowerCamelCase__				=      False
   lowerCamelCase__				=      True





   def      _snake_case		(       self					:			Dict			):
       super().setUp()

       SCREAMING_SNAKE_CASE										=   [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "こんにちは",
           "こん",
           "にちは",
           "ばんは",
           "##こん",
           "##にちは",
           "##ばんは",
           "世界",
           "##世界",
           "、",
           "##、",
           "。",
           "##。",
       ]

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES["vocab_file"]			)
       with open(self.vocab_file     , "w"     , encoding="utf-8"			) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)





   def      _snake_case		(       self					:			Tuple     , __lowerCamelCase					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。 \nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   "こんにちは 、 世界 。 こんばんは 、 世界 。"
       return input_text, output_text





   def      _snake_case		(       self					:			str     , __lowerCamelCase					:			Any			):
       SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   self.get_input_output_texts(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.encode(__lowerCamelCase     , add_special_tokens=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.decode(__lowerCamelCase     , clean_up_tokenization_spaces=__lowerCamelCase			)
       return text, ids





   def      _snake_case		(       self					:			Optional[int]			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			List[Any]			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			List[Any]			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file			)

       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"			)
       self.assertListEqual(__lowerCamelCase     , ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 12, 10, 14, 4, 9, 12, 10, 14]			)





   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file     , word_tokenizer_type="mecab"			)
       self.assertIsNotNone(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。\nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize(__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 12, 10, 14, 4, 9, 12, 10, 14]			)

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , "tokenizer.bin"			)
       with open(__lowerCamelCase     , "wb"			) as handle:
           pickle.dump(__lowerCamelCase     , __lowerCamelCase			)

       with open(__lowerCamelCase     , "rb"			) as handle:
           SCREAMING_SNAKE_CASE										=   pickle.load(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer_new.tokenize(__lowerCamelCase			)

       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)





   def      _snake_case		(       self					:			Optional[Any]			):
       SCREAMING_SNAKE_CASE										=   MecabTokenizer(mecab_dic="ipadic"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   def      _snake_case		(       self					:			int			):
       try:
           SCREAMING_SNAKE_CASE										=   MecabTokenizer(mecab_dic="unidic_lite"			)
       except ModuleNotFoundError:
           return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   def      _snake_case		(       self					:			Optional[Any]			):
       try:
           SCREAMING_SNAKE_CASE										=   MecabTokenizer(mecab_dic="unidic"			)
       except ModuleNotFoundError:
           return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   def      _snake_case		(       self					:			Any			):
       SCREAMING_SNAKE_CASE										=   MecabTokenizer(do_lower_case=__lowerCamelCase     , mecab_dic="ipadic"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   def      _snake_case		(       self					:			Union[str, Any]			):
       try:
           SCREAMING_SNAKE_CASE										=   MecabTokenizer(
               do_lower_case=__lowerCamelCase     , normalize_text=__lowerCamelCase     , mecab_option="-d /usr/local/lib/mecab/dic/jumandic"			)
       except RuntimeError:
           # if dict doesn't exist in the system, previous code raises this error.
           return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]     , )





   def      _snake_case		(       self					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   MecabTokenizer(normalize_text=__lowerCamelCase     , mecab_dic="ipadic"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]     , )





   @require_sudachi
   def      _snake_case		(       self					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file     , word_tokenizer_type="sudachi"			)
       self.assertIsNotNone(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。\nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize(__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 12, 10, 14, 4, 9, 12, 10, 14]			)

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , "tokenizer.bin"			)
       with open(__lowerCamelCase     , "wb"			) as handle:
           pickle.dump(__lowerCamelCase     , __lowerCamelCase			)

       with open(__lowerCamelCase     , "rb"			) as handle:
           SCREAMING_SNAKE_CASE										=   pickle.load(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer_new.tokenize(__lowerCamelCase			)

       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)





   @require_sudachi
   def      _snake_case		(       self					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]     , )





   @require_sudachi
   def      _snake_case		(       self					:			int			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(sudachi_dict_type="core"     , sudachi_split_mode="A"			)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"			)     , ["外国", "人", "参政", "権"]			)





   @require_sudachi
   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(sudachi_dict_type="core"     , sudachi_split_mode="B"			)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"			)     , ["外国人", "参政権"]			)





   @require_sudachi
   def      _snake_case		(       self					:			int			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(sudachi_dict_type="core"     , sudachi_split_mode="C"			)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"			)     , ["外国人参政権"]			)





   @require_sudachi
   def      _snake_case		(       self					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(do_lower_case=__lowerCamelCase     , sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]     , )





   @require_sudachi
   def      _snake_case		(       self					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(normalize_text=__lowerCamelCase     , sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]     , )





   @require_sudachi
   def      _snake_case		(       self					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(trim_whitespace=__lowerCamelCase     , sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file     , word_tokenizer_type="jumanpp"			)
       self.assertIsNotNone(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。\nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize(__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 12, 10, 14, 4, 9, 12, 10, 14]			)

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , "tokenizer.bin"			)
       with open(__lowerCamelCase     , "wb"			) as handle:
           pickle.dump(__lowerCamelCase     , __lowerCamelCase			)

       with open(__lowerCamelCase     , "rb"			) as handle:
           SCREAMING_SNAKE_CASE										=   pickle.load(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer_new.tokenize(__lowerCamelCase			)

       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)





   @require_jumanpp
   def      _snake_case		(       self					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer(do_lower_case=__lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			Optional[Any]			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer(normalize_text=__lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			Dict			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer(trim_whitespace=__lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"			)     , ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]     , )





   def      _snake_case		(       self					:			int			):
       SCREAMING_SNAKE_CASE										=   ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

       SCREAMING_SNAKE_CASE										=   {}
       for i, token in enumerate(__lowerCamelCase			):
           SCREAMING_SNAKE_CASE										=   i
       SCREAMING_SNAKE_CASE										=   WordpieceTokenizer(vocab=__lowerCamelCase     , unk_token="[UNK]"			)

       self.assertListEqual(tokenizer.tokenize(""			)     , []			)

       self.assertListEqual(tokenizer.tokenize("こんにちは"			)     , ["こんにちは"]			)

       self.assertListEqual(tokenizer.tokenize("こんばんは"			)     , ["こん", "##ばんは"]			)

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"			)     , ["こん", "##ばんは", "[UNK]", "こんにちは"]			)





   def      _snake_case		(       self					:			Any			):
       SCREAMING_SNAKE_CASE										=   BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"			)
       SCREAMING_SNAKE_CASE										=   tokenizer.subword_tokenizer

       SCREAMING_SNAKE_CASE										=   subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"			)
       self.assertListEqual(__lowerCamelCase     , ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]			)

       SCREAMING_SNAKE_CASE										=   subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"			)
       self.assertListEqual(__lowerCamelCase     , ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]			)





   def      _snake_case		(       self					:			Optional[Any]			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"			)

       SCREAMING_SNAKE_CASE										=   tokenizer.encode("ありがとう。"     , add_special_tokens=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.encode("どういたしまして。"     , add_special_tokens=__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer.build_inputs_with_special_tokens(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.build_inputs_with_special_tokens(__lowerCamelCase     , __lowerCamelCase			)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			_SCREAMING_SNAKE_CASE (   __snake_case					,       unittest.TestCase		):







   '''simple docstring'''




   lowerCamelCase__				=      BertJapaneseTokenizer
   lowerCamelCase__				=      False





   def      _snake_case		(       self					:			Optional[Any]			):
       super().setUp()

       SCREAMING_SNAKE_CASE										=   ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES["vocab_file"]			)
       with open(self.vocab_file     , "w"     , encoding="utf-8"			) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)





   def      _snake_case		(       self					:			str     , **__lowerCamelCase					:			Optional[int]			):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname     , subword_tokenizer_type="character"     , **__lowerCamelCase			)





   def      _snake_case		(       self					:			Dict     , __lowerCamelCase					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。 \nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
       return input_text, output_text





   def      _snake_case		(       self					:			Tuple			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			Tuple			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			Any			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file     , subword_tokenizer_type="character"			)

       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"			)
       self.assertListEqual(
           __lowerCamelCase     , ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]			)
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]			)





   def      _snake_case		(       self					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       SCREAMING_SNAKE_CASE										=   {}
       for i, token in enumerate(__lowerCamelCase			):
           SCREAMING_SNAKE_CASE										=   i
       SCREAMING_SNAKE_CASE										=   CharacterTokenizer(vocab=__lowerCamelCase     , unk_token="[UNK]"			)

       self.assertListEqual(tokenizer.tokenize(""			)     , []			)

       self.assertListEqual(tokenizer.tokenize("こんにちは"			)     , ["こ", "ん", "に", "ち", "は"]			)

       self.assertListEqual(tokenizer.tokenize("こんにちほ"			)     , ["こ", "ん", "に", "ち", "[UNK]"]			)





   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"			)

       SCREAMING_SNAKE_CASE										=   tokenizer.encode("ありがとう。"     , add_special_tokens=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.encode("どういたしまして。"     , add_special_tokens=__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer.build_inputs_with_special_tokens(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.build_inputs_with_special_tokens(__lowerCamelCase     , __lowerCamelCase			)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			_SCREAMING_SNAKE_CASE (   unittest.TestCase		):







   '''simple docstring'''




   def      _snake_case		(       self					:			str			):
       SCREAMING_SNAKE_CASE										=   "cl-tohoku/bert-base-japanese"
       SCREAMING_SNAKE_CASE										=   AutoTokenizer.from_pretrained(__lowerCamelCase			)
       self.assertIsInstance(__lowerCamelCase     , __lowerCamelCase			)





class 			_SCREAMING_SNAKE_CASE (   unittest.TestCase		):







   '''simple docstring'''




   def      _snake_case		(       self					:			Dict			):
       SCREAMING_SNAKE_CASE										=   "cl-tohoku/bert-base-japanese"
       with self.assertLogs("transformers"     , level="WARNING"			) as cm:
           BertTokenizer.from_pretrained(__lowerCamelCase			)
           self.assertTrue(
               cm.records[0].message.startswith(
                   "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                   " is called from."			)			)
       SCREAMING_SNAKE_CASE										=   "bert-base-cased"
       with self.assertLogs("transformers"     , level="WARNING"			) as cm:
           BertJapaneseTokenizer.from_pretrained(__lowerCamelCase			)
           self.assertTrue(
               cm.records[0].message.startswith(
                   "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                   " is called from."			)			)
"""simple docstring"""



from __future__ import annotations

import inspect
import unittest
from typing import List, Tuple

from transformers import RegNetConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST, TFRegNetForImageClassification, TFRegNetModel


if is_vision_available():
				from PIL import Image

				from transformers import AutoImageProcessor
class 						__UpperCAmelCase    :







			'''simple docstring'''





			def __init__(			self   ,      snake_case_   ,      snake_case_=3   ,      snake_case_=32   ,      snake_case_=3   ,      snake_case_=10   ,      snake_case_=[10, 20, 30, 40]   ,      snake_case_=[1, 1, 2, 1]   ,      snake_case_=True   ,      snake_case_=True   ,      snake_case_="relu"   ,      snake_case_=3   ,      snake_case_=None   ,      ):







										'''simple docstring'''


										A__       :     Tuple								=			parent
										A__       :     int								=			batch_size
										A__       :     List[Any]								=			image_size
										A__       :     Any								=			num_channels
										A__       :     Tuple								=			embeddings_size
										A__       :     Union[str, Any]								=			hidden_sizes
										A__       :     Tuple								=			depths
										A__       :     Union[str, Any]								=			is_training
										A__       :     Any								=			use_labels
										A__       :     Optional[Any]								=			hidden_act
										A__       :     str								=			num_labels
										A__       :     Optional[Any]								=			scope
										A__       :     List[Any]								=			len(snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     List[Any]								=			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

										A__       :     str								=			None
										if self.use_labels:
																	A__       :     Tuple								=			ids_tensor([self.batch_size]   ,      self.num_labels							)

										A__       :     Dict								=			self.get_config()
										return config, pixel_values, labels





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return RegNetConfig(
										    num_channels=self.num_channels   ,      embeddings_size=self.embeddings_size   ,      hidden_sizes=self.hidden_sizes   ,      depths=self.depths   ,      hidden_act=self.hidden_act   ,      num_labels=self.num_labels   ,      )





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Union[str, Any]								=			TFRegNetModel(config=snake_case_							)
										A__       :     Optional[int]								=			model(snake_case_   ,      training=snake_case_							)
										# expected last hidden states: B, C, H // 32, W // 32
										self.parent.assertEqual(
										    result.last_hidden_state.shape   ,      (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)   ,      )





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Optional[int]								=			self.num_labels
										A__       :     Tuple								=			TFRegNetForImageClassification(snake_case_							)
										A__       :     Optional[int]								=			model(snake_case_   ,      labels=snake_case_   ,      training=snake_case_							)
										self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.num_labels)							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     List[str]								=			self.prepare_config_and_inputs()
										A__				,							A__				,							A__       :     Optional[Any]								=			config_and_inputs
										A__       :     str								=			{"""pixel_values""": pixel_values}
										return config, inputs_dict





@require_tf
class 						__UpperCAmelCase    (__A							,   __A							,   unittest.TestCase     ):







			'''simple docstring'''
			_UpperCamelCase  :						str					     =				(TFRegNetModel, TFRegNetForImageClassification) if is_tf_available() else ()
			_UpperCamelCase  :						Tuple					     =				(
			    {'feature-extraction': TFRegNetModel, 'image-classification': TFRegNetForImageClassification}
			    if is_tf_available()
			    else {}
			)

			_UpperCamelCase  :						str					     =				False
			_UpperCamelCase  :						int					     =				False
			_UpperCamelCase  :						Tuple					     =				False
			_UpperCamelCase  :						Any					     =				False
			_UpperCamelCase  :						Union[str, Any]					     =				False





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Dict								=			TFRegNetModelTester(self							)
										A__       :     str								=			ConfigTester(self   ,      config_class=snake_case_   ,      has_text_modality=snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return





			@unittest.skip(reason="""RegNet does not use inputs_embeds"""							)
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										pass





			@unittest.skipIf(
			    not is_tf_available() or len(tf.config.list_physical_devices("""GPU"""							)							) == 0   ,      reason="""TF does not support backprop for grouped convolutions on CPU."""   ,      )
			@slow
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										super().test_keras_fit()





			@unittest.skip(reason="""RegNet does not support input and output embeddings"""							)
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										pass





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__				,							A__       :     int								=			self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
																	A__       :     int								=			model_class(snake_case_							)
																	A__       :     Optional[int]								=			inspect.signature(model.call							)
																	# signature.parameters is an OrderedDict => so arg_names order is deterministic
																	A__       :     Union[str, Any]								=			[*signature.parameters.keys()]

																	A__       :     Tuple								=			["""pixel_values"""]
																	self.assertListEqual(arg_names[:1]   ,      snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Union[str, Any]								=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										def check_hidden_states_output(snake_case_   ,      snake_case_   ,      snake_case_							):
																	A__       :     str								=			model_class(snake_case_							)
																	A__       :     Union[str, Any]								=			model(**self._prepare_for_class(snake_case_   ,      snake_case_							)   ,      training=snake_case_							)

																	A__       :     Optional[int]								=			outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

																	A__       :     Union[str, Any]								=			self.model_tester.num_stages
																	self.assertEqual(len(snake_case_							)   ,      expected_num_stages + 1							)

																	# RegNet's feature maps are of shape (batch_size, num_channels, height, width)
																	self.assertListEqual(
																	    list(hidden_states[0].shape[-2:]							)   ,      [self.model_tester.image_size // 2, self.model_tester.image_size // 2]   ,      )

										A__				,							A__       :     int								=			self.model_tester.prepare_config_and_inputs_for_common()
										A__       :     Optional[int]								=			["""basic""", """bottleneck"""]
										for model_class in self.all_model_classes:
																	for layer_type in layers_type:
																								A__       :     Optional[Any]								=			layer_type
																								A__       :     Dict								=			True
																								check_hidden_states_output(snake_case_   ,      snake_case_   ,      snake_case_							)

																								# check that output_hidden_states also work using config
																								del inputs_dict["output_hidden_states"]
																								A__       :     List[Any]								=			True

																								check_hidden_states_output(snake_case_   ,      snake_case_   ,      snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__				,							A__       :     Optional[Any]								=			self.model_tester.prepare_config_and_inputs_for_common()

										def check_equivalence(snake_case_   ,      snake_case_   ,      snake_case_   ,      snake_case_={}							):
																	A__       :     List[str]								=			model(snake_case_   ,      return_dict=snake_case_   ,      **snake_case_							)
																	A__       :     Union[str, Any]								=			model(snake_case_   ,      return_dict=snake_case_   ,      **snake_case_							).to_tuple()

																	def recursive_check(snake_case_   ,      snake_case_							):
																								if isinstance(snake_case_   ,      (List, Tuple)							):
																															for tuple_iterable_value, dict_iterable_value in zip(snake_case_   ,      snake_case_							):
																																						recursive_check(snake_case_   ,      snake_case_							)
																								elif tuple_object is None:
																															return
																								else:
																															self.assertTrue(
																															    all(tf.equal(snake_case_   ,      snake_case_							)							)   ,      msg=(
																															        """Tuple and dict output are not equal. Difference:"""
																															        F''' {tf.math.reduce_max(tf.abs(tuple_object - dict_object							)							)}'''
																															    )   ,      )

																								recursive_check(snake_case_   ,      snake_case_							)

										for model_class in self.all_model_classes:
																	A__       :     Any								=			model_class(snake_case_							)

																	A__       :     Dict								=			self._prepare_for_class(snake_case_   ,      snake_case_							)
																	A__       :     List[Any]								=			self._prepare_for_class(snake_case_   ,      snake_case_							)
																	check_equivalence(snake_case_   ,      snake_case_   ,      snake_case_							)

																	A__       :     Tuple								=			self._prepare_for_class(snake_case_   ,      snake_case_   ,      return_labels=snake_case_							)
																	A__       :     List[str]								=			self._prepare_for_class(snake_case_   ,      snake_case_   ,      return_labels=snake_case_							)
																	check_equivalence(snake_case_   ,      snake_case_   ,      snake_case_							)

																	A__       :     Optional[int]								=			self._prepare_for_class(snake_case_   ,      snake_case_							)
																	A__       :     List[str]								=			self._prepare_for_class(snake_case_   ,      snake_case_							)
																	check_equivalence(snake_case_   ,      snake_case_   ,      snake_case_   ,      {"""output_hidden_states""": True}							)

																	A__       :     Dict								=			self._prepare_for_class(snake_case_   ,      snake_case_   ,      return_labels=snake_case_							)
																	A__       :     Union[str, Any]								=			self._prepare_for_class(snake_case_   ,      snake_case_   ,      return_labels=snake_case_							)
																	check_equivalence(snake_case_   ,      snake_case_   ,      snake_case_   ,      {"""output_hidden_states""": True}							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Tuple								=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_image_classification(*snake_case_							)





			@slow
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										for model_name in TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	A__       :     Any								=			TFRegNetModel.from_pretrained(snake_case_							)
																	self.assertIsNotNone(snake_case_							)





def  _A(		):
							A__       :     Optional[int]								=			Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""							)
							return image



@require_tf
@require_vision
class 						__UpperCAmelCase    (unittest.TestCase     ):







			'''simple docstring'''





			@cached_property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return (
										    AutoImageProcessor.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]							)
										    if is_vision_available()
										    else None
										)





			@slow
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     int								=			TFRegNetForImageClassification.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]							)

										A__       :     str								=			self.default_image_processor
										A__       :     Any								=			prepare_img()
										A__       :     Tuple								=			image_processor(images=snake_case_   ,      return_tensors="""tf"""							)

										# forward pass
										A__       :     List[Any]								=			model(**snake_case_   ,      training=snake_case_							)

										# verify the logits
										A__       :     Tuple								=			tf.TensorShape((1, 1_000)							)
										self.assertEqual(outputs.logits.shape   ,      snake_case_							)

										A__       :     Tuple								=			tf.constant([-0.41_80, -1.50_51, -3.48_36]							)

										tf.debugging.assert_near(outputs.logits[0, :3]   ,      snake_case_   ,      atol=1E-4							)





