



'''simple docstring'''

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: str									=      BertJapaneseTokenizer
			a_	: List[str]									=      False
			a_	: List[Any]									=      True






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					super().setUp()

					snake_case__	     =[
					    '[UNK]',
					    '[CLS]',
					    '[SEP]',
					    'こんにちは',
					    'こん',
					    'にちは',
					    'ばんは',
					    '##こん',
					    '##にちは',
					    '##ばんは',
					    '世界',
					    '##世界',
					    '、',
					    '##、',
					    '。',
					    '##。',
					]

					snake_case__	     =os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file					,   'w'					,   encoding='utf-8'						) as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							str:
					snake_case__	     ='こんにちは、世界。 \nこんばんは、世界。'
					snake_case__	     ='こんにちは 、 世界 。 こんばんは 、 世界 。'
					return input_text, output_text






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Any:
					snake_case__     ,	snake_case__	     =self.get_input_output_texts(_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode(_UpperCAmelCase					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.decode(_UpperCAmelCase					,   clean_up_tokenization_spaces=_UpperCAmelCase						)
					return text, ids






			def 						_lowercase  (     self						)							->							Dict:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							List[str]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							str:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class(self.vocab_file						)

					snake_case__	     =tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='mecab'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =MecabTokenizer(mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							str:
					try:
							snake_case__	     =MecabTokenizer(mecab_dic='unidic_lite'						)
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							List[str]:
					try:
							snake_case__	     =MecabTokenizer(mecab_dic='unidic'						)
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =MecabTokenizer(do_lower_case=_UpperCAmelCase					,   mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							List[Any]:
					try:
							snake_case__	     =MecabTokenizer(
							    do_lower_case=_UpperCAmelCase					,   normalize_text=_UpperCAmelCase					,   mecab_option='-d /usr/local/lib/mecab/dic/jumandic'						)
					except RuntimeError:
							# if dict doesn't exist in the system, previous code raises this error.
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =MecabTokenizer(normalize_text=_UpperCAmelCase					,   mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='sudachi'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Union[str, Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='A'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国', '人', '参政', '権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='B'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国人', '参政権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='C'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国人参政権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =SudachiTokenizer(do_lower_case=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Dict:
					snake_case__	     =SudachiTokenizer(normalize_text=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =SudachiTokenizer(trim_whitespace=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='jumanpp'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			@require_jumanpp
			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =JumanppTokenizer(do_lower_case=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =JumanppTokenizer(normalize_text=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =JumanppTokenizer(trim_whitespace=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							str:
					snake_case__	     =JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'						)					,   ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']					,   )






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

					snake_case__	     ={}
					for i, token in enumerate(_UpperCAmelCase						):
							snake_case__	     =i
					snake_case__	     =WordpieceTokenizer(vocab=_UpperCAmelCase					,   unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)					,   []						)

					self.assertListEqual(tokenizer.tokenize('こんにちは'						)					,   ['こんにちは']						)

					self.assertListEqual(tokenizer.tokenize('こんばんは'						)					,   ['こん', '##ばんは']						)

					self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'						)					,   ['こん', '##ばんは', '[UNK]', 'こんにちは']						)






			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'						)
					snake_case__	     =tokenizer.subword_tokenizer

					snake_case__	     =subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']						)

					snake_case__	     =subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']						)






			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'						)

					snake_case__	     =tokenizer.encode('ありがとう。'					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode('どういたしまして。'					,   add_special_tokens=_UpperCAmelCase						)

					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase					,   _UpperCAmelCase						)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: int									=      BertJapaneseTokenizer
			a_	: Optional[Any]									=      False






			def 						_lowercase  (     self						)							->							Any:
					super().setUp()

					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

					snake_case__	     =os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file					,   'w'					,   encoding='utf-8'						) as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)






			def 						_lowercase  (     self					,   **_UpperCAmelCase						)							->							Union[str, Any]:
					return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,   subword_tokenizer_type='character'					,   **_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Optional[Any]:
					snake_case__	     ='こんにちは、世界。 \nこんばんは、世界。'
					snake_case__	     ='こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
					return input_text, output_text






			def 						_lowercase  (     self						)							->							List[Any]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Tuple:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   subword_tokenizer_type='character'						)

					snake_case__	     =tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'						)
					self.assertListEqual(
					    _UpperCAmelCase					,   ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']						)
					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

					snake_case__	     ={}
					for i, token in enumerate(_UpperCAmelCase						):
							snake_case__	     =i
					snake_case__	     =CharacterTokenizer(vocab=_UpperCAmelCase					,   unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)					,   []						)

					self.assertListEqual(tokenizer.tokenize('こんにちは'						)					,   ['こ', 'ん', 'に', 'ち', 'は']						)

					self.assertListEqual(tokenizer.tokenize('こんにちほ'						)					,   ['こ', 'ん', 'に', 'ち', '[UNK]']						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'						)

					snake_case__	     =tokenizer.encode('ありがとう。'					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode('どういたしまして。'					,   add_special_tokens=_UpperCAmelCase						)

					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase					,   _UpperCAmelCase						)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     ='cl-tohoku/bert-base-japanese'
					snake_case__	     =AutoTokenizer.from_pretrained(_UpperCAmelCase						)
					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)





class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     ='cl-tohoku/bert-base-japanese'
					with self.assertLogs('transformers'					,   level='WARNING'						) as cm:
							BertTokenizer.from_pretrained(_UpperCAmelCase						)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
							        ' is called from.'						)						)
					snake_case__	     ='bert-base-cased'
					with self.assertLogs('transformers'					,   level='WARNING'						) as cm:
							BertJapaneseTokenizer.from_pretrained(_UpperCAmelCase						)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
							        ' is called from.'						)						)









'''simple docstring'''

import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    MODEL_FOR_PRETRAINING_MAPPING,
			    AlbertForMaskedLM,
			    AlbertForMultipleChoice,
			    AlbertForPreTraining,
			    AlbertForQuestionAnswering,
			    AlbertForSequenceClassification,
			    AlbertForTokenClassification,
			    AlbertModel,
			)
			from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST

class 							a__:






			def __init__(     self					,   _UpperCAmelCase					,   _UpperCAmelCase=13					,   _UpperCAmelCase=7					,   _UpperCAmelCase=True					,   _UpperCAmelCase=True					,   _UpperCAmelCase=True					,   _UpperCAmelCase=True					,   _UpperCAmelCase=99					,   _UpperCAmelCase=16					,   _UpperCAmelCase=36					,   _UpperCAmelCase=6					,   _UpperCAmelCase=6					,   _UpperCAmelCase=6					,   _UpperCAmelCase=37					,   _UpperCAmelCase="gelu"					,   _UpperCAmelCase=0.1					,   _UpperCAmelCase=0.1					,   _UpperCAmelCase=512					,   _UpperCAmelCase=16					,   _UpperCAmelCase=2					,   _UpperCAmelCase=0.02					,   _UpperCAmelCase=3					,   _UpperCAmelCase=4					,   _UpperCAmelCase=None					,   )							->							Dict:
					snake_case__	     =parent
					snake_case__	     =batch_size
					snake_case__	     =seq_length
					snake_case__	     =is_training
					snake_case__	     =use_input_mask
					snake_case__	     =use_token_type_ids
					snake_case__	     =use_labels
					snake_case__	     =vocab_size
					snake_case__	     =embedding_size
					snake_case__	     =hidden_size
					snake_case__	     =num_hidden_layers
					snake_case__	     =num_hidden_groups
					snake_case__	     =num_attention_heads
					snake_case__	     =intermediate_size
					snake_case__	     =hidden_act
					snake_case__	     =hidden_dropout_prob
					snake_case__	     =attention_probs_dropout_prob
					snake_case__	     =max_position_embeddings
					snake_case__	     =type_vocab_size
					snake_case__	     =type_sequence_label_size
					snake_case__	     =initializer_range
					snake_case__	     =num_labels
					snake_case__	     =num_choices
					snake_case__	     =scope






			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =ids_tensor([self.batch_size, self.seq_length]					,   self.vocab_size						)

					snake_case__	     =None
					if self.use_input_mask:
							snake_case__	     =random_attention_mask([self.batch_size, self.seq_length]						)

					snake_case__	     =None
					if self.use_token_type_ids:
							snake_case__	     =ids_tensor([self.batch_size, self.seq_length]					,   self.type_vocab_size						)

					snake_case__	     =None
					snake_case__	     =None
					snake_case__	     =None
					if self.use_labels:
							snake_case__	     =ids_tensor([self.batch_size]					,   self.type_sequence_label_size						)
							snake_case__	     =ids_tensor([self.batch_size, self.seq_length]					,   self.num_labels						)
							snake_case__	     =ids_tensor([self.batch_size]					,   self.num_choices						)

					snake_case__	     =self.get_config()

					return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					return AlbertConfig(
					    vocab_size=self.vocab_size					,   hidden_size=self.hidden_size					,   num_hidden_layers=self.num_hidden_layers					,   num_attention_heads=self.num_attention_heads					,   intermediate_size=self.intermediate_size					,   hidden_act=self.hidden_act					,   hidden_dropout_prob=self.hidden_dropout_prob					,   attention_probs_dropout_prob=self.attention_probs_dropout_prob					,   max_position_embeddings=self.max_position_embeddings					,   type_vocab_size=self.type_vocab_size					,   initializer_range=self.initializer_range					,   num_hidden_groups=self.num_hidden_groups					,   )






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase						)							->							List[str]:
					snake_case__	     =AlbertModel(config=_UpperCAmelCase						)
					model.to(_UpperCAmelCase						)
					model.eval()
					snake_case__	     =model(_UpperCAmelCase					,   attention_mask=_UpperCAmelCase					,   token_type_ids=_UpperCAmelCase						)
					snake_case__	     =model(_UpperCAmelCase					,   token_type_ids=_UpperCAmelCase						)
					snake_case__	     =model(_UpperCAmelCase						)
					self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, self.seq_length, self.hidden_size)						)
					self.parent.assertEqual(result.pooler_output.shape					,   (self.batch_size, self.hidden_size)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase						)							->							Dict:
					snake_case__	     =AlbertForPreTraining(config=_UpperCAmelCase						)
					model.to(_UpperCAmelCase						)
					model.eval()
					snake_case__	     =model(
					    _UpperCAmelCase					,   attention_mask=_UpperCAmelCase					,   token_type_ids=_UpperCAmelCase					,   labels=_UpperCAmelCase					,   sentence_order_label=_UpperCAmelCase					,   )
					self.parent.assertEqual(result.prediction_logits.shape					,   (self.batch_size, self.seq_length, self.vocab_size)						)
					self.parent.assertEqual(result.sop_logits.shape					,   (self.batch_size, config.num_labels)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase						)							->							List[str]:
					snake_case__	     =AlbertForMaskedLM(config=_UpperCAmelCase						)
					model.to(_UpperCAmelCase						)
					model.eval()
					snake_case__	     =model(_UpperCAmelCase					,   attention_mask=_UpperCAmelCase					,   token_type_ids=_UpperCAmelCase					,   labels=_UpperCAmelCase						)
					self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.seq_length, self.vocab_size)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase						)							->							List[str]:
					snake_case__	     =AlbertForQuestionAnswering(config=_UpperCAmelCase						)
					model.to(_UpperCAmelCase						)
					model.eval()
					snake_case__	     =model(
					    _UpperCAmelCase					,   attention_mask=_UpperCAmelCase					,   token_type_ids=_UpperCAmelCase					,   start_positions=_UpperCAmelCase					,   end_positions=_UpperCAmelCase					,   )
					self.parent.assertEqual(result.start_logits.shape					,   (self.batch_size, self.seq_length)						)
					self.parent.assertEqual(result.end_logits.shape					,   (self.batch_size, self.seq_length)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase						)							->							Optional[int]:
					snake_case__	     =self.num_labels
					snake_case__	     =AlbertForSequenceClassification(_UpperCAmelCase						)
					model.to(_UpperCAmelCase						)
					model.eval()
					snake_case__	     =model(_UpperCAmelCase					,   attention_mask=_UpperCAmelCase					,   token_type_ids=_UpperCAmelCase					,   labels=_UpperCAmelCase						)
					self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.num_labels)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase						)							->							Any:
					snake_case__	     =self.num_labels
					snake_case__	     =AlbertForTokenClassification(config=_UpperCAmelCase						)
					model.to(_UpperCAmelCase						)
					model.eval()
					snake_case__	     =model(_UpperCAmelCase					,   attention_mask=_UpperCAmelCase					,   token_type_ids=_UpperCAmelCase					,   labels=_UpperCAmelCase						)
					self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.seq_length, self.num_labels)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase						)							->							Dict:
					snake_case__	     =self.num_choices
					snake_case__	     =AlbertForMultipleChoice(config=_UpperCAmelCase						)
					model.to(_UpperCAmelCase						)
					model.eval()
					snake_case__	     =input_ids.unsqueeze(1						).expand(-1					,   self.num_choices					,   -1						).contiguous()
					snake_case__	     =token_type_ids.unsqueeze(1						).expand(-1					,   self.num_choices					,   -1						).contiguous()
					snake_case__	     =input_mask.unsqueeze(1						).expand(-1					,   self.num_choices					,   -1						).contiguous()
					snake_case__	     =model(
					    _UpperCAmelCase					,   attention_mask=_UpperCAmelCase					,   token_type_ids=_UpperCAmelCase					,   labels=_UpperCAmelCase					,   )
					self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.num_choices)						)






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =self.prepare_config_and_inputs()
					(
					    (
					    snake_case__
					)     ,	(
					    snake_case__
					)     ,	(
					    snake_case__
					)     ,	(
					    snake_case__
					)     ,	(
					    snake_case__
					)     ,	(
					    snake_case__
					)     ,	(
					    snake_case__
					)     ,	
					)	     =config_and_inputs
					snake_case__	     ={'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
					return config, inputs_dict





@require_torch
class 							a__( snake_case__   ,      snake_case__   ,      unittest.TestCase     ):
			a_	: Optional[int]									=      (
			    (
			        AlbertModel,
			        AlbertForPreTraining,
			        AlbertForMaskedLM,
			        AlbertForMultipleChoice,
			        AlbertForSequenceClassification,
			        AlbertForTokenClassification,
			        AlbertForQuestionAnswering,
			    )
			    if is_torch_available()
			    else ()
			)
			a_	: List[str]									=      (
			    {
			        '''feature-extraction''': AlbertModel,
			        '''fill-mask''': AlbertForMaskedLM,
			        '''question-answering''': AlbertForQuestionAnswering,
			        '''text-classification''': AlbertForSequenceClassification,
			        '''token-classification''': AlbertForTokenClassification,
			        '''zero-shot''': AlbertForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			a_	: int									=      True






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase=False						)							->							Any:
					snake_case__	     =super()._prepare_for_class(_UpperCAmelCase					,   _UpperCAmelCase					,   return_labels=_UpperCAmelCase						)

					if return_labels:
							if model_class in get_values(_UpperCAmelCase						):
									snake_case__	     =torch.zeros(
									    (self.model_tester.batch_size, self.model_tester.seq_length)					,   dtype=torch.long					,   device=_UpperCAmelCase						)
									snake_case__	     =torch.zeros(
									    self.model_tester.batch_size					,   dtype=torch.long					,   device=_UpperCAmelCase						)
					return inputs_dict






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =AlbertModelTester(self						)
					snake_case__	     =ConfigTester(self					,   config_class=_UpperCAmelCase					,   hidden_size=37						)






			def 						_lowercase  (     self						)							->							Dict:
					self.config_tester.run_common_tests()






			def 						_lowercase  (     self						)							->							str:
					snake_case__	     =self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(*_UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_pretraining(*_UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							List[str]:
					snake_case__	     =self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							List[str]:
					snake_case__	     =self.model_tester.prepare_config_and_inputs()
					for type in ["absolute", "relative_key", "relative_key_query"]:
							snake_case__	     =type
							self.model_tester.create_and_check_model(*_UpperCAmelCase						)






			@slow
			def 						_lowercase  (     self						)							->							Union[str, Any]:
					for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							snake_case__	     =AlbertModel.from_pretrained(_UpperCAmelCase						)
							self.assertIsNotNone(_UpperCAmelCase						)





@require_torch
class 							a__( unittest.TestCase     ):






			@slow
			def 						_lowercase  (     self						)							->							str:
					snake_case__	     =AlbertModel.from_pretrained('albert-base-v2'						)
					snake_case__	     =torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]						)
					snake_case__	     =torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]						)
					with torch.no_grad():
							snake_case__	     =model(_UpperCAmelCase					,   attention_mask=_UpperCAmelCase						)[0]
					snake_case__	     =torch.Size((1, 11, 768)						)
					self.assertEqual(output.shape					,   _UpperCAmelCase						)
					snake_case__	     =torch.tensor(
					    [[[-0.6_513, 1.5_035, -0.2_766], [-0.6_515, 1.5_046, -0.2_780], [-0.6_512, 1.5_049, -0.2_784]]]						)

					self.assertTrue(torch.allclose(output[:, 1:4, 1:4]					,   _UpperCAmelCase					,   atol=1E-4						)						)




