





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    AlbertForMaskedLM,
							    AlbertForMultipleChoice,
							    AlbertForPreTraining,
							    AlbertForQuestionAnswering,
							    AlbertForSequenceClassification,
							    AlbertForTokenClassification,
							    AlbertModel,
							)
							from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST





class 				a			:






				def __init__(  self			, _lowerCAmelCase			, _lowerCAmelCase=13			, _lowerCAmelCase=7			, _lowerCAmelCase=True			, _lowerCAmelCase=True			, _lowerCAmelCase=True			, _lowerCAmelCase=True			, _lowerCAmelCase=99			, _lowerCAmelCase=16			, _lowerCAmelCase=36			, _lowerCAmelCase=6			, _lowerCAmelCase=6			, _lowerCAmelCase=6			, _lowerCAmelCase=37			, _lowerCAmelCase="gelu"			, _lowerCAmelCase=0.1			, _lowerCAmelCase=0.1			, _lowerCAmelCase=512			, _lowerCAmelCase=16			, _lowerCAmelCase=2			, _lowerCAmelCase=0.02			, _lowerCAmelCase=3			, _lowerCAmelCase=4			, _lowerCAmelCase=None			, ):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   parent
										__SCREAMING_SNAKE_CASE:				Tuple           =   batch_size
										__SCREAMING_SNAKE_CASE:				List[str]           =   seq_length
										__SCREAMING_SNAKE_CASE:				Tuple           =   is_training
										__SCREAMING_SNAKE_CASE:				int           =   use_input_mask
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   use_token_type_ids
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   use_labels
										__SCREAMING_SNAKE_CASE:				List[str]           =   vocab_size
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   embedding_size
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   hidden_size
										__SCREAMING_SNAKE_CASE:				int           =   num_hidden_layers
										__SCREAMING_SNAKE_CASE:				int           =   num_hidden_groups
										__SCREAMING_SNAKE_CASE:				List[Any]           =   num_attention_heads
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   intermediate_size
										__SCREAMING_SNAKE_CASE:				Dict           =   hidden_act
										__SCREAMING_SNAKE_CASE:				Dict           =   hidden_dropout_prob
										__SCREAMING_SNAKE_CASE:				Dict           =   attention_probs_dropout_prob
										__SCREAMING_SNAKE_CASE:				Dict           =   max_position_embeddings
										__SCREAMING_SNAKE_CASE:				int           =   type_vocab_size
										__SCREAMING_SNAKE_CASE:				Any           =   type_sequence_label_size
										__SCREAMING_SNAKE_CASE:				int           =   initializer_range
										__SCREAMING_SNAKE_CASE:				List[str]           =   num_labels
										__SCREAMING_SNAKE_CASE:				List[str]           =   num_choices
										__SCREAMING_SNAKE_CASE:				Any           =   scope






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   ids_tensor([self.batch_size, self.seq_length]			, self.vocab_size					)

										__SCREAMING_SNAKE_CASE:				Tuple           =   None
										if self.use_input_mask:
																__SCREAMING_SNAKE_CASE:				Dict           =   random_attention_mask([self.batch_size, self.seq_length]					)

										__SCREAMING_SNAKE_CASE:				int           =   None
										if self.use_token_type_ids:
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   ids_tensor([self.batch_size, self.seq_length]			, self.type_vocab_size					)

										__SCREAMING_SNAKE_CASE:				str           =   None
										__SCREAMING_SNAKE_CASE:				List[str]           =   None
										__SCREAMING_SNAKE_CASE:				int           =   None
										if self.use_labels:
																__SCREAMING_SNAKE_CASE:				List[Any]           =   ids_tensor([self.batch_size]			, self.type_sequence_label_size					)
																__SCREAMING_SNAKE_CASE:				int           =   ids_tensor([self.batch_size, self.seq_length]			, self.num_labels					)
																__SCREAMING_SNAKE_CASE:				List[Any]           =   ids_tensor([self.batch_size]			, self.num_choices					)

										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   self.get_config()

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






				def 			snake_case_    (  self					):

										"""simple docstring"""




										return AlbertConfig(
										    vocab_size=self.vocab_size			, hidden_size=self.hidden_size			, num_hidden_layers=self.num_hidden_layers			, num_attention_heads=self.num_attention_heads			, intermediate_size=self.intermediate_size			, hidden_act=self.hidden_act			, hidden_dropout_prob=self.hidden_dropout_prob			, attention_probs_dropout_prob=self.attention_probs_dropout_prob			, max_position_embeddings=self.max_position_embeddings			, type_vocab_size=self.type_vocab_size			, initializer_range=self.initializer_range			, num_hidden_groups=self.num_hidden_groups			, )






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   AlbertModel(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Dict           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   model(_lowerCAmelCase			, token_type_ids=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Dict           =   model(_lowerCAmelCase					)
										self.parent.assertEqual(result.last_hidden_state.shape			, (self.batch_size, self.seq_length, self.hidden_size)					)
										self.parent.assertEqual(result.pooler_output.shape			, (self.batch_size, self.hidden_size)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   AlbertForPreTraining(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   model(
										    _lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase			, sentence_order_label=_lowerCAmelCase			, )
										self.parent.assertEqual(result.prediction_logits.shape			, (self.batch_size, self.seq_length, self.vocab_size)					)
										self.parent.assertEqual(result.sop_logits.shape			, (self.batch_size, config.num_labels)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   AlbertForMaskedLM(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				str           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase					)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.vocab_size)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   AlbertForQuestionAnswering(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Dict           =   model(
										    _lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, start_positions=_lowerCAmelCase			, end_positions=_lowerCAmelCase			, )
										self.parent.assertEqual(result.start_logits.shape			, (self.batch_size, self.seq_length)					)
										self.parent.assertEqual(result.end_logits.shape			, (self.batch_size, self.seq_length)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   self.num_labels
										__SCREAMING_SNAKE_CASE:				Any           =   AlbertForSequenceClassification(_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Tuple           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase					)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.num_labels)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   self.num_labels
										__SCREAMING_SNAKE_CASE:				Tuple           =   AlbertForTokenClassification(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				str           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase					)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.num_labels)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   self.num_choices
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   AlbertForMultipleChoice(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				List[str]           =   input_ids.unsqueeze(1					).expand(-1			, self.num_choices			, -1					).contiguous()
										__SCREAMING_SNAKE_CASE:				Any           =   token_type_ids.unsqueeze(1					).expand(-1			, self.num_choices			, -1					).contiguous()
										__SCREAMING_SNAKE_CASE:				Dict           =   input_mask.unsqueeze(1					).expand(-1			, self.num_choices			, -1					).contiguous()
										__SCREAMING_SNAKE_CASE:				Tuple           =   model(
										    _lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase			, )
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.num_choices)					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   self.prepare_config_and_inputs()
										(
										    (
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,
										):				int           =   config_and_inputs
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
										return config, inputs_dict





@require_torch
class 				a			(      __lowercase   ,__lowercase   ,unittest.TestCase			):
				SCREAMING_SNAKE_CASE__     :   Union[str, Any]		      =     (
				    (
				        AlbertModel,
				        AlbertForPreTraining,
				        AlbertForMaskedLM,
				        AlbertForMultipleChoice,
				        AlbertForSequenceClassification,
				        AlbertForTokenClassification,
				        AlbertForQuestionAnswering,
				    )
				    if is_torch_available()
				    else ()
				)
				SCREAMING_SNAKE_CASE__     :   Tuple		      =     (
				    {
				        '''feature-extraction''': AlbertModel,
				        '''fill-mask''': AlbertForMaskedLM,
				        '''question-answering''': AlbertForQuestionAnswering,
				        '''text-classification''': AlbertForSequenceClassification,
				        '''token-classification''': AlbertForTokenClassification,
				        '''zero-shot''': AlbertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				SCREAMING_SNAKE_CASE__     :   Dict		      =     True






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase=False					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   super()._prepare_for_class(_lowerCAmelCase			, _lowerCAmelCase			, return_labels=_lowerCAmelCase					)

										if return_labels:
																if model_class in get_values(_lowerCAmelCase					):
																						__SCREAMING_SNAKE_CASE:				List[str]           =   torch.zeros(
																						    (self.model_tester.batch_size, self.model_tester.seq_length)			, dtype=torch.long			, device=_lowerCAmelCase					)
																						__SCREAMING_SNAKE_CASE:				Dict           =   torch.zeros(
																						    self.model_tester.batch_size			, dtype=torch.long			, device=_lowerCAmelCase					)
										return inputs_dict






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   AlbertModelTester(self					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   ConfigTester(self			, config_class=_lowerCAmelCase			, hidden_size=37					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										self.config_tester.run_common_tests()






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_pretraining(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
																__SCREAMING_SNAKE_CASE:				str           =   type
																self.model_tester.create_and_check_model(*_lowerCAmelCase					)






				@slow
				def 			snake_case_    (  self					):

										"""simple docstring"""




										for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   AlbertModel.from_pretrained(_lowerCAmelCase					)
																self.assertIsNotNone(_lowerCAmelCase					)





@require_torch
class 				a			(      unittest.TestCase			):






				@slow
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   AlbertModel.from_pretrained('''albert-base-v2'''					)
										__SCREAMING_SNAKE_CASE:				Any           =   torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]					)
										__SCREAMING_SNAKE_CASE:				Dict           =   torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]					)
										with torch.no_grad():
																__SCREAMING_SNAKE_CASE:				Dict           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase					)[0]
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   torch.Size((1, 11, 768)					)
										self.assertEqual(output.shape			, _lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Tuple           =   torch.tensor(
										    [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]					)

										self.assertTrue(torch.allclose(output[:, 1:4, 1:4]			, _lowerCAmelCase			, atol=1e-4					)					)










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCAmelCase							:       str					 =	{
    """configuration_pegasus_x""": ["""PEGASUS_X_PRETRAINED_CONFIG_ARCHIVE_MAP""", """PegasusXConfig"""],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase							:       Optional[Any]					 =	[
							    """PEGASUS_X_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """PegasusXForConditionalGeneration""",
							    """PegasusXModel""",
							    """PegasusXPreTrainedModel""",
							]


if TYPE_CHECKING:
							from .configuration_pegasus_x import PEGASUS_X_PRETRAINED_CONFIG_ARCHIVE_MAP, PegasusXConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_pegasus_x import (
														    PEGASUS_X_PRETRAINED_MODEL_ARCHIVE_LIST,
														    PegasusXForConditionalGeneration,
														    PegasusXModel,
														    PegasusXPreTrainedModel,
														)


else:
							import sys

							lowerCAmelCase							:       str					 =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



