





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           AlbertForMaskedLM,
           AlbertForMultipleChoice,
           AlbertForPreTraining,
           AlbertForQuestionAnswering,
           AlbertForSequenceClassification,
           AlbertForTokenClassification,
           AlbertModel,
       )
       from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class     SCREAMING_SNAKE_CASE       :


   def __init__(   self      :    Optional[Any]      ,					a      :    Dict      ,					a      :    Optional[int]=13      ,					a      :    List[str]=7      ,					a      :    int=True      ,					a      :    List[Any]=True      ,					a      :    List[Any]=True      ,					a      :    Tuple=True      ,					a      :    Union[str, Any]=99      ,					a      :    str=16      ,					a      :    Union[str, Any]=36      ,					a      :    Optional[Any]=6      ,					a      :    Dict=6      ,					a      :    Optional[int]=6      ,					a      :    int=37      ,					a      :    str="gelu"      ,					a      :    str=0.1      ,					a      :    List[str]=0.1      ,					a      :    Optional[int]=512      ,					a      :    Optional[Any]=16      ,					a      :    Union[str, Any]=2      ,					a      :    Union[str, Any]=0.02      ,					a      :    Dict=3      ,					a      :    Optional[Any]=4      ,					a      :    List[str]=None      ,					)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  parent
     lowercase__           =  batch_size
     lowercase__           =  seq_length
     lowercase__           =  is_training
     lowercase__           =  use_input_mask
     lowercase__           =  use_token_type_ids
     lowercase__           =  use_labels
     lowercase__           =  vocab_size
     lowercase__           =  embedding_size
     lowercase__           =  hidden_size
     lowercase__           =  num_hidden_layers
     lowercase__           =  num_hidden_groups
     lowercase__           =  num_attention_heads
     lowercase__           =  intermediate_size
     lowercase__           =  hidden_act
     lowercase__           =  hidden_dropout_prob
     lowercase__           =  attention_probs_dropout_prob
     lowercase__           =  max_position_embeddings
     lowercase__           =  type_vocab_size
     lowercase__           =  type_sequence_label_size
     lowercase__           =  initializer_range
     lowercase__           =  num_labels
     lowercase__           =  num_choices
     lowercase__           =  scope


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  ids_tensor([self.batch_size, self.seq_length]      ,					self.vocab_size	)

     lowercase__           =  None
     if self.use_input_mask:
       lowercase__           =  random_attention_mask([self.batch_size, self.seq_length]	)

     lowercase__           =  None
     if self.use_token_type_ids:
       lowercase__           =  ids_tensor([self.batch_size, self.seq_length]      ,					self.type_vocab_size	)

     lowercase__           =  None
     lowercase__           =  None
     lowercase__           =  None
     if self.use_labels:
       lowercase__           =  ids_tensor([self.batch_size]      ,					self.type_sequence_label_size	)
       lowercase__           =  ids_tensor([self.batch_size, self.seq_length]      ,					self.num_labels	)
       lowercase__           =  ids_tensor([self.batch_size]      ,					self.num_choices	)

     lowercase__           =  self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict	)->       int:





     """simple docstring"""







     return AlbertConfig(
         vocab_size=self.vocab_size      ,					hidden_size=self.hidden_size      ,					num_hidden_layers=self.num_hidden_layers      ,					num_attention_heads=self.num_attention_heads      ,					intermediate_size=self.intermediate_size      ,					hidden_act=self.hidden_act      ,					hidden_dropout_prob=self.hidden_dropout_prob      ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,					max_position_embeddings=self.max_position_embeddings      ,					type_vocab_size=self.type_vocab_size      ,					initializer_range=self.initializer_range      ,					num_hidden_groups=self.num_hidden_groups      ,					)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int      ,					a      :    str      ,					a      :    List[Any]      ,					a      :    Optional[Any]      ,					a      :    Optional[int]      ,					a      :    Tuple      ,					a      :    int      ,					a      :    Union[str, Any]	)->       Any:





     """simple docstring"""







     lowercase__           =  AlbertModel(config=_snake_case	)
     model.to(_snake_case	)
     model.eval()
     lowercase__           =  model(_snake_case      ,					attention_mask=_snake_case      ,					token_type_ids=_snake_case	)
     lowercase__           =  model(_snake_case      ,					token_type_ids=_snake_case	)
     lowercase__           =  model(_snake_case	)
     self.parent.assertEqual(result.last_hidden_state.shape      ,					(self.batch_size, self.seq_length, self.hidden_size)	)
     self.parent.assertEqual(result.pooler_output.shape      ,					(self.batch_size, self.hidden_size)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    Optional[Any]      ,					a      :    Any      ,					a      :    str      ,					a      :    Tuple      ,					a      :    List[Any]      ,					a      :    Union[str, Any]      ,					a      :    List[str]	)->       str:





     """simple docstring"""







     lowercase__           =  AlbertForPreTraining(config=_snake_case	)
     model.to(_snake_case	)
     model.eval()
     lowercase__           =  model(
         _snake_case      ,					attention_mask=_snake_case      ,					token_type_ids=_snake_case      ,					labels=_snake_case      ,					sentence_order_label=_snake_case      ,					)
     self.parent.assertEqual(result.prediction_logits.shape      ,					(self.batch_size, self.seq_length, self.vocab_size)	)
     self.parent.assertEqual(result.sop_logits.shape      ,					(self.batch_size, config.num_labels)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict      ,					a      :    Union[str, Any]      ,					a      :    Union[str, Any]      ,					a      :    Optional[int]      ,					a      :    str      ,					a      :    Tuple      ,					a      :    Any      ,					a      :    Tuple	)->       Dict:





     """simple docstring"""







     lowercase__           =  AlbertForMaskedLM(config=_snake_case	)
     model.to(_snake_case	)
     model.eval()
     lowercase__           =  model(_snake_case      ,					attention_mask=_snake_case      ,					token_type_ids=_snake_case      ,					labels=_snake_case	)
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.seq_length, self.vocab_size)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]      ,					a      :    Union[str, Any]      ,					a      :    List[Any]      ,					a      :    Tuple      ,					a      :    str      ,					a      :    Optional[int]      ,					a      :    Tuple      ,					a      :    List[Any]	)->       Union[str, Any]:





     """simple docstring"""







     lowercase__           =  AlbertForQuestionAnswering(config=_snake_case	)
     model.to(_snake_case	)
     model.eval()
     lowercase__           =  model(
         _snake_case      ,					attention_mask=_snake_case      ,					token_type_ids=_snake_case      ,					start_positions=_snake_case      ,					end_positions=_snake_case      ,					)
     self.parent.assertEqual(result.start_logits.shape      ,					(self.batch_size, self.seq_length)	)
     self.parent.assertEqual(result.end_logits.shape      ,					(self.batch_size, self.seq_length)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]      ,					a      :    Any      ,					a      :    Optional[Any]      ,					a      :    List[str]      ,					a      :    Optional[Any]      ,					a      :    int      ,					a      :    Union[str, Any]      ,					a      :    Optional[int]	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  self.num_labels
     lowercase__           =  AlbertForSequenceClassification(_snake_case	)
     model.to(_snake_case	)
     model.eval()
     lowercase__           =  model(_snake_case      ,					attention_mask=_snake_case      ,					token_type_ids=_snake_case      ,					labels=_snake_case	)
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.num_labels)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    str      ,					a      :    Optional[int]      ,					a      :    Optional[int]      ,					a      :    Dict      ,					a      :    Union[str, Any]      ,					a      :    Tuple      ,					a      :    Union[str, Any]      ,					a      :    Optional[int]	)->       Optional[Any]:





     """simple docstring"""







     lowercase__           =  self.num_labels
     lowercase__           =  AlbertForTokenClassification(config=_snake_case	)
     model.to(_snake_case	)
     model.eval()
     lowercase__           =  model(_snake_case      ,					attention_mask=_snake_case      ,					token_type_ids=_snake_case      ,					labels=_snake_case	)
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.seq_length, self.num_labels)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    str      ,					a      :    Optional[Any]      ,					a      :    Tuple      ,					a      :    Any      ,					a      :    int      ,					a      :    List[str]      ,					a      :    List[Any]      ,					a      :    Tuple	)->       Any:





     """simple docstring"""







     lowercase__           =  self.num_choices
     lowercase__           =  AlbertForMultipleChoice(config=_snake_case	)
     model.to(_snake_case	)
     model.eval()
     lowercase__           =  input_ids.unsqueeze(1	).expand(-1      ,					self.num_choices      ,					-1	).contiguous()
     lowercase__           =  token_type_ids.unsqueeze(1	).expand(-1      ,					self.num_choices      ,					-1	).contiguous()
     lowercase__           =  input_mask.unsqueeze(1	).expand(-1      ,					self.num_choices      ,					-1	).contiguous()
     lowercase__           =  model(
         _snake_case      ,					attention_mask=_snake_case      ,					token_type_ids=_snake_case      ,					labels=_snake_case      ,					)
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.num_choices)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict	)->       int:





     """simple docstring"""







     lowercase__           =  self.prepare_config_and_inputs()
     (
         (
         lowercase__					
     )      ,   (
         lowercase__					
     )      ,   (
         lowercase__					
     )      ,   (
         lowercase__					
     )      ,   (
         lowercase__					
     )      ,   (
         lowercase__					
     )      ,   (
         lowercase__					
     )      ,   					
     )           =  config_and_inputs
     lowercase__           =  {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
     return config, inputs_dict






@require_torch
class     SCREAMING_SNAKE_CASE       (UpperCAmelCase      ,     UpperCAmelCase      ,     unittest.TestCase				):
   _UpperCamelCase   :		List[Any]			 =	(
       (
           AlbertModel,
           AlbertForPreTraining,
           AlbertForMaskedLM,
           AlbertForMultipleChoice,
           AlbertForSequenceClassification,
           AlbertForTokenClassification,
           AlbertForQuestionAnswering,
       )
       if is_torch_available()
       else ()
   )
   _UpperCamelCase   :		Any			 =	(
       {
           'feature-extraction': AlbertModel,
           'fill-mask': AlbertForMaskedLM,
           'question-answering': AlbertForQuestionAnswering,
           'text-classification': AlbertForSequenceClassification,
           'token-classification': AlbertForTokenClassification,
           'zero-shot': AlbertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   _UpperCamelCase   :		int			 =	True


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Union[str, Any]      ,					a      :    Any      ,					a      :    List[str]      ,					a      :    int=False	)->       Optional[Any]:





     """simple docstring"""







     lowercase__           =  super()._prepare_for_class(_snake_case      ,					_snake_case      ,					return_labels=_snake_case	)

     if return_labels:
       if model_class in get_values(_snake_case	):
         lowercase__           =  torch.zeros(
             (self.model_tester.batch_size, self.model_tester.seq_length)      ,					dtype=torch.long      ,					device=_snake_case	)
         lowercase__           =  torch.zeros(
             self.model_tester.batch_size      ,					dtype=torch.long      ,					device=_snake_case	)
     return inputs_dict


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple	)->       Dict:





     """simple docstring"""







     lowercase__           =  AlbertModelTester(self	)
     lowercase__           =  ConfigTester(self      ,					config_class=_snake_case      ,					hidden_size=37	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any	)->       Tuple:





     """simple docstring"""







     self.config_tester.run_common_tests()


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any	)->       Tuple:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*_snake_case	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       int:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_pretraining(*_snake_case	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       Union[str, Any]:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*_snake_case	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*_snake_case	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]	)->       Dict:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*_snake_case	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]	)->       Dict:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*_snake_case	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]	)->       Optional[Any]:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     for type in ["absolute", "relative_key", "relative_key_query"]:
       lowercase__           =  type
       self.model_tester.create_and_check_model(*_snake_case	)


   @slow
   def 	SCREAMING_SNAKE_CASE_							(   self      :    str	)->       List[str]:





     """simple docstring"""







     for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       lowercase__           =  AlbertModel.from_pretrained(_snake_case	)
       self.assertIsNotNone(_snake_case	)






@require_torch
class     SCREAMING_SNAKE_CASE       (unittest.TestCase				):


   @slow
   def 	SCREAMING_SNAKE_CASE_							(   self      :    str	)->       Union[str, Any]:





     """simple docstring"""







     lowercase__           =  AlbertModel.from_pretrained('albert-base-v2'	)
     lowercase__           =  torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]	)
     lowercase__           =  torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
     with torch.no_grad():
       lowercase__           =  model(_snake_case      ,					attention_mask=_snake_case	)[0]
     lowercase__           =  torch.Size((1, 11, 768)	)
     self.assertEqual(output.shape      ,					_snake_case	)
     lowercase__           =  torch.tensor(
         [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]	)

     self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,					_snake_case      ,					atol=1E-4	)	)









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_          =		logging.get_logger(__name__)

lowercase_          =		{
    """transfo-xl-wt103""": """https://huggingface.co/transfo-xl-wt103/resolve/main/config.json""",
}
class     SCREAMING_SNAKE_CASE       (UpperCAmelCase				):
   _UpperCamelCase   :		Optional[Any]			 =	'transfo-xl'
   _UpperCamelCase   :		Any			 =	['mems']
   _UpperCamelCase   :		Any			 =	{
       'n_token': 'vocab_size',
       'hidden_size': 'd_model',
       'num_attention_heads': 'n_head',
       'num_hidden_layers': 'n_layer',
   }


   def __init__(   self      :    Optional[Any]      ,					a      :    Optional[int]=267_735      ,					a      :    str=[20_000, 40_000, 200_000]      ,					a      :    str=1_024      ,					a      :    str=1_024      ,					a      :    int=16      ,					a      :    Optional[int]=64      ,					a      :    Optional[int]=4_096      ,					a      :    int=4      ,					a      :    Tuple=False      ,					a      :    Any=18      ,					a      :    Tuple=1_600      ,					a      :    Union[str, Any]=1_000      ,					a      :    str=True      ,					a      :    Dict=True      ,					a      :    Any=0      ,					a      :    List[Any]=-1      ,					a      :    List[Any]=True      ,					a      :    Tuple=0.1      ,					a      :    List[Any]=0.0      ,					a      :    Optional[Any]=True      ,					a      :    int="normal"      ,					a      :    Optional[Any]=0.01      ,					a      :    str=0.01      ,					a      :    List[Any]=0.02      ,					a      :    List[Any]=1E-5      ,					a      :    Optional[Any]=0      ,					**a      :    Optional[int]      ,					)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  vocab_size
     lowercase__           =  []
     self.cutoffs.extend(a	)
     if proj_share_all_but_first:
       lowercase__           =  [False] + [True] * len(self.cutoffs	)
     else:
       lowercase__           =  [False] + [False] * len(self.cutoffs	)
     lowercase__           =  d_model
     lowercase__           =  d_embed
     lowercase__           =  d_head
     lowercase__           =  d_inner
     lowercase__           =  div_val
     lowercase__           =  pre_lnorm
     lowercase__           =  n_layer
     lowercase__           =  n_head
     lowercase__           =  mem_len
     lowercase__           =  same_length
     lowercase__           =  attn_type
     lowercase__           =  clamp_len
     lowercase__           =  sample_softmax
     lowercase__           =  adaptive
     lowercase__           =  dropout
     lowercase__           =  dropatt
     lowercase__           =  untie_r
     lowercase__           =  init
     lowercase__           =  init_range
     lowercase__           =  proj_init_std
     lowercase__           =  init_std
     lowercase__           =  layer_norm_epsilon
     super().__init__(eos_token_id=a      ,					**a	)


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]	)->       Union[str, Any]:





     """simple docstring"""







     logger.info(f"""The model {self.model_type} is one of the few models that has no sequence length limit."""	)
     return -1


   @max_position_embeddings.setter
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    Optional[int]	)->       Optional[int]:





     """simple docstring"""







     raise NotImplementedError(
         f"""The model {self.model_type} is one of the few models that has no sequence length limit."""	)


