


'''simple docstring'''




from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


UpperCAmelCase_      :		List[str]          =		logging.get_logger(__name__)

UpperCAmelCase_      :		Tuple          =		{
    "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/config.json",
    # See all BlenderbotSmall models at https://huggingface.co/models?filter=blenderbot_small
}







class 					a			(				snake_case__  ):
 '''simple docstring'''




 __lowerCAmelCase						: Optional[Any]							  =		"""blenderbot-small"""
 __lowerCAmelCase						: Optional[Any]							  =		["""past_key_values"""]
 __lowerCAmelCase						: Optional[int]							  =		{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}




 def __init__(    self		,      lowerCamelCase_=5_0_2_6_5		,      lowerCamelCase_=5_1_2		,      lowerCamelCase_=8		,      lowerCamelCase_=2_0_4_8		,      lowerCamelCase_=1_6		,      lowerCamelCase_=8		,      lowerCamelCase_=2_0_4_8		,      lowerCamelCase_=1_6		,      lowerCamelCase_=0.0		,      lowerCamelCase_=0.0		,      lowerCamelCase_=True		,      lowerCamelCase_=True		,      lowerCamelCase_="gelu"		,      lowerCamelCase_=5_1_2		,      lowerCamelCase_=0.1		,      lowerCamelCase_=0.0		,      lowerCamelCase_=0.0		,      lowerCamelCase_=0.02		,      lowerCamelCase_=1		,      lowerCamelCase_=False		,      lowerCamelCase_=0		,      lowerCamelCase_=1		,      lowerCamelCase_=2		,      lowerCamelCase_=2		,      **lowerCamelCase_		,      )      ->    Optional[Any]:
    _a					:  Tuple						=  vocab_size
    _a					:  Tuple						=  max_position_embeddings
    _a					:  Dict						=  d_model
    _a					:  str						=  encoder_ffn_dim
    _a					:  List[str]						=  encoder_layers
    _a					:  Optional[int]						=  encoder_attention_heads
    _a					:  Union[str, Any]						=  decoder_ffn_dim
    _a					:  List[str]						=  decoder_layers
    _a					:  List[str]						=  decoder_attention_heads
    _a					:  str						=  dropout
    _a					:  Dict						=  attention_dropout
    _a					:  int						=  activation_dropout
    _a					:  Optional[int]						=  activation_function
    _a					:  Dict						=  init_std
    _a					:  int						=  encoder_layerdrop
    _a					:  Tuple						=  decoder_layerdrop
    _a					:  Optional[int]						=  use_cache
    _a					:  List[Any]						=  encoder_layers
    _a					:  Dict						=  scale_embedding  # scale factor will be sqrt(d_model) if True

    super().__init__(
        pad_token_id=lowerCamelCase_		,      bos_token_id=lowerCamelCase_		,      eos_token_id=lowerCamelCase_		,      is_encoder_decoder=lowerCamelCase_		,      decoder_start_token_id=lowerCamelCase_		,      forced_eos_token_id=lowerCamelCase_		,      **lowerCamelCase_		,      )







class 					a			(				snake_case__  ):
 '''simple docstring'''




 @property
 def   __UpperCamelCase   (    self      )      ->    Mapping[str, Mapping[int, str]]:
    if self.task in ["default", "seq2seq-lm"]:
       _a					:  List[str]						=  OrderedDict(
           [
               ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
               ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
           ]      )

       if self.use_past:
          _a					:  Any						=  {0: 'batch'}
          _a					:  List[str]						=  {0: 'batch', 1: 'past_decoder_sequence + sequence'}
       else:
          _a					:  Tuple						=  {0: 'batch', 1: 'decoder_sequence'}
          _a					:  int						=  {0: 'batch', 1: 'decoder_sequence'}

       if self.use_past:
          self.fill_with_past_key_values_(lowerCamelCase_		,      direction='inputs'      )
    elif self.task == "causal-lm":
       # TODO: figure this case out.
       _a					:  Any						=  OrderedDict(
           [
               ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
               ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
           ]      )
       if self.use_past:
          _a		,						_a					:  Optional[int]						=  self.num_layers
          for i in range(lowerCamelCase_      ):
             _a					:  Dict						=  {0: 'batch', 2: 'past_sequence + sequence'}
             _a					:  Union[str, Any]						=  {0: 'batch', 2: 'past_sequence + sequence'}
    else:
       _a					:  str						=  OrderedDict(
           [
               ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
               ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
               ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
               ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
           ]      )

    return common_inputs




 @property
 def   __UpperCamelCase   (    self      )      ->    Mapping[str, Mapping[int, str]]:
    if self.task in ["default", "seq2seq-lm"]:
       _a					:  str						=  super().outputs
    else:
       _a					:  Any						=  super(lowerCamelCase_		,      self      ).outputs
       if self.use_past:
          _a		,						_a					:  Tuple						=  self.num_layers
          for i in range(lowerCamelCase_      ):
             _a					:  Optional[Any]						=  {0: 'batch', 2: 'past_sequence + sequence'}
             _a					:  Any						=  {0: 'batch', 2: 'past_sequence + sequence'}
    return common_outputs




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_ = -1		,      lowerCamelCase_ = -1		,      lowerCamelCase_ = False		,      lowerCamelCase_ = None		,      )      ->    Mapping[str, Any]:
    _a					:  int						=  self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
        lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )

    # Generate decoder inputs
    _a					:  Tuple						=  seq_length if not self.use_past else 1
    _a					:  List[str]						=  self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
        lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )
    _a					:  int						=  {F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
    _a					:  Any						=  dict(**lowerCamelCase_		,      **lowerCamelCase_      )

    if self.use_past:
       if not is_torch_available():
          raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'      )
       else:
          import torch
       _a		,						_a					:  Dict						=  common_inputs['input_ids'].shape
       _a					:  Union[str, Any]						=  common_inputs['decoder_input_ids'].shape[1]
       _a		,						_a					:  List[Any]						=  self.num_attention_heads
       _a					:  Optional[Any]						=  (
           batch,
           num_encoder_attention_heads,
           encoder_seq_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )
       _a					:  int						=  decoder_seq_length + 3
       _a					:  int						=  (
           batch,
           num_decoder_attention_heads,
           decoder_past_length,
           self._config.hidden_size // num_decoder_attention_heads,
       )

       _a					:  List[str]						=  torch.cat(
           [common_inputs['decoder_attention_mask'], torch.ones(lowerCamelCase_		,      lowerCamelCase_      )]		,      dim=1      )

       _a					:  str						=  []
       # If the number of encoder and decoder layers are present in the model configuration, both are considered
       _a		,						_a					:  List[str]						=  self.num_layers
       _a					:  List[Any]						=  min(lowerCamelCase_		,      lowerCamelCase_      )
       _a					:  Union[str, Any]						=  max(lowerCamelCase_		,      lowerCamelCase_      ) - min_num_layers
       _a					:  List[str]						=  'encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

       for _ in range(lowerCamelCase_      ):
          common_inputs["past_key_values"].append(
              (
                  torch.zeros(lowerCamelCase_      ),
                  torch.zeros(lowerCamelCase_      ),
                  torch.zeros(lowerCamelCase_      ),
                  torch.zeros(lowerCamelCase_      ),
              )      )
       # TODO: test this.
       _a					:  Optional[int]						=  encoder_shape if remaining_side_name == 'encoder' else decoder_shape
       for _ in range(lowerCamelCase_		,      lowerCamelCase_      ):
          common_inputs["past_key_values"].append((torch.zeros(lowerCamelCase_      ), torch.zeros(lowerCamelCase_      ))      )
    return common_inputs




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_ = -1		,      lowerCamelCase_ = -1		,      lowerCamelCase_ = False		,      lowerCamelCase_ = None		,      )      ->    Mapping[str, Any]:
    _a					:  Optional[int]						=  self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
        lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )

    if self.use_past:
       if not is_torch_available():
          raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'      )
       else:
          import torch
       _a		,						_a					:  Optional[int]						=  common_inputs['input_ids'].shape
       # Not using the same length for past_key_values
       _a					:  Optional[Any]						=  seqlen + 2
       _a		,						_a					:  int						=  self.num_layers
       _a		,						_a					:  Tuple						=  self.num_attention_heads
       _a					:  str						=  (
           batch,
           num_encoder_attention_heads,
           past_key_values_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )

       _a					:  Tuple						=  common_inputs['attention_mask'].dtype
       _a					:  Dict						=  torch.cat(
           [common_inputs['attention_mask'], torch.ones(lowerCamelCase_		,      lowerCamelCase_		,      dtype=lowerCamelCase_      )]		,      dim=1      )
       _a					:  List[Any]						=  [
           (torch.zeros(lowerCamelCase_      ), torch.zeros(lowerCamelCase_      )) for _ in range(lowerCamelCase_      )
       ]
    return common_inputs




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_ = -1		,      lowerCamelCase_ = -1		,      lowerCamelCase_ = False		,      lowerCamelCase_ = None		,      )      ->    Mapping[str, Any]:
    # Copied from OnnxConfig.generate_dummy_inputs
    # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
    # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
    _a					:  int						=  compute_effective_axis_dimension(
        lowerCamelCase_		,      fixed_dimension=OnnxConfig.default_fixed_batch		,      num_token_to_add=0      )

    # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
    _a					:  List[str]						=  tokenizer.num_special_tokens_to_add(lowerCamelCase_      )
    _a					:  Optional[Any]						=  compute_effective_axis_dimension(
        lowerCamelCase_		,      fixed_dimension=OnnxConfig.default_fixed_sequence		,      num_token_to_add=lowerCamelCase_      )

    # Generate dummy inputs according to compute batch and sequence
    _a					:  Optional[int]						=  [' '.join([tokenizer.unk_token]      ) * seq_length] * batch_size
    _a					:  str						=  dict(tokenizer(lowerCamelCase_		,      return_tensors=lowerCamelCase_      )      )
    return common_inputs




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_ = -1		,      lowerCamelCase_ = -1		,      lowerCamelCase_ = False		,      lowerCamelCase_ = None		,      )      ->    Mapping[str, Any]:
    if self.task in ["default", "seq2seq-lm"]:
       _a					:  Optional[int]						=  self._generate_dummy_inputs_for_default_and_seqaseq_lm(
           lowerCamelCase_		,      batch_size=lowerCamelCase_		,      seq_length=lowerCamelCase_		,      is_pair=lowerCamelCase_		,      framework=lowerCamelCase_      )

    elif self.task == "causal-lm":
       _a					:  Union[str, Any]						=  self._generate_dummy_inputs_for_causal_lm(
           lowerCamelCase_		,      batch_size=lowerCamelCase_		,      seq_length=lowerCamelCase_		,      is_pair=lowerCamelCase_		,      framework=lowerCamelCase_      )
    else:
       _a					:  Optional[Any]						=  self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           lowerCamelCase_		,      batch_size=lowerCamelCase_		,      seq_length=lowerCamelCase_		,      is_pair=lowerCamelCase_		,      framework=lowerCamelCase_      )

    return common_inputs




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Optional[int]:
    if self.task in ["default", "seq2seq-lm"]:
       _a					:  Any						=  super()._flatten_past_key_values_(lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )
    else:
       _a					:  List[str]						=  super(lowerCamelCase_		,      self      )._flatten_past_key_values_(
           lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )






'''simple docstring'''




import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     AlbertForMaskedLM,
     AlbertForMultipleChoice,
     AlbertForPreTraining,
     AlbertForQuestionAnswering,
     AlbertForSequenceClassification,
     AlbertForTokenClassification,
     AlbertModel,
 )
 from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class 					a			:
 '''simple docstring'''




 def __init__(    self		,      lowerCamelCase_		,      lowerCamelCase_=1_3		,      lowerCamelCase_=7		,      lowerCamelCase_=True		,      lowerCamelCase_=True		,      lowerCamelCase_=True		,      lowerCamelCase_=True		,      lowerCamelCase_=9_9		,      lowerCamelCase_=1_6		,      lowerCamelCase_=3_6		,      lowerCamelCase_=6		,      lowerCamelCase_=6		,      lowerCamelCase_=6		,      lowerCamelCase_=3_7		,      lowerCamelCase_="gelu"		,      lowerCamelCase_=0.1		,      lowerCamelCase_=0.1		,      lowerCamelCase_=5_1_2		,      lowerCamelCase_=1_6		,      lowerCamelCase_=2		,      lowerCamelCase_=0.02		,      lowerCamelCase_=3		,      lowerCamelCase_=4		,      lowerCamelCase_=None		,      )      ->    Dict:
    _a					:  Dict						=  parent
    _a					:  Tuple						=  batch_size
    _a					:  Optional[int]						=  seq_length
    _a					:  List[str]						=  is_training
    _a					:  Tuple						=  use_input_mask
    _a					:  Dict						=  use_token_type_ids
    _a					:  List[str]						=  use_labels
    _a					:  Optional[int]						=  vocab_size
    _a					:  int						=  embedding_size
    _a					:  Tuple						=  hidden_size
    _a					:  str						=  num_hidden_layers
    _a					:  List[str]						=  num_hidden_groups
    _a					:  Union[str, Any]						=  num_attention_heads
    _a					:  Dict						=  intermediate_size
    _a					:  List[str]						=  hidden_act
    _a					:  Tuple						=  hidden_dropout_prob
    _a					:  Any						=  attention_probs_dropout_prob
    _a					:  Tuple						=  max_position_embeddings
    _a					:  Dict						=  type_vocab_size
    _a					:  Tuple						=  type_sequence_label_size
    _a					:  Dict						=  initializer_range
    _a					:  str						=  num_labels
    _a					:  Optional[Any]						=  num_choices
    _a					:  Optional[int]						=  scope




 def   __UpperCamelCase   (    self      )      ->    Optional[int]:
    _a					:  Optional[int]						=  ids_tensor([self.batch_size, self.seq_length]		,      self.vocab_size      )

    _a					:  List[str]						=  None
    if self.use_input_mask:
       _a					:  Tuple						=  random_attention_mask([self.batch_size, self.seq_length]      )

    _a					:  Dict						=  None
    if self.use_token_type_ids:
       _a					:  Any						=  ids_tensor([self.batch_size, self.seq_length]		,      self.type_vocab_size      )

    _a					:  List[str]						=  None
    _a					:  Optional[int]						=  None
    _a					:  List[Any]						=  None
    if self.use_labels:
       _a					:  Optional[int]						=  ids_tensor([self.batch_size]		,      self.type_sequence_label_size      )
       _a					:  Any						=  ids_tensor([self.batch_size, self.seq_length]		,      self.num_labels      )
       _a					:  str						=  ids_tensor([self.batch_size]		,      self.num_choices      )

    _a					:  int						=  self.get_config()

    return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




 def   __UpperCamelCase   (    self      )      ->    Any:
    return AlbertConfig(
        vocab_size=self.vocab_size		,      hidden_size=self.hidden_size		,      num_hidden_layers=self.num_hidden_layers		,      num_attention_heads=self.num_attention_heads		,      intermediate_size=self.intermediate_size		,      hidden_act=self.hidden_act		,      hidden_dropout_prob=self.hidden_dropout_prob		,      attention_probs_dropout_prob=self.attention_probs_dropout_prob		,      max_position_embeddings=self.max_position_embeddings		,      type_vocab_size=self.type_vocab_size		,      initializer_range=self.initializer_range		,      num_hidden_groups=self.num_hidden_groups		,      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    List[str]:
    _a					:  str						=  AlbertModel(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  Tuple						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_      )
    _a					:  Dict						=  model(lowerCamelCase_		,      token_type_ids=lowerCamelCase_      )
    _a					:  List[str]						=  model(lowerCamelCase_      )
    self.parent.assertEqual(result.last_hidden_state.shape		,      (self.batch_size, self.seq_length, self.hidden_size)      )
    self.parent.assertEqual(result.pooler_output.shape		,      (self.batch_size, self.hidden_size)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Any:
    _a					:  Optional[int]						=  AlbertForPreTraining(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  Optional[Any]						=  model(
        lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_		,      sentence_order_label=lowerCamelCase_		,      )
    self.parent.assertEqual(result.prediction_logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size)      )
    self.parent.assertEqual(result.sop_logits.shape		,      (self.batch_size, config.num_labels)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Tuple:
    _a					:  Union[str, Any]						=  AlbertForMaskedLM(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  int						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_      )
    self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Tuple:
    _a					:  Union[str, Any]						=  AlbertForQuestionAnswering(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  Tuple						=  model(
        lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      start_positions=lowerCamelCase_		,      end_positions=lowerCamelCase_		,      )
    self.parent.assertEqual(result.start_logits.shape		,      (self.batch_size, self.seq_length)      )
    self.parent.assertEqual(result.end_logits.shape		,      (self.batch_size, self.seq_length)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    int:
    _a					:  str						=  self.num_labels
    _a					:  str						=  AlbertForSequenceClassification(lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  str						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_      )
    self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_labels)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Optional[int]:
    _a					:  Optional[int]						=  self.num_labels
    _a					:  Any						=  AlbertForTokenClassification(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  List[Any]						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_      )
    self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.num_labels)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Union[str, Any]:
    _a					:  Optional[int]						=  self.num_choices
    _a					:  Any						=  AlbertForMultipleChoice(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  str						=  input_ids.unsqueeze(1      ).expand(-1		,      self.num_choices		,      -1      ).contiguous()
    _a					:  Dict						=  token_type_ids.unsqueeze(1      ).expand(-1		,      self.num_choices		,      -1      ).contiguous()
    _a					:  List[Any]						=  input_mask.unsqueeze(1      ).expand(-1		,      self.num_choices		,      -1      ).contiguous()
    _a					:  Optional[Any]						=  model(
        lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_		,      )
    self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_choices)      )




 def   __UpperCamelCase   (    self      )      ->    Optional[Any]:
    _a					:  Optional[int]						=  self.prepare_config_and_inputs()
    (
        (
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						
    )					:  Optional[int]						=  config_and_inputs
    _a					:  Optional[Any]						=  {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
    return config, inputs_dict







@require_torch
class 					a			(				snake_case__			,							snake_case__			,							unittest.TestCase  ):
 '''simple docstring'''




 __lowerCAmelCase						: str							  =		(
     (
         AlbertModel,
         AlbertForPreTraining,
         AlbertForMaskedLM,
         AlbertForMultipleChoice,
         AlbertForSequenceClassification,
         AlbertForTokenClassification,
         AlbertForQuestionAnswering,
     )
     if is_torch_available()
     else ()
 )
 __lowerCAmelCase						: List[Any]							  =		(
     {
         """feature-extraction""": AlbertModel,
         """fill-mask""": AlbertForMaskedLM,
         """question-answering""": AlbertForQuestionAnswering,
         """text-classification""": AlbertForSequenceClassification,
         """token-classification""": AlbertForTokenClassification,
         """zero-shot""": AlbertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 __lowerCAmelCase						: Optional[int]							  =		True




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_=False      )      ->    List[str]:
    _a					:  Optional[Any]						=  super()._prepare_for_class(lowerCamelCase_		,      lowerCamelCase_		,      return_labels=lowerCamelCase_      )

    if return_labels:
       if model_class in get_values(lowerCamelCase_      ):
          _a					:  str						=  torch.zeros(
              (self.model_tester.batch_size, self.model_tester.seq_length)		,      dtype=torch.long		,      device=lowerCamelCase_      )
          _a					:  int						=  torch.zeros(
              self.model_tester.batch_size		,      dtype=torch.long		,      device=lowerCamelCase_      )
    return inputs_dict




 def   __UpperCamelCase   (    self      )      ->    List[Any]:
    _a					:  int						=  AlbertModelTester(self      )
    _a					:  Union[str, Any]						=  ConfigTester(self		,      config_class=lowerCamelCase_		,      hidden_size=3_7      )




 def   __UpperCamelCase   (    self      )      ->    Tuple:
    self.config_tester.run_common_tests()




 def   __UpperCamelCase   (    self      )      ->    Optional[int]:
    _a					:  Tuple						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Tuple:
    _a					:  str						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_pretraining(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Any:
    _a					:  Optional[int]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Union[str, Any]:
    _a					:  Any						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Tuple:
    _a					:  Union[str, Any]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Union[str, Any]:
    _a					:  Optional[int]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Dict:
    _a					:  str						=  self.model_tester.prepare_config_and_inputs()
    for type in ["absolute", "relative_key", "relative_key_query"]:
       _a					:  Optional[Any]						=  type
       self.model_tester.create_and_check_model(*lowerCamelCase_      )




 @slow
 def   __UpperCamelCase   (    self      )      ->    Optional[Any]:
    for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       _a					:  int						=  AlbertModel.from_pretrained(lowerCamelCase_      )
       self.assertIsNotNone(lowerCamelCase_      )







@require_torch
class 					a			(				unittest.TestCase  ):
 '''simple docstring'''




 @slow
 def   __UpperCamelCase   (    self      )      ->    Union[str, Any]:
    _a					:  Dict						=  AlbertModel.from_pretrained('albert-base-v2'      )
    _a					:  Optional[Any]						=  torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]      )
    _a					:  List[str]						=  torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
    with torch.no_grad():
       _a					:  int						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_      )[0]
    _a					:  str						=  torch.Size((1, 1_1, 7_6_8)      )
    self.assertEqual(output.shape		,      lowerCamelCase_      )
    _a					:  int						=  torch.tensor(
        [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]      )

    self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,      lowerCamelCase_		,      atol=1e-4      )      )


