






'''simple docstring'''


import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_PRETRAINING_MAPPING,
         NezhaForMaskedLM,
         NezhaForMultipleChoice,
         NezhaForNextSentencePrediction,
         NezhaForPreTraining,
         NezhaForQuestionAnswering,
         NezhaForSequenceClassification,
         NezhaForTokenClassification,
         NezhaModel,
     )
     from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST
class    _SCREAMING_SNAKE_CASE							:


   '''simple docstring'''





   def __init__(   self							:						int ,      lowercase							:						Any ,      lowercase							:						Optional[Any]=1_3 ,      lowercase							:						Union[str, Any]=7 ,      lowercase							:						Tuple=True ,      lowercase							:						Optional[int]=True ,      lowercase							:						List[Any]=True ,      lowercase							:						Dict=True ,      lowercase							:						Tuple=9_9 ,      lowercase							:						int=3_2 ,      lowercase							:						Dict=5 ,      lowercase							:						Any=4 ,      lowercase							:						Optional[Any]=3_7 ,      lowercase							:						Any="gelu" ,      lowercase							:						Union[str, Any]=0.1 ,      lowercase							:						Union[str, Any]=0.1 ,      lowercase							:						Tuple=1_2_8 ,      lowercase							:						Optional[int]=3_2 ,      lowercase							:						Optional[Any]=1_6 ,      lowercase							:						List[Any]=2 ,      lowercase							:						Optional[Any]=0.0_2 ,      lowercase							:						Optional[int]=3 ,      lowercase							:						List[Any]=4 ,      lowercase							:						Any=None ,      )							->    int:

         '''simple docstring'''







         UpperCamelCase__         = parent
         UpperCamelCase__         = batch_size
         UpperCamelCase__         = seq_length
         UpperCamelCase__         = is_training
         UpperCamelCase__         = use_input_mask
         UpperCamelCase__         = use_token_type_ids
         UpperCamelCase__         = use_labels
         UpperCamelCase__         = vocab_size
         UpperCamelCase__         = hidden_size
         UpperCamelCase__         = num_hidden_layers
         UpperCamelCase__         = num_attention_heads
         UpperCamelCase__         = intermediate_size
         UpperCamelCase__         = hidden_act
         UpperCamelCase__         = hidden_dropout_prob
         UpperCamelCase__         = attention_probs_dropout_prob
         UpperCamelCase__         = max_position_embeddings
         UpperCamelCase__         = type_vocab_size
         UpperCamelCase__         = type_sequence_label_size
         UpperCamelCase__         = initializer_range
         UpperCamelCase__         = num_labels
         UpperCamelCase__         = num_choices
         UpperCamelCase__         = scope






   def 				A      (   self							:						int			)							->    str:

         '''simple docstring'''







         UpperCamelCase__         = ids_tensor([self.batch_size, self.seq_length] ,      self.vocab_size			)

         UpperCamelCase__         = None
         if self.use_input_mask:
               UpperCamelCase__         = random_attention_mask([self.batch_size, self.seq_length]			)

         UpperCamelCase__         = None
         if self.use_token_type_ids:
               UpperCamelCase__         = ids_tensor([self.batch_size, self.seq_length] ,      self.type_vocab_size			)

         UpperCamelCase__         = None
         UpperCamelCase__         = None
         UpperCamelCase__         = None
         if self.use_labels:
               UpperCamelCase__         = ids_tensor([self.batch_size] ,      self.type_sequence_label_size			)
               UpperCamelCase__         = ids_tensor([self.batch_size, self.seq_length] ,      self.num_labels			)
               UpperCamelCase__         = ids_tensor([self.batch_size] ,      self.num_choices			)

         UpperCamelCase__         = self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






   def 				A      (   self							:						Optional[int]			)							->    Dict:

         '''simple docstring'''







         return NezhaConfig(
             vocab_size=self.vocab_size ,      hidden_size=self.hidden_size ,      num_hidden_layers=self.num_hidden_layers ,      num_attention_heads=self.num_attention_heads ,      intermediate_size=self.intermediate_size ,      hidden_act=self.hidden_act ,      hidden_dropout_prob=self.hidden_dropout_prob ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob ,      max_position_embeddings=self.max_position_embeddings ,      type_vocab_size=self.type_vocab_size ,      is_decoder=_a ,      initializer_range=self.initializer_range ,      )






   def 				A      (   self							:						Optional[int]			)							->    str:

         '''simple docstring'''







         (
             UpperCamelCase__		
         )         = self.prepare_config_and_inputs()

         UpperCamelCase__         = True
         UpperCamelCase__         = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
         UpperCamelCase__         = ids_tensor([self.batch_size, self.seq_length] ,      vocab_size=2			)

         return (
             config,
             input_ids,
             token_type_ids,
             input_mask,
             sequence_labels,
             token_labels,
             choice_labels,
             encoder_hidden_states,
             encoder_attention_mask,
         )






   def 				A      (   self							:						Dict ,      lowercase							:						Optional[Any] ,      lowercase							:						str ,      lowercase							:						Tuple ,      lowercase							:						Optional[Any] ,      lowercase							:						Optional[Any] ,      lowercase							:						List[str] ,      lowercase							:						List[Any]			)							->    List[str]:

         '''simple docstring'''







         UpperCamelCase__         = NezhaModel(config=_a			)
         model.to(_a			)
         model.eval()
         UpperCamelCase__         = model(_a ,      attention_mask=_a ,      token_type_ids=_a			)
         UpperCamelCase__         = model(_a ,      token_type_ids=_a			)
         UpperCamelCase__         = model(_a			)
         self.parent.assertEqual(result.last_hidden_state.shape ,      (self.batch_size, self.seq_length, self.hidden_size)			)
         self.parent.assertEqual(result.pooler_output.shape ,      (self.batch_size, self.hidden_size)			)






   def 				A      (   self							:						int ,      lowercase							:						List[str] ,      lowercase							:						Any ,      lowercase							:						Dict ,      lowercase							:						Tuple ,      lowercase							:						Optional[Any] ,      lowercase							:						List[Any] ,      lowercase							:						Any ,      lowercase							:						Optional[Any] ,      lowercase							:						Tuple ,      )							->    Optional[int]:

         '''simple docstring'''







         UpperCamelCase__         = True
         UpperCamelCase__         = NezhaModel(_a			)
         model.to(_a			)
         model.eval()
         UpperCamelCase__         = model(
             _a ,      attention_mask=_a ,      token_type_ids=_a ,      encoder_hidden_states=_a ,      encoder_attention_mask=_a ,      )
         UpperCamelCase__         = model(
             _a ,      attention_mask=_a ,      token_type_ids=_a ,      encoder_hidden_states=_a ,      )
         UpperCamelCase__         = model(_a ,      attention_mask=_a ,      token_type_ids=_a			)
         self.parent.assertEqual(result.last_hidden_state.shape ,      (self.batch_size, self.seq_length, self.hidden_size)			)
         self.parent.assertEqual(result.pooler_output.shape ,      (self.batch_size, self.hidden_size)			)






   def 				A      (   self							:						str ,      lowercase							:						List[str] ,      lowercase							:						Tuple ,      lowercase							:						Optional[Any] ,      lowercase							:						Tuple ,      lowercase							:						Tuple ,      lowercase							:						Union[str, Any] ,      lowercase							:						int			)							->    Optional[int]:

         '''simple docstring'''







         UpperCamelCase__         = NezhaForMaskedLM(config=_a			)
         model.to(_a			)
         model.eval()
         UpperCamelCase__         = model(_a ,      attention_mask=_a ,      token_type_ids=_a ,      labels=_a			)
         self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.seq_length, self.vocab_size)			)






   def 				A      (   self							:						Union[str, Any] ,      lowercase							:						str ,      lowercase							:						str ,      lowercase							:						Dict ,      lowercase							:						Dict ,      lowercase							:						str ,      lowercase							:						Union[str, Any] ,      lowercase							:						int			)							->    Dict:

         '''simple docstring'''







         UpperCamelCase__         = NezhaForNextSentencePrediction(config=_a			)
         model.to(_a			)
         model.eval()
         UpperCamelCase__         = model(
             _a ,      attention_mask=_a ,      token_type_ids=_a ,      labels=_a ,      )
         self.parent.assertEqual(result.logits.shape ,      (self.batch_size, 2)			)






   def 				A      (   self							:						Optional[Any] ,      lowercase							:						List[Any] ,      lowercase							:						List[str] ,      lowercase							:						Union[str, Any] ,      lowercase							:						Union[str, Any] ,      lowercase							:						Dict ,      lowercase							:						int ,      lowercase							:						Optional[Any]			)							->    List[Any]:

         '''simple docstring'''







         UpperCamelCase__         = NezhaForPreTraining(config=_a			)
         model.to(_a			)
         model.eval()
         UpperCamelCase__         = model(
             _a ,      attention_mask=_a ,      token_type_ids=_a ,      labels=_a ,      next_sentence_label=_a ,      )
         self.parent.assertEqual(result.prediction_logits.shape ,      (self.batch_size, self.seq_length, self.vocab_size)			)
         self.parent.assertEqual(result.seq_relationship_logits.shape ,      (self.batch_size, 2)			)






   def 				A      (   self							:						Any ,      lowercase							:						str ,      lowercase							:						Dict ,      lowercase							:						Optional[int] ,      lowercase							:						Union[str, Any] ,      lowercase							:						int ,      lowercase							:						List[Any] ,      lowercase							:						Any			)							->    Dict:

         '''simple docstring'''







         UpperCamelCase__         = NezhaForQuestionAnswering(config=_a			)
         model.to(_a			)
         model.eval()
         UpperCamelCase__         = model(
             _a ,      attention_mask=_a ,      token_type_ids=_a ,      start_positions=_a ,      end_positions=_a ,      )
         self.parent.assertEqual(result.start_logits.shape ,      (self.batch_size, self.seq_length)			)
         self.parent.assertEqual(result.end_logits.shape ,      (self.batch_size, self.seq_length)			)






   def 				A      (   self							:						Union[str, Any] ,      lowercase							:						Optional[int] ,      lowercase							:						Optional[Any] ,      lowercase							:						int ,      lowercase							:						str ,      lowercase							:						List[str] ,      lowercase							:						List[str] ,      lowercase							:						Any			)							->    Optional[int]:

         '''simple docstring'''







         UpperCamelCase__         = self.num_labels
         UpperCamelCase__         = NezhaForSequenceClassification(_a			)
         model.to(_a			)
         model.eval()
         UpperCamelCase__         = model(_a ,      attention_mask=_a ,      token_type_ids=_a ,      labels=_a			)
         self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.num_labels)			)






   def 				A      (   self							:						List[str] ,      lowercase							:						Any ,      lowercase							:						Dict ,      lowercase							:						Dict ,      lowercase							:						Dict ,      lowercase							:						Union[str, Any] ,      lowercase							:						Tuple ,      lowercase							:						Tuple			)							->    int:

         '''simple docstring'''







         UpperCamelCase__         = self.num_labels
         UpperCamelCase__         = NezhaForTokenClassification(config=_a			)
         model.to(_a			)
         model.eval()
         UpperCamelCase__         = model(_a ,      attention_mask=_a ,      token_type_ids=_a ,      labels=_a			)
         self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.seq_length, self.num_labels)			)






   def 				A      (   self							:						Optional[int] ,      lowercase							:						Optional[int] ,      lowercase							:						Optional[Any] ,      lowercase							:						List[str] ,      lowercase							:						Optional[Any] ,      lowercase							:						Optional[Any] ,      lowercase							:						Any ,      lowercase							:						Tuple			)							->    Tuple:

         '''simple docstring'''







         UpperCamelCase__         = self.num_choices
         UpperCamelCase__         = NezhaForMultipleChoice(config=_a			)
         model.to(_a			)
         model.eval()
         UpperCamelCase__         = input_ids.unsqueeze(1			).expand(-1 ,      self.num_choices ,      -1			).contiguous()
         UpperCamelCase__         = token_type_ids.unsqueeze(1			).expand(-1 ,      self.num_choices ,      -1			).contiguous()
         UpperCamelCase__         = input_mask.unsqueeze(1			).expand(-1 ,      self.num_choices ,      -1			).contiguous()
         UpperCamelCase__         = model(
             _a ,      attention_mask=_a ,      token_type_ids=_a ,      labels=_a ,      )
         self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.num_choices)			)







   def 				A      (   self							:						int			)							->    Tuple:

         '''simple docstring'''







         UpperCamelCase__         = self.prepare_config_and_inputs()
         (
             UpperCamelCase__		
         )         = config_and_inputs
         UpperCamelCase__         = {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
         return config, inputs_dict
@require_torch
class    _SCREAMING_SNAKE_CASE							(		UpperCamelCase_       ,UpperCamelCase_       ,UpperCamelCase_       ,unittest.TestCase	):


   '''simple docstring'''





   __a							:							Union[str, Any]					     =					(
       (
           NezhaModel,
           NezhaForMaskedLM,
           NezhaForMultipleChoice,
           NezhaForNextSentencePrediction,
           NezhaForPreTraining,
           NezhaForQuestionAnswering,
           NezhaForSequenceClassification,
           NezhaForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   __a							:							List[Any]					     =					(
       {
           """feature-extraction""": NezhaModel,
           """fill-mask""": NezhaForMaskedLM,
           """question-answering""": NezhaForQuestionAnswering,
           """text-classification""": NezhaForSequenceClassification,
           """token-classification""": NezhaForTokenClassification,
           """zero-shot""": NezhaForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   __a							:							Optional[Any]					     =					True






   def 				A      (   self							:						Union[str, Any] ,      lowercase							:						Any ,      lowercase							:						Optional[Any] ,      lowercase							:						Any=False			)							->    Tuple:

         '''simple docstring'''







         UpperCamelCase__         = super()._prepare_for_class(_a ,      _a ,      return_labels=_a			)

         if return_labels:
               if model_class in get_values(_a			):
                     UpperCamelCase__         = torch.zeros(
                         (self.model_tester.batch_size, self.model_tester.seq_length) ,      dtype=torch.long ,      device=_a			)
                     UpperCamelCase__         = torch.zeros(
                         self.model_tester.batch_size ,      dtype=torch.long ,      device=_a			)
         return inputs_dict






   def 				A      (   self							:						List[Any]			)							->    Any:

         '''simple docstring'''







         UpperCamelCase__         = NezhaModelTester(self			)
         UpperCamelCase__         = ConfigTester(self ,      config_class=_a ,      hidden_size=3_7			)






   def 				A      (   self							:						int			)							->    Optional[int]:

         '''simple docstring'''







         self.config_tester.run_common_tests()






   def 				A      (   self							:						str			)							->    int:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_a			)






   def 				A      (   self							:						Tuple			)							->    Union[str, Any]:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs_for_decoder()
         self.model_tester.create_and_check_model_as_decoder(*_a			)






   def 				A      (   self							:						List[str]			)							->    List[Any]:

         '''simple docstring'''







         (
             UpperCamelCase__		
         )         = self.model_tester.prepare_config_and_inputs_for_decoder()

         UpperCamelCase__         = None

         self.model_tester.create_and_check_model_as_decoder(
             _a ,      _a ,      _a ,      _a ,      _a ,      _a ,      _a ,      _a ,      _a ,      )






   def 				A      (   self							:						Optional[int]			)							->    int:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*_a			)






   def 				A      (   self							:						int			)							->    Optional[Any]:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*_a			)






   def 				A      (   self							:						Optional[Any]			)							->    str:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_next_sequence_prediction(*_a			)






   def 				A      (   self							:						Dict			)							->    Union[str, Any]:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_pretraining(*_a			)






   def 				A      (   self							:						List[Any]			)							->    int:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*_a			)






   def 				A      (   self							:						Optional[Any]			)							->    List[Any]:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*_a			)






   def 				A      (   self							:						Optional[int]			)							->    List[str]:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*_a			)






   @slow
   def 				A      (   self							:						Union[str, Any]			)							->    Optional[Any]:

         '''simple docstring'''







         for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               UpperCamelCase__         = NezhaModel.from_pretrained(_a			)
               self.assertIsNotNone(_a			)







   @slow
   @require_torch_gpu
   def 				A      (   self							:						Optional[Any]			)							->    Any:

         '''simple docstring'''







         UpperCamelCase__         = self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes:
               # NezhaForMultipleChoice behaves incorrectly in JIT environments.
               if model_class == NezhaForMultipleChoice:
                     return

               UpperCamelCase__         = True
               UpperCamelCase__         = model_class(config=_a			)

               UpperCamelCase__         = self._prepare_for_class(_a ,      _a			)
               UpperCamelCase__         = torch.jit.trace(
                   _a ,      (inputs_dict["""input_ids"""].to("""cpu"""			), inputs_dict["""attention_mask"""].to("""cpu"""			))			)

               with tempfile.TemporaryDirectory() as tmp:
                     torch.jit.save(_a ,      os.path.join(_a ,      """bert.pt"""			)			)
                     UpperCamelCase__         = torch.jit.load(os.path.join(_a ,      """bert.pt"""			) ,      map_location=_a			)
                     loaded(inputs_dict["""input_ids"""].to(_a			) ,      inputs_dict["""attention_mask"""].to(_a			)			)
@require_torch
class    _SCREAMING_SNAKE_CASE							(		unittest.TestCase	):


   '''simple docstring'''





   @slow
   def 				A      (   self							:						str			)							->    Any:

         '''simple docstring'''







         UpperCamelCase__         = NezhaModel.from_pretrained("""sijunhe/nezha-cn-base"""			)
         UpperCamelCase__         = torch.tensor([[0, 1, 2, 3, 4, 5]]			)
         UpperCamelCase__         = torch.tensor([[0, 1, 1, 1, 1, 1]]			)
         with torch.no_grad():
               UpperCamelCase__         = model(_a ,      attention_mask=_a			)[0]
         UpperCamelCase__         = torch.Size((1, 6, 7_6_8)			)
         self.assertEqual(output.shape ,      _a			)
         UpperCamelCase__         = torch.tensor([[[0.0_6_8_5, 0.2_4_4_1, 0.1_1_0_2], [0.0_6_0_0, 0.1_9_0_6, 0.1_3_4_9], [0.0_2_2_1, 0.0_8_1_9, 0.0_5_8_6]]]			)

         self.assertTrue(torch.allclose(output[:, 1:4, 1:4] ,      _a ,      atol=1e-4			)			)







   @slow
   def 				A      (   self							:						Tuple			)							->    Optional[Any]:

         '''simple docstring'''







         UpperCamelCase__         = NezhaForMaskedLM.from_pretrained("""sijunhe/nezha-cn-base"""			)
         UpperCamelCase__         = torch.tensor([[0, 1, 2, 3, 4, 5]]			)
         UpperCamelCase__         = torch.tensor([[1, 1, 1, 1, 1, 1]]			)
         with torch.no_grad():
               UpperCamelCase__         = model(_a ,      attention_mask=_a			)[0]
         UpperCamelCase__         = torch.Size((1, 6, 2_1_1_2_8)			)
         self.assertEqual(output.shape ,      _a			)
         UpperCamelCase__         = torch.tensor(
             [[-2.7_9_3_9, -1.7_9_0_2, -2.2_1_8_9], [-2.8_5_8_5, -1.8_9_0_8, -2.3_7_2_3], [-2.6_4_9_9, -1.7_7_5_0, -2.2_5_5_8]]			)

         self.assertTrue(torch.allclose(output[:, 1:4, 1:4] ,      _a ,      atol=1e-4			)			)








'''simple docstring'''


import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    AlbertForMaskedLM,
					    AlbertForMultipleChoice,
					    AlbertForPreTraining,
					    AlbertForQuestionAnswering,
					    AlbertForSequenceClassification,
					    AlbertForTokenClassification,
					    AlbertModel,
					)
					from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class    _SCREAMING_SNAKE_CASE							:


			'''simple docstring'''





			def __init__(   self							:						Optional[int] ,      lowercase							:						List[str] ,      lowercase							:						List[Any]=1_3 ,      lowercase							:						Union[str, Any]=7 ,      lowercase							:						Dict=True ,      lowercase							:						Optional[int]=True ,      lowercase							:						List[Any]=True ,      lowercase							:						Dict=True ,      lowercase							:						List[str]=9_9 ,      lowercase							:						Dict=1_6 ,      lowercase							:						Dict=3_6 ,      lowercase							:						str=6 ,      lowercase							:						List[Any]=6 ,      lowercase							:						int=6 ,      lowercase							:						Union[str, Any]=3_7 ,      lowercase							:						Union[str, Any]="gelu" ,      lowercase							:						List[Any]=0.1 ,      lowercase							:						List[str]=0.1 ,      lowercase							:						str=5_1_2 ,      lowercase							:						Any=1_6 ,      lowercase							:						str=2 ,      lowercase							:						List[Any]=0.0_2 ,      lowercase							:						Tuple=3 ,      lowercase							:						Dict=4 ,      lowercase							:						Dict=None ,      )							->    List[Any]:

									'''simple docstring'''







									UpperCamelCase__         = parent
									UpperCamelCase__         = batch_size
									UpperCamelCase__         = seq_length
									UpperCamelCase__         = is_training
									UpperCamelCase__         = use_input_mask
									UpperCamelCase__         = use_token_type_ids
									UpperCamelCase__         = use_labels
									UpperCamelCase__         = vocab_size
									UpperCamelCase__         = embedding_size
									UpperCamelCase__         = hidden_size
									UpperCamelCase__         = num_hidden_layers
									UpperCamelCase__         = num_hidden_groups
									UpperCamelCase__         = num_attention_heads
									UpperCamelCase__         = intermediate_size
									UpperCamelCase__         = hidden_act
									UpperCamelCase__         = hidden_dropout_prob
									UpperCamelCase__         = attention_probs_dropout_prob
									UpperCamelCase__         = max_position_embeddings
									UpperCamelCase__         = type_vocab_size
									UpperCamelCase__         = type_sequence_label_size
									UpperCamelCase__         = initializer_range
									UpperCamelCase__         = num_labels
									UpperCamelCase__         = num_choices
									UpperCamelCase__         = scope






			def 				A      (   self							:						List[str]			)							->    str:

									'''simple docstring'''







									UpperCamelCase__         = ids_tensor([self.batch_size, self.seq_length] ,      self.vocab_size			)

									UpperCamelCase__         = None
									if self.use_input_mask:
															UpperCamelCase__         = random_attention_mask([self.batch_size, self.seq_length]			)

									UpperCamelCase__         = None
									if self.use_token_type_ids:
															UpperCamelCase__         = ids_tensor([self.batch_size, self.seq_length] ,      self.type_vocab_size			)

									UpperCamelCase__         = None
									UpperCamelCase__         = None
									UpperCamelCase__         = None
									if self.use_labels:
															UpperCamelCase__         = ids_tensor([self.batch_size] ,      self.type_sequence_label_size			)
															UpperCamelCase__         = ids_tensor([self.batch_size, self.seq_length] ,      self.num_labels			)
															UpperCamelCase__         = ids_tensor([self.batch_size] ,      self.num_choices			)

									UpperCamelCase__         = self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






			def 				A      (   self							:						Optional[int]			)							->    str:

									'''simple docstring'''







									return AlbertConfig(
									    vocab_size=self.vocab_size ,      hidden_size=self.hidden_size ,      num_hidden_layers=self.num_hidden_layers ,      num_attention_heads=self.num_attention_heads ,      intermediate_size=self.intermediate_size ,      hidden_act=self.hidden_act ,      hidden_dropout_prob=self.hidden_dropout_prob ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob ,      max_position_embeddings=self.max_position_embeddings ,      type_vocab_size=self.type_vocab_size ,      initializer_range=self.initializer_range ,      num_hidden_groups=self.num_hidden_groups ,      )






			def 				A      (   self							:						List[str] ,      lowercase							:						Union[str, Any] ,      lowercase							:						Optional[Any] ,      lowercase							:						Dict ,      lowercase							:						str ,      lowercase							:						int ,      lowercase							:						List[str] ,      lowercase							:						Optional[Any]			)							->    Optional[int]:

									'''simple docstring'''







									UpperCamelCase__         = AlbertModel(config=lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = model(lowercase ,      attention_mask=lowercase ,      token_type_ids=lowercase			)
									UpperCamelCase__         = model(lowercase ,      token_type_ids=lowercase			)
									UpperCamelCase__         = model(lowercase			)
									self.parent.assertEqual(result.last_hidden_state.shape ,      (self.batch_size, self.seq_length, self.hidden_size)			)
									self.parent.assertEqual(result.pooler_output.shape ,      (self.batch_size, self.hidden_size)			)






			def 				A      (   self							:						int ,      lowercase							:						List[str] ,      lowercase							:						Optional[int] ,      lowercase							:						Optional[Any] ,      lowercase							:						Optional[Any] ,      lowercase							:						Optional[int] ,      lowercase							:						List[Any] ,      lowercase							:						int			)							->    Optional[int]:

									'''simple docstring'''







									UpperCamelCase__         = AlbertForPreTraining(config=lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = model(
									    lowercase ,      attention_mask=lowercase ,      token_type_ids=lowercase ,      labels=lowercase ,      sentence_order_label=lowercase ,      )
									self.parent.assertEqual(result.prediction_logits.shape ,      (self.batch_size, self.seq_length, self.vocab_size)			)
									self.parent.assertEqual(result.sop_logits.shape ,      (self.batch_size, config.num_labels)			)






			def 				A      (   self							:						Optional[int] ,      lowercase							:						Dict ,      lowercase							:						List[Any] ,      lowercase							:						Any ,      lowercase							:						List[str] ,      lowercase							:						int ,      lowercase							:						Any ,      lowercase							:						Dict			)							->    Any:

									'''simple docstring'''







									UpperCamelCase__         = AlbertForMaskedLM(config=lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = model(lowercase ,      attention_mask=lowercase ,      token_type_ids=lowercase ,      labels=lowercase			)
									self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.seq_length, self.vocab_size)			)






			def 				A      (   self							:						Optional[Any] ,      lowercase							:						Any ,      lowercase							:						Union[str, Any] ,      lowercase							:						List[Any] ,      lowercase							:						List[Any] ,      lowercase							:						List[str] ,      lowercase							:						Optional[Any] ,      lowercase							:						Optional[Any]			)							->    List[Any]:

									'''simple docstring'''







									UpperCamelCase__         = AlbertForQuestionAnswering(config=lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = model(
									    lowercase ,      attention_mask=lowercase ,      token_type_ids=lowercase ,      start_positions=lowercase ,      end_positions=lowercase ,      )
									self.parent.assertEqual(result.start_logits.shape ,      (self.batch_size, self.seq_length)			)
									self.parent.assertEqual(result.end_logits.shape ,      (self.batch_size, self.seq_length)			)






			def 				A      (   self							:						Any ,      lowercase							:						Optional[Any] ,      lowercase							:						Any ,      lowercase							:						Dict ,      lowercase							:						Any ,      lowercase							:						Optional[int] ,      lowercase							:						str ,      lowercase							:						Dict			)							->    List[Any]:

									'''simple docstring'''







									UpperCamelCase__         = self.num_labels
									UpperCamelCase__         = AlbertForSequenceClassification(lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = model(lowercase ,      attention_mask=lowercase ,      token_type_ids=lowercase ,      labels=lowercase			)
									self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.num_labels)			)






			def 				A      (   self							:						List[str] ,      lowercase							:						int ,      lowercase							:						Any ,      lowercase							:						Tuple ,      lowercase							:						List[str] ,      lowercase							:						Optional[int] ,      lowercase							:						List[Any] ,      lowercase							:						Dict			)							->    Optional[int]:

									'''simple docstring'''







									UpperCamelCase__         = self.num_labels
									UpperCamelCase__         = AlbertForTokenClassification(config=lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = model(lowercase ,      attention_mask=lowercase ,      token_type_ids=lowercase ,      labels=lowercase			)
									self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.seq_length, self.num_labels)			)






			def 				A      (   self							:						List[Any] ,      lowercase							:						Union[str, Any] ,      lowercase							:						List[Any] ,      lowercase							:						Any ,      lowercase							:						List[str] ,      lowercase							:						Optional[Any] ,      lowercase							:						List[str] ,      lowercase							:						List[Any]			)							->    str:

									'''simple docstring'''







									UpperCamelCase__         = self.num_choices
									UpperCamelCase__         = AlbertForMultipleChoice(config=lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = input_ids.unsqueeze(1			).expand(-1 ,      self.num_choices ,      -1			).contiguous()
									UpperCamelCase__         = token_type_ids.unsqueeze(1			).expand(-1 ,      self.num_choices ,      -1			).contiguous()
									UpperCamelCase__         = input_mask.unsqueeze(1			).expand(-1 ,      self.num_choices ,      -1			).contiguous()
									UpperCamelCase__         = model(
									    lowercase ,      attention_mask=lowercase ,      token_type_ids=lowercase ,      labels=lowercase ,      )
									self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.num_choices)			)







			def 				A      (   self							:						Optional[int]			)							->    List[str]:

									'''simple docstring'''







									UpperCamelCase__         = self.prepare_config_and_inputs()
									(
									    (
									    UpperCamelCase__
									)    ,   (
									    UpperCamelCase__
									)    ,   (
									    UpperCamelCase__
									)    ,   (
									    UpperCamelCase__
									)    ,   (
									    UpperCamelCase__
									)    ,   (
									    UpperCamelCase__
									)    ,   (
									    UpperCamelCase__
									)    ,   
									)         = config_and_inputs
									UpperCamelCase__         = {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
									return config, inputs_dict
@require_torch
class    _SCREAMING_SNAKE_CASE							(		SCREAMING_SNAKE_CASE       ,SCREAMING_SNAKE_CASE       ,unittest.TestCase	):


			'''simple docstring'''





			__a							:							Any					     =					(
			    (
			        AlbertModel,
			        AlbertForPreTraining,
			        AlbertForMaskedLM,
			        AlbertForMultipleChoice,
			        AlbertForSequenceClassification,
			        AlbertForTokenClassification,
			        AlbertForQuestionAnswering,
			    )
			    if is_torch_available()
			    else ()
			)
			__a							:							List[Any]					     =					(
			    {
			        "feature-extraction": AlbertModel,
			        "fill-mask": AlbertForMaskedLM,
			        "question-answering": AlbertForQuestionAnswering,
			        "text-classification": AlbertForSequenceClassification,
			        "token-classification": AlbertForTokenClassification,
			        "zero-shot": AlbertForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			__a							:							List[str]					     =					True






			def 				A      (   self							:						Any ,      lowercase							:						int ,      lowercase							:						Dict ,      lowercase							:						Optional[Any]=False			)							->    Tuple:

									'''simple docstring'''







									UpperCamelCase__         = super()._prepare_for_class(lowercase ,      lowercase ,      return_labels=lowercase			)

									if return_labels:
															if model_class in get_values(lowercase			):
																					UpperCamelCase__         = torch.zeros(
																					    (self.model_tester.batch_size, self.model_tester.seq_length) ,      dtype=torch.long ,      device=lowercase			)
																					UpperCamelCase__         = torch.zeros(
																					    self.model_tester.batch_size ,      dtype=torch.long ,      device=lowercase			)
									return inputs_dict






			def 				A      (   self							:						Any			)							->    Optional[int]:

									'''simple docstring'''







									UpperCamelCase__         = AlbertModelTester(self			)
									UpperCamelCase__         = ConfigTester(self ,      config_class=lowercase ,      hidden_size=3_7			)






			def 				A      (   self							:						int			)							->    List[str]:

									'''simple docstring'''







									self.config_tester.run_common_tests()






			def 				A      (   self							:						str			)							->    List[Any]:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*lowercase			)






			def 				A      (   self							:						int			)							->    Dict:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_pretraining(*lowercase			)






			def 				A      (   self							:						Tuple			)							->    int:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*lowercase			)






			def 				A      (   self							:						Any			)							->    int:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*lowercase			)






			def 				A      (   self							:						List[str]			)							->    int:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*lowercase			)






			def 				A      (   self							:						int			)							->    Optional[int]:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*lowercase			)






			def 				A      (   self							:						Tuple			)							->    List[str]:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
															UpperCamelCase__         = type
															self.model_tester.create_and_check_model(*lowercase			)







			@slow
			def 				A      (   self							:						Optional[int]			)							->    Any:

									'''simple docstring'''







									for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															UpperCamelCase__         = AlbertModel.from_pretrained(lowercase			)
															self.assertIsNotNone(lowercase			)
@require_torch
class    _SCREAMING_SNAKE_CASE							(		unittest.TestCase	):


			'''simple docstring'''





			@slow
			def 				A      (   self							:						List[Any]			)							->    Optional[int]:

									'''simple docstring'''







									UpperCamelCase__         = AlbertModel.from_pretrained("""albert-base-v2"""			)
									UpperCamelCase__         = torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]			)
									UpperCamelCase__         = torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]			)
									with torch.no_grad():
															UpperCamelCase__         = model(lowercase ,      attention_mask=lowercase			)[0]
									UpperCamelCase__         = torch.Size((1, 1_1, 7_6_8)			)
									self.assertEqual(output.shape ,      lowercase			)
									UpperCamelCase__         = torch.tensor(
									    [[[-0.6_5_1_3, 1.5_0_3_5, -0.2_7_6_6], [-0.6_5_1_5, 1.5_0_4_6, -0.2_7_8_0], [-0.6_5_1_2, 1.5_0_4_9, -0.2_7_8_4]]]			)

									self.assertTrue(torch.allclose(output[:, 1:4, 1:4] ,      lowercase ,      atol=1e-4			)			)
