


import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
    import jax.numpy as jnp

    from transformers.models.albert.modeling_flax_albert import (
        FlaxAlbertForMaskedLM,
        FlaxAlbertForMultipleChoice,
        FlaxAlbertForPreTraining,
        FlaxAlbertForQuestionAnswering,
        FlaxAlbertForSequenceClassification,
        FlaxAlbertForTokenClassification,
        FlaxAlbertModel,
    )

class 							lowercase  (						unittest.TestCase			):







     def __init__(						self :						Dict   ,   _lowercase :						Optional[int]   ,   _lowercase :						List[str]=13   ,   _lowercase :						Dict=7   ,   _lowercase :						Any=True   ,   _lowercase :						Union[str, Any]=True   ,   _lowercase :						int=True   ,   _lowercase :						Union[str, Any]=True   ,   _lowercase :						List[str]=99   ,   _lowercase :						Optional[Any]=32   ,   _lowercase :						Optional[Any]=5   ,   _lowercase :						Optional[int]=4   ,   _lowercase :						Optional[int]=37   ,   _lowercase :						Tuple="gelu"   ,   _lowercase :						Optional[Any]=0.1   ,   _lowercase :						List[str]=0.1   ,   _lowercase :						Tuple=5_12   ,   _lowercase :						Tuple=16   ,   _lowercase :						Union[str, Any]=2   ,   _lowercase :						int=0.02   ,   _lowercase :						Union[str, Any]=4   ,   ):
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		parent
           SCREAMING_SNAKE_CASE__       :  Dict						     =		batch_size
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		seq_length
           SCREAMING_SNAKE_CASE__       :  str						     =		is_training
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		use_attention_mask
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		use_token_type_ids
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		use_labels
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		vocab_size
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		hidden_size
           SCREAMING_SNAKE_CASE__       :  Any						     =		num_hidden_layers
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		num_attention_heads
           SCREAMING_SNAKE_CASE__       :  int						     =		intermediate_size
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		hidden_act
           SCREAMING_SNAKE_CASE__       :  str						     =		hidden_dropout_prob
           SCREAMING_SNAKE_CASE__       :  str						     =		attention_probs_dropout_prob
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		max_position_embeddings
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		type_vocab_size
           SCREAMING_SNAKE_CASE__       :  str						     =		type_sequence_label_size
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		initializer_range
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		num_choices







     def 							lowercase__       (						self :						Any      ):
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		ids_tensor([self.batch_size, self.seq_length]   ,   self.vocab_size      )

           SCREAMING_SNAKE_CASE__       :  Tuple						     =		None
           if self.use_attention_mask:
                 SCREAMING_SNAKE_CASE__       :  Dict						     =		random_attention_mask([self.batch_size, self.seq_length]      )

           SCREAMING_SNAKE_CASE__       :  List[str]						     =		None
           if self.use_token_type_ids:
                 SCREAMING_SNAKE_CASE__       :  List[Any]						     =		ids_tensor([self.batch_size, self.seq_length]   ,   self.type_vocab_size      )

           SCREAMING_SNAKE_CASE__       :  Any						     =		AlbertConfig(
               vocab_size=self.vocab_size   ,   hidden_size=self.hidden_size   ,   num_hidden_layers=self.num_hidden_layers   ,   num_attention_heads=self.num_attention_heads   ,   intermediate_size=self.intermediate_size   ,   hidden_act=self.hidden_act   ,   hidden_dropout_prob=self.hidden_dropout_prob   ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,   max_position_embeddings=self.max_position_embeddings   ,   type_vocab_size=self.type_vocab_size   ,   is_decoder=_lowercase   ,   initializer_range=self.initializer_range   ,   )

           return config, input_ids, token_type_ids, attention_mask







     def 							lowercase__       (						self :						Tuple      ):
           SCREAMING_SNAKE_CASE__       :  Dict						     =		self.prepare_config_and_inputs()
           SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__       :  int						     =		config_and_inputs
           SCREAMING_SNAKE_CASE__       :  Dict						     =		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
           return config, inputs_dict




@require_flax
class 							lowercase  (						_UpperCAmelCase    ,       unittest.TestCase			):
     lowerCamelCase			:			Dict      = (
         (
             FlaxAlbertModel,
             FlaxAlbertForPreTraining,
             FlaxAlbertForMaskedLM,
             FlaxAlbertForMultipleChoice,
             FlaxAlbertForQuestionAnswering,
             FlaxAlbertForSequenceClassification,
             FlaxAlbertForTokenClassification,
             FlaxAlbertForQuestionAnswering,
         )
         if is_flax_available()
         else ()
     )







     def 							lowercase__       (						self :						str      ):
           SCREAMING_SNAKE_CASE__       :  Any						     =		FlaxAlbertModelTester(self      )







     @slow
     def 							lowercase__       (						self :						List[str]      ):
           for model_class_name in self.all_model_classes:
                 SCREAMING_SNAKE_CASE__       :  str						     =		model_class_name.from_pretrained('''albert-base-v2'''      )
                 SCREAMING_SNAKE_CASE__       :  List[str]						     =		model(np.ones((1, 1)      )      )
                 self.assertIsNotNone(_lowercase      )




@require_flax
class 							lowercase  (						unittest.TestCase			):







     @slow
     def 							lowercase__       (						self :						Optional[Any]      ):
           SCREAMING_SNAKE_CASE__       :  Dict						     =		FlaxAlbertModel.from_pretrained('''albert-base-v2'''      )
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		np.array([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]      )
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		model(_lowercase   ,   attention_mask=_lowercase      )[0]
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		(1, 11, 7_68)
           self.assertEqual(output.shape   ,   _lowercase      )
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		np.array(
               [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]      )

           self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]   ,   _lowercase   ,   atol=1E-4      )      )













'''simple docstring'''

from typing import Optional, Union

import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models.modeling_utils import ModelMixin






class   UpperCAmelCase      (			UpperCamelCase__     ,			UpperCamelCase__							):
 @register_to_config
 def __init__(							self :List[str]		,			lowercase_ :int = 7_68		,			)->				str:
     super().__init__()

     A__							=     nn.Parameter(torch.zeros(1		,			lowercase_				)				)
     A__							=     nn.Parameter(torch.ones(1		,			lowercase_				)				)
 def 	UpperCAmelCase_				(							self :List[Any]		,			lowercase_ :Optional[Union[str, torch.device]] = None		,			lowercase_ :Optional[torch.dtype] = None		,			)->				Tuple:
     A__							=     nn.Parameter(self.mean.to(lowercase_				).to(lowercase_				)				)
     A__							=     nn.Parameter(self.std.to(lowercase_				).to(lowercase_				)				)
     return self
 def 	UpperCAmelCase_				(							self :Optional[int]		,			lowercase_ :str				)->				Union[str, Any]:
     A__							=     (embeds - self.mean) * 1.0 / self.std
     return embeds



 def 	UpperCAmelCase_				(							self :Any		,			lowercase_ :List[Any]				)->				Tuple:
     A__							=     (embeds * self.std) + self.mean
     return embeds



