




import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
     import jax.numpy as jnp

     from transformers.models.albert.modeling_flax_albert import (
         FlaxAlbertForMaskedLM,
         FlaxAlbertForMultipleChoice,
         FlaxAlbertForPreTraining,
         FlaxAlbertForQuestionAnswering,
         FlaxAlbertForSequenceClassification,
         FlaxAlbertForTokenClassification,
         FlaxAlbertModel,
     )

class     __a(	unittest.TestCase  ):






    """simple docstring"""





    def __init__( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE=13  ,_SCREAMING_SNAKE_CASE=7  ,_SCREAMING_SNAKE_CASE=True  ,_SCREAMING_SNAKE_CASE=True  ,_SCREAMING_SNAKE_CASE=True  ,_SCREAMING_SNAKE_CASE=True  ,_SCREAMING_SNAKE_CASE=99  ,_SCREAMING_SNAKE_CASE=32  ,_SCREAMING_SNAKE_CASE=5  ,_SCREAMING_SNAKE_CASE=4  ,_SCREAMING_SNAKE_CASE=37  ,_SCREAMING_SNAKE_CASE="gelu"  ,_SCREAMING_SNAKE_CASE=0.1  ,_SCREAMING_SNAKE_CASE=0.1  ,_SCREAMING_SNAKE_CASE=512  ,_SCREAMING_SNAKE_CASE=16  ,_SCREAMING_SNAKE_CASE=2  ,_SCREAMING_SNAKE_CASE=0.02  ,_SCREAMING_SNAKE_CASE=4  ,)					->						List[str]:
         UpperCAmelCase_     :				Dict  =     parent
         UpperCAmelCase_     :				List[str]  =     batch_size
         UpperCAmelCase_     :				Optional[Any]  =     seq_length
         UpperCAmelCase_     :				str  =     is_training
         UpperCAmelCase_     :				Union[str, Any]  =     use_attention_mask
         UpperCAmelCase_     :				Optional[int]  =     use_token_type_ids
         UpperCAmelCase_     :				int  =     use_labels
         UpperCAmelCase_     :				str  =     vocab_size
         UpperCAmelCase_     :				Optional[int]  =     hidden_size
         UpperCAmelCase_     :				Optional[Any]  =     num_hidden_layers
         UpperCAmelCase_     :				Optional[Any]  =     num_attention_heads
         UpperCAmelCase_     :				Union[str, Any]  =     intermediate_size
         UpperCAmelCase_     :				Optional[Any]  =     hidden_act
         UpperCAmelCase_     :				Optional[int]  =     hidden_dropout_prob
         UpperCAmelCase_     :				List[Any]  =     attention_probs_dropout_prob
         UpperCAmelCase_     :				Optional[Any]  =     max_position_embeddings
         UpperCAmelCase_     :				Optional[int]  =     type_vocab_size
         UpperCAmelCase_     :				Tuple  =     type_sequence_label_size
         UpperCAmelCase_     :				Optional[Any]  =     initializer_range
         UpperCAmelCase_     :				Tuple  =     num_choices





    def 				a__ ( self    )					->						List[Any]:
         UpperCAmelCase_     :				int  =     ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size    )

         UpperCAmelCase_     :				Optional[int]  =     None
         if self.use_attention_mask:
              UpperCAmelCase_     :				Union[str, Any]  =     random_attention_mask([self.batch_size, self.seq_length]    )

         UpperCAmelCase_     :				Union[str, Any]  =     None
         if self.use_token_type_ids:
              UpperCAmelCase_     :				Dict  =     ids_tensor([self.batch_size, self.seq_length]  ,self.type_vocab_size    )

         UpperCAmelCase_     :				Any  =     AlbertConfig(
             vocab_size=self.vocab_size  ,hidden_size=self.hidden_size  ,num_hidden_layers=self.num_hidden_layers  ,num_attention_heads=self.num_attention_heads  ,intermediate_size=self.intermediate_size  ,hidden_act=self.hidden_act  ,hidden_dropout_prob=self.hidden_dropout_prob  ,attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,max_position_embeddings=self.max_position_embeddings  ,type_vocab_size=self.type_vocab_size  ,is_decoder=_lowerCAmelCase  ,initializer_range=self.initializer_range  ,)

         return config, input_ids, token_type_ids, attention_mask





    def 				a__ ( self    )					->						Optional[int]:
         UpperCAmelCase_     :				str  =     self.prepare_config_and_inputs()
         UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_     :				Union[str, Any]  =     config_and_inputs
         UpperCAmelCase_     :				int  =     {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
         return config, inputs_dict


@require_flax
class     __a(	_SCREAMING_SNAKE_CASE							,		unittest.TestCase  ):






    """simple docstring"""





    lowerCAmelCase         = (
        (
            FlaxAlbertModel,
            FlaxAlbertForPreTraining,
            FlaxAlbertForMaskedLM,
            FlaxAlbertForMultipleChoice,
            FlaxAlbertForQuestionAnswering,
            FlaxAlbertForSequenceClassification,
            FlaxAlbertForTokenClassification,
            FlaxAlbertForQuestionAnswering,
        )
        if is_flax_available()
        else ()
    )





    def 				a__ ( self    )					->						Dict:
         UpperCAmelCase_     :				Union[str, Any]  =     FlaxAlbertModelTester(self    )





    @slow
    def 				a__ ( self    )					->						List[Any]:
         for model_class_name in self.all_model_classes:
              UpperCAmelCase_     :				int  =     model_class_name.from_pretrained('''albert-base-v2'''    )
              UpperCAmelCase_     :				Dict  =     model(np.ones((1, 1)    )    )
              self.assertIsNotNone(_lowerCAmelCase    )


@require_flax
class     __a(	unittest.TestCase  ):






    """simple docstring"""





    @slow
    def 				a__ ( self    )					->						int:
         UpperCAmelCase_     :				int  =     FlaxAlbertModel.from_pretrained('''albert-base-v2'''    )
         UpperCAmelCase_     :				str  =     np.array([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]    )
         UpperCAmelCase_     :				Dict  =     np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]    )
         UpperCAmelCase_     :				Union[str, Any]  =     model(_lowerCAmelCase  ,attention_mask=_lowerCAmelCase    )[0]
         UpperCAmelCase_     :				Any  =     (1, 11, 768)
         self.assertEqual(output.shape  ,_lowerCAmelCase    )
         UpperCAmelCase_     :				Tuple  =     np.array(
             [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]    )

         self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]  ,_lowerCAmelCase  ,atol=1e-4    )    )





from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
   from ...onnx.config import PatchingSpec
   from ...tokenization_utils_base import PreTrainedTokenizerBase


lowerCamelCase__					:  Optional[int]      							=				logging.get_logger(__name__)

lowerCamelCase__					:  Dict      							=				{
    'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json',
    'allenai/longformer-large-4096': 'https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json',
    'allenai/longformer-large-4096-finetuned-triviaqa': (
        'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json'
    ),
    'allenai/longformer-base-4096-extra.pos.embd.only': (
        'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json'
    ),
    'allenai/longformer-large-4096-extra.pos.embd.only': (
        'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json'
    ),
}





class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE   ):


 '''simple docstring'''







 lowercase_							       =       "longformer"



 def __init__(				self    :						Union[str, Any]							,							_lowerCAmelCase    :						Union[List[int], int] = 512							,							_lowerCAmelCase    :						int = 2							,							_lowerCAmelCase    :						int = 1							,							_lowerCAmelCase    :						int = 0							,							_lowerCAmelCase    :						int = 2							,							_lowerCAmelCase    :						int = 30_522							,							_lowerCAmelCase    :						int = 768							,							_lowerCAmelCase    :						int = 12							,							_lowerCAmelCase    :						int = 12							,							_lowerCAmelCase    :						int = 3_072							,							_lowerCAmelCase    :						str = "gelu"							,							_lowerCAmelCase    :						float = 0.1							,							_lowerCAmelCase    :						float = 0.1							,							_lowerCAmelCase    :						int = 512							,							_lowerCAmelCase    :						int = 2							,							_lowerCAmelCase    :						float = 0.02							,							_lowerCAmelCase    :						float = 1E-12							,							_lowerCAmelCase    :						bool = False							,							**_lowerCAmelCase    :						Union[str, Any]							,							):
       super().__init__(pad_token_id=_lowerCAmelCase							,							**_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     attention_window
       SCREAMING_SNAKE_CASE_      		=     sep_token_id
       SCREAMING_SNAKE_CASE_      		=     bos_token_id
       SCREAMING_SNAKE_CASE_      		=     eos_token_id
       SCREAMING_SNAKE_CASE_      		=     vocab_size
       SCREAMING_SNAKE_CASE_      		=     hidden_size
       SCREAMING_SNAKE_CASE_      		=     num_hidden_layers
       SCREAMING_SNAKE_CASE_      		=     num_attention_heads
       SCREAMING_SNAKE_CASE_      		=     hidden_act
       SCREAMING_SNAKE_CASE_      		=     intermediate_size
       SCREAMING_SNAKE_CASE_      		=     hidden_dropout_prob
       SCREAMING_SNAKE_CASE_      		=     attention_probs_dropout_prob
       SCREAMING_SNAKE_CASE_      		=     max_position_embeddings
       SCREAMING_SNAKE_CASE_      		=     type_vocab_size
       SCREAMING_SNAKE_CASE_      		=     initializer_range
       SCREAMING_SNAKE_CASE_      		=     layer_norm_eps
       SCREAMING_SNAKE_CASE_      		=     onnx_export





class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE   ):


 '''simple docstring'''







 def __init__(				self    :						Optional[Any]							,							_lowerCAmelCase    :						"PretrainedConfig"							,							_lowerCAmelCase    :						str = "default"							,							_lowerCAmelCase    :						"List[PatchingSpec]" = None		):
       super().__init__(_lowerCAmelCase							,							_lowerCAmelCase							,							_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     True



 @property
 def  lowerCAmelCase_    (				self    :						Any		):
       if self.task == "multiple-choice":
             SCREAMING_SNAKE_CASE_      		=     {0: 'batch', 1: 'choice', 2: 'sequence'}
       else:
             SCREAMING_SNAKE_CASE_      		=     {0: 'batch', 1: 'sequence'}
       return OrderedDict(
           [
               ('input_ids', dynamic_axis),
               ('attention_mask', dynamic_axis),
               ('global_attention_mask', dynamic_axis),
           ]		)



 @property
 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       SCREAMING_SNAKE_CASE_      		=     super().outputs
       if self.task == "default":
             SCREAMING_SNAKE_CASE_      		=     {0: 'batch'}
       return outputs



 @property
 def  lowerCAmelCase_    (				self    :						str		):
       return 1E-4



 @property
 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       # needs to be >= 14 to support tril operator
       return max(super().default_onnx_opset							,							14		)




 def  lowerCAmelCase_    (				self    :						str							,							_lowerCAmelCase    :						"PreTrainedTokenizerBase"							,							_lowerCAmelCase    :						int = -1							,							_lowerCAmelCase    :						int = -1							,							_lowerCAmelCase    :						bool = False							,							_lowerCAmelCase    :						Optional[TensorType] = None							,							):
       SCREAMING_SNAKE_CASE_      		=     super().generate_dummy_inputs(
           preprocessor=_lowerCAmelCase							,							batch_size=_lowerCAmelCase							,							seq_length=_lowerCAmelCase							,							is_pair=_lowerCAmelCase							,							framework=_lowerCAmelCase		)
       import torch

       # for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
       # makes the export fail randomly
       SCREAMING_SNAKE_CASE_      		=     torch.zeros_like(inputs['input_ids']		)
       # make every second token global
       SCREAMING_SNAKE_CASE_      		=     1

       return inputs