





import unittest

from transformers import BigBirdConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
				import jax

				from transformers.models.big_bird.modeling_flax_big_bird import (
				    FlaxBigBirdForCausalLM,
				    FlaxBigBirdForMaskedLM,
				    FlaxBigBirdForMultipleChoice,
				    FlaxBigBirdForPreTraining,
				    FlaxBigBirdForQuestionAnswering,
				    FlaxBigBirdForSequenceClassification,
				    FlaxBigBirdForTokenClassification,
				    FlaxBigBirdModel,
				)
class        snake_case_		(			unittest.TestCase):







		def __init__(		self			,     __lowercase			,     __lowercase=2			,     __lowercase=5_6			,     __lowercase=True			,     __lowercase=True			,     __lowercase=True			,     __lowercase=True			,     __lowercase=9_9			,     __lowercase=3_2			,     __lowercase=2			,     __lowercase=2			,     __lowercase=7			,     __lowercase="gelu_new"			,     __lowercase=0.1			,     __lowercase=0.1			,     __lowercase=5_1_2			,     __lowercase=1_6			,     __lowercase=2			,     __lowercase=0.0_2			,     __lowercase=4			,     __lowercase="block_sparse"			,     __lowercase=True			,     __lowercase=False			,     __lowercase=2			,     __lowercase=3			,     )				->   Any:
				lowerCamelCase   : str								=parent
				lowerCamelCase   : List[str]								=batch_size
				lowerCamelCase   : str								=seq_length
				lowerCamelCase   : Union[str, Any]								=is_training
				lowerCamelCase   : List[str]								=use_attention_mask
				lowerCamelCase   : Any								=use_token_type_ids
				lowerCamelCase   : Tuple								=use_labels
				lowerCamelCase   : List[str]								=vocab_size
				lowerCamelCase   : List[Any]								=hidden_size
				lowerCamelCase   : Union[str, Any]								=num_hidden_layers
				lowerCamelCase   : Dict								=num_attention_heads
				lowerCamelCase   : List[Any]								=intermediate_size
				lowerCamelCase   : Optional[Any]								=hidden_act
				lowerCamelCase   : Tuple								=hidden_dropout_prob
				lowerCamelCase   : str								=attention_probs_dropout_prob
				lowerCamelCase   : str								=max_position_embeddings
				lowerCamelCase   : Tuple								=type_vocab_size
				lowerCamelCase   : int								=type_sequence_label_size
				lowerCamelCase   : Optional[int]								=initializer_range
				lowerCamelCase   : Union[str, Any]								=num_choices

				lowerCamelCase   : Optional[Any]								=rescale_embeddings
				lowerCamelCase   : str								=attention_type
				lowerCamelCase   : Dict								=use_bias
				lowerCamelCase   : Optional[int]								=block_size
				lowerCamelCase   : List[Any]								=num_random_blocks







		def 			__lowercase   (		self   )				->   Tuple:
				lowerCamelCase   : Any								=ids_tensor([self.batch_size, self.seq_length]			,     self.vocab_size   )

				lowerCamelCase   : List[str]								=None
				if self.use_attention_mask:
						lowerCamelCase   : Optional[Any]								=random_attention_mask([self.batch_size, self.seq_length]   )

				lowerCamelCase   : Dict								=None
				if self.use_token_type_ids:
						lowerCamelCase   : str								=ids_tensor([self.batch_size, self.seq_length]			,     self.type_vocab_size   )

				lowerCamelCase   : str								=BigBirdConfig(
				    vocab_size=self.vocab_size			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     max_position_embeddings=self.max_position_embeddings			,     type_vocab_size=self.type_vocab_size			,     is_decoder=__lowercase			,     initializer_range=self.initializer_range			,     attention_type=self.attention_type			,     block_size=self.block_size			,     num_random_blocks=self.num_random_blocks			,     use_bias=self.use_bias			,     rescale_embeddings=self.rescale_embeddings			,     )

				return config, input_ids, token_type_ids, attention_mask







		def 			__lowercase   (		self   )				->   Optional[Any]:
				lowerCamelCase   : str								=self.prepare_config_and_inputs()
				lowerCamelCase   : str								=config_and_inputs
				lowerCamelCase   : Optional[int]								={
				    '''input_ids''': input_ids,
				    '''token_type_ids''': token_type_ids,
				    '''attention_mask''': attention_mask,
				}
				return config, inputs_dict

@require_flax
class        snake_case_		(			_A						,					unittest.TestCase):
		lowerCamelCase    :List[str]						=     (
		    (
		        FlaxBigBirdForCausalLM,
		        FlaxBigBirdModel,
		        FlaxBigBirdForPreTraining,
		        FlaxBigBirdForMaskedLM,
		        FlaxBigBirdForMultipleChoice,
		        FlaxBigBirdForQuestionAnswering,
		        FlaxBigBirdForSequenceClassification,
		        FlaxBigBirdForTokenClassification,
		    )
		    if is_flax_available()
		    else ()
		)

		lowerCamelCase    :List[str]						=     False
		lowerCamelCase    :str						=     False







		def 			__lowercase   (		self   )				->   Union[str, Any]:
				lowerCamelCase   : Dict								=FlaxBigBirdModelTester(self   )







		@slow
		# copied from `test_modeling_flax_common` because it takes much longer than other models
		def 			__lowercase   (		self   )				->   Any:
				super().test_from_pretrained_save_pretrained()







		@slow
		# copied from `test_modeling_flax_common` because it takes much longer than other models
		def 			__lowercase   (		self   )				->   Dict:
				super().test_from_pretrained_with_no_automatic_init()







		@slow
		# copied from `test_modeling_flax_common` because it takes much longer than other models
		def 			__lowercase   (		self   )				->   Union[str, Any]:
				super().test_no_automatic_init()







		@slow
		# copied from `test_modeling_flax_common` because it takes much longer than other models
		def 			__lowercase   (		self   )				->   Tuple:
				super().test_hidden_states_output()







		@slow
		def 			__lowercase   (		self   )				->   Optional[int]:
				for model_class_name in self.all_model_classes:
						lowerCamelCase   : Optional[Any]								=model_class_name.from_pretrained('''google/bigbird-roberta-base'''   )
						self.assertIsNotNone(__lowercase   )







		def 			__lowercase   (		self   )				->   Optional[int]:
				if self.test_attn_probs:
						super().test_attention_outputs()







		@slow
		# copied from `test_modeling_flax_common` because it takes much longer than other models
		def 			__lowercase   (		self   )				->   int:
				lowerCamelCase   : List[str]								=self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
						with self.subTest(model_class.__name__   ):
								lowerCamelCase   : Optional[int]								=self._prepare_for_class(__lowercase			,     __lowercase   )
								lowerCamelCase   : Tuple								=model_class(__lowercase   )

								@jax.jit
								def model_jitted(__lowercase			,     __lowercase=None			,     **__lowercase   ):
										return model(input_ids=__lowercase			,     attention_mask=__lowercase			,     **__lowercase   )

								with self.subTest('''JIT Enabled'''   ):
										lowerCamelCase   : Tuple								=model_jitted(**__lowercase   ).to_tuple()

								with self.subTest('''JIT Disabled'''   ):
										with jax.disable_jit():
												lowerCamelCase   : List[Any]								=model_jitted(**__lowercase   ).to_tuple()

								self.assertEqual(len(__lowercase   )			,     len(__lowercase   )   )
								for jitted_output, output in zip(__lowercase			,     __lowercase   ):
										self.assertEqual(jitted_output.shape			,     output.shape   )







		def 			__lowercase   (		self			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase=1e-5			,     __lowercase="outputs"			,     __lowercase=None   )				->   Optional[int]:
				# `bigbird_block_sparse_attention` in `FlaxBigBird` returns `attention_probs = None`, while in PyTorch version,
				# an effort was done to return `attention_probs` (yet to be verified).
				if name.startswith('''outputs.attentions'''   ):
						return
				else:
						super().check_pt_flax_outputs(__lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase   )








from typing import Optional, Union

import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models.modeling_utils import ModelMixin
class        snake_case_		(			_A						,					_A):







		@register_to_config
		def __init__(		self			,     __lowercase = 7_6_8			,     )				->   List[Any]:
				super().__init__()

				lowerCamelCase   : str								=nn.Parameter(torch.zeros(1			,     __lowercase   )   )
				lowerCamelCase   : Union[str, Any]								=nn.Parameter(torch.ones(1			,     __lowercase   )   )







		def 			__lowercase   (		self			,     __lowercase = None			,     __lowercase = None			,     )				->   List[str]:
				lowerCamelCase   : Tuple								=nn.Parameter(self.mean.to(__lowercase   ).to(__lowercase   )   )
				lowerCamelCase   : Dict								=nn.Parameter(self.std.to(__lowercase   ).to(__lowercase   )   )
				return self







		def 			__lowercase   (		self			,     __lowercase   )				->   Optional[Any]:
				lowerCamelCase   : Any								=(embeds - self.mean) * 1.0 / self.std
				return embeds







		def 			__lowercase   (		self			,     __lowercase   )				->   Optional[int]:
				lowerCamelCase   : Any								=(embeds * self.std) + self.mean
				return embeds

