






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


a_							=    {'''configuration_vit_mae''': ['''VIT_MAE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ViTMAEConfig''']}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a_							=    [
	    '''VIT_MAE_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''ViTMAEForPreTraining''',
	    '''ViTMAELayer''',
	    '''ViTMAEModel''',
	    '''ViTMAEPreTrainedModel''',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a_							=    [
	    '''TFViTMAEForPreTraining''',
	    '''TFViTMAEModel''',
	    '''TFViTMAEPreTrainedModel''',
	]

if TYPE_CHECKING:
	from .configuration_vit_mae import VIT_MAE_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTMAEConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_vit_mae import (
		    VIT_MAE_PRETRAINED_MODEL_ARCHIVE_LIST,
		    ViTMAEForPreTraining,
		    ViTMAELayer,
		    ViTMAEModel,
		    ViTMAEPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_vit_mae import TFViTMAEForPreTraining, TFViTMAEModel, TFViTMAEPreTrainedModel


else:
	import sys

	a_							=    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)














import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    NystromformerForMaskedLM,
	    NystromformerForMultipleChoice,
	    NystromformerForQuestionAnswering,
	    NystromformerForSequenceClassification,
	    NystromformerForTokenClassification,
	    NystromformerModel,
	)
	from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


class      lowercase__     :

	def __init__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase=13		,    __UpperCAmelCase=7		,    __UpperCAmelCase=True		,    __UpperCAmelCase=True		,    __UpperCAmelCase=True		,    __UpperCAmelCase=True		,    __UpperCAmelCase=99		,    __UpperCAmelCase=32		,    __UpperCAmelCase=5		,    __UpperCAmelCase=4		,    __UpperCAmelCase=37		,    __UpperCAmelCase="gelu"		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=512		,    __UpperCAmelCase=16		,    __UpperCAmelCase=2		,    __UpperCAmelCase=0.02		,    __UpperCAmelCase=3		,    __UpperCAmelCase=4		,    __UpperCAmelCase=None		,    )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						parent
							lowerCAmelCase__       =						batch_size
							lowerCAmelCase__       =						seq_length
							lowerCAmelCase__       =						is_training
							lowerCAmelCase__       =						use_input_mask
							lowerCAmelCase__       =						use_token_type_ids
							lowerCAmelCase__       =						use_labels
							lowerCAmelCase__       =						vocab_size
							lowerCAmelCase__       =						hidden_size
							lowerCAmelCase__       =						num_hidden_layers
							lowerCAmelCase__       =						num_attention_heads
							lowerCAmelCase__       =						intermediate_size
							lowerCAmelCase__       =						hidden_act
							lowerCAmelCase__       =						hidden_dropout_prob
							lowerCAmelCase__       =						attention_probs_dropout_prob
							lowerCAmelCase__       =						max_position_embeddings
							lowerCAmelCase__       =						type_vocab_size
							lowerCAmelCase__       =						type_sequence_label_size
							lowerCAmelCase__       =						initializer_range
							lowerCAmelCase__       =						num_labels
							lowerCAmelCase__       =						num_choices
							lowerCAmelCase__       =						scope

	def 		UpperCAmelCase  (							self       )->					Optional[Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						ids_tensor([self.batch_size, self.seq_length]		,    self.vocab_size       )

							lowerCAmelCase__       =						None
							if self.use_input_mask:
													lowerCAmelCase__       =						random_attention_mask([self.batch_size, self.seq_length]       )

							lowerCAmelCase__       =						None
							if self.use_token_type_ids:
													lowerCAmelCase__       =						ids_tensor([self.batch_size, self.seq_length]		,    self.type_vocab_size       )

							lowerCAmelCase__       =						None
							lowerCAmelCase__       =						None
							lowerCAmelCase__       =						None
							if self.use_labels:
													lowerCAmelCase__       =						ids_tensor([self.batch_size]		,    self.type_sequence_label_size       )
													lowerCAmelCase__       =						ids_tensor([self.batch_size, self.seq_length]		,    self.num_labels       )
													lowerCAmelCase__       =						ids_tensor([self.batch_size]		,    self.num_choices       )

							lowerCAmelCase__       =						self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

	def 		UpperCAmelCase  (							self       )->					Optional[Any]:







							'''simple docstring'''





							return NystromformerConfig(
							    vocab_size=self.vocab_size		,    hidden_size=self.hidden_size		,    num_hidden_layers=self.num_hidden_layers		,    num_attention_heads=self.num_attention_heads		,    intermediate_size=self.intermediate_size		,    hidden_act=self.hidden_act		,    hidden_dropout_prob=self.hidden_dropout_prob		,    attention_probs_dropout_prob=self.attention_probs_dropout_prob		,    max_position_embeddings=self.max_position_embeddings		,    type_vocab_size=self.type_vocab_size		,    is_decoder=__UpperCAmelCase		,    initializer_range=self.initializer_range		,    )

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						NystromformerModel(config=__UpperCAmelCase       )
							model.to(__UpperCAmelCase       )
							model.eval()
							lowerCAmelCase__       =						model(__UpperCAmelCase		,    attention_mask=__UpperCAmelCase		,    token_type_ids=__UpperCAmelCase       )
							lowerCAmelCase__       =						model(__UpperCAmelCase		,    token_type_ids=__UpperCAmelCase       )
							lowerCAmelCase__       =						model(__UpperCAmelCase       )
							self.parent.assertEqual(result.last_hidden_state.shape		,    (self.batch_size, self.seq_length, self.hidden_size)       )

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase       )->					str:







							'''simple docstring'''





							lowerCAmelCase__       =						NystromformerForMaskedLM(config=__UpperCAmelCase       )
							model.to(__UpperCAmelCase       )
							model.eval()
							lowerCAmelCase__       =						model(__UpperCAmelCase		,    attention_mask=__UpperCAmelCase		,    token_type_ids=__UpperCAmelCase		,    labels=__UpperCAmelCase       )
							self.parent.assertEqual(result.logits.shape		,    (self.batch_size, self.seq_length, self.vocab_size)       )

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						NystromformerForQuestionAnswering(config=__UpperCAmelCase       )
							model.to(__UpperCAmelCase       )
							model.eval()
							lowerCAmelCase__       =						model(
							    __UpperCAmelCase		,    attention_mask=__UpperCAmelCase		,    token_type_ids=__UpperCAmelCase		,    start_positions=__UpperCAmelCase		,    end_positions=__UpperCAmelCase		,    )
							self.parent.assertEqual(result.start_logits.shape		,    (self.batch_size, self.seq_length)       )
							self.parent.assertEqual(result.end_logits.shape		,    (self.batch_size, self.seq_length)       )

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase       )->					Dict:







							'''simple docstring'''





							lowerCAmelCase__       =						self.num_labels
							lowerCAmelCase__       =						NystromformerForSequenceClassification(__UpperCAmelCase       )
							model.to(__UpperCAmelCase       )
							model.eval()
							lowerCAmelCase__       =						model(__UpperCAmelCase		,    attention_mask=__UpperCAmelCase		,    token_type_ids=__UpperCAmelCase		,    labels=__UpperCAmelCase       )
							self.parent.assertEqual(result.logits.shape		,    (self.batch_size, self.num_labels)       )

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase       )->					str:







							'''simple docstring'''





							lowerCAmelCase__       =						self.num_labels
							lowerCAmelCase__       =						NystromformerForTokenClassification(config=__UpperCAmelCase       )
							model.to(__UpperCAmelCase       )
							model.eval()
							lowerCAmelCase__       =						model(__UpperCAmelCase		,    attention_mask=__UpperCAmelCase		,    token_type_ids=__UpperCAmelCase		,    labels=__UpperCAmelCase       )
							self.parent.assertEqual(result.logits.shape		,    (self.batch_size, self.seq_length, self.num_labels)       )

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase       )->					Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__       =						self.num_choices
							lowerCAmelCase__       =						NystromformerForMultipleChoice(config=__UpperCAmelCase       )
							model.to(__UpperCAmelCase       )
							model.eval()
							lowerCAmelCase__       =						input_ids.unsqueeze(1       ).expand(-1		,    self.num_choices		,    -1       ).contiguous()
							lowerCAmelCase__       =						token_type_ids.unsqueeze(1       ).expand(-1		,    self.num_choices		,    -1       ).contiguous()
							lowerCAmelCase__       =						input_mask.unsqueeze(1       ).expand(-1		,    self.num_choices		,    -1       ).contiguous()
							lowerCAmelCase__       =						model(
							    __UpperCAmelCase		,    attention_mask=__UpperCAmelCase		,    token_type_ids=__UpperCAmelCase		,    labels=__UpperCAmelCase		,    )
							self.parent.assertEqual(result.logits.shape		,    (self.batch_size, self.num_choices)       )


	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						self.prepare_config_and_inputs()
							(
							    (
							    lowerCAmelCase__
							)      ,							(
							    lowerCAmelCase__
							)      ,							(
							    lowerCAmelCase__
							)      ,							(
							    lowerCAmelCase__
							)      ,							(
							    lowerCAmelCase__
							)      ,							(
							    lowerCAmelCase__
							)      ,							(
							    lowerCAmelCase__
							)      ,							
							)       =						config_and_inputs
							lowerCAmelCase__       =						{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
							return config, inputs_dict





@require_torch
class      lowercase__     (			_UpperCAmelCase,					_UpperCAmelCase,					unittest.TestCase	):
	a_         =(
	    (
	        NystromformerModel,
	        NystromformerForMaskedLM,
	        NystromformerForMultipleChoice,
	        NystromformerForQuestionAnswering,
	        NystromformerForSequenceClassification,
	        NystromformerForTokenClassification,
	    )
	    if is_torch_available()
	    else ()
	)
	a_         =(
	    {
	        """feature-extraction""": NystromformerModel,
	        """fill-mask""": NystromformerForMaskedLM,
	        """question-answering""": NystromformerForQuestionAnswering,
	        """text-classification""": NystromformerForSequenceClassification,
	        """token-classification""": NystromformerForTokenClassification,
	        """zero-shot""": NystromformerForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	a_         =False
	a_         =False

	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						NystromformerModelTester(self       )
							lowerCAmelCase__       =						ConfigTester(self		,    config_class=__UpperCAmelCase		,    hidden_size=37       )

	def 		UpperCAmelCase  (							self       )->					Dict:







							'''simple docstring'''





							self.config_tester.run_common_tests()

	def 		UpperCAmelCase  (							self       )->					List[Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__UpperCAmelCase       )

	def 		UpperCAmelCase  (							self       )->					Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
													lowerCAmelCase__       =						type
													self.model_tester.create_and_check_model(*__UpperCAmelCase       )

	def 		UpperCAmelCase  (							self       )->					List[Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*__UpperCAmelCase       )

	def 		UpperCAmelCase  (							self       )->					Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__       =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*__UpperCAmelCase       )

	def 		UpperCAmelCase  (							self       )->					Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__UpperCAmelCase       )

	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*__UpperCAmelCase       )

	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__UpperCAmelCase       )


	@slow
	def 		UpperCAmelCase  (							self       )->					Optional[int]:







							'''simple docstring'''





							for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													lowerCAmelCase__       =						NystromformerModel.from_pretrained(__UpperCAmelCase       )
													self.assertIsNotNone(__UpperCAmelCase       )





@require_torch
class      lowercase__     (			unittest.TestCase	):

	@slow
	def 		UpperCAmelCase  (							self       )->					Dict:







							'''simple docstring'''





							lowerCAmelCase__       =						NystromformerModel.from_pretrained("uw-madison/nystromformer-512"       )
							lowerCAmelCase__       =						torch.tensor([[0, 1, 2, 3, 4, 5]]       )

							with torch.no_grad():
													lowerCAmelCase__       =						model(__UpperCAmelCase       )[0]

							lowerCAmelCase__       =						torch.Size((1, 6, 768)       )
							self.assertEqual(output.shape		,    __UpperCAmelCase       )

							lowerCAmelCase__       =						torch.tensor(
							    [[[-0.4_532, -0.0_936, 0.5_137], [-0.2_676, 0.0_628, 0.6_186], [-0.3_629, -0.1_726, 0.4_716]]]       )

							self.assertTrue(torch.allclose(output[:, :3, :3]		,    __UpperCAmelCase		,    atol=1E-4       )       )


	@slow
	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						"the [MASK] of Belgium is Brussels"

							lowerCAmelCase__       =						AutoTokenizer.from_pretrained("uw-madison/nystromformer-512"       )
							lowerCAmelCase__       =						NystromformerForMaskedLM.from_pretrained("uw-madison/nystromformer-512"       )

							lowerCAmelCase__       =						tokenizer(__UpperCAmelCase		,    return_tensors="pt"       )

							with torch.no_grad():
													lowerCAmelCase__       =						model(encoding.input_ids       ).logits

							lowerCAmelCase__       =						token_logits[:, 2, :].argmax(-1       )[0]

							self.assertEqual(tokenizer.decode(__UpperCAmelCase       )		,    "capital"       )






