


import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    NystromformerForMaskedLM,
			    NystromformerForMultipleChoice,
			    NystromformerForQuestionAnswering,
			    NystromformerForSequenceClassification,
			    NystromformerForTokenClassification,
			    NystromformerModel,
			)
			from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST

class        a	:




							"""simple docstring"""


							def __init__(							self :						List[str]					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						List[Any]=13					,			lowerCamelCase :						str=7					,			lowerCamelCase :						Optional[Any]=True					,			lowerCamelCase :						List[str]=True					,			lowerCamelCase :						str=True					,			lowerCamelCase :						int=True					,			lowerCamelCase :						List[Any]=99					,			lowerCamelCase :						str=32					,			lowerCamelCase :						Optional[int]=5					,			lowerCamelCase :						Tuple=4					,			lowerCamelCase :						str=37					,			lowerCamelCase :						Union[str, Any]="gelu"					,			lowerCamelCase :						Any=0.1					,			lowerCamelCase :						str=0.1					,			lowerCamelCase :						str=512					,			lowerCamelCase :						Any=16					,			lowerCamelCase :						Tuple=2					,			lowerCamelCase :						Any=0.02					,			lowerCamelCase :						List[str]=3					,			lowerCamelCase :						Optional[int]=4					,			lowerCamelCase :						List[Any]=None					,			)			->					Optional[Any]:
														__snake_case	:    Tuple											=     parent
														__snake_case	:    Optional[int]											=     batch_size
														__snake_case	:    Optional[int]											=     seq_length
														__snake_case	:    Any											=     is_training
														__snake_case	:    List[str]											=     use_input_mask
														__snake_case	:    List[Any]											=     use_token_type_ids
														__snake_case	:    Tuple											=     use_labels
														__snake_case	:    List[Any]											=     vocab_size
														__snake_case	:    int											=     hidden_size
														__snake_case	:    Optional[int]											=     num_hidden_layers
														__snake_case	:    int											=     num_attention_heads
														__snake_case	:    int											=     intermediate_size
														__snake_case	:    Any											=     hidden_act
														__snake_case	:    Optional[Any]											=     hidden_dropout_prob
														__snake_case	:    Any											=     attention_probs_dropout_prob
														__snake_case	:    Any											=     max_position_embeddings
														__snake_case	:    int											=     type_vocab_size
														__snake_case	:    Tuple											=     type_sequence_label_size
														__snake_case	:    Dict											=     initializer_range
														__snake_case	:    str											=     num_labels
														__snake_case	:    Union[str, Any]											=     num_choices
														__snake_case	:    Dict											=     scope

							def 					__snake_case      (							self :						str       )			->					Optional[int]:
														__snake_case	:    str											=     ids_tensor([self.batch_size, self.seq_length]					,			self.vocab_size       )

														__snake_case	:    str											=     None
														if self.use_input_mask:
																					__snake_case	:    int											=     random_attention_mask([self.batch_size, self.seq_length]       )

														__snake_case	:    str											=     None
														if self.use_token_type_ids:
																					__snake_case	:    Union[str, Any]											=     ids_tensor([self.batch_size, self.seq_length]					,			self.type_vocab_size       )

														__snake_case	:    Union[str, Any]											=     None
														__snake_case	:    Optional[int]											=     None
														__snake_case	:    Tuple											=     None
														if self.use_labels:
																					__snake_case	:    List[Any]											=     ids_tensor([self.batch_size]					,			self.type_sequence_label_size       )
																					__snake_case	:    Dict											=     ids_tensor([self.batch_size, self.seq_length]					,			self.num_labels       )
																					__snake_case	:    Tuple											=     ids_tensor([self.batch_size]					,			self.num_choices       )

														__snake_case	:    str											=     self.get_config()

														return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

							def 					__snake_case      (							self :						Dict       )			->					str:
														return NystromformerConfig(
														    vocab_size=self.vocab_size					,			hidden_size=self.hidden_size					,			num_hidden_layers=self.num_hidden_layers					,			num_attention_heads=self.num_attention_heads					,			intermediate_size=self.intermediate_size					,			hidden_act=self.hidden_act					,			hidden_dropout_prob=self.hidden_dropout_prob					,			attention_probs_dropout_prob=self.attention_probs_dropout_prob					,			max_position_embeddings=self.max_position_embeddings					,			type_vocab_size=self.type_vocab_size					,			is_decoder=A__					,			initializer_range=self.initializer_range					,			)

							def 					__snake_case      (							self :						List[str]					,			lowerCamelCase :						List[Any]					,			lowerCamelCase :						Any					,			lowerCamelCase :						str					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						int       )			->					str:
														__snake_case	:    List[str]											=     NystromformerModel(config=A__       )
														model.to(A__       )
														model.eval()
														__snake_case	:    int											=     model(A__					,			attention_mask=A__					,			token_type_ids=A__       )
														__snake_case	:    Tuple											=     model(A__					,			token_type_ids=A__       )
														__snake_case	:    Optional[int]											=     model(A__       )
														self.parent.assertEqual(result.last_hidden_state.shape					,			(self.batch_size, self.seq_length, self.hidden_size)       )

							def 					__snake_case      (							self :						str					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						List[str]					,			lowerCamelCase :						int					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						int					,			lowerCamelCase :						int       )			->					Optional[int]:
														__snake_case	:    List[Any]											=     NystromformerForMaskedLM(config=A__       )
														model.to(A__       )
														model.eval()
														__snake_case	:    List[str]											=     model(A__					,			attention_mask=A__					,			token_type_ids=A__					,			labels=A__       )
														self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.vocab_size)       )

							def 					__snake_case      (							self :						Optional[int]					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						int					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						Any					,			lowerCamelCase :						List[str]					,			lowerCamelCase :						int       )			->					List[Any]:
														__snake_case	:    List[str]											=     NystromformerForQuestionAnswering(config=A__       )
														model.to(A__       )
														model.eval()
														__snake_case	:    Optional[Any]											=     model(
														    A__					,			attention_mask=A__					,			token_type_ids=A__					,			start_positions=A__					,			end_positions=A__					,			)
														self.parent.assertEqual(result.start_logits.shape					,			(self.batch_size, self.seq_length)       )
														self.parent.assertEqual(result.end_logits.shape					,			(self.batch_size, self.seq_length)       )

							def 					__snake_case      (							self :						Optional[int]					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						int					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						Any       )			->					Dict:
														__snake_case	:    Any											=     self.num_labels
														__snake_case	:    List[str]											=     NystromformerForSequenceClassification(A__       )
														model.to(A__       )
														model.eval()
														__snake_case	:    Optional[Any]											=     model(A__					,			attention_mask=A__					,			token_type_ids=A__					,			labels=A__       )
														self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_labels)       )

							def 					__snake_case      (							self :						Any					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						Any					,			lowerCamelCase :						List[Any]					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						Any					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						List[str]       )			->					Any:
														__snake_case	:    Optional[Any]											=     self.num_labels
														__snake_case	:    Optional[int]											=     NystromformerForTokenClassification(config=A__       )
														model.to(A__       )
														model.eval()
														__snake_case	:    Union[str, Any]											=     model(A__					,			attention_mask=A__					,			token_type_ids=A__					,			labels=A__       )
														self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.num_labels)       )

							def 					__snake_case      (							self :						Optional[Any]					,			lowerCamelCase :						Dict					,			lowerCamelCase :						Any					,			lowerCamelCase :						List[Any]					,			lowerCamelCase :						int					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						int       )			->					int:
														__snake_case	:    int											=     self.num_choices
														__snake_case	:    Dict											=     NystromformerForMultipleChoice(config=A__       )
														model.to(A__       )
														model.eval()
														__snake_case	:    str											=     input_ids.unsqueeze(1       ).expand(-1					,			self.num_choices					,			-1       ).contiguous()
														__snake_case	:    Tuple											=     token_type_ids.unsqueeze(1       ).expand(-1					,			self.num_choices					,			-1       ).contiguous()
														__snake_case	:    Optional[int]											=     input_mask.unsqueeze(1       ).expand(-1					,			self.num_choices					,			-1       ).contiguous()
														__snake_case	:    Any											=     model(
														    A__					,			attention_mask=A__					,			token_type_ids=A__					,			labels=A__					,			)
														self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_choices)       )






							def 					__snake_case      (							self :						Dict       )			->					List[Any]:
														__snake_case	:    Optional[int]											=     self.prepare_config_and_inputs()
														(
														    (
														    __snake_case		
														)				,    (
														    __snake_case		
														)				,    (
														    __snake_case		
														)				,    (
														    __snake_case		
														)				,    (
														    __snake_case		
														)				,    (
														    __snake_case		
														)				,    (
														    __snake_case		
														)				,    		
														)	:    List[str]											=     config_and_inputs
														__snake_case	:    Optional[Any]											=     {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
														return config, inputs_dict

@require_torch
class        a	(__a  ,       __a  ,       unittest.TestCase   ):




							"""simple docstring"""


							__UpperCAmelCase					:							Optional[Any]  						=						(
							    (
							        NystromformerModel,
							        NystromformerForMaskedLM,
							        NystromformerForMultipleChoice,
							        NystromformerForQuestionAnswering,
							        NystromformerForSequenceClassification,
							        NystromformerForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							__UpperCAmelCase					:							List[Any]  						=						(
							    {
							        "feature-extraction": NystromformerModel,
							        "fill-mask": NystromformerForMaskedLM,
							        "question-answering": NystromformerForQuestionAnswering,
							        "text-classification": NystromformerForSequenceClassification,
							        "token-classification": NystromformerForTokenClassification,
							        "zero-shot": NystromformerForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							__UpperCAmelCase					:							Any  						=						False
							__UpperCAmelCase					:							List[Any]  						=						False

							def 					__snake_case      (							self :						Optional[int]       )			->					Union[str, Any]:
														__snake_case	:    Tuple											=     NystromformerModelTester(self       )
														__snake_case	:    str											=     ConfigTester(self					,			config_class=A__					,			hidden_size=37       )

							def 					__snake_case      (							self :						str       )			->					Any:
														self.config_tester.run_common_tests()

							def 					__snake_case      (							self :						Tuple       )			->					Union[str, Any]:
														__snake_case	:    Optional[int]											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*A__       )

							def 					__snake_case      (							self :						List[str]       )			->					str:
														__snake_case	:    Any											=     self.model_tester.prepare_config_and_inputs()
														for type in ["absolute", "relative_key", "relative_key_query"]:
																					__snake_case	:    List[Any]											=     type
																					self.model_tester.create_and_check_model(*A__       )

							def 					__snake_case      (							self :						int       )			->					List[Any]:
														__snake_case	:    Dict											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_masked_lm(*A__       )

							def 					__snake_case      (							self :						Any       )			->					Optional[Any]:
														__snake_case	:    Optional[Any]											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_multiple_choice(*A__       )

							def 					__snake_case      (							self :						Any       )			->					Tuple:
														__snake_case	:    Dict											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_question_answering(*A__       )

							def 					__snake_case      (							self :						int       )			->					Optional[Any]:
														__snake_case	:    List[Any]											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_sequence_classification(*A__       )

							def 					__snake_case      (							self :						Any       )			->					Dict:
														__snake_case	:    List[str]											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_token_classification(*A__       )






							@slow
							def 					__snake_case      (							self :						Optional[int]       )			->					Tuple:
														for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																					__snake_case	:    Optional[int]											=     NystromformerModel.from_pretrained(A__       )
																					self.assertIsNotNone(A__       )

@require_torch
class        a	(unittest.TestCase   ):




							"""simple docstring"""


							@slow
							def 					__snake_case      (							self :						int       )			->					List[str]:
														__snake_case	:    Dict											=     NystromformerModel.from_pretrained("uw-madison/nystromformer-512"       )
														__snake_case	:    Union[str, Any]											=     torch.tensor([[0, 1, 2, 3, 4, 5]]       )

														with torch.no_grad():
																					__snake_case	:    Optional[int]											=     model(A__       )[0]

														__snake_case	:    Any											=     torch.Size((1, 6, 768)       )
														self.assertEqual(output.shape					,			A__       )

														__snake_case	:    Optional[Any]											=     torch.tensor(
														    [[[-0.45_32, -0.09_36, 0.51_37], [-0.26_76, 0.06_28, 0.61_86], [-0.36_29, -0.17_26, 0.47_16]]]       )

														self.assertTrue(torch.allclose(output[:, :3, :3]					,			A__					,			atol=1E-4       )       )






							@slow
							def 					__snake_case      (							self :						List[str]       )			->					Union[str, Any]:
														__snake_case	:    Dict											=     "the [MASK] of Belgium is Brussels"

														__snake_case	:    str											=     AutoTokenizer.from_pretrained("uw-madison/nystromformer-512"       )
														__snake_case	:    str											=     NystromformerForMaskedLM.from_pretrained("uw-madison/nystromformer-512"       )

														__snake_case	:    Tuple											=     tokenizer(A__					,			return_tensors="pt"       )

														with torch.no_grad():
																					__snake_case	:    Any											=     model(encoding.input_ids       ).logits

														__snake_case	:    Tuple											=     token_logits[:, 2, :].argmax(-1       )[0]

														self.assertEqual(tokenizer.decode(A__       )					,			"capital"       )





import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case  :	Optional[Any]			      =       logging.get_logger(__name__)

_snake_case  :	str			      =       {
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}

class        a	(_lowerCAmelCase   ):




							"""simple docstring"""


							__UpperCAmelCase					:							List[Any]  						=						"xlnet"
							__UpperCAmelCase					:							Dict  						=						["mems"]
							__UpperCAmelCase					:							Tuple  						=						{
							    "n_token": "vocab_size",  # Backward compatibility
							    "hidden_size": "d_model",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}

							def __init__(							self :						List[Any]					,			lowerCamelCase :						Optional[int]=32000					,			lowerCamelCase :						str=1024					,			lowerCamelCase :						int=24					,			lowerCamelCase :						List[Any]=16					,			lowerCamelCase :						Any=4096					,			lowerCamelCase :						Optional[Any]="gelu"					,			lowerCamelCase :						Optional[Any]=True					,			lowerCamelCase :						Optional[int]="bi"					,			lowerCamelCase :						Union[str, Any]=0.02					,			lowerCamelCase :						Union[str, Any]=1E-12					,			lowerCamelCase :						Dict=0.1					,			lowerCamelCase :						int=512					,			lowerCamelCase :						str=None					,			lowerCamelCase :						Dict=True					,			lowerCamelCase :						str=False					,			lowerCamelCase :						Any=False					,			lowerCamelCase :						Optional[Any]=-1					,			lowerCamelCase :						List[Any]=False					,			lowerCamelCase :						Optional[int]="last"					,			lowerCamelCase :						Dict=True					,			lowerCamelCase :						Union[str, Any]="tanh"					,			lowerCamelCase :						int=0.1					,			lowerCamelCase :						Any=5					,			lowerCamelCase :						List[str]=5					,			lowerCamelCase :						List[Any]=5					,			lowerCamelCase :						Tuple=1					,			lowerCamelCase :						Dict=2					,			**lowerCamelCase :						List[str]					,			)			->					List[Any]:
														__snake_case	:    Optional[Any]											=     vocab_size
														__snake_case	:    Any											=     d_model
														__snake_case	:    int											=     n_layer
														__snake_case	:    Optional[int]											=     n_head
														if d_model % n_head != 0:
																					raise ValueError(F'\'d_model % n_head\' ({d_model % n_head}) should be equal to 0'       )
														if "d_head" in kwargs:
																					if kwargs["d_head"] != d_model // n_head:
																												raise ValueError(
																												    F'`d_head` ({kwargs["d_head"]}) should be equal to `d_model // n_head` ({d_model // n_head})'       )
														__snake_case	:    Any											=     d_model // n_head
														__snake_case	:    Tuple											=     ff_activation
														__snake_case	:    int											=     d_inner
														__snake_case	:    Optional[Any]											=     untie_r
														__snake_case	:    Optional[Any]											=     attn_type

														__snake_case	:    List[str]											=     initializer_range
														__snake_case	:    Tuple											=     layer_norm_eps

														__snake_case	:    str											=     dropout
														__snake_case	:    Any											=     mem_len
														__snake_case	:    Tuple											=     reuse_len
														__snake_case	:    Union[str, Any]											=     bi_data
														__snake_case	:    List[Any]											=     clamp_len
														__snake_case	:    Optional[int]											=     same_length

														__snake_case	:    List[str]											=     summary_type
														__snake_case	:    Any											=     summary_use_proj
														__snake_case	:    Any											=     summary_activation
														__snake_case	:    Optional[int]											=     summary_last_dropout
														__snake_case	:    Dict											=     start_n_top
														__snake_case	:    Optional[Any]											=     end_n_top

														__snake_case	:    List[str]											=     bos_token_id
														__snake_case	:    Any											=     pad_token_id
														__snake_case	:    Optional[Any]											=     eos_token_id

														if "use_cache" in kwargs:
																					warnings.warn(
																					    "The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`"
																					    " instead."					,			lowerCamelCase					,			)
																					__snake_case	:    Any											=     kwargs["use_cache"]

														__snake_case	:    int											=     use_mems_eval
														__snake_case	:    Optional[int]											=     use_mems_train
														super().__init__(pad_token_id=lowerCamelCase					,			bos_token_id=lowerCamelCase					,			eos_token_id=lowerCamelCase					,			**lowerCamelCase       )

							@property
							def 					__snake_case      (							self :						Any       )			->					Optional[int]:
														logger.info(F'The model {self.model_type} is one of the few models that has no sequence length limit.'       )
														return -1






							@max_position_embeddings.setter
							def 					__snake_case      (							self :						Union[str, Any]					,			lowerCamelCase :						Any       )			->					Optional[Any]:
														# Message copied from Transformer-XL documentation
														raise NotImplementedError(
														    F'The model {self.model_type} is one of the few models that has no sequence length limit.'       )

