





import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    NezhaForMaskedLM,
					    NezhaForMultipleChoice,
					    NezhaForNextSentencePrediction,
					    NezhaForPreTraining,
					    NezhaForQuestionAnswering,
					    NezhaForSequenceClassification,
					    NezhaForTokenClassification,
					    NezhaModel,
					)
					from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST

class        __SCREAMING_SNAKE_CASE					:






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase=13  , __lowerCAmelCase=7  , __lowerCAmelCase=True  , __lowerCAmelCase=True  , __lowerCAmelCase=True  , __lowerCAmelCase=True  , __lowerCAmelCase=99  , __lowerCAmelCase=32  , __lowerCAmelCase=5  , __lowerCAmelCase=4  , __lowerCAmelCase=37  , __lowerCAmelCase="gelu"  , __lowerCAmelCase=0.1  , __lowerCAmelCase=0.1  , __lowerCAmelCase=128  , __lowerCAmelCase=32  , __lowerCAmelCase=16  , __lowerCAmelCase=2  , __lowerCAmelCase=0.02  , __lowerCAmelCase=3  , __lowerCAmelCase=4  , __lowerCAmelCase=None  , ):
														UpperCamelCase__				     =			parent
														UpperCamelCase__				     =			batch_size
														UpperCamelCase__				     =			seq_length
														UpperCamelCase__				     =			is_training
														UpperCamelCase__				     =			use_input_mask
														UpperCamelCase__				     =			use_token_type_ids
														UpperCamelCase__				     =			use_labels
														UpperCamelCase__				     =			vocab_size
														UpperCamelCase__				     =			hidden_size
														UpperCamelCase__				     =			num_hidden_layers
														UpperCamelCase__				     =			num_attention_heads
														UpperCamelCase__				     =			intermediate_size
														UpperCamelCase__				     =			hidden_act
														UpperCamelCase__				     =			hidden_dropout_prob
														UpperCamelCase__				     =			attention_probs_dropout_prob
														UpperCamelCase__				     =			max_position_embeddings
														UpperCamelCase__				     =			type_vocab_size
														UpperCamelCase__				     =			type_sequence_label_size
														UpperCamelCase__				     =			initializer_range
														UpperCamelCase__				     =			num_labels
														UpperCamelCase__				     =			num_choices
														UpperCamelCase__				     =			scope






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			ids_tensor([self.batch_size, self.seq_length]  , self.vocab_size  )

														UpperCamelCase__				     =			None
														if self.use_input_mask:
																					UpperCamelCase__				     =			random_attention_mask([self.batch_size, self.seq_length]  )

														UpperCamelCase__				     =			None
														if self.use_token_type_ids:
																					UpperCamelCase__				     =			ids_tensor([self.batch_size, self.seq_length]  , self.type_vocab_size  )

														UpperCamelCase__				     =			None
														UpperCamelCase__				     =			None
														UpperCamelCase__				     =			None
														if self.use_labels:
																					UpperCamelCase__				     =			ids_tensor([self.batch_size]  , self.type_sequence_label_size  )
																					UpperCamelCase__				     =			ids_tensor([self.batch_size, self.seq_length]  , self.num_labels  )
																					UpperCamelCase__				     =			ids_tensor([self.batch_size]  , self.num_choices  )

														UpperCamelCase__				     =			self.get_config()

														return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






							def 				_lowerCamelCase		(						self  ):
														return NezhaConfig(
														    vocab_size=self.vocab_size  , hidden_size=self.hidden_size  , num_hidden_layers=self.num_hidden_layers  , num_attention_heads=self.num_attention_heads  , intermediate_size=self.intermediate_size  , hidden_act=self.hidden_act  , hidden_dropout_prob=self.hidden_dropout_prob  , attention_probs_dropout_prob=self.attention_probs_dropout_prob  , max_position_embeddings=self.max_position_embeddings  , type_vocab_size=self.type_vocab_size  , is_decoder=lowerCAmelCase_  , initializer_range=self.initializer_range  , )






							def 				_lowerCamelCase		(						self  ):
														(
														    (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     						
														)				     =			self.prepare_config_and_inputs()

														UpperCamelCase__				     =			True
														UpperCamelCase__				     =			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]  )
														UpperCamelCase__				     =			ids_tensor([self.batch_size, self.seq_length]  , vocab_size=2  )

														return (
														    config,
														    input_ids,
														    token_type_ids,
														    input_mask,
														    sequence_labels,
														    token_labels,
														    choice_labels,
														    encoder_hidden_states,
														    encoder_attention_mask,
														)






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			NezhaModel(config=lowerCAmelCase_  )
														model.to(lowerCAmelCase_  )
														model.eval()
														UpperCamelCase__				     =			model(lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  )
														UpperCamelCase__				     =			model(lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  )
														UpperCamelCase__				     =			model(lowerCAmelCase_  )
														self.parent.assertEqual(result.last_hidden_state.shape  , (self.batch_size, self.seq_length, self.hidden_size)  )
														self.parent.assertEqual(result.pooler_output.shape  , (self.batch_size, self.hidden_size)  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , ):
														UpperCamelCase__				     =			True
														UpperCamelCase__				     =			NezhaModel(lowerCAmelCase_  )
														model.to(lowerCAmelCase_  )
														model.eval()
														UpperCamelCase__				     =			model(
														    lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  , encoder_hidden_states=lowerCAmelCase_  , encoder_attention_mask=lowerCAmelCase_  , )
														UpperCamelCase__				     =			model(
														    lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  , encoder_hidden_states=lowerCAmelCase_  , )
														UpperCamelCase__				     =			model(lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  )
														self.parent.assertEqual(result.last_hidden_state.shape  , (self.batch_size, self.seq_length, self.hidden_size)  )
														self.parent.assertEqual(result.pooler_output.shape  , (self.batch_size, self.hidden_size)  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			NezhaForMaskedLM(config=lowerCAmelCase_  )
														model.to(lowerCAmelCase_  )
														model.eval()
														UpperCamelCase__				     =			model(lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  , labels=lowerCAmelCase_  )
														self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.seq_length, self.vocab_size)  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			NezhaForNextSentencePrediction(config=lowerCAmelCase_  )
														model.to(lowerCAmelCase_  )
														model.eval()
														UpperCamelCase__				     =			model(
														    lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  , labels=lowerCAmelCase_  , )
														self.parent.assertEqual(result.logits.shape  , (self.batch_size, 2)  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			NezhaForPreTraining(config=lowerCAmelCase_  )
														model.to(lowerCAmelCase_  )
														model.eval()
														UpperCamelCase__				     =			model(
														    lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  , labels=lowerCAmelCase_  , next_sentence_label=lowerCAmelCase_  , )
														self.parent.assertEqual(result.prediction_logits.shape  , (self.batch_size, self.seq_length, self.vocab_size)  )
														self.parent.assertEqual(result.seq_relationship_logits.shape  , (self.batch_size, 2)  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			NezhaForQuestionAnswering(config=lowerCAmelCase_  )
														model.to(lowerCAmelCase_  )
														model.eval()
														UpperCamelCase__				     =			model(
														    lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  , start_positions=lowerCAmelCase_  , end_positions=lowerCAmelCase_  , )
														self.parent.assertEqual(result.start_logits.shape  , (self.batch_size, self.seq_length)  )
														self.parent.assertEqual(result.end_logits.shape  , (self.batch_size, self.seq_length)  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			self.num_labels
														UpperCamelCase__				     =			NezhaForSequenceClassification(lowerCAmelCase_  )
														model.to(lowerCAmelCase_  )
														model.eval()
														UpperCamelCase__				     =			model(lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  , labels=lowerCAmelCase_  )
														self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.num_labels)  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			self.num_labels
														UpperCamelCase__				     =			NezhaForTokenClassification(config=lowerCAmelCase_  )
														model.to(lowerCAmelCase_  )
														model.eval()
														UpperCamelCase__				     =			model(lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  , labels=lowerCAmelCase_  )
														self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.seq_length, self.num_labels)  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			self.num_choices
														UpperCamelCase__				     =			NezhaForMultipleChoice(config=lowerCAmelCase_  )
														model.to(lowerCAmelCase_  )
														model.eval()
														UpperCamelCase__				     =			input_ids.unsqueeze(1  ).expand(-1  , self.num_choices  , -1  ).contiguous()
														UpperCamelCase__				     =			token_type_ids.unsqueeze(1  ).expand(-1  , self.num_choices  , -1  ).contiguous()
														UpperCamelCase__				     =			input_mask.unsqueeze(1  ).expand(-1  , self.num_choices  , -1  ).contiguous()
														UpperCamelCase__				     =			model(
														    lowerCAmelCase_  , attention_mask=lowerCAmelCase_  , token_type_ids=lowerCAmelCase_  , labels=lowerCAmelCase_  , )
														self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.num_choices)  )







							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.prepare_config_and_inputs()
														(
														    (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     						
														)				     =			config_and_inputs
														UpperCamelCase__				     =			{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
														return config, inputs_dict






@require_torch
class        __SCREAMING_SNAKE_CASE					(     __a   ,		__a   ,		__a   ,		unittest.TestCase       ):
							snake_case     :      List[Any]						     =       (
							    (
							        NezhaModel,
							        NezhaForMaskedLM,
							        NezhaForMultipleChoice,
							        NezhaForNextSentencePrediction,
							        NezhaForPreTraining,
							        NezhaForQuestionAnswering,
							        NezhaForSequenceClassification,
							        NezhaForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							snake_case     :      List[Any]						     =       (
							    {
							        "feature-extraction": NezhaModel,
							        "fill-mask": NezhaForMaskedLM,
							        "question-answering": NezhaForQuestionAnswering,
							        "text-classification": NezhaForSequenceClassification,
							        "token-classification": NezhaForTokenClassification,
							        "zero-shot": NezhaForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							snake_case     :      List[str]						     =       True






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase=False  ):
														UpperCamelCase__				     =			super()._prepare_for_class(lowerCAmelCase_  , lowerCAmelCase_  , return_labels=lowerCAmelCase_  )

														if return_labels:
																					if model_class in get_values(lowerCAmelCase_  ):
																												UpperCamelCase__				     =			torch.zeros(
																												    (self.model_tester.batch_size, self.model_tester.seq_length)  , dtype=torch.long  , device=lowerCAmelCase_  )
																												UpperCamelCase__				     =			torch.zeros(
																												    self.model_tester.batch_size  , dtype=torch.long  , device=lowerCAmelCase_  )
														return inputs_dict






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			NezhaModelTester(self  )
														UpperCamelCase__				     =			ConfigTester(self  , config_class=lowerCAmelCase_  , hidden_size=37  )






							def 				_lowerCamelCase		(						self  ):
														self.config_tester.run_common_tests()






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*lowerCAmelCase_  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs_for_decoder()
														self.model_tester.create_and_check_model_as_decoder(*lowerCAmelCase_  )






							def 				_lowerCamelCase		(						self  ):
														(
														    (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     (
														    UpperCamelCase__						
														)     ,     						
														)				     =			self.model_tester.prepare_config_and_inputs_for_decoder()

														UpperCamelCase__				     =			None

														self.model_tester.create_and_check_model_as_decoder(
														    lowerCAmelCase_  , lowerCAmelCase_  , lowerCAmelCase_  , lowerCAmelCase_  , lowerCAmelCase_  , lowerCAmelCase_  , lowerCAmelCase_  , lowerCAmelCase_  , lowerCAmelCase_  , )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase_  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_next_sequence_prediction(*lowerCAmelCase_  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase_  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase_  )






							@slow
							def 				_lowerCamelCase		(						self  ):







														for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																					UpperCamelCase__				     =			NezhaModel.from_pretrained(lowerCAmelCase_  )
																					self.assertIsNotNone(lowerCAmelCase_  )







							@slow
							@require_torch_gpu
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__     ,     UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs_for_common()
														for model_class in self.all_model_classes:
																					# NezhaForMultipleChoice behaves incorrectly in JIT environments.
																					if model_class == NezhaForMultipleChoice:
																												return

																					UpperCamelCase__				     =			True
																					UpperCamelCase__				     =			model_class(config=lowerCAmelCase_  )

																					UpperCamelCase__				     =			self._prepare_for_class(lowerCAmelCase_  , lowerCAmelCase_  )
																					UpperCamelCase__				     =			torch.jit.trace(
																					    lowerCAmelCase_  , (inputs_dict["""input_ids"""].to("""cpu"""  ), inputs_dict["""attention_mask"""].to("""cpu"""  ))  )

																					with tempfile.TemporaryDirectory() as tmp:
																												torch.jit.save(lowerCAmelCase_  , os.path.join(lowerCAmelCase_  , """bert.pt"""  )  )
																												UpperCamelCase__				     =			torch.jit.load(os.path.join(lowerCAmelCase_  , """bert.pt"""  )  , map_location=lowerCAmelCase_  )
																												loaded(inputs_dict["""input_ids"""].to(lowerCAmelCase_  )  , inputs_dict["""attention_mask"""].to(lowerCAmelCase_  )  )






@require_torch
class        __SCREAMING_SNAKE_CASE					(     unittest.TestCase       ):






							@slow
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			NezhaModel.from_pretrained("""sijunhe/nezha-cn-base"""  )
														UpperCamelCase__				     =			torch.tensor([[0, 1, 2, 3, 4, 5]]  )
														UpperCamelCase__				     =			torch.tensor([[0, 1, 1, 1, 1, 1]]  )
														with torch.no_grad():
																					UpperCamelCase__				     =			model(lowerCAmelCase_  , attention_mask=lowerCAmelCase_  )[0]
														UpperCamelCase__				     =			torch.Size((1, 6, 768)  )
														self.assertEqual(output.shape  , lowerCAmelCase_  )
														UpperCamelCase__				     =			torch.tensor([[[0.0685, 0.2441, 0.1102], [0.0600, 0.1906, 0.1349], [0.0221, 0.0819, 0.0586]]]  )

														self.assertTrue(torch.allclose(output[:, 1:4, 1:4]  , lowerCAmelCase_  , atol=1E-4  )  )







							@slow
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			NezhaForMaskedLM.from_pretrained("""sijunhe/nezha-cn-base"""  )
														UpperCamelCase__				     =			torch.tensor([[0, 1, 2, 3, 4, 5]]  )
														UpperCamelCase__				     =			torch.tensor([[1, 1, 1, 1, 1, 1]]  )
														with torch.no_grad():
																					UpperCamelCase__				     =			model(lowerCAmelCase_  , attention_mask=lowerCAmelCase_  )[0]
														UpperCamelCase__				     =			torch.Size((1, 6, 21128)  )
														self.assertEqual(output.shape  , lowerCAmelCase_  )
														UpperCamelCase__				     =			torch.tensor(
														    [[-2.7939, -1.7902, -2.2189], [-2.8585, -1.8908, -2.3723], [-2.6499, -1.7750, -2.2558]]  )

														self.assertTrue(torch.allclose(output[:, 1:4, 1:4]  , lowerCAmelCase_  , atol=1E-4  )  )









import inspect
import unittest

class        __SCREAMING_SNAKE_CASE					(     unittest.TestCase       ):






							def 				_lowerCamelCase		(						self  ):
														try:
																					import diffusers  # noqa: F401
														except ImportError:
																					assert False







							def 				_lowerCamelCase		(						self  ):
														import diffusers
														from diffusers.dependency_versions_table import deps

														UpperCamelCase__				     =			inspect.getmembers(__lowerCAmelCase  , inspect.isclass  )

														for cls_name, cls_module in all_classes:
																					if "dummy_" in cls_module.__module__:
																												for backend in cls_module._backends:
																																			if backend == "k_diffusion":
																																										UpperCamelCase__				     =			"""k-diffusion"""
																																			elif backend == "invisible_watermark":
																																										UpperCamelCase__				     =			"""invisible-watermark"""
																																			assert backend in deps, f"""{backend} is not in the deps table!"""




