


"""simple docstring"""



import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    NezhaForMaskedLM,
		    NezhaForMultipleChoice,
		    NezhaForNextSentencePrediction,
		    NezhaForPreTraining,
		    NezhaForQuestionAnswering,
		    NezhaForSequenceClassification,
		    NezhaForTokenClassification,
		    NezhaModel,
		)
		from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST







class    __a						:







				def __init__( self      ,	UpperCAmelCase      ,	UpperCAmelCase=13      ,	UpperCAmelCase=7      ,	UpperCAmelCase=True      ,	UpperCAmelCase=True      ,	UpperCAmelCase=True      ,	UpperCAmelCase=True      ,	UpperCAmelCase=99      ,	UpperCAmelCase=32      ,	UpperCAmelCase=5      ,	UpperCAmelCase=4      ,	UpperCAmelCase=37      ,	UpperCAmelCase="gelu"      ,	UpperCAmelCase=0.1      ,	UpperCAmelCase=0.1      ,	UpperCAmelCase=128      ,	UpperCAmelCase=32      ,	UpperCAmelCase=16      ,	UpperCAmelCase=2      ,	UpperCAmelCase=0.0_2      ,	UpperCAmelCase=3      ,	UpperCAmelCase=4      ,	UpperCAmelCase=None      ,	):

					'''simple docstring'''






					lowerCAmelCase_      							=			parent
					lowerCAmelCase_      							=			batch_size
					lowerCAmelCase_      							=			seq_length
					lowerCAmelCase_      							=			is_training
					lowerCAmelCase_      							=			use_input_mask
					lowerCAmelCase_      							=			use_token_type_ids
					lowerCAmelCase_      							=			use_labels
					lowerCAmelCase_      							=			vocab_size
					lowerCAmelCase_      							=			hidden_size
					lowerCAmelCase_      							=			num_hidden_layers
					lowerCAmelCase_      							=			num_attention_heads
					lowerCAmelCase_      							=			intermediate_size
					lowerCAmelCase_      							=			hidden_act
					lowerCAmelCase_      							=			hidden_dropout_prob
					lowerCAmelCase_      							=			attention_probs_dropout_prob
					lowerCAmelCase_      							=			max_position_embeddings
					lowerCAmelCase_      							=			type_vocab_size
					lowerCAmelCase_      							=			type_sequence_label_size
					lowerCAmelCase_      							=			initializer_range
					lowerCAmelCase_      							=			num_labels
					lowerCAmelCase_      							=			num_choices
					lowerCAmelCase_      							=			scope







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			ids_tensor([self.batch_size, self.seq_length]      ,	self.vocab_size   )

					lowerCAmelCase_      							=			None
					if self.use_input_mask:
						lowerCAmelCase_      							=			random_attention_mask([self.batch_size, self.seq_length]   )

					lowerCAmelCase_      							=			None
					if self.use_token_type_ids:
						lowerCAmelCase_      							=			ids_tensor([self.batch_size, self.seq_length]      ,	self.type_vocab_size   )

					lowerCAmelCase_      							=			None
					lowerCAmelCase_      							=			None
					lowerCAmelCase_      							=			None
					if self.use_labels:
						lowerCAmelCase_      							=			ids_tensor([self.batch_size]      ,	self.type_sequence_label_size   )
						lowerCAmelCase_      							=			ids_tensor([self.batch_size, self.seq_length]      ,	self.num_labels   )
						lowerCAmelCase_      							=			ids_tensor([self.batch_size]      ,	self.num_choices   )

					lowerCAmelCase_      							=			self.get_config()

					return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					return NezhaConfig(
					    vocab_size=self.vocab_size      ,	hidden_size=self.hidden_size      ,	num_hidden_layers=self.num_hidden_layers      ,	num_attention_heads=self.num_attention_heads      ,	intermediate_size=self.intermediate_size      ,	hidden_act=self.hidden_act      ,	hidden_dropout_prob=self.hidden_dropout_prob      ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,	max_position_embeddings=self.max_position_embeddings      ,	type_vocab_size=self.type_vocab_size      ,	is_decoder=UpperCAmelCase      ,	initializer_range=self.initializer_range      ,	)







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					(
					    (
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	
					)      							=			self.prepare_config_and_inputs()

					lowerCAmelCase_      							=			True
					lowerCAmelCase_      							=			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
					lowerCAmelCase_      							=			ids_tensor([self.batch_size, self.seq_length]      ,	vocab_size=2   )

					return (
					    config,
					    input_ids,
					    token_type_ids,
					    input_mask,
					    sequence_labels,
					    token_labels,
					    choice_labels,
					    encoder_hidden_states,
					    encoder_attention_mask,
					)







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			NezhaModel(config=UpperCAmelCase   )
					model.to(UpperCAmelCase   )
					model.eval()
					lowerCAmelCase_      							=			model(UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase   )
					lowerCAmelCase_      							=			model(UpperCAmelCase      ,	token_type_ids=UpperCAmelCase   )
					lowerCAmelCase_      							=			model(UpperCAmelCase   )
					self.parent.assertEqual(result.last_hidden_state.shape      ,	(self.batch_size, self.seq_length, self.hidden_size)   )
					self.parent.assertEqual(result.pooler_output.shape      ,	(self.batch_size, self.hidden_size)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	):

					'''simple docstring'''






					lowerCAmelCase_      							=			True
					lowerCAmelCase_      							=			NezhaModel(UpperCAmelCase   )
					model.to(UpperCAmelCase   )
					model.eval()
					lowerCAmelCase_      							=			model(
					    UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase      ,	encoder_hidden_states=UpperCAmelCase      ,	encoder_attention_mask=UpperCAmelCase      ,	)
					lowerCAmelCase_      							=			model(
					    UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase      ,	encoder_hidden_states=UpperCAmelCase      ,	)
					lowerCAmelCase_      							=			model(UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase   )
					self.parent.assertEqual(result.last_hidden_state.shape      ,	(self.batch_size, self.seq_length, self.hidden_size)   )
					self.parent.assertEqual(result.pooler_output.shape      ,	(self.batch_size, self.hidden_size)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			NezhaForMaskedLM(config=UpperCAmelCase   )
					model.to(UpperCAmelCase   )
					model.eval()
					lowerCAmelCase_      							=			model(UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase      ,	labels=UpperCAmelCase   )
					self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.seq_length, self.vocab_size)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			NezhaForNextSentencePrediction(config=UpperCAmelCase   )
					model.to(UpperCAmelCase   )
					model.eval()
					lowerCAmelCase_      							=			model(
					    UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase      ,	labels=UpperCAmelCase      ,	)
					self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, 2)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			NezhaForPreTraining(config=UpperCAmelCase   )
					model.to(UpperCAmelCase   )
					model.eval()
					lowerCAmelCase_      							=			model(
					    UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase      ,	labels=UpperCAmelCase      ,	next_sentence_label=UpperCAmelCase      ,	)
					self.parent.assertEqual(result.prediction_logits.shape      ,	(self.batch_size, self.seq_length, self.vocab_size)   )
					self.parent.assertEqual(result.seq_relationship_logits.shape      ,	(self.batch_size, 2)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			NezhaForQuestionAnswering(config=UpperCAmelCase   )
					model.to(UpperCAmelCase   )
					model.eval()
					lowerCAmelCase_      							=			model(
					    UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase      ,	start_positions=UpperCAmelCase      ,	end_positions=UpperCAmelCase      ,	)
					self.parent.assertEqual(result.start_logits.shape      ,	(self.batch_size, self.seq_length)   )
					self.parent.assertEqual(result.end_logits.shape      ,	(self.batch_size, self.seq_length)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.num_labels
					lowerCAmelCase_      							=			NezhaForSequenceClassification(UpperCAmelCase   )
					model.to(UpperCAmelCase   )
					model.eval()
					lowerCAmelCase_      							=			model(UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase      ,	labels=UpperCAmelCase   )
					self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.num_labels)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.num_labels
					lowerCAmelCase_      							=			NezhaForTokenClassification(config=UpperCAmelCase   )
					model.to(UpperCAmelCase   )
					model.eval()
					lowerCAmelCase_      							=			model(UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase      ,	labels=UpperCAmelCase   )
					self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.seq_length, self.num_labels)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.num_choices
					lowerCAmelCase_      							=			NezhaForMultipleChoice(config=UpperCAmelCase   )
					model.to(UpperCAmelCase   )
					model.eval()
					lowerCAmelCase_      							=			input_ids.unsqueeze(1   ).expand(-1      ,	self.num_choices      ,	-1   ).contiguous()
					lowerCAmelCase_      							=			token_type_ids.unsqueeze(1   ).expand(-1      ,	self.num_choices      ,	-1   ).contiguous()
					lowerCAmelCase_      							=			input_mask.unsqueeze(1   ).expand(-1      ,	self.num_choices      ,	-1   ).contiguous()
					lowerCAmelCase_      							=			model(
					    UpperCAmelCase      ,	attention_mask=UpperCAmelCase      ,	token_type_ids=UpperCAmelCase      ,	labels=UpperCAmelCase      ,	)
					self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.num_choices)   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.prepare_config_and_inputs()
					(
					    (
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	
					)      							=			config_and_inputs
					lowerCAmelCase_      							=			{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
					return config, inputs_dict







@require_torch
class    __a						( __snake_case					,			__snake_case					,			__snake_case					,			unittest.TestCase							):
				lowerCamelCase     :       str			       =(
				    (
				        NezhaModel,
				        NezhaForMaskedLM,
				        NezhaForMultipleChoice,
				        NezhaForNextSentencePrediction,
				        NezhaForPreTraining,
				        NezhaForQuestionAnswering,
				        NezhaForSequenceClassification,
				        NezhaForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				lowerCamelCase     :       Optional[Any]			       =(
				    {
				        'feature-extraction': NezhaModel,
				        'fill-mask': NezhaForMaskedLM,
				        'question-answering': NezhaForQuestionAnswering,
				        'text-classification': NezhaForSequenceClassification,
				        'token-classification': NezhaForTokenClassification,
				        'zero-shot': NezhaForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				lowerCamelCase     :       Optional[int]			       =True







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase=False   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			super()._prepare_for_class(UpperCAmelCase      ,	UpperCAmelCase      ,	return_labels=UpperCAmelCase   )

					if return_labels:
						if model_class in get_values(UpperCAmelCase   ):
							lowerCAmelCase_      							=			torch.zeros(
							    (self.model_tester.batch_size, self.model_tester.seq_length)      ,	dtype=torch.long      ,	device=UpperCAmelCase   )
							lowerCAmelCase_      							=			torch.zeros(
							    self.model_tester.batch_size      ,	dtype=torch.long      ,	device=UpperCAmelCase   )
					return inputs_dict







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			NezhaModelTester(self   )
					lowerCAmelCase_      							=			ConfigTester(self      ,	config_class=UpperCAmelCase      ,	hidden_size=37   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					self.config_tester.run_common_tests()







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(*UpperCAmelCase   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs_for_decoder()
					self.model_tester.create_and_check_model_as_decoder(*UpperCAmelCase   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					(
					    (
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	(
					    lowerCAmelCase_
					)							,	
					)      							=			self.model_tester.prepare_config_and_inputs_for_decoder()

					lowerCAmelCase_      							=			None

					self.model_tester.create_and_check_model_as_decoder(
					    UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	)







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_next_sequence_prediction(*UpperCAmelCase   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_pretraining(*UpperCAmelCase   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase   )







				@slow
				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowerCAmelCase_      							=			NezhaModel.from_pretrained(UpperCAmelCase   )
						self.assertIsNotNone(UpperCAmelCase   )







				@slow
				@require_torch_gpu
				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_							,	lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs_for_common()
					for model_class in self.all_model_classes:
						# NezhaForMultipleChoice behaves incorrectly in JIT environments.
						if model_class == NezhaForMultipleChoice:
							return

						lowerCAmelCase_      							=			True
						lowerCAmelCase_      							=			model_class(config=UpperCAmelCase   )

						lowerCAmelCase_      							=			self._prepare_for_class(UpperCAmelCase      ,	UpperCAmelCase   )
						lowerCAmelCase_      							=			torch.jit.trace(
						    UpperCAmelCase      ,	(inputs_dict['''input_ids'''].to('''cpu'''   ), inputs_dict['''attention_mask'''].to('''cpu'''   ))   )

						with tempfile.TemporaryDirectory() as tmp:
							torch.jit.save(UpperCAmelCase      ,	os.path.join(UpperCAmelCase      ,	'''bert.pt'''   )   )
							lowerCAmelCase_      							=			torch.jit.load(os.path.join(UpperCAmelCase      ,	'''bert.pt'''   )      ,	map_location=UpperCAmelCase   )
							loaded(inputs_dict['''input_ids'''].to(UpperCAmelCase   )      ,	inputs_dict['''attention_mask'''].to(UpperCAmelCase   )   )







@require_torch
class    __a						( unittest.TestCase							):







				@slow
				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			NezhaModel.from_pretrained('''sijunhe/nezha-cn-base'''   )
					lowerCAmelCase_      							=			torch.tensor([[0, 1, 2, 3, 4, 5]]   )
					lowerCAmelCase_      							=			torch.tensor([[0, 1, 1, 1, 1, 1]]   )
					with torch.no_grad():
						lowerCAmelCase_      							=			model(UpperCAmelCase      ,	attention_mask=UpperCAmelCase   )[0]
					lowerCAmelCase_      							=			torch.Size((1, 6, 768)   )
					self.assertEqual(output.shape      ,	UpperCAmelCase   )
					lowerCAmelCase_      							=			torch.tensor([[[0.0_6_8_5, 0.2_4_4_1, 0.1_1_0_2], [0.0_6_0_0, 0.1_9_0_6, 0.1_3_4_9], [0.0_2_2_1, 0.0_8_1_9, 0.0_5_8_6]]]   )

					self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,	UpperCAmelCase      ,	atol=1E-4   )   )







				@slow
				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			NezhaForMaskedLM.from_pretrained('''sijunhe/nezha-cn-base'''   )
					lowerCAmelCase_      							=			torch.tensor([[0, 1, 2, 3, 4, 5]]   )
					lowerCAmelCase_      							=			torch.tensor([[1, 1, 1, 1, 1, 1]]   )
					with torch.no_grad():
						lowerCAmelCase_      							=			model(UpperCAmelCase      ,	attention_mask=UpperCAmelCase   )[0]
					lowerCAmelCase_      							=			torch.Size((1, 6, 2_1128)   )
					self.assertEqual(output.shape      ,	UpperCAmelCase   )
					lowerCAmelCase_      							=			torch.tensor(
					    [[-2.7_9_3_9, -1.7_9_0_2, -2.2_1_8_9], [-2.8_5_8_5, -1.8_9_0_8, -2.3_7_2_3], [-2.6_4_9_9, -1.7_7_5_0, -2.2_5_5_8]]   )

					self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,	UpperCAmelCase      ,	atol=1E-4   )   )



"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase_							=							{
    'configuration_mobilebert': [
        'MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'MobileBertConfig',
        'MobileBertOnnxConfig',
    ],
    'tokenization_mobilebert': ['MobileBertTokenizer'],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase_							=							['MobileBertTokenizerFast']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase_							=							[
		    'MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'MobileBertForMaskedLM',
		    'MobileBertForMultipleChoice',
		    'MobileBertForNextSentencePrediction',
		    'MobileBertForPreTraining',
		    'MobileBertForQuestionAnswering',
		    'MobileBertForSequenceClassification',
		    'MobileBertForTokenClassification',
		    'MobileBertLayer',
		    'MobileBertModel',
		    'MobileBertPreTrainedModel',
		    'load_tf_weights_in_mobilebert',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase_							=							[
		    'TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'TFMobileBertForMaskedLM',
		    'TFMobileBertForMultipleChoice',
		    'TFMobileBertForNextSentencePrediction',
		    'TFMobileBertForPreTraining',
		    'TFMobileBertForQuestionAnswering',
		    'TFMobileBertForSequenceClassification',
		    'TFMobileBertForTokenClassification',
		    'TFMobileBertMainLayer',
		    'TFMobileBertModel',
		    'TFMobileBertPreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_mobilebert import (
		    MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    MobileBertConfig,
		    MobileBertOnnxConfig,
		)
		from .tokenization_mobilebert import MobileBertTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_mobilebert_fast import MobileBertTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_mobilebert import (
				    MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    MobileBertForMaskedLM,
				    MobileBertForMultipleChoice,
				    MobileBertForNextSentencePrediction,
				    MobileBertForPreTraining,
				    MobileBertForQuestionAnswering,
				    MobileBertForSequenceClassification,
				    MobileBertForTokenClassification,
				    MobileBertLayer,
				    MobileBertModel,
				    MobileBertPreTrainedModel,
				    load_tf_weights_in_mobilebert,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_mobilebert import (
				    TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFMobileBertForMaskedLM,
				    TFMobileBertForMultipleChoice,
				    TFMobileBertForNextSentencePrediction,
				    TFMobileBertForPreTraining,
				    TFMobileBertForQuestionAnswering,
				    TFMobileBertForSequenceClassification,
				    TFMobileBertForTokenClassification,
				    TFMobileBertMainLayer,
				    TFMobileBertModel,
				    TFMobileBertPreTrainedModel,
				)

else:
		import sys

		lowercase_							=							_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)