




import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    MobileBertForMaskedLM,
					    MobileBertForMultipleChoice,
					    MobileBertForNextSentencePrediction,
					    MobileBertForPreTraining,
					    MobileBertForQuestionAnswering,
					    MobileBertForSequenceClassification,
					    MobileBertForTokenClassification,
					    MobileBertModel,
					)







class        __lowerCAmelCase					:

			"""simple docstring"""







			def __init__( self     ,		lowerCamelCase__     ,		lowerCamelCase__=13     ,		lowerCamelCase__=7     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=99     ,		lowerCamelCase__=64     ,		lowerCamelCase__=32     ,		lowerCamelCase__=5     ,		lowerCamelCase__=4     ,		lowerCamelCase__=37     ,		lowerCamelCase__="gelu"     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=512     ,		lowerCamelCase__=16     ,		lowerCamelCase__=2     ,		lowerCamelCase__=0.02     ,		lowerCamelCase__=3     ,		lowerCamelCase__=4     ,		lowerCamelCase__=None     ,		)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       parent
							__lowerCamelCase									=       batch_size
							__lowerCamelCase									=       seq_length
							__lowerCamelCase									=       is_training
							__lowerCamelCase									=       use_input_mask
							__lowerCamelCase									=       use_token_type_ids
							__lowerCamelCase									=       use_labels
							__lowerCamelCase									=       vocab_size
							__lowerCamelCase									=       hidden_size
							__lowerCamelCase									=       embedding_size
							__lowerCamelCase									=       num_hidden_layers
							__lowerCamelCase									=       num_attention_heads
							__lowerCamelCase									=       intermediate_size
							__lowerCamelCase									=       hidden_act
							__lowerCamelCase									=       hidden_dropout_prob
							__lowerCamelCase									=       attention_probs_dropout_prob
							__lowerCamelCase									=       max_position_embeddings
							__lowerCamelCase									=       type_vocab_size
							__lowerCamelCase									=       type_sequence_label_size
							__lowerCamelCase									=       initializer_range
							__lowerCamelCase									=       num_labels
							__lowerCamelCase									=       num_choices
							__lowerCamelCase									=       scope


			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       ids_tensor([self.batch_size, self.seq_length]     ,		self.vocab_size				)

							__lowerCamelCase									=       None
							if self.use_input_mask:
											__lowerCamelCase									=       random_attention_mask([self.batch_size, self.seq_length]				)

							__lowerCamelCase									=       None
							if self.use_token_type_ids:
											__lowerCamelCase									=       ids_tensor([self.batch_size, self.seq_length]     ,		self.type_vocab_size				)

							__lowerCamelCase									=       None
							__lowerCamelCase									=       None
							__lowerCamelCase									=       None
							if self.use_labels:
											__lowerCamelCase									=       ids_tensor([self.batch_size]     ,		self.type_sequence_label_size				)
											__lowerCamelCase									=       ids_tensor([self.batch_size, self.seq_length]     ,		self.num_labels				)
											__lowerCamelCase									=       ids_tensor([self.batch_size]     ,		self.num_choices				)

							__lowerCamelCase									=       self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							return MobileBertConfig(
							    vocab_size=self.vocab_size     ,		hidden_size=self.hidden_size     ,		num_hidden_layers=self.num_hidden_layers     ,		num_attention_heads=self.num_attention_heads     ,		intermediate_size=self.intermediate_size     ,		embedding_size=self.embedding_size     ,		hidden_act=self.hidden_act     ,		hidden_dropout_prob=self.hidden_dropout_prob     ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,		max_position_embeddings=self.max_position_embeddings     ,		type_vocab_size=self.type_vocab_size     ,		is_decoder=lowerCamelCase__     ,		initializer_range=self.initializer_range     ,		)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       MobileBertModel(config=lowerCamelCase__				)
							model.to(lowerCamelCase__				)
							model.eval()
							__lowerCamelCase									=       model(lowerCamelCase__     ,		attention_mask=lowerCamelCase__     ,		token_type_ids=lowerCamelCase__				)
							__lowerCamelCase									=       model(lowerCamelCase__     ,		token_type_ids=lowerCamelCase__				)
							__lowerCamelCase									=       model(lowerCamelCase__				)

							self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, self.seq_length, self.hidden_size)				)
							self.parent.assertEqual(result.pooler_output.shape     ,		(self.batch_size, self.hidden_size)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Dict:



							'''simple docstring'''
							__lowerCamelCase									=       MobileBertForMaskedLM(config=lowerCamelCase__				)
							model.to(lowerCamelCase__				)
							model.eval()
							__lowerCamelCase									=       model(lowerCamelCase__     ,		attention_mask=lowerCamelCase__     ,		token_type_ids=lowerCamelCase__     ,		labels=lowerCamelCase__				)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.seq_length, self.vocab_size)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Dict:



							'''simple docstring'''
							__lowerCamelCase									=       MobileBertForNextSentencePrediction(config=lowerCamelCase__				)
							model.to(lowerCamelCase__				)
							model.eval()
							__lowerCamelCase									=       model(
							    lowerCamelCase__     ,		attention_mask=lowerCamelCase__     ,		token_type_ids=lowerCamelCase__     ,		labels=lowerCamelCase__     ,		)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, 2)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       MobileBertForPreTraining(config=lowerCamelCase__				)
							model.to(lowerCamelCase__				)
							model.eval()
							__lowerCamelCase									=       model(
							    lowerCamelCase__     ,		attention_mask=lowerCamelCase__     ,		token_type_ids=lowerCamelCase__     ,		labels=lowerCamelCase__     ,		next_sentence_label=lowerCamelCase__     ,		)
							self.parent.assertEqual(result.prediction_logits.shape     ,		(self.batch_size, self.seq_length, self.vocab_size)				)
							self.parent.assertEqual(result.seq_relationship_logits.shape     ,		(self.batch_size, 2)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       MobileBertForQuestionAnswering(config=lowerCamelCase__				)
							model.to(lowerCamelCase__				)
							model.eval()
							__lowerCamelCase									=       model(
							    lowerCamelCase__     ,		attention_mask=lowerCamelCase__     ,		token_type_ids=lowerCamelCase__     ,		start_positions=lowerCamelCase__     ,		end_positions=lowerCamelCase__     ,		)
							self.parent.assertEqual(result.start_logits.shape     ,		(self.batch_size, self.seq_length)				)
							self.parent.assertEqual(result.end_logits.shape     ,		(self.batch_size, self.seq_length)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       self.num_labels
							__lowerCamelCase									=       MobileBertForSequenceClassification(lowerCamelCase__				)
							model.to(lowerCamelCase__				)
							model.eval()
							__lowerCamelCase									=       model(lowerCamelCase__     ,		attention_mask=lowerCamelCase__     ,		token_type_ids=lowerCamelCase__     ,		labels=lowerCamelCase__				)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.num_labels)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       self.num_labels
							__lowerCamelCase									=       MobileBertForTokenClassification(config=lowerCamelCase__				)
							model.to(lowerCamelCase__				)
							model.eval()
							__lowerCamelCase									=       model(lowerCamelCase__     ,		attention_mask=lowerCamelCase__     ,		token_type_ids=lowerCamelCase__     ,		labels=lowerCamelCase__				)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.seq_length, self.num_labels)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Dict:



							'''simple docstring'''
							__lowerCamelCase									=       self.num_choices
							__lowerCamelCase									=       MobileBertForMultipleChoice(config=lowerCamelCase__				)
							model.to(lowerCamelCase__				)
							model.eval()
							__lowerCamelCase									=       input_ids.unsqueeze(1				).expand(-1     ,		self.num_choices     ,		-1				).contiguous()
							__lowerCamelCase									=       token_type_ids.unsqueeze(1				).expand(-1     ,		self.num_choices     ,		-1				).contiguous()
							__lowerCamelCase									=       input_mask.unsqueeze(1				).expand(-1     ,		self.num_choices     ,		-1				).contiguous()
							__lowerCamelCase									=       model(
							    lowerCamelCase__     ,		attention_mask=lowerCamelCase__     ,		token_type_ids=lowerCamelCase__     ,		labels=lowerCamelCase__     ,		)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.num_choices)				)






			def  lowercase_       ( self				)       -> List[str]:



							'''simple docstring'''
							__lowerCamelCase									=       self.prepare_config_and_inputs()
							(
							    (
							    __lowerCamelCase
							)			,		(
							    __lowerCamelCase
							)			,		(
							    __lowerCamelCase
							)			,		(
							    __lowerCamelCase
							)			,		(
							    __lowerCamelCase
							)			,		(
							    __lowerCamelCase
							)			,		(
							    __lowerCamelCase
							)			,		
							)									=       config_and_inputs
							__lowerCamelCase									=       {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict







@require_torch
class        __lowerCAmelCase					(			__magic_name__     ,							__magic_name__     ,							unittest.TestCase					):

			"""simple docstring"""







			snake_case_  						=			(
			    (
			        MobileBertModel,
			        MobileBertForMaskedLM,
			        MobileBertForMultipleChoice,
			        MobileBertForNextSentencePrediction,
			        MobileBertForPreTraining,
			        MobileBertForQuestionAnswering,
			        MobileBertForSequenceClassification,
			        MobileBertForTokenClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			snake_case_  						=			(
			    {
			        '''feature-extraction''': MobileBertModel,
			        '''fill-mask''': MobileBertForMaskedLM,
			        '''question-answering''': MobileBertForQuestionAnswering,
			        '''text-classification''': MobileBertForSequenceClassification,
			        '''token-classification''': MobileBertForTokenClassification,
			        '''zero-shot''': MobileBertForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			snake_case_  						=			True


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__=False				)       -> List[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       super()._prepare_for_class(lowerCamelCase__     ,		lowerCamelCase__     ,		return_labels=lowerCamelCase__				)

							if return_labels:
											if model_class in get_values(lowerCamelCase__				):
															__lowerCamelCase									=       torch.zeros(
															    (self.model_tester.batch_size, self.model_tester.seq_length)     ,		dtype=torch.long     ,		device=lowerCamelCase__				)
															__lowerCamelCase									=       torch.zeros(
															    self.model_tester.batch_size     ,		dtype=torch.long     ,		device=lowerCamelCase__				)
							return inputs_dict


			def  lowercase_       ( self				)       -> List[str]:



							'''simple docstring'''
							__lowerCamelCase									=       MobileBertModelTester(self				)
							__lowerCamelCase									=       ConfigTester(self     ,		config_class=lowerCamelCase__     ,		hidden_size=37				)


			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							self.config_tester.run_common_tests()


			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_model(*lowerCamelCase__				)


			def  lowercase_       ( self				)       -> List[str]:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_masked_lm(*lowerCamelCase__				)


			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_multiple_choice(*lowerCamelCase__				)


			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*lowerCamelCase__				)


			def  lowercase_       ( self				)       -> List[str]:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_pretraining(*lowerCamelCase__				)


			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_question_answering(*lowerCamelCase__				)


			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_sequence_classification(*lowerCamelCase__				)






			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_token_classification(*lowerCamelCase__				)







def      lowerCamelCase_						(			UpperCamelCase__							: Any )	->			List[Any]:





				"""simple docstring"""


				return torch.tensor(
				    UpperCamelCase__					,	dtype=torch.long					,	device=UpperCamelCase__					,	)


__A								=       1e-3







@require_torch
@require_sentencepiece
@require_tokenizers
class        __lowerCAmelCase					(			unittest.TestCase					):

			"""simple docstring"""







			@slow
			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							__lowerCamelCase									=       MobileBertModel.from_pretrained('google/mobilebert-uncased'				).to(lowerCamelCase__				)
							__lowerCamelCase									=       _long_tensor([[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]				)
							with torch.no_grad():
											__lowerCamelCase									=       model(lowerCamelCase__				)[0]
							__lowerCamelCase									=       torch.Size((1, 9, 512)				)
							self.assertEqual(output.shape     ,		lowerCamelCase__				)
							__lowerCamelCase									=       torch.tensor(
							    [
							        [
							            [-2.4_73_65_26e07, 8.2_69_16_56e04, 1.6_52_18_38e05],
							            [-5.7_54_17_04e-01, 3.9_05_60_22e00, 4.4_01_15_07e00],
							            [2.6_04_73_59e00, 1.5_67_76_52e00, -1.7_32_41_88e-01],
							        ]
							    ]     ,		device=lowerCamelCase__     ,		)

							# MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
							# ~1 difference, it's therefore not a good idea to measure using addition.
							# Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
							# result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
							__lowerCamelCase									=       torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE				)
							__lowerCamelCase									=       torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE				)

							self.assertTrue(lower_bound and upper_bound				)








def      lowerCamelCase_						(			)	->			List[str]:





				"""simple docstring"""


				__lowerCamelCase									=       0
				for i in range(1					,	1001 ):
								total += i**i
				return str(UpperCamelCase__ )[-10:]


if __name__ == "__main__":
					print(solution())


