





import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       MODEL_FOR_PRETRAINING_MAPPING,
       MobileBertForMaskedLM,
       MobileBertForMultipleChoice,
       MobileBertForNextSentencePrediction,
       MobileBertForPreTraining,
       MobileBertForQuestionAnswering,
       MobileBertForSequenceClassification,
       MobileBertForTokenClassification,
       MobileBertModel,
   )




class 			__UpperCamelCase    :





     def __init__(     self				,							_UpperCamelCase				,							_UpperCamelCase=13				,							_UpperCamelCase=7				,							_UpperCamelCase=True				,							_UpperCamelCase=True				,							_UpperCamelCase=True				,							_UpperCamelCase=True				,							_UpperCamelCase=99				,							_UpperCamelCase=64				,							_UpperCamelCase=32				,							_UpperCamelCase=5				,							_UpperCamelCase=4				,							_UpperCamelCase=37				,							_UpperCamelCase="gelu"				,							_UpperCamelCase=0.1				,							_UpperCamelCase=0.1				,							_UpperCamelCase=512				,							_UpperCamelCase=16				,							_UpperCamelCase=2				,							_UpperCamelCase=0.02				,							_UpperCamelCase=3				,							_UpperCamelCase=4				,							_UpperCamelCase=None				,							):
        _UpperCAmelCase           =   parent
        _UpperCAmelCase           =   batch_size
        _UpperCAmelCase           =   seq_length
        _UpperCAmelCase           =   is_training
        _UpperCAmelCase           =   use_input_mask
        _UpperCAmelCase           =   use_token_type_ids
        _UpperCAmelCase           =   use_labels
        _UpperCAmelCase           =   vocab_size
        _UpperCAmelCase           =   hidden_size
        _UpperCAmelCase           =   embedding_size
        _UpperCAmelCase           =   num_hidden_layers
        _UpperCAmelCase           =   num_attention_heads
        _UpperCAmelCase           =   intermediate_size
        _UpperCAmelCase           =   hidden_act
        _UpperCAmelCase           =   hidden_dropout_prob
        _UpperCAmelCase           =   attention_probs_dropout_prob
        _UpperCAmelCase           =   max_position_embeddings
        _UpperCAmelCase           =   type_vocab_size
        _UpperCAmelCase           =   type_sequence_label_size
        _UpperCAmelCase           =   initializer_range
        _UpperCAmelCase           =   num_labels
        _UpperCAmelCase           =   num_choices
        _UpperCAmelCase           =   scope





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   ids_tensor([self.batch_size, self.seq_length]				,							self.vocab_size					)

        _UpperCAmelCase           =   None
        if self.use_input_mask:
           _UpperCAmelCase           =   random_attention_mask([self.batch_size, self.seq_length]					)

        _UpperCAmelCase           =   None
        if self.use_token_type_ids:
           _UpperCAmelCase           =   ids_tensor([self.batch_size, self.seq_length]				,							self.type_vocab_size					)

        _UpperCAmelCase           =   None
        _UpperCAmelCase           =   None
        _UpperCAmelCase           =   None
        if self.use_labels:
           _UpperCAmelCase           =   ids_tensor([self.batch_size]				,							self.type_sequence_label_size					)
           _UpperCAmelCase           =   ids_tensor([self.batch_size, self.seq_length]				,							self.num_labels					)
           _UpperCAmelCase           =   ids_tensor([self.batch_size]				,							self.num_choices					)

        _UpperCAmelCase           =   self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





     def 		UpperCamelCase(     self					):
        return MobileBertConfig(
            vocab_size=self.vocab_size				,							hidden_size=self.hidden_size				,							num_hidden_layers=self.num_hidden_layers				,							num_attention_heads=self.num_attention_heads				,							intermediate_size=self.intermediate_size				,							embedding_size=self.embedding_size				,							hidden_act=self.hidden_act				,							hidden_dropout_prob=self.hidden_dropout_prob				,							attention_probs_dropout_prob=self.attention_probs_dropout_prob				,							max_position_embeddings=self.max_position_embeddings				,							type_vocab_size=self.type_vocab_size				,							is_decoder=_UpperCamelCase				,							initializer_range=self.initializer_range				,							)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   MobileBertModel(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(_UpperCamelCase				,							attention_mask=_UpperCamelCase				,							token_type_ids=_UpperCamelCase					)
        _UpperCAmelCase           =   model(_UpperCamelCase				,							token_type_ids=_UpperCamelCase					)
        _UpperCAmelCase           =   model(_UpperCamelCase					)

        self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)					)
        self.parent.assertEqual(result.pooler_output.shape				,							(self.batch_size, self.hidden_size)					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   MobileBertForMaskedLM(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(_UpperCamelCase				,							attention_mask=_UpperCamelCase				,							token_type_ids=_UpperCamelCase				,							labels=_UpperCamelCase					)
        self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.vocab_size)					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   MobileBertForNextSentencePrediction(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(
            _UpperCamelCase				,							attention_mask=_UpperCamelCase				,							token_type_ids=_UpperCamelCase				,							labels=_UpperCamelCase				,							)
        self.parent.assertEqual(result.logits.shape				,							(self.batch_size, 2)					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   MobileBertForPreTraining(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(
            _UpperCamelCase				,							attention_mask=_UpperCamelCase				,							token_type_ids=_UpperCamelCase				,							labels=_UpperCamelCase				,							next_sentence_label=_UpperCamelCase				,							)
        self.parent.assertEqual(result.prediction_logits.shape				,							(self.batch_size, self.seq_length, self.vocab_size)					)
        self.parent.assertEqual(result.seq_relationship_logits.shape				,							(self.batch_size, 2)					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   MobileBertForQuestionAnswering(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(
            _UpperCamelCase				,							attention_mask=_UpperCamelCase				,							token_type_ids=_UpperCamelCase				,							start_positions=_UpperCamelCase				,							end_positions=_UpperCamelCase				,							)
        self.parent.assertEqual(result.start_logits.shape				,							(self.batch_size, self.seq_length)					)
        self.parent.assertEqual(result.end_logits.shape				,							(self.batch_size, self.seq_length)					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   self.num_labels
        _UpperCAmelCase           =   MobileBertForSequenceClassification(_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(_UpperCamelCase				,							attention_mask=_UpperCamelCase				,							token_type_ids=_UpperCamelCase				,							labels=_UpperCamelCase					)
        self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.num_labels)					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   self.num_labels
        _UpperCAmelCase           =   MobileBertForTokenClassification(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(_UpperCamelCase				,							attention_mask=_UpperCamelCase				,							token_type_ids=_UpperCamelCase				,							labels=_UpperCamelCase					)
        self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.num_labels)					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   self.num_choices
        _UpperCAmelCase           =   MobileBertForMultipleChoice(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   input_ids.unsqueeze(1					).expand(-1				,							self.num_choices				,							-1					).contiguous()
        _UpperCAmelCase           =   token_type_ids.unsqueeze(1					).expand(-1				,							self.num_choices				,							-1					).contiguous()
        _UpperCAmelCase           =   input_mask.unsqueeze(1					).expand(-1				,							self.num_choices				,							-1					).contiguous()
        _UpperCAmelCase           =   model(
            _UpperCamelCase				,							attention_mask=_UpperCamelCase				,							token_type_ids=_UpperCamelCase				,							labels=_UpperCamelCase				,							)
        self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.num_choices)					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.prepare_config_and_inputs()
        (
            (
            _UpperCAmelCase
        )       ,      (
            _UpperCAmelCase
        )       ,      (
            _UpperCAmelCase
        )       ,      (
            _UpperCAmelCase
        )       ,      (
            _UpperCAmelCase
        )       ,      (
            _UpperCAmelCase
        )       ,      (
            _UpperCAmelCase
        )       ,      
        )           =   config_and_inputs
        _UpperCAmelCase           =   {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
        return config, inputs_dict




@require_torch
class 			__UpperCamelCase    (							A__	, A__	, unittest.TestCase    ):
     __A	:  int							  =   (
         (
             MobileBertModel,
             MobileBertForMaskedLM,
             MobileBertForMultipleChoice,
             MobileBertForNextSentencePrediction,
             MobileBertForPreTraining,
             MobileBertForQuestionAnswering,
             MobileBertForSequenceClassification,
             MobileBertForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     __A	:  Union[str, Any]							  =   (
         {
             """feature-extraction""": MobileBertModel,
             """fill-mask""": MobileBertForMaskedLM,
             """question-answering""": MobileBertForQuestionAnswering,
             """text-classification""": MobileBertForSequenceClassification,
             """token-classification""": MobileBertForTokenClassification,
             """zero-shot""": MobileBertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     __A	:  Optional[Any]							  =   True





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase=False					):
        _UpperCAmelCase           =   super()._prepare_for_class(_UpperCamelCase				,							_UpperCamelCase				,							return_labels=_UpperCamelCase					)

        if return_labels:
           if model_class in get_values(_UpperCamelCase					):
              _UpperCAmelCase           =   torch.zeros(
                  (self.model_tester.batch_size, self.model_tester.seq_length)				,							dtype=torch.long				,							device=_UpperCamelCase					)
              _UpperCAmelCase           =   torch.zeros(
                  self.model_tester.batch_size				,							dtype=torch.long				,							device=_UpperCamelCase					)
        return inputs_dict





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   MobileBertModelTester(self					)
        _UpperCAmelCase           =   ConfigTester(self				,							config_class=_UpperCamelCase				,							hidden_size=37					)





     def 		UpperCamelCase(     self					):
        self.config_tester.run_common_tests()





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mobilebert_model(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mobilebert_for_masked_lm(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mobilebert_for_multiple_choice(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mobilebert_for_pretraining(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mobilebert_for_question_answering(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mobilebert_for_sequence_classification(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mobilebert_for_token_classification(*_UpperCamelCase					)




def      A__       (    SCREAMING_SNAKE_CASE_   :      Union[str, Any]						)  ->  Optional[int]:






   """simple docstring"""







   return torch.tensor(
       SCREAMING_SNAKE_CASE_				,	dtype=torch.long				,	device=SCREAMING_SNAKE_CASE_				,	)


UpperCAmelCase_           =	1e-3




@require_torch
@require_sentencepiece
@require_tokenizers
class 			__UpperCamelCase    (							unittest.TestCase    ):





     @slow
     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   MobileBertModel.from_pretrained('''google/mobilebert-uncased'''					).to(_UpperCamelCase					)
        _UpperCAmelCase           =   _long_tensor([[101, 7110, 1005, 1056, 2023, 11333, 17413, 1029, 102]]					)
        with torch.no_grad():
           _UpperCAmelCase           =   model(_UpperCamelCase					)[0]
        _UpperCAmelCase           =   torch.Size((1, 9, 512)					)
        self.assertEqual(output.shape				,							_UpperCamelCase					)
        _UpperCAmelCase           =   torch.tensor(
            [
                [
                    [-2.4736526e07, 8.2691656e04, 1.6521838e05],
                    [-5.7541704e-01, 3.9056022e00, 4.4011507e00],
                    [2.6047359e00, 1.5677652e00, -1.7324188e-01],
                ]
            ]				,							device=_UpperCamelCase				,							)

        # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
        # ~1 difference, it's therefore not a good idea to measure using addition.
        # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
        # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
        _UpperCAmelCase           =   torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE					)
        _UpperCAmelCase           =   torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE					)

        self.assertTrue(lower_bound and upper_bound					)



from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase		   =			logging.get_logger(__name__)

lowerCamelCase		   =			{
    'sayakpaul/vit-msn-base': 'https://huggingface.co/sayakpaul/vit-msn-base/resolve/main/config.json',
    # See all ViT MSN models at https://huggingface.co/models?filter=vit_msn
}




class A    (  UpperCamelCase_	):
      UpperCamelCase__				:    Union[str, Any] ='vit_msn'






      def __init__(   self		:	Union[str, Any]     ,  lowercase_		:	List[str]=768     ,  lowercase_		:	Optional[int]=12     ,  lowercase_		:	List[str]=12     ,  lowercase_		:	List[Any]=3072     ,  lowercase_		:	int="gelu"     ,  lowercase_		:	List[Any]=0.0     ,  lowercase_		:	Optional[int]=0.0     ,  lowercase_		:	Dict=0.02     ,  lowercase_		:	Any=1E-06     ,  lowercase_		:	Union[str, Any]=224     ,  lowercase_		:	Optional[int]=16     ,  lowercase_		:	List[Any]=3     ,  lowercase_		:	Any=True     ,  **lowercase_		:	Tuple     ,  )   ->		Dict:
             """simple docstring"""




             super().__init__(**lowercase_					)

             _lowerCamelCase			:						Dict            =hidden_size
             _lowerCamelCase			:						Any            =num_hidden_layers
             _lowerCamelCase			:						Union[str, Any]            =num_attention_heads
             _lowerCamelCase			:						List[Any]            =intermediate_size
             _lowerCamelCase			:						Tuple            =hidden_act
             _lowerCamelCase			:						Any            =hidden_dropout_prob
             _lowerCamelCase			:						Dict            =attention_probs_dropout_prob
             _lowerCamelCase			:						Any            =initializer_range
             _lowerCamelCase			:						List[Any]            =layer_norm_eps
             _lowerCamelCase			:						Optional[int]            =image_size
             _lowerCamelCase			:						Optional[int]            =patch_size
             _lowerCamelCase			:						List[Any]            =num_channels
             _lowerCamelCase			:						Optional[Any]            =qkv_bias


