






from __future__ import annotations

import copy
import inspect
import json
import math
import os
import tempfile
import unittest
from importlib import import_module

import numpy as np

from transformers import ViTMAEConfig
from transformers.file_utils import cached_property, is_tf_available, is_vision_available
from transformers.testing_utils import require_tf, require_vision, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers import TFViTMAEForPreTraining, TFViTMAEModel


if is_vision_available():
   from PIL import Image

   from transformers import ViTImageProcessor



class 				_A			:
    """simple docstring"""

    def __init__(				self      :     Any ,     __SCREAMING_SNAKE_CASE      :     Any ,     __SCREAMING_SNAKE_CASE      :     str=13 ,     __SCREAMING_SNAKE_CASE      :     Tuple=30 ,     __SCREAMING_SNAKE_CASE      :     List[Any]=2 ,     __SCREAMING_SNAKE_CASE      :     Optional[Any]=3 ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]=True ,     __SCREAMING_SNAKE_CASE      :     List[Any]=True ,     __SCREAMING_SNAKE_CASE      :     Any=32 ,     __SCREAMING_SNAKE_CASE      :     Optional[int]=2 ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]=4 ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]=37 ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]="gelu" ,     __SCREAMING_SNAKE_CASE      :     Tuple=0.1 ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]=0.1 ,     __SCREAMING_SNAKE_CASE      :     str=10 ,     __SCREAMING_SNAKE_CASE      :     Dict=0.02 ,     __SCREAMING_SNAKE_CASE      :     int=3 ,     __SCREAMING_SNAKE_CASE      :     str=0.6 ,     __SCREAMING_SNAKE_CASE      :     int=None ,     )      ->							Dict:
           __UpperCAmelCase     				=parent
           __UpperCAmelCase     				=batch_size
           __UpperCAmelCase     				=image_size
           __UpperCAmelCase     				=patch_size
           __UpperCAmelCase     				=num_channels
           __UpperCAmelCase     				=is_training
           __UpperCAmelCase     				=use_labels
           __UpperCAmelCase     				=hidden_size
           __UpperCAmelCase     				=num_hidden_layers
           __UpperCAmelCase     				=num_attention_heads
           __UpperCAmelCase     				=intermediate_size
           __UpperCAmelCase     				=hidden_act
           __UpperCAmelCase     				=hidden_dropout_prob
           __UpperCAmelCase     				=attention_probs_dropout_prob
           __UpperCAmelCase     				=type_sequence_label_size
           __UpperCAmelCase     				=initializer_range
           __UpperCAmelCase     				=mask_ratio
           __UpperCAmelCase     				=scope

           # in ViTMAE, the expected sequence length = (num_patches + 1) * (1 - config.mask_ratio), rounded above
           # (we add 1 for the [CLS] token)
           __UpperCAmelCase     				=(image_size // patch_size) ** 2
           __UpperCAmelCase     				=int(math.ceil((1 - mask_ratio) * (num_patches + 1)      )      )



    def _a							(				self      :     Optional[int]      )      ->							List[Any]:
           __UpperCAmelCase     				=floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]      )

           __UpperCAmelCase     				=None
           if self.use_labels:
                  __UpperCAmelCase     				=ids_tensor([self.batch_size] ,     self.type_sequence_label_size      )

           __UpperCAmelCase     				=self.get_config()

           return config, pixel_values, labels



    def _a							(				self      :     Optional[int]      )      ->							Tuple:
           return ViTMAEConfig(
               image_size=self.image_size ,     patch_size=self.patch_size ,     num_channels=self.num_channels ,     hidden_size=self.hidden_size ,     num_hidden_layers=self.num_hidden_layers ,     num_attention_heads=self.num_attention_heads ,     intermediate_size=self.intermediate_size ,     decoder_hidden_size=self.hidden_size ,     decoder_num_hidden_layers=self.num_hidden_layers ,     decoder_num_attention_heads=self.num_attention_heads ,     decoder_intermediate_size=self.intermediate_size ,     hidden_act=self.hidden_act ,     hidden_dropout_prob=self.hidden_dropout_prob ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob ,     is_decoder=lowerCAmelCase_ ,     initializer_range=self.initializer_range ,     mask_ratio=self.mask_ratio ,     )



    def _a							(				self      :     Tuple ,     __SCREAMING_SNAKE_CASE      :     Dict ,     __SCREAMING_SNAKE_CASE      :     Dict ,     __SCREAMING_SNAKE_CASE      :     List[Any]      )      ->							Optional[Any]:
           __UpperCAmelCase     				=TFViTMAEModel(config=lowerCAmelCase_      )
           __UpperCAmelCase     				=model(lowerCAmelCase_ ,     training=lowerCAmelCase_      )
           self.parent.assertEqual(result.last_hidden_state.shape ,     (self.batch_size, self.seq_length, self.hidden_size)      )



    def _a							(				self      :     Union[str, Any] ,     __SCREAMING_SNAKE_CASE      :     List[Any] ,     __SCREAMING_SNAKE_CASE      :     int ,     __SCREAMING_SNAKE_CASE      :     Optional[Any]      )      ->							str:
           __UpperCAmelCase     				=TFViTMAEForPreTraining(lowerCAmelCase_      )
           __UpperCAmelCase     				=model(lowerCAmelCase_ ,     training=lowerCAmelCase_      )
           # expected sequence length = num_patches
           __UpperCAmelCase     				=(self.image_size // self.patch_size) ** 2
           __UpperCAmelCase     				=self.patch_size**2 * self.num_channels
           self.parent.assertEqual(result.logits.shape ,     (self.batch_size, num_patches, expected_num_channels)      )

           # test greyscale images
           __UpperCAmelCase     				=1
           __UpperCAmelCase     				=TFViTMAEForPreTraining(lowerCAmelCase_      )

           __UpperCAmelCase     				=floats_tensor([self.batch_size, 1, self.image_size, self.image_size]      )
           __UpperCAmelCase     				=model(lowerCAmelCase_ ,     training=lowerCAmelCase_      )
           __UpperCAmelCase     				=self.patch_size**2
           self.parent.assertEqual(result.logits.shape ,     (self.batch_size, num_patches, expected_num_channels)      )



    def _a							(				self      :     Tuple      )      ->							int:
           __UpperCAmelCase     				=self.prepare_config_and_inputs()
           (__UpperCAmelCase)     				=config_and_inputs
           __UpperCAmelCase     				={"""pixel_values""": pixel_values}
           return config, inputs_dict




@require_tf
class 				_A			(     _SCREAMING_SNAKE_CASE					,					_SCREAMING_SNAKE_CASE					,					unittest.TestCase				):
    """simple docstring"""

    lowerCamelCase	:							List[str]				= (TFViTMAEModel, TFViTMAEForPreTraining) if is_tf_available() else ()
    lowerCamelCase	:							Dict				= {"""feature-extraction""": TFViTMAEModel} if is_tf_available() else {}

    lowerCamelCase	:							Tuple				= False
    lowerCamelCase	:							List[str]				= False
    lowerCamelCase	:							Tuple				= False
    lowerCamelCase	:							Any				= False



    def _a							(				self      :     Tuple      )      ->							int:
           __UpperCAmelCase     				=TFViTMAEModelTester(self      )
           __UpperCAmelCase     				=ConfigTester(self ,     config_class=lowerCAmelCase_ ,     has_text_modality=lowerCAmelCase_ ,     hidden_size=37      )



    def _a							(				self      :     List[Any]      )      ->							int:
           self.config_tester.run_common_tests()



    @unittest.skip(reason="""ViTMAE does not use inputs_embeds"""      )
    def _a							(				self      :     List[str]      )      ->							str:
           pass



    def _a							(				self      :     Optional[Any]      )      ->							Union[str, Any]:
           __UpperCAmelCase     				=self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                  __UpperCAmelCase     				=model_class(lowerCAmelCase_      )
                  self.assertIsInstance(model.get_input_embeddings() ,     (tf.keras.layers.Layer)      )
                  __UpperCAmelCase     				=model.get_output_embeddings()
                  self.assertTrue(x is None or isinstance(lowerCAmelCase_ ,     tf.keras.layers.Layer      )      )



    def _a							(				self      :     str      )      ->							List[Any]:
           __UpperCAmelCase     				=self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                  __UpperCAmelCase     				=model_class(lowerCAmelCase_      )
                  __UpperCAmelCase     				=inspect.signature(model.call      )
                  # signature.parameters is an OrderedDict => so arg_names order is deterministic
                  __UpperCAmelCase     				=[*signature.parameters.keys()]

                  __UpperCAmelCase     				=["""pixel_values"""]
                  self.assertListEqual(arg_names[:1] ,     lowerCAmelCase_      )



    def _a							(				self      :     Any      )      ->							Optional[Any]:
           __UpperCAmelCase     				=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*lowerCAmelCase_      )



    def _a							(				self      :     int      )      ->							int:
           __UpperCAmelCase     				=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase_      )



    def _a							(				self      :     str      )      ->							List[Any]:
           # make the mask reproducible
           np.random.seed(2      )

           __UpperCAmelCase     				=self.model_tester.prepare_config_and_inputs_for_common()

           __UpperCAmelCase     				=int((config.image_size // config.patch_size) ** 2      )
           __UpperCAmelCase     				=np.random.uniform(size=(self.model_tester.batch_size, num_patches)      )

           for model_class in self.all_model_classes:
                  __UpperCAmelCase     				=model_class(lowerCAmelCase_      )
                  __UpperCAmelCase     				=self._prepare_for_class(lowerCAmelCase_ ,     lowerCAmelCase_      )

                  __UpperCAmelCase     				=model(lowerCAmelCase_ ,     noise=lowerCAmelCase_      )

                  __UpperCAmelCase     				=copy.deepcopy(self._prepare_for_class(lowerCAmelCase_ ,     lowerCAmelCase_      )      )
                  __UpperCAmelCase     				=model(**lowerCAmelCase_ ,     noise=lowerCAmelCase_      )
                  __UpperCAmelCase     				=outputs_dict[0].numpy()
                  __UpperCAmelCase     				=outputs_keywords[0].numpy()

                  self.assertLess(np.sum(np.abs(output_dict - output_keywords      )      ) ,     1e-6      )



    def _a							(				self      :     List[str]      )      ->							str:
           # make the mask reproducible
           np.random.seed(2      )

           __UpperCAmelCase     				=self.model_tester.prepare_config_and_inputs_for_common()

           __UpperCAmelCase     				=int((config.image_size // config.patch_size) ** 2      )
           __UpperCAmelCase     				=np.random.uniform(size=(self.model_tester.batch_size, num_patches)      )

           def prepare_numpy_arrays(__SCREAMING_SNAKE_CASE      :     Dict      ):
                  __UpperCAmelCase     				={}
                  for k, v in inputs_dict.items():
                         if tf.is_tensor(lowerCAmelCase_      ):
                                __UpperCAmelCase     				=v.numpy()
                         else:
                                __UpperCAmelCase     				=np.array(lowerCAmelCase_      )

                  return inputs_np_dict

           for model_class in self.all_model_classes:
                  __UpperCAmelCase     				=model_class(lowerCAmelCase_      )

                  __UpperCAmelCase     				=self._prepare_for_class(lowerCAmelCase_ ,     lowerCAmelCase_      )
                  __UpperCAmelCase     				=prepare_numpy_arrays(lowerCAmelCase_      )

                  __UpperCAmelCase     				=model(lowerCAmelCase_ ,     noise=lowerCAmelCase_      )
                  __UpperCAmelCase     				=model(**lowerCAmelCase_ ,     noise=lowerCAmelCase_      )
                  self.assert_outputs_same(lowerCAmelCase_ ,     lowerCAmelCase_      )



    def _a							(				self      :     List[str] ,     __SCREAMING_SNAKE_CASE      :     List[Any] ,     __SCREAMING_SNAKE_CASE      :     Tuple ,     __SCREAMING_SNAKE_CASE      :     Dict      )      ->							Tuple:
           # make masks reproducible
           np.random.seed(2      )

           __UpperCAmelCase     				=int((tf_model.config.image_size // tf_model.config.patch_size) ** 2      )
           __UpperCAmelCase     				=np.random.uniform(size=(self.model_tester.batch_size, num_patches)      )
           __UpperCAmelCase     				=tf.constant(lowerCAmelCase_      )

           # Add `noise` argument.
           # PT inputs will be prepared in `super().check_pt_tf_models()` with this added `noise` argument
           __UpperCAmelCase     				=tf_noise

           super().check_pt_tf_models(lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_      )



    def _a							(				self      :     Tuple      )      ->							Any:
           # make mask reproducible
           np.random.seed(2      )

           __UpperCAmelCase     				=self.model_tester.prepare_config_and_inputs_for_common()

           __UpperCAmelCase     				={
               module_member
               for model_class in self.all_model_classes
               for module in (import_module(model_class.__module__      ),)
               for module_member_name in dir(lowerCAmelCase_      )
               if module_member_name.endswith("""MainLayer"""      )
               # This condition is required, since `modeling_tf_clip.py` has 3 classes whose names end with `MainLayer`.
               and module_member_name[: -len("""MainLayer"""      )] == model_class.__name__[: -len("""Model"""      )]
               for module_member in (getattr(lowerCAmelCase_ ,     lowerCAmelCase_      ),)
               if isinstance(lowerCAmelCase_ ,     lowerCAmelCase_      )
               and tf.keras.layers.Layer in module_member.__bases__
               and getattr(lowerCAmelCase_ ,     """_keras_serializable""" ,     lowerCAmelCase_      )
           }

           __UpperCAmelCase     				=int((config.image_size // config.patch_size) ** 2      )
           __UpperCAmelCase     				=np.random.uniform(size=(self.model_tester.batch_size, num_patches)      )
           __UpperCAmelCase     				=tf.convert_to_tensor(lowerCAmelCase_      )
           inputs_dict.update({"""noise""": noise}      )

           for main_layer_class in tf_main_layer_classes:
                  __UpperCAmelCase     				=main_layer_class(lowerCAmelCase_      )

                  __UpperCAmelCase     				={
                      name: tf.keras.Input(tensor.shape[1:] ,     dtype=tensor.dtype      ) for name, tensor in inputs_dict.items()
                  }

                  __UpperCAmelCase     				=tf.keras.Model(lowerCAmelCase_ ,     outputs=main_layer(lowerCAmelCase_      )      )
                  __UpperCAmelCase     				=model(lowerCAmelCase_      )

                  with tempfile.TemporaryDirectory() as tmpdirname:
                         __UpperCAmelCase     				=os.path.join(lowerCAmelCase_ ,     """keras_model.h5"""      )
                         model.save(lowerCAmelCase_      )
                         __UpperCAmelCase     				=tf.keras.models.load_model(
                             lowerCAmelCase_ ,     custom_objects={main_layer_class.__name__: main_layer_class}      )
                         assert isinstance(lowerCAmelCase_ ,     tf.keras.Model      )
                         __UpperCAmelCase     				=model(lowerCAmelCase_      )
                         self.assert_outputs_same(lowerCAmelCase_ ,     lowerCAmelCase_      )



    @slow
    def _a							(				self      :     Optional[Any]      )      ->							Any:
           # make mask reproducible
           np.random.seed(2      )

           __UpperCAmelCase     				=self.model_tester.prepare_config_and_inputs_for_common()

           __UpperCAmelCase     				=int((config.image_size // config.patch_size) ** 2      )
           __UpperCAmelCase     				=np.random.uniform(size=(self.model_tester.batch_size, num_patches)      )

           for model_class in self.all_model_classes:
                  __UpperCAmelCase     				=model_class(lowerCAmelCase_      )
                  __UpperCAmelCase     				=self._prepare_for_class(lowerCAmelCase_ ,     lowerCAmelCase_      )
                  __UpperCAmelCase     				=model(lowerCAmelCase_ ,     noise=lowerCAmelCase_      )

                  if model_class.__name__ == "TFViTMAEModel":
                         __UpperCAmelCase     				=outputs.last_hidden_state.numpy()
                         __UpperCAmelCase     				=0
                  else:
                         __UpperCAmelCase     				=outputs.logits.numpy()
                         __UpperCAmelCase     				=0

                  with tempfile.TemporaryDirectory() as tmpdirname:
                         model.save_pretrained(lowerCAmelCase_ ,     saved_model=lowerCAmelCase_      )
                         __UpperCAmelCase     				=model_class.from_pretrained(lowerCAmelCase_      )
                         __UpperCAmelCase     				=model(lowerCAmelCase_ ,     noise=lowerCAmelCase_      )

                         if model_class.__name__ == "TFViTMAEModel":
                                __UpperCAmelCase     				=after_outputs["""last_hidden_state"""].numpy()
                                __UpperCAmelCase     				=0
                         else:
                                __UpperCAmelCase     				=after_outputs["""logits"""].numpy()
                                __UpperCAmelCase     				=0

                         __UpperCAmelCase     				=np.amax(np.abs(out_a - out_a      )      )
                         self.assertLessEqual(lowerCAmelCase_ ,     1e-5      )



    def _a							(				self      :     List[Any]      )      ->							str:
           # make mask reproducible
           np.random.seed(2      )

           __UpperCAmelCase     				=self.model_tester.prepare_config_and_inputs_for_common()

           __UpperCAmelCase     				=int((config.image_size // config.patch_size) ** 2      )
           __UpperCAmelCase     				=np.random.uniform(size=(self.model_tester.batch_size, num_patches)      )

           for model_class in self.all_model_classes:
                  __UpperCAmelCase     				=model_class(lowerCAmelCase_      )
                  __UpperCAmelCase     				=self._prepare_for_class(lowerCAmelCase_ ,     lowerCAmelCase_      )

                  __UpperCAmelCase     				=model(lowerCAmelCase_ ,     noise=lowerCAmelCase_      )
                  __UpperCAmelCase     				=model.get_config()
                  # make sure that returned config is jsonifiable, which is required by keras
                  json.dumps(lowerCAmelCase_      )
                  __UpperCAmelCase     				=model_class.from_config(model.get_config()      )
                  # make sure it also accepts a normal config
                  __UpperCAmelCase     				=model_class.from_config(model.config      )
                  __UpperCAmelCase     				=new_model(lowerCAmelCase_      )  # Build model
                  new_model.set_weights(model.get_weights()      )
                  __UpperCAmelCase     				=new_model(lowerCAmelCase_ ,     noise=lowerCAmelCase_      )

                  self.assert_outputs_same(lowerCAmelCase_ ,     lowerCAmelCase_      )



    @unittest.skip(
        reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load
    to get deterministic results."""      )
    def _a							(				self      :     int      )      ->							str:
           pass



    @unittest.skip(reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load"""      )
    def _a							(				self      :     str      )      ->							Optional[int]:
           pass



    @slow
    def _a							(				self      :     Any      )      ->							int:
           __UpperCAmelCase     				=TFViTMAEModel.from_pretrained("""google/vit-base-patch16-224"""      )
           self.assertIsNotNone(lowerCAmelCase_      )







def       lowercase__	(					)	->				List[Any]:







       """simple docstring"""






       __UpperCAmelCase     				=Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""		)
       return image





@require_tf
@require_vision
class 				_A			(     unittest.TestCase				):
    """simple docstring"""

    @cached_property
    def _a							(				self      :     Dict      )      ->							Optional[Any]:
           return ViTImageProcessor.from_pretrained("""facebook/vit-mae-base"""      ) if is_vision_available() else None



    @slow
    def _a							(				self      :     Tuple      )      ->							Union[str, Any]:
           # make random mask reproducible across the PT and TF model
           np.random.seed(2      )

           __UpperCAmelCase     				=TFViTMAEForPreTraining.from_pretrained("""facebook/vit-mae-base"""      )

           __UpperCAmelCase     				=self.default_image_processor
           __UpperCAmelCase     				=prepare_img()
           __UpperCAmelCase     				=image_processor(images=lowerCAmelCase_ ,     return_tensors="""tf"""      )

           # prepare a noise vector that will be also used for testing the TF model
           # (this way we can ensure that the PT and TF models operate on the same inputs)
           __UpperCAmelCase     				=ViTMAEConfig()
           __UpperCAmelCase     				=int((vit_mae_config.image_size // vit_mae_config.patch_size) ** 2      )
           __UpperCAmelCase     				=np.random.uniform(size=(1, num_patches)      )

           # forward pass
           __UpperCAmelCase     				=model(**lowerCAmelCase_ ,     noise=lowerCAmelCase_      )

           # verify the logits
           __UpperCAmelCase     				=tf.convert_to_tensor([1, 196, 768]      )
           self.assertEqual(outputs.logits.shape ,     lowerCAmelCase_      )

           __UpperCAmelCase     				=tf.convert_to_tensor(
               [[-0.0_548, -1.7_023, -0.9_325], [0.3_721, -0.5_670, -0.2_233], [0.8_235, -1.3_878, -0.3_524]]      )

           tf.debugging.assert_near(outputs.logits[0, :3, :3] ,     lowerCAmelCase_ ,     atol=1e-4      )












"""simple docstring"""
import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    MobileBertForMaskedLM,
							    MobileBertForMultipleChoice,
							    MobileBertForNextSentencePrediction,
							    MobileBertForPreTraining,
							    MobileBertForQuestionAnswering,
							    MobileBertForSequenceClassification,
							    MobileBertForTokenClassification,
							    MobileBertModel,
							)



class   SCREAMING_SNAKE_CASE	:





	"""simple docstring"""




	def __init__(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_=13  ,  lowerCAmelCase_=7  ,  lowerCAmelCase_=True  ,  lowerCAmelCase_=True  ,  lowerCAmelCase_=True  ,  lowerCAmelCase_=True  ,  lowerCAmelCase_=99  ,  lowerCAmelCase_=64  ,  lowerCAmelCase_=32  ,  lowerCAmelCase_=5  ,  lowerCAmelCase_=4  ,  lowerCAmelCase_=37  ,  lowerCAmelCase_="gelu"  ,  lowerCAmelCase_=0.1  ,  lowerCAmelCase_=0.1  ,  lowerCAmelCase_=512  ,  lowerCAmelCase_=16  ,  lowerCAmelCase_=2  ,  lowerCAmelCase_=0.02  ,  lowerCAmelCase_=3  ,  lowerCAmelCase_=4  ,  lowerCAmelCase_=None  ,  ):
		A_		:    Optional[Any]						     = parent
		A_		:    List[str]						     = batch_size
		A_		:    List[str]						     = seq_length
		A_		:    List[str]						     = is_training
		A_		:    Optional[Any]						     = use_input_mask
		A_		:    Dict						     = use_token_type_ids
		A_		:    Any						     = use_labels
		A_		:    Tuple						     = vocab_size
		A_		:    Optional[Any]						     = hidden_size
		A_		:    Tuple						     = embedding_size
		A_		:    Tuple						     = num_hidden_layers
		A_		:    Any						     = num_attention_heads
		A_		:    Optional[int]						     = intermediate_size
		A_		:    Union[str, Any]						     = hidden_act
		A_		:    Optional[int]						     = hidden_dropout_prob
		A_		:    Optional[Any]						     = attention_probs_dropout_prob
		A_		:    str						     = max_position_embeddings
		A_		:    str						     = type_vocab_size
		A_		:    Optional[Any]						     = type_sequence_label_size
		A_		:    List[str]						     = initializer_range
		A_		:    str						     = num_labels
		A_		:    Dict						     = num_choices
		A_		:    Any						     = scope





	def  lowerCamelCase(self				):
		A_		:    Any						     = ids_tensor([self.batch_size, self.seq_length]  ,  self.vocab_size				)

		A_		:    Optional[int]						     = None
		if self.use_input_mask:
			A_		:    str						     = random_attention_mask([self.batch_size, self.seq_length]				)

		A_		:    int						     = None
		if self.use_token_type_ids:
			A_		:    Union[str, Any]						     = ids_tensor([self.batch_size, self.seq_length]  ,  self.type_vocab_size				)

		A_		:    Union[str, Any]						     = None
		A_		:    Union[str, Any]						     = None
		A_		:    List[str]						     = None
		if self.use_labels:
			A_		:    Dict						     = ids_tensor([self.batch_size]  ,  self.type_sequence_label_size				)
			A_		:    Union[str, Any]						     = ids_tensor([self.batch_size, self.seq_length]  ,  self.num_labels				)
			A_		:    Dict						     = ids_tensor([self.batch_size]  ,  self.num_choices				)

		A_		:    Tuple						     = self.get_config()

		return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





	def  lowerCamelCase(self				):
		return MobileBertConfig(
		    vocab_size=self.vocab_size  ,  hidden_size=self.hidden_size  ,  num_hidden_layers=self.num_hidden_layers  ,  num_attention_heads=self.num_attention_heads  ,  intermediate_size=self.intermediate_size  ,  embedding_size=self.embedding_size  ,  hidden_act=self.hidden_act  ,  hidden_dropout_prob=self.hidden_dropout_prob  ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,  max_position_embeddings=self.max_position_embeddings  ,  type_vocab_size=self.type_vocab_size  ,  is_decoder=lowerCAmelCase_  ,  initializer_range=self.initializer_range  ,  )





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    str						     = MobileBertModel(config=lowerCAmelCase_				)
		model.to(lowerCAmelCase_				)
		model.eval()
		A_		:    Optional[int]						     = model(lowerCAmelCase_  ,  attention_mask=lowerCAmelCase_  ,  token_type_ids=lowerCAmelCase_				)
		A_		:    Union[str, Any]						     = model(lowerCAmelCase_  ,  token_type_ids=lowerCAmelCase_				)
		A_		:    int						     = model(lowerCAmelCase_				)

		self.parent.assertEqual(result.last_hidden_state.shape  ,  (self.batch_size, self.seq_length, self.hidden_size)				)
		self.parent.assertEqual(result.pooler_output.shape  ,  (self.batch_size, self.hidden_size)				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    Optional[int]						     = MobileBertForMaskedLM(config=lowerCAmelCase_				)
		model.to(lowerCAmelCase_				)
		model.eval()
		A_		:    List[Any]						     = model(lowerCAmelCase_  ,  attention_mask=lowerCAmelCase_  ,  token_type_ids=lowerCAmelCase_  ,  labels=lowerCAmelCase_				)
		self.parent.assertEqual(result.logits.shape  ,  (self.batch_size, self.seq_length, self.vocab_size)				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    Optional[int]						     = MobileBertForNextSentencePrediction(config=lowerCAmelCase_				)
		model.to(lowerCAmelCase_				)
		model.eval()
		A_		:    Tuple						     = model(
		    lowerCAmelCase_  ,  attention_mask=lowerCAmelCase_  ,  token_type_ids=lowerCAmelCase_  ,  labels=lowerCAmelCase_  ,  )
		self.parent.assertEqual(result.logits.shape  ,  (self.batch_size, 2)				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    int						     = MobileBertForPreTraining(config=lowerCAmelCase_				)
		model.to(lowerCAmelCase_				)
		model.eval()
		A_		:    Union[str, Any]						     = model(
		    lowerCAmelCase_  ,  attention_mask=lowerCAmelCase_  ,  token_type_ids=lowerCAmelCase_  ,  labels=lowerCAmelCase_  ,  next_sentence_label=lowerCAmelCase_  ,  )
		self.parent.assertEqual(result.prediction_logits.shape  ,  (self.batch_size, self.seq_length, self.vocab_size)				)
		self.parent.assertEqual(result.seq_relationship_logits.shape  ,  (self.batch_size, 2)				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    Union[str, Any]						     = MobileBertForQuestionAnswering(config=lowerCAmelCase_				)
		model.to(lowerCAmelCase_				)
		model.eval()
		A_		:    Tuple						     = model(
		    lowerCAmelCase_  ,  attention_mask=lowerCAmelCase_  ,  token_type_ids=lowerCAmelCase_  ,  start_positions=lowerCAmelCase_  ,  end_positions=lowerCAmelCase_  ,  )
		self.parent.assertEqual(result.start_logits.shape  ,  (self.batch_size, self.seq_length)				)
		self.parent.assertEqual(result.end_logits.shape  ,  (self.batch_size, self.seq_length)				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    int						     = self.num_labels
		A_		:    Optional[int]						     = MobileBertForSequenceClassification(lowerCAmelCase_				)
		model.to(lowerCAmelCase_				)
		model.eval()
		A_		:    int						     = model(lowerCAmelCase_  ,  attention_mask=lowerCAmelCase_  ,  token_type_ids=lowerCAmelCase_  ,  labels=lowerCAmelCase_				)
		self.parent.assertEqual(result.logits.shape  ,  (self.batch_size, self.num_labels)				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    str						     = self.num_labels
		A_		:    int						     = MobileBertForTokenClassification(config=lowerCAmelCase_				)
		model.to(lowerCAmelCase_				)
		model.eval()
		A_		:    str						     = model(lowerCAmelCase_  ,  attention_mask=lowerCAmelCase_  ,  token_type_ids=lowerCAmelCase_  ,  labels=lowerCAmelCase_				)
		self.parent.assertEqual(result.logits.shape  ,  (self.batch_size, self.seq_length, self.num_labels)				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    str						     = self.num_choices
		A_		:    int						     = MobileBertForMultipleChoice(config=lowerCAmelCase_				)
		model.to(lowerCAmelCase_				)
		model.eval()
		A_		:    Tuple						     = input_ids.unsqueeze(1				).expand(-1  ,  self.num_choices  ,  -1				).contiguous()
		A_		:    str						     = token_type_ids.unsqueeze(1				).expand(-1  ,  self.num_choices  ,  -1				).contiguous()
		A_		:    Optional[int]						     = input_mask.unsqueeze(1				).expand(-1  ,  self.num_choices  ,  -1				).contiguous()
		A_		:    Union[str, Any]						     = model(
		    lowerCAmelCase_  ,  attention_mask=lowerCAmelCase_  ,  token_type_ids=lowerCAmelCase_  ,  labels=lowerCAmelCase_  ,  )
		self.parent.assertEqual(result.logits.shape  ,  (self.batch_size, self.num_choices)				)





	def  lowerCamelCase(self				):
		A_		:    Optional[Any]						     = self.prepare_config_and_inputs()
		(
		    (
		    A_
		)     ,  (
		    A_
		)     ,  (
		    A_
		)     ,  (
		    A_
		)     ,  (
		    A_
		)     ,  (
		    A_
		)     ,  (
		    A_
		)     ,  
		)		:    Dict						     = config_and_inputs
		A_		:    Any						     = {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
		return config, inputs_dict






@require_torch
class   SCREAMING_SNAKE_CASE	(					_SCREAMING_SNAKE_CASE							,							_SCREAMING_SNAKE_CASE							,							unittest.TestCase    ):





	"""simple docstring"""




	_A							:					Dict 			=					(
	    (
	        MobileBertModel,
	        MobileBertForMaskedLM,
	        MobileBertForMultipleChoice,
	        MobileBertForNextSentencePrediction,
	        MobileBertForPreTraining,
	        MobileBertForQuestionAnswering,
	        MobileBertForSequenceClassification,
	        MobileBertForTokenClassification,
	    )
	    if is_torch_available()
	    else ()
	)
	_A							:					Union[str, Any] 			=					(
	    {
	        """feature-extraction""": MobileBertModel,
	        """fill-mask""": MobileBertForMaskedLM,
	        """question-answering""": MobileBertForQuestionAnswering,
	        """text-classification""": MobileBertForSequenceClassification,
	        """token-classification""": MobileBertForTokenClassification,
	        """zero-shot""": MobileBertForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	_A							:					Union[str, Any] 			=					True





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_  ,  lowerCAmelCase_=False				):
		A_		:    Optional[int]						     = super()._prepare_for_class(lowerCAmelCase_  ,  lowerCAmelCase_  ,  return_labels=lowerCAmelCase_				)

		if return_labels:
			if model_class in get_values(lowerCAmelCase_				):
				A_		:    List[Any]						     = torch.zeros(
				    (self.model_tester.batch_size, self.model_tester.seq_length)  ,  dtype=torch.long  ,  device=lowerCAmelCase_				)
				A_		:    Optional[int]						     = torch.zeros(
				    self.model_tester.batch_size  ,  dtype=torch.long  ,  device=lowerCAmelCase_				)
		return inputs_dict





	def  lowerCamelCase(self				):
		A_		:    Optional[int]						     = MobileBertModelTester(self				)
		A_		:    Optional[Any]						     = ConfigTester(self  ,  config_class=lowerCAmelCase_  ,  hidden_size=37				)





	def  lowerCamelCase(self				):
		self.config_tester.run_common_tests()





	def  lowerCamelCase(self				):
		A_		:    Optional[Any]						     = self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_mobilebert_model(*lowerCAmelCase_				)





	def  lowerCamelCase(self				):
		A_		:    Tuple						     = self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_mobilebert_for_masked_lm(*lowerCAmelCase_				)





	def  lowerCamelCase(self				):
		A_		:    Optional[Any]						     = self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_mobilebert_for_multiple_choice(*lowerCAmelCase_				)





	def  lowerCamelCase(self				):
		A_		:    Optional[int]						     = self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*lowerCAmelCase_				)





	def  lowerCamelCase(self				):
		A_		:    Dict						     = self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_mobilebert_for_pretraining(*lowerCAmelCase_				)





	def  lowerCamelCase(self				):
		A_		:    Any						     = self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_mobilebert_for_question_answering(*lowerCAmelCase_				)





	def  lowerCamelCase(self				):
		A_		:    Dict						     = self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_mobilebert_for_sequence_classification(*lowerCAmelCase_				)





	def  lowerCamelCase(self				):
		A_		:    Optional[Any]						     = self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_mobilebert_for_token_classification(*lowerCAmelCase_				)






def       __UpperCamelCase						(		snake_case__       ):
	return torch.tensor(
	    snake_case__ , dtype=torch.long , device=snake_case__ , )


_lowerCAmelCase       =				1e-3




@require_torch
@require_sentencepiece
@require_tokenizers
class   SCREAMING_SNAKE_CASE	(					unittest.TestCase    ):





	"""simple docstring"""




	@slow
	def  lowerCamelCase(self				):
		A_		:    Union[str, Any]						     = MobileBertModel.from_pretrained("""google/mobilebert-uncased"""				).to(lowerCAmelCase_				)
		A_		:    Any						     = _long_tensor([[101, 7110, 1005, 1056, 2023, 11333, 17413, 1029, 102]]				)
		with torch.no_grad():
			A_		:    Dict						     = model(lowerCAmelCase_				)[0]
		A_		:    Any						     = torch.Size((1, 9, 512)				)
		self.assertEqual(output.shape  ,  lowerCAmelCase_				)
		A_		:    Optional[int]						     = torch.tensor(
		    [
		        [
		            [-2.4_7_3_6_5_2_6e0_7, 8.2_6_9_1_6_5_6e0_4, 1.6_5_2_1_8_3_8e0_5],
		            [-5.7_5_4_1_7_0_4e-0_1, 3.9_0_5_6_0_2_2e0_0, 4.4_0_1_1_5_0_7e0_0],
		            [2.6_0_4_7_3_5_9e0_0, 1.5_6_7_7_6_5_2e0_0, -1.7_3_2_4_1_8_8e-0_1],
		        ]
		    ]  ,  device=lowerCAmelCase_  ,  )

		# MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
		# ~1 difference, it's therefore not a good idea to measure using addition.
		# Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
		# result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
		A_		:    List[str]						     = torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE				)
		A_		:    Any						     = torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE				)

		self.assertTrue(lower_bound and upper_bound				)

