
"""simple docstring"""




import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import BlenderbotSmallConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
  import os

  # The slow tests are often failing with OOM error on GPU
  # This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
  # but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
  A__   :					List[Any]							=   'platform'

  import jax
  import jax.numpy as jnp

  from transformers.models.blenderbot_small.modeling_flax_blenderbot_small import (
      FlaxBlenderbotSmallForConditionalGeneration,
      FlaxBlenderbotSmallModel,
      shift_tokens_right,
  )




def  _lowerCAmelCase		(		_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase=None ,				_UpperCamelCase=None ,				_UpperCamelCase=None ,				_UpperCamelCase=None ,				_UpperCamelCase=None ,				_UpperCamelCase=None ,				):






  """simple docstring"""



  if attention_mask is None:
    _lowercase:			Tuple       =     np.where(input_ids != config.pad_token_id ,				1 ,				0						)
  if decoder_attention_mask is None:
    _lowercase:			Tuple       =     np.where(decoder_input_ids != config.pad_token_id ,				1 ,				0						)
  if head_mask is None:
    _lowercase:			Dict       =     np.ones((config.encoder_layers, config.encoder_attention_heads)						)
  if decoder_head_mask is None:
    _lowercase:			Union[str, Any]       =     np.ones((config.decoder_layers, config.decoder_attention_heads)						)
  if cross_attn_head_mask is None:
    _lowercase:			int       =     np.ones((config.decoder_layers, config.decoder_attention_heads)						)
  return {
      "input_ids": input_ids,
      "decoder_input_ids": decoder_input_ids,
      "attention_mask": attention_mask,
      "decoder_attention_mask": attention_mask,
  }


class 			__magic_name__       :
       def __init__(     self ,				A_ ,				A_=13 ,				A_=7 ,				A_=True ,				A_=False ,				A_=99 ,				A_=16 ,				A_=2 ,				A_=4 ,				A_=4 ,				A_="gelu" ,				A_=0.1 ,				A_=0.1 ,				A_=32 ,				A_=2 ,				A_=1 ,				A_=0 ,				A_=0.02 ,				)     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			str       =     parent
         _lowercase:			Dict       =     batch_size
         _lowercase:			Union[str, Any]       =     seq_length
         _lowercase:			List[Any]       =     is_training
         _lowercase:			List[str]       =     use_labels
         _lowercase:			Union[str, Any]       =     vocab_size
         _lowercase:			Optional[int]       =     hidden_size
         _lowercase:			List[Any]       =     num_hidden_layers
         _lowercase:			Union[str, Any]       =     num_attention_heads
         _lowercase:			Optional[Any]       =     intermediate_size
         _lowercase:			Any       =     hidden_act
         _lowercase:			Dict       =     hidden_dropout_prob
         _lowercase:			Union[str, Any]       =     attention_probs_dropout_prob
         _lowercase:			Tuple       =     max_position_embeddings
         _lowercase:			Union[str, Any]       =     eos_token_id
         _lowercase:			int       =     pad_token_id
         _lowercase:			List[Any]       =     bos_token_id
         _lowercase:			str       =     initializer_range
       def 							lowercase_							(     self       )     ->	Optional[Any]:





         """simple docstring"""




         _lowercase:			Dict       =     np.clip(ids_tensor([self.batch_size, self.seq_length - 1] ,				self.vocab_size       ) ,				3 ,				self.vocab_size       )
         _lowercase:			Optional[int]       =     np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1) ,				dtype=np.intaa       )) ,				-1       )

         _lowercase:			Optional[int]       =     shift_tokens_right(A_ ,				1 ,				2       )

         _lowercase:			Optional[Any]       =     BlenderbotSmallConfig(
             vocab_size=self.vocab_size ,				d_model=self.hidden_size ,				encoder_layers=self.num_hidden_layers ,				decoder_layers=self.num_hidden_layers ,				encoder_attention_heads=self.num_attention_heads ,				decoder_attention_heads=self.num_attention_heads ,				encoder_ffn_dim=self.intermediate_size ,				decoder_ffn_dim=self.intermediate_size ,				dropout=self.hidden_dropout_prob ,				attention_dropout=self.attention_probs_dropout_prob ,				max_position_embeddings=self.max_position_embeddings ,				eos_token_id=self.eos_token_id ,				bos_token_id=self.bos_token_id ,				pad_token_id=self.pad_token_id ,				initializer_range=self.initializer_range ,				use_cache=A_ ,				)
         _lowercase:			int       =     prepare_blenderbot_inputs_dict(A_ ,				A_ ,				A_       )
         return config, inputs_dict
       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         _lowercase ,				_lowercase:			Optional[Any]       =     self.prepare_config_and_inputs()
         return config, inputs_dict
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			str       =     20
         _lowercase:			int       =     model_class_name(A_       )

         _lowercase:			Optional[int]       =     model.encode(inputs_dict['''input_ids''']       )

         _lowercase ,				_lowercase:			Dict       =     (
             inputs_dict['''decoder_input_ids'''],
             inputs_dict['''decoder_attention_mask'''],
         )

         _lowercase:			Optional[int]       =     model.init_cache(decoder_input_ids.shape[0] ,				A_ ,				A_       )
         _lowercase:			List[Any]       =     jnp.ones((decoder_input_ids.shape[0], max_decoder_length) ,				dtype='''i4'''       )

         _lowercase:			Dict       =     jnp.broadcast_to(
             jnp.arange(decoder_input_ids.shape[-1] - 1       )[None, :] ,				(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1) ,				)
         _lowercase:			Optional[int]       =     model.decode(
             decoder_input_ids[:, :-1] ,				A_ ,				decoder_attention_mask=A_ ,				past_key_values=A_ ,				decoder_position_ids=A_ ,				)

         _lowercase:			Optional[Any]       =     jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]] ,				dtype='''i4'''       )
         _lowercase:			int       =     model.decode(
             decoder_input_ids[:, -1:] ,				A_ ,				decoder_attention_mask=A_ ,				past_key_values=outputs_cache.past_key_values ,				decoder_position_ids=A_ ,				)

         _lowercase:			Any       =     model.decode(A_ ,				A_       )

         _lowercase:			str       =     np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])       )       )
         self.parent.assertTrue(diff < 1E-3 ,				msg=f'''Max diff is {diff}'''       )



       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_       )     ->	int:





         """simple docstring"""




         _lowercase:			Optional[int]       =     20
         _lowercase:			Dict       =     model_class_name(A_       )

         _lowercase:			Optional[Any]       =     model.encode(inputs_dict['''input_ids''']       )

         _lowercase ,				_lowercase:			List[Any]       =     (
             inputs_dict['''decoder_input_ids'''],
             inputs_dict['''decoder_attention_mask'''],
         )

         _lowercase:			Union[str, Any]       =     jnp.concatenate(
             [
                 decoder_attention_mask,
                 jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])       ),
             ] ,				axis=-1 ,				)

         _lowercase:			List[Any]       =     model.init_cache(decoder_input_ids.shape[0] ,				A_ ,				A_       )
         _lowercase:			List[str]       =     jnp.broadcast_to(
             jnp.arange(decoder_input_ids.shape[-1] - 1       )[None, :] ,				(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1) ,				)

         _lowercase:			Any       =     model.decode(
             decoder_input_ids[:, :-1] ,				A_ ,				decoder_attention_mask=A_ ,				past_key_values=A_ ,				decoder_position_ids=A_ ,				)
         _lowercase:			Union[str, Any]       =     jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]] ,				dtype='''i4'''       )
         _lowercase:			Optional[Any]       =     model.decode(
             decoder_input_ids[:, -1:] ,				A_ ,				past_key_values=outputs_cache.past_key_values ,				decoder_attention_mask=A_ ,				decoder_position_ids=A_ ,				)

         _lowercase:			int       =     model.decode(A_ ,				A_ ,				decoder_attention_mask=A_       )

         _lowercase:			Any       =     np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])       )       )
         self.parent.assertTrue(diff < 1E-3 ,				msg=f'''Max diff is {diff}'''       )


@require_flax
class 			__magic_name__       (       unittest.TestCase ):
       UpperCamelCase_   =		99
       def 							lowercase_							(     self       )     ->	str:





         """simple docstring"""




         _lowercase:			List[Any]       =     np.array(
             [
                 [71, 82, 18, 33, 46, 91, 2],
                 [68, 34, 26, 58, 30, 82, 2],
                 [5, 97, 17, 39, 94, 40, 2],
                 [76, 83, 94, 25, 70, 78, 2],
                 [87, 59, 41, 35, 48, 66, 2],
                 [55, 13, 16, 58, 5, 2, 1],  # note padding
                 [64, 27, 31, 51, 12, 75, 2],
                 [52, 64, 86, 17, 83, 39, 2],
                 [48, 61, 9, 24, 71, 82, 2],
                 [26, 1, 60, 48, 22, 13, 2],
                 [21, 5, 62, 28, 14, 76, 2],
                 [45, 98, 37, 86, 59, 48, 2],
                 [70, 70, 50, 9, 28, 0, 2],
             ] ,				dtype=np.intaa ,				)

         _lowercase:			int       =     input_ids.shape[0]
         _lowercase:			str       =     BlenderbotSmallConfig(
             vocab_size=self.vocab_size ,				d_model=24 ,				encoder_layers=2 ,				decoder_layers=2 ,				encoder_attention_heads=2 ,				decoder_attention_heads=2 ,				encoder_ffn_dim=32 ,				decoder_ffn_dim=32 ,				max_position_embeddings=48 ,				eos_token_id=2 ,				pad_token_id=1 ,				bos_token_id=0 ,				)
         return config, input_ids, batch_size
       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         _lowercase ,				_lowercase ,				_lowercase:			List[str]       =     self._get_config_and_data()
         _lowercase:			str       =     FlaxBlenderbotSmallForConditionalGeneration(A_       )
         _lowercase:			Dict       =     lm_model(input_ids=A_       )
         _lowercase:			Optional[Any]       =     (batch_size, input_ids.shape[1], config.vocab_size)
         self.assertEqual(outputs['''logits'''].shape ,				A_       )
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         _lowercase:			Tuple       =     BlenderbotSmallConfig(
             vocab_size=self.vocab_size ,				d_model=14 ,				encoder_layers=2 ,				decoder_layers=2 ,				encoder_attention_heads=2 ,				decoder_attention_heads=2 ,				encoder_ffn_dim=8 ,				decoder_ffn_dim=8 ,				max_position_embeddings=48 ,				)
         _lowercase:			List[Any]       =     FlaxBlenderbotSmallForConditionalGeneration(A_       )
         _lowercase:			Tuple       =     np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]] ,				dtype=np.intaa       )
         _lowercase:			Optional[int]       =     np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]] ,				dtype=np.intaa       )
         _lowercase:			int       =     lm_model(input_ids=A_ ,				decoder_input_ids=A_       )
         _lowercase:			Optional[int]       =     (*summary.shape, config.vocab_size)
         self.assertEqual(outputs['''logits'''].shape ,				A_       )



       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         _lowercase:			Dict       =     np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]] ,				dtype=np.intaa       )
         _lowercase:			Any       =     shift_tokens_right(A_ ,				1 ,				2       )
         _lowercase:			Any       =     np.equal(A_ ,				1       ).astype(np.floataa       ).sum()
         _lowercase:			Dict       =     np.equal(A_ ,				1       ).astype(np.floataa       ).sum()
         self.assertEqual(shifted.shape ,				input_ids.shape       )
         self.assertEqual(A_ ,				n_pad_before - 1       )
         self.assertTrue(np.equal(shifted[:, 0] ,				2       ).all()       )


@require_flax
class 			__magic_name__       (       SCREAMING_SNAKE_CASE__   ,					unittest.TestCase   ,					SCREAMING_SNAKE_CASE__ ):
       UpperCamelCase_   =		True
       UpperCamelCase_   =		(
           (
               FlaxBlenderbotSmallModel,
               FlaxBlenderbotSmallForConditionalGeneration,
           )
           if is_flax_available()
           else ()
       )
       UpperCamelCase_   =		(FlaxBlenderbotSmallForConditionalGeneration,) if is_flax_available() else ()
       def 							lowercase_							(     self       )     ->	Tuple:





         """simple docstring"""




         _lowercase:			Dict       =     FlaxBlenderbotSmallModelTester(self       )
       def 							lowercase_							(     self       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase ,				_lowercase:			str       =     self.model_tester.prepare_config_and_inputs()
         for model_class in self.all_model_classes:
           self.model_tester.check_use_cache_forward(A_ ,				A_ ,				A_       )
       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         _lowercase ,				_lowercase:			Tuple       =     self.model_tester.prepare_config_and_inputs()
         for model_class in self.all_model_classes:
           self.model_tester.check_use_cache_forward_with_attn_mask(A_ ,				A_ ,				A_       )
       def 							lowercase_							(     self       )     ->	Any:





         """simple docstring"""




         _lowercase ,				_lowercase:			Union[str, Any]       =     self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
           with self.subTest(model_class.__name__       ):
             _lowercase:			Union[str, Any]       =     self._prepare_for_class(A_ ,				A_       )
             _lowercase:			int       =     model_class(A_       )

             @jax.jit
             def encode_jitted(A_ ,				A_=None ,				**A_       ):
               return model.encode(input_ids=A_ ,				attention_mask=A_       )

             with self.subTest('''JIT Enabled'''       ):
               _lowercase:			int       =     encode_jitted(**A_       ).to_tuple()

             with self.subTest('''JIT Disabled'''       ):
               with jax.disable_jit():
                 _lowercase:			List[Any]       =     encode_jitted(**A_       ).to_tuple()

             self.assertEqual(len(A_       ) ,				len(A_       )       )
             for jitted_output, output in zip(A_ ,				A_       ):
               self.assertEqual(jitted_output.shape ,				output.shape       )
       def 							lowercase_							(     self       )     ->	Any:





         """simple docstring"""




         _lowercase ,				_lowercase:			Tuple       =     self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
           with self.subTest(model_class.__name__       ):
             _lowercase:			Any       =     model_class(A_       )
             _lowercase:			Dict       =     model.encode(inputs_dict['''input_ids'''] ,				inputs_dict['''attention_mask''']       )

             _lowercase:			List[str]       =     {
                 '''decoder_input_ids''': inputs_dict['''decoder_input_ids'''],
                 '''decoder_attention_mask''': inputs_dict['''decoder_attention_mask'''],
                 '''encoder_outputs''': encoder_outputs,
             }

             @jax.jit
             def decode_jitted(A_ ,				A_ ,				A_       ):
               return model.decode(
                   decoder_input_ids=A_ ,				decoder_attention_mask=A_ ,				encoder_outputs=A_ ,				)

             with self.subTest('''JIT Enabled'''       ):
               _lowercase:			int       =     decode_jitted(**A_       ).to_tuple()

             with self.subTest('''JIT Disabled'''       ):
               with jax.disable_jit():
                 _lowercase:			str       =     decode_jitted(**A_       ).to_tuple()

             self.assertEqual(len(A_       ) ,				len(A_       )       )
             for jitted_output, output in zip(A_ ,				A_       ):
               self.assertEqual(jitted_output.shape ,				output.shape       )



       @slow
       def 							lowercase_							(     self       )     ->	Tuple:





         """simple docstring"""




         for model_class_name in self.all_model_classes:
           _lowercase:			List[Any]       =     model_class_name.from_pretrained('''facebook/blenderbot_small-90M'''       )
           # FlaxBlenderbotForSequenceClassification expects eos token in input_ids
           _lowercase:			Optional[int]       =     np.ones((1, 1)       ) * model.config.eos_token_id
           _lowercase:			Union[str, Any]       =     model(A_       )
           self.assertIsNotNone(A_       )


"""simple docstring"""




import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      MobileBertForMaskedLM,
      MobileBertForMultipleChoice,
      MobileBertForNextSentencePrediction,
      MobileBertForPreTraining,
      MobileBertForQuestionAnswering,
      MobileBertForSequenceClassification,
      MobileBertForTokenClassification,
      MobileBertModel,
  )


class 			__magic_name__       :
       def __init__(     self ,				A_ ,				A_=13 ,				A_=7 ,				A_=True ,				A_=True ,				A_=True ,				A_=True ,				A_=99 ,				A_=64 ,				A_=32 ,				A_=5 ,				A_=4 ,				A_=37 ,				A_="gelu" ,				A_=0.1 ,				A_=0.1 ,				A_=512 ,				A_=16 ,				A_=2 ,				A_=0.02 ,				A_=3 ,				A_=4 ,				A_=None ,				)     ->	Dict:





         """simple docstring"""




         _lowercase:			List[Any]       =     parent
         _lowercase:			str       =     batch_size
         _lowercase:			List[str]       =     seq_length
         _lowercase:			Any       =     is_training
         _lowercase:			Union[str, Any]       =     use_input_mask
         _lowercase:			Any       =     use_token_type_ids
         _lowercase:			int       =     use_labels
         _lowercase:			Any       =     vocab_size
         _lowercase:			Optional[int]       =     hidden_size
         _lowercase:			Optional[int]       =     embedding_size
         _lowercase:			Union[str, Any]       =     num_hidden_layers
         _lowercase:			Optional[int]       =     num_attention_heads
         _lowercase:			Optional[int]       =     intermediate_size
         _lowercase:			List[str]       =     hidden_act
         _lowercase:			Dict       =     hidden_dropout_prob
         _lowercase:			Any       =     attention_probs_dropout_prob
         _lowercase:			Tuple       =     max_position_embeddings
         _lowercase:			List[str]       =     type_vocab_size
         _lowercase:			Any       =     type_sequence_label_size
         _lowercase:			int       =     initializer_range
         _lowercase:			Any       =     num_labels
         _lowercase:			int       =     num_choices
         _lowercase:			List[str]       =     scope
       def 							lowercase_							(     self       )     ->	Optional[Any]:





         """simple docstring"""




         _lowercase:			Tuple       =     ids_tensor([self.batch_size, self.seq_length] ,				self.vocab_size       )

         _lowercase:			Optional[Any]       =     None
         if self.use_input_mask:
           _lowercase:			Any       =     random_attention_mask([self.batch_size, self.seq_length]       )

         _lowercase:			Tuple       =     None
         if self.use_token_type_ids:
           _lowercase:			str       =     ids_tensor([self.batch_size, self.seq_length] ,				self.type_vocab_size       )

         _lowercase:			Dict       =     None
         _lowercase:			Tuple       =     None
         _lowercase:			List[str]       =     None
         if self.use_labels:
           _lowercase:			str       =     ids_tensor([self.batch_size] ,				self.type_sequence_label_size       )
           _lowercase:			int       =     ids_tensor([self.batch_size, self.seq_length] ,				self.num_labels       )
           _lowercase:			Dict       =     ids_tensor([self.batch_size] ,				self.num_choices       )

         _lowercase:			str       =     self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
       def 							lowercase_							(     self       )     ->	List[Any]:





         """simple docstring"""




         return MobileBertConfig(
             vocab_size=self.vocab_size ,				hidden_size=self.hidden_size ,				num_hidden_layers=self.num_hidden_layers ,				num_attention_heads=self.num_attention_heads ,				intermediate_size=self.intermediate_size ,				embedding_size=self.embedding_size ,				hidden_act=self.hidden_act ,				hidden_dropout_prob=self.hidden_dropout_prob ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob ,				max_position_embeddings=self.max_position_embeddings ,				type_vocab_size=self.type_vocab_size ,				is_decoder=A_ ,				initializer_range=self.initializer_range ,				)
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_       )     ->	Any:





         """simple docstring"""




         _lowercase:			Tuple       =     MobileBertModel(config=A_       )
         model.to(A_       )
         model.eval()
         _lowercase:			Tuple       =     model(A_ ,				attention_mask=A_ ,				token_type_ids=A_       )
         _lowercase:			Dict       =     model(A_ ,				token_type_ids=A_       )
         _lowercase:			Optional[int]       =     model(A_       )

         self.parent.assertEqual(result.last_hidden_state.shape ,				(self.batch_size, self.seq_length, self.hidden_size)       )
         self.parent.assertEqual(result.pooler_output.shape ,				(self.batch_size, self.hidden_size)       )
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_       )     ->	int:





         """simple docstring"""




         _lowercase:			int       =     MobileBertForMaskedLM(config=A_       )
         model.to(A_       )
         model.eval()
         _lowercase:			Optional[int]       =     model(A_ ,				attention_mask=A_ ,				token_type_ids=A_ ,				labels=A_       )
         self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.seq_length, self.vocab_size)       )
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_       )     ->	int:





         """simple docstring"""




         _lowercase:			Any       =     MobileBertForNextSentencePrediction(config=A_       )
         model.to(A_       )
         model.eval()
         _lowercase:			int       =     model(
             A_ ,				attention_mask=A_ ,				token_type_ids=A_ ,				labels=A_ ,				)
         self.parent.assertEqual(result.logits.shape ,				(self.batch_size, 2)       )
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			str       =     MobileBertForPreTraining(config=A_       )
         model.to(A_       )
         model.eval()
         _lowercase:			List[str]       =     model(
             A_ ,				attention_mask=A_ ,				token_type_ids=A_ ,				labels=A_ ,				next_sentence_label=A_ ,				)
         self.parent.assertEqual(result.prediction_logits.shape ,				(self.batch_size, self.seq_length, self.vocab_size)       )
         self.parent.assertEqual(result.seq_relationship_logits.shape ,				(self.batch_size, 2)       )
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_       )     ->	List[str]:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     MobileBertForQuestionAnswering(config=A_       )
         model.to(A_       )
         model.eval()
         _lowercase:			Dict       =     model(
             A_ ,				attention_mask=A_ ,				token_type_ids=A_ ,				start_positions=A_ ,				end_positions=A_ ,				)
         self.parent.assertEqual(result.start_logits.shape ,				(self.batch_size, self.seq_length)       )
         self.parent.assertEqual(result.end_logits.shape ,				(self.batch_size, self.seq_length)       )
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_       )     ->	List[Any]:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     self.num_labels
         _lowercase:			int       =     MobileBertForSequenceClassification(A_       )
         model.to(A_       )
         model.eval()
         _lowercase:			List[Any]       =     model(A_ ,				attention_mask=A_ ,				token_type_ids=A_ ,				labels=A_       )
         self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.num_labels)       )
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_       )     ->	Tuple:





         """simple docstring"""




         _lowercase:			Dict       =     self.num_labels
         _lowercase:			Optional[Any]       =     MobileBertForTokenClassification(config=A_       )
         model.to(A_       )
         model.eval()
         _lowercase:			Optional[int]       =     model(A_ ,				attention_mask=A_ ,				token_type_ids=A_ ,				labels=A_       )
         self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.seq_length, self.num_labels)       )
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_ ,				A_       )     ->	List[str]:





         """simple docstring"""




         _lowercase:			Union[str, Any]       =     self.num_choices
         _lowercase:			List[Any]       =     MobileBertForMultipleChoice(config=A_       )
         model.to(A_       )
         model.eval()
         _lowercase:			Any       =     input_ids.unsqueeze(1       ).expand(-1 ,				self.num_choices ,				-1       ).contiguous()
         _lowercase:			Tuple       =     token_type_ids.unsqueeze(1       ).expand(-1 ,				self.num_choices ,				-1       ).contiguous()
         _lowercase:			List[str]       =     input_mask.unsqueeze(1       ).expand(-1 ,				self.num_choices ,				-1       ).contiguous()
         _lowercase:			Any       =     model(
             A_ ,				attention_mask=A_ ,				token_type_ids=A_ ,				labels=A_ ,				)
         self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.num_choices)       )



       def 							lowercase_							(     self       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			Any       =     self.prepare_config_and_inputs()
         (
             (
             _lowercase
         ) ,				(
             _lowercase
         ) ,				(
             _lowercase
         ) ,				(
             _lowercase
         ) ,				(
             _lowercase
         ) ,				(
             _lowercase
         ) ,				(
             _lowercase
         ) ,				
         ):			Dict       =     config_and_inputs
         _lowercase:			int       =     {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict


@require_torch
class 			__magic_name__       (       SCREAMING_SNAKE_CASE__   ,					SCREAMING_SNAKE_CASE__   ,					unittest.TestCase ):
       UpperCamelCase_   =		(
           (
               MobileBertModel,
               MobileBertForMaskedLM,
               MobileBertForMultipleChoice,
               MobileBertForNextSentencePrediction,
               MobileBertForPreTraining,
               MobileBertForQuestionAnswering,
               MobileBertForSequenceClassification,
               MobileBertForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       UpperCamelCase_   =		(
           {
               '''feature-extraction''': MobileBertModel,
               '''fill-mask''': MobileBertForMaskedLM,
               '''question-answering''': MobileBertForQuestionAnswering,
               '''text-classification''': MobileBertForSequenceClassification,
               '''token-classification''': MobileBertForTokenClassification,
               '''zero-shot''': MobileBertForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       UpperCamelCase_   =		True
       def 							lowercase_							(     self ,				A_ ,				A_ ,				A_=False       )     ->	str:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     super()._prepare_for_class(A_ ,				A_ ,				return_labels=A_       )

         if return_labels:
           if model_class in get_values(A_       ):
             _lowercase:			Tuple       =     torch.zeros(
                 (self.model_tester.batch_size, self.model_tester.seq_length) ,				dtype=torch.long ,				device=A_       )
             _lowercase:			Tuple       =     torch.zeros(
                 self.model_tester.batch_size ,				dtype=torch.long ,				device=A_       )
         return inputs_dict
       def 							lowercase_							(     self       )     ->	Tuple:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     MobileBertModelTester(self       )
         _lowercase:			str       =     ConfigTester(self ,				config_class=A_ ,				hidden_size=37       )
       def 							lowercase_							(     self       )     ->	Any:





         """simple docstring"""




         self.config_tester.run_common_tests()
       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         _lowercase:			str       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_mobilebert_model(*A_       )
       def 							lowercase_							(     self       )     ->	Optional[Any]:





         """simple docstring"""




         _lowercase:			Union[str, Any]       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_mobilebert_for_masked_lm(*A_       )
       def 							lowercase_							(     self       )     ->	Any:





         """simple docstring"""




         _lowercase:			List[str]       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_mobilebert_for_multiple_choice(*A_       )
       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         _lowercase:			Any       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*A_       )
       def 							lowercase_							(     self       )     ->	Optional[Any]:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_mobilebert_for_pretraining(*A_       )
       def 							lowercase_							(     self       )     ->	Tuple:





         """simple docstring"""




         _lowercase:			List[Any]       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_mobilebert_for_question_answering(*A_       )
       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         _lowercase:			str       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_mobilebert_for_sequence_classification(*A_       )



       def 							lowercase_							(     self       )     ->	List[Any]:





         """simple docstring"""




         _lowercase:			Dict       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_mobilebert_for_token_classification(*A_       )




def  _lowerCAmelCase		(		_UpperCamelCase						):






  """simple docstring"""



  return torch.tensor(
      _UpperCamelCase ,				dtype=torch.long ,				device=_UpperCamelCase ,				)


A__   :					Optional[Any]							=   1e-3







@require_torch
@require_sentencepiece
@require_tokenizers
class 			__magic_name__       (       unittest.TestCase ):
       @slow
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         _lowercase:			Optional[int]       =     MobileBertModel.from_pretrained('''google/mobilebert-uncased'''       ).to(A_       )
         _lowercase:			str       =     _long_tensor([[101, 7110, 1005, 1056, 2023, 1_1333, 1_7413, 1029, 102]]       )
         with torch.no_grad():
           _lowercase:			Dict       =     model(A_       )[0]
         _lowercase:			List[Any]       =     torch.Size((1, 9, 512)       )
         self.assertEqual(output.shape ,				A_       )
         _lowercase:			Tuple       =     torch.tensor(
             [
                 [
                     [-2.4_73_65_26E07, 8.2_69_16_56E04, 1.6_52_18_38E05],
                     [-5.7_54_17_04E-01, 3.9_05_60_22E00, 4.4_01_15_07E00],
                     [2.6_04_73_59E00, 1.5_67_76_52E00, -1.7_32_41_88E-01],
                 ]
             ] ,				device=A_ ,				)

         # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
         # ~1 difference, it's therefore not a good idea to measure using addition.
         # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
         # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
         _lowercase:			Tuple       =     torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE       )
         _lowercase:			Dict       =     torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE       )

         self.assertTrue(lower_bound and upper_bound       )
