
"""simple docstring"""


import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          MobileBertForMaskedLM,
          MobileBertForMultipleChoice,
          MobileBertForNextSentencePrediction,
          MobileBertForPreTraining,
          MobileBertForQuestionAnswering,
          MobileBertForSequenceClassification,
          MobileBertForTokenClassification,
          MobileBertModel,
      )



class   __a  :






       def __init__(  self :							Any       ,      UpperCAmelCase_ :							Tuple       ,      UpperCAmelCase_ :							Union[str, Any]=13       ,      UpperCAmelCase_ :							str=7       ,      UpperCAmelCase_ :							Optional[Any]=True       ,      UpperCAmelCase_ :							Dict=True       ,      UpperCAmelCase_ :							Tuple=True       ,      UpperCAmelCase_ :							int=True       ,      UpperCAmelCase_ :							Dict=99       ,      UpperCAmelCase_ :							List[str]=64       ,      UpperCAmelCase_ :							Any=32       ,      UpperCAmelCase_ :							Tuple=5       ,      UpperCAmelCase_ :							str=4       ,      UpperCAmelCase_ :							int=37       ,      UpperCAmelCase_ :							Optional[int]="gelu"       ,      UpperCAmelCase_ :							Optional[Any]=0.1       ,      UpperCAmelCase_ :							Tuple=0.1       ,      UpperCAmelCase_ :							int=512       ,      UpperCAmelCase_ :							str=16       ,      UpperCAmelCase_ :							int=2       ,      UpperCAmelCase_ :							int=0.02       ,      UpperCAmelCase_ :							Union[str, Any]=3       ,      UpperCAmelCase_ :							str=4       ,      UpperCAmelCase_ :							List[Any]=None       ,      )->  List[str]:







              """simple docstring"""
              UpperCamelCase					 =		parent
              UpperCamelCase					 =		batch_size
              UpperCamelCase					 =		seq_length
              UpperCamelCase					 =		is_training
              UpperCamelCase					 =		use_input_mask
              UpperCamelCase					 =		use_token_type_ids
              UpperCamelCase					 =		use_labels
              UpperCamelCase					 =		vocab_size
              UpperCamelCase					 =		hidden_size
              UpperCamelCase					 =		embedding_size
              UpperCamelCase					 =		num_hidden_layers
              UpperCamelCase					 =		num_attention_heads
              UpperCamelCase					 =		intermediate_size
              UpperCamelCase					 =		hidden_act
              UpperCamelCase					 =		hidden_dropout_prob
              UpperCamelCase					 =		attention_probs_dropout_prob
              UpperCamelCase					 =		max_position_embeddings
              UpperCamelCase					 =		type_vocab_size
              UpperCamelCase					 =		type_sequence_label_size
              UpperCamelCase					 =		initializer_range
              UpperCamelCase					 =		num_labels
              UpperCamelCase					 =		num_choices
              UpperCamelCase					 =		scope






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[int]			)->  Any:







              """simple docstring"""
              UpperCamelCase					 =		ids_tensor([self.batch_size, self.seq_length]       ,      self.vocab_size			)

              UpperCamelCase					 =		None
              if self.use_input_mask:
                     UpperCamelCase					 =		random_attention_mask([self.batch_size, self.seq_length]			)

              UpperCamelCase					 =		None
              if self.use_token_type_ids:
                     UpperCamelCase					 =		ids_tensor([self.batch_size, self.seq_length]       ,      self.type_vocab_size			)

              UpperCamelCase					 =		None
              UpperCamelCase					 =		None
              UpperCamelCase					 =		None
              if self.use_labels:
                     UpperCamelCase					 =		ids_tensor([self.batch_size]       ,      self.type_sequence_label_size			)
                     UpperCamelCase					 =		ids_tensor([self.batch_size, self.seq_length]       ,      self.num_labels			)
                     UpperCamelCase					 =		ids_tensor([self.batch_size]       ,      self.num_choices			)

              UpperCamelCase					 =		self.get_config()

              return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[int]			)->  Union[str, Any]:







              """simple docstring"""
              return MobileBertConfig(
                  vocab_size=self.vocab_size       ,      hidden_size=self.hidden_size       ,      num_hidden_layers=self.num_hidden_layers       ,      num_attention_heads=self.num_attention_heads       ,      intermediate_size=self.intermediate_size       ,      embedding_size=self.embedding_size       ,      hidden_act=self.hidden_act       ,      hidden_dropout_prob=self.hidden_dropout_prob       ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,      max_position_embeddings=self.max_position_embeddings       ,      type_vocab_size=self.type_vocab_size       ,      is_decoder=UpperCAmelCase_       ,      initializer_range=self.initializer_range       ,      )






       def 	_SCREAMING_SNAKE_CASE       (  self :							Any       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							List[str]       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							Union[str, Any]       ,      UpperCAmelCase_ :							Any       ,      UpperCAmelCase_ :							Optional[int]       ,      UpperCAmelCase_ :							List[Any]			)->  int:







              """simple docstring"""
              UpperCamelCase					 =		MobileBertModel(config=UpperCAmelCase_			)
              model.to(UpperCAmelCase_			)
              model.eval()
              UpperCamelCase					 =		model(UpperCAmelCase_       ,      attention_mask=UpperCAmelCase_       ,      token_type_ids=UpperCAmelCase_			)
              UpperCamelCase					 =		model(UpperCAmelCase_       ,      token_type_ids=UpperCAmelCase_			)
              UpperCamelCase					 =		model(UpperCAmelCase_			)

              self.parent.assertEqual(result.last_hidden_state.shape       ,      (self.batch_size, self.seq_length, self.hidden_size)			)
              self.parent.assertEqual(result.pooler_output.shape       ,      (self.batch_size, self.hidden_size)			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							str       ,      UpperCAmelCase_ :							List[str]       ,      UpperCAmelCase_ :							Tuple       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							Optional[int]       ,      UpperCAmelCase_ :							List[Any]       ,      UpperCAmelCase_ :							int       ,      UpperCAmelCase_ :							List[Any]			)->  str:







              """simple docstring"""
              UpperCamelCase					 =		MobileBertForMaskedLM(config=UpperCAmelCase_			)
              model.to(UpperCAmelCase_			)
              model.eval()
              UpperCamelCase					 =		model(UpperCAmelCase_       ,      attention_mask=UpperCAmelCase_       ,      token_type_ids=UpperCAmelCase_       ,      labels=UpperCAmelCase_			)
              self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.seq_length, self.vocab_size)			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Union[str, Any]       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							Any       ,      UpperCAmelCase_ :							Any       ,      UpperCAmelCase_ :							Optional[int]       ,      UpperCAmelCase_ :							Dict       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							int			)->  int:







              """simple docstring"""
              UpperCamelCase					 =		MobileBertForNextSentencePrediction(config=UpperCAmelCase_			)
              model.to(UpperCAmelCase_			)
              model.eval()
              UpperCamelCase					 =		model(
                  UpperCAmelCase_       ,      attention_mask=UpperCAmelCase_       ,      token_type_ids=UpperCAmelCase_       ,      labels=UpperCAmelCase_       ,      )
              self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, 2)			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							str       ,      UpperCAmelCase_ :							Union[str, Any]       ,      UpperCAmelCase_ :							List[Any]       ,      UpperCAmelCase_ :							Any       ,      UpperCAmelCase_ :							str       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							Any       ,      UpperCAmelCase_ :							Optional[int]			)->  Optional[int]:







              """simple docstring"""
              UpperCamelCase					 =		MobileBertForPreTraining(config=UpperCAmelCase_			)
              model.to(UpperCAmelCase_			)
              model.eval()
              UpperCamelCase					 =		model(
                  UpperCAmelCase_       ,      attention_mask=UpperCAmelCase_       ,      token_type_ids=UpperCAmelCase_       ,      labels=UpperCAmelCase_       ,      next_sentence_label=UpperCAmelCase_       ,      )
              self.parent.assertEqual(result.prediction_logits.shape       ,      (self.batch_size, self.seq_length, self.vocab_size)			)
              self.parent.assertEqual(result.seq_relationship_logits.shape       ,      (self.batch_size, 2)			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[Any]       ,      UpperCAmelCase_ :							Tuple       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							Tuple       ,      UpperCAmelCase_ :							int       ,      UpperCAmelCase_ :							Union[str, Any]       ,      UpperCAmelCase_ :							Dict       ,      UpperCAmelCase_ :							Tuple			)->  Union[str, Any]:







              """simple docstring"""
              UpperCamelCase					 =		MobileBertForQuestionAnswering(config=UpperCAmelCase_			)
              model.to(UpperCAmelCase_			)
              model.eval()
              UpperCamelCase					 =		model(
                  UpperCAmelCase_       ,      attention_mask=UpperCAmelCase_       ,      token_type_ids=UpperCAmelCase_       ,      start_positions=UpperCAmelCase_       ,      end_positions=UpperCAmelCase_       ,      )
              self.parent.assertEqual(result.start_logits.shape       ,      (self.batch_size, self.seq_length)			)
              self.parent.assertEqual(result.end_logits.shape       ,      (self.batch_size, self.seq_length)			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							int       ,      UpperCAmelCase_ :							List[Any]       ,      UpperCAmelCase_ :							List[Any]       ,      UpperCAmelCase_ :							List[Any]       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							Optional[int]       ,      UpperCAmelCase_ :							List[str]       ,      UpperCAmelCase_ :							Optional[Any]			)->  Any:







              """simple docstring"""
              UpperCamelCase					 =		self.num_labels
              UpperCamelCase					 =		MobileBertForSequenceClassification(UpperCAmelCase_			)
              model.to(UpperCAmelCase_			)
              model.eval()
              UpperCamelCase					 =		model(UpperCAmelCase_       ,      attention_mask=UpperCAmelCase_       ,      token_type_ids=UpperCAmelCase_       ,      labels=UpperCAmelCase_			)
              self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.num_labels)			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Dict       ,      UpperCAmelCase_ :							Optional[int]       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							Any       ,      UpperCAmelCase_ :							Optional[Any]       ,      UpperCAmelCase_ :							str       ,      UpperCAmelCase_ :							List[Any]       ,      UpperCAmelCase_ :							List[Any]			)->  Union[str, Any]:







              """simple docstring"""
              UpperCamelCase					 =		self.num_labels
              UpperCamelCase					 =		MobileBertForTokenClassification(config=UpperCAmelCase_			)
              model.to(UpperCAmelCase_			)
              model.eval()
              UpperCamelCase					 =		model(UpperCAmelCase_       ,      attention_mask=UpperCAmelCase_       ,      token_type_ids=UpperCAmelCase_       ,      labels=UpperCAmelCase_			)
              self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.seq_length, self.num_labels)			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							List[str]       ,      UpperCAmelCase_ :							Tuple       ,      UpperCAmelCase_ :							List[Any]       ,      UpperCAmelCase_ :							List[Any]       ,      UpperCAmelCase_ :							List[Any]       ,      UpperCAmelCase_ :							str       ,      UpperCAmelCase_ :							Tuple       ,      UpperCAmelCase_ :							List[Any]			)->  Optional[int]:







              """simple docstring"""
              UpperCamelCase					 =		self.num_choices
              UpperCamelCase					 =		MobileBertForMultipleChoice(config=UpperCAmelCase_			)
              model.to(UpperCAmelCase_			)
              model.eval()
              UpperCamelCase					 =		input_ids.unsqueeze(1			).expand(-1       ,      self.num_choices       ,      -1			).contiguous()
              UpperCamelCase					 =		token_type_ids.unsqueeze(1			).expand(-1       ,      self.num_choices       ,      -1			).contiguous()
              UpperCamelCase					 =		input_mask.unsqueeze(1			).expand(-1       ,      self.num_choices       ,      -1			).contiguous()
              UpperCamelCase					 =		model(
                  UpperCAmelCase_       ,      attention_mask=UpperCAmelCase_       ,      token_type_ids=UpperCAmelCase_       ,      labels=UpperCAmelCase_       ,      )
              self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.num_choices)			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[int]			)->  Tuple:







              """simple docstring"""
              UpperCamelCase					 =		self.prepare_config_and_inputs()
              (
                  (
                  UpperCamelCase
              ) ,       (
                  UpperCamelCase
              ) ,       (
                  UpperCamelCase
              ) ,       (
                  UpperCamelCase
              ) ,       (
                  UpperCamelCase
              ) ,       (
                  UpperCamelCase
              ) ,       (
                  UpperCamelCase
              ) ,       
              )					 =		config_and_inputs
              UpperCamelCase					 =		{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
              return config, inputs_dict





@require_torch
class   __a  (      _lowerCAmelCase    ,     _lowerCAmelCase    ,     unittest.TestCase	):
       UpperCamelCase_		:					Optional[int]   =			(
           (
               MobileBertModel,
               MobileBertForMaskedLM,
               MobileBertForMultipleChoice,
               MobileBertForNextSentencePrediction,
               MobileBertForPreTraining,
               MobileBertForQuestionAnswering,
               MobileBertForSequenceClassification,
               MobileBertForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       UpperCamelCase_		:					Dict   =			(
           {
               '''feature-extraction''': MobileBertModel,
               '''fill-mask''': MobileBertForMaskedLM,
               '''question-answering''': MobileBertForQuestionAnswering,
               '''text-classification''': MobileBertForSequenceClassification,
               '''token-classification''': MobileBertForTokenClassification,
               '''zero-shot''': MobileBertForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       UpperCamelCase_		:					List[str]   =			True






       def 	_SCREAMING_SNAKE_CASE       (  self :							str       ,      UpperCAmelCase_ :							Any       ,      UpperCAmelCase_ :							str       ,      UpperCAmelCase_ :							Tuple=False			)->  str:







              """simple docstring"""
              UpperCamelCase					 =		super()._prepare_for_class(UpperCAmelCase_       ,      UpperCAmelCase_       ,      return_labels=UpperCAmelCase_			)

              if return_labels:
                     if model_class in get_values(UpperCAmelCase_			):
                            UpperCamelCase					 =		torch.zeros(
                                (self.model_tester.batch_size, self.model_tester.seq_length)       ,      dtype=torch.long       ,      device=UpperCAmelCase_			)
                            UpperCamelCase					 =		torch.zeros(
                                self.model_tester.batch_size       ,      dtype=torch.long       ,      device=UpperCAmelCase_			)
              return inputs_dict






       def 	_SCREAMING_SNAKE_CASE       (  self :							List[Any]			)->  Tuple:







              """simple docstring"""
              UpperCamelCase					 =		MobileBertModelTester(self			)
              UpperCamelCase					 =		ConfigTester(self       ,      config_class=UpperCAmelCase_       ,      hidden_size=37			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Dict			)->  str:







              """simple docstring"""
              self.config_tester.run_common_tests()






       def 	_SCREAMING_SNAKE_CASE       (  self :							Tuple			)->  Tuple:







              """simple docstring"""
              UpperCamelCase					 =		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_model(*UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[int]			)->  int:







              """simple docstring"""
              UpperCamelCase					 =		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_masked_lm(*UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							int			)->  List[Any]:







              """simple docstring"""
              UpperCamelCase					 =		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_multiple_choice(*UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Union[str, Any]			)->  str:







              """simple docstring"""
              UpperCamelCase					 =		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							List[str]			)->  Optional[Any]:







              """simple docstring"""
              UpperCamelCase					 =		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_pretraining(*UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							str			)->  int:







              """simple docstring"""
              UpperCamelCase					 =		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_question_answering(*UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[Any]			)->  Dict:







              """simple docstring"""
              UpperCamelCase					 =		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_sequence_classification(*UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[int]			)->  int:







              """simple docstring"""
              UpperCamelCase					 =		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_token_classification(*UpperCAmelCase_			)





def 	lowerCamelCase__			(							UpperCAmelCase_				)->							List[str]:







       """simple docstring"""
       return torch.tensor(
           UpperCAmelCase_		,      dtype=torch.long		,      device=UpperCAmelCase_		,      )


SCREAMING_SNAKE_CASE      =     1E-3



@require_torch
@require_sentencepiece
@require_tokenizers
class   __a  (      unittest.TestCase	):






       @slow
       def 	_SCREAMING_SNAKE_CASE       (  self :							List[str]			)->  Optional[Any]:







              """simple docstring"""
              UpperCamelCase					 =		MobileBertModel.from_pretrained("google/mobilebert-uncased"			).to(UpperCAmelCase_			)
              UpperCamelCase					 =		_long_tensor([[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]			)
              with torch.no_grad():
                     UpperCamelCase					 =		model(UpperCAmelCase_			)[0]
              UpperCamelCase					 =		torch.Size((1, 9, 512)			)
              self.assertEqual(output.shape       ,      UpperCAmelCase_			)
              UpperCamelCase					 =		torch.tensor(
                  [
                      [
                          [-2.4_736_526e07, 8.2_691_656e04, 1.6_521_838e05],
                          [-5.7_541_704e-01, 3.9_056_022e00, 4.4_011_507e00],
                          [2.6_047_359e00, 1.5_677_652e00, -1.7_324_188e-01],
                      ]
                  ]       ,      device=UpperCAmelCase_       ,      )

              # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
              # ~1 difference, it's therefore not a good idea to measure using addition.
              # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
              # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
              UpperCamelCase					 =		torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE			)
              UpperCamelCase					 =		torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE			)

              self.assertTrue(lower_bound and upper_bound			)







"""simple docstring"""


import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
      import torch

      from transformers import XLMRobertaModel



@require_sentencepiece
@require_tokenizers
@require_torch
class   __a  (      unittest.TestCase	):






       @slow
       def 	_SCREAMING_SNAKE_CASE       (  self :							List[str]			)->  Optional[int]:







              """simple docstring"""
              UpperCamelCase					 =		XLMRobertaModel.from_pretrained("xlm-roberta-base"			)
              UpperCamelCase					 =		torch.tensor([[0, 581, 10_269, 83, 99_942, 136, 60_742, 23, 70, 80_583, 18_276, 2]]			)
              # The dog is cute and lives in the garden house

              UpperCamelCase					 =		torch.Size((1, 12, 768)			)  # batch_size, sequence_length, embedding_vector_dim
              UpperCamelCase					 =		torch.tensor(
                  [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]]			)
              #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
              #  xlmr.eval()
              #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
              with torch.no_grad():
                     UpperCamelCase					 =		model(UpperCAmelCase_			)["last_hidden_state"].detach()
              self.assertEqual(output.shape       ,      UpperCAmelCase_			)
              # compare the actual values for a slice of last dim
              self.assertTrue(torch.allclose(output[:, :, -1]       ,      UpperCAmelCase_       ,      atol=1e-3			)			)






       @slow
       def 	_SCREAMING_SNAKE_CASE       (  self :							Union[str, Any]			)->  Optional[int]:







              """simple docstring"""
              UpperCamelCase					 =		XLMRobertaModel.from_pretrained("xlm-roberta-large"			)
              UpperCamelCase					 =		torch.tensor([[0, 581, 10_269, 83, 99_942, 136, 60_742, 23, 70, 80_583, 18_276, 2]]			)
              # The dog is cute and lives in the garden house

              UpperCamelCase					 =		torch.Size((1, 12, 1_024)			)  # batch_size, sequence_length, embedding_vector_dim
              UpperCamelCase					 =		torch.tensor(
                  [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]]			)
              #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
              #  xlmr.eval()
              #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
              with torch.no_grad():
                     UpperCamelCase					 =		model(UpperCAmelCase_			)["last_hidden_state"].detach()
              self.assertEqual(output.shape       ,      UpperCAmelCase_			)
              # compare the actual values for a slice of last dim
              self.assertTrue(torch.allclose(output[:, :, -1]       ,      UpperCAmelCase_       ,      atol=1e-3			)			)





