






import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    RemBertConfig,
				    RemBertForCausalLM,
				    RemBertForMaskedLM,
				    RemBertForMultipleChoice,
				    RemBertForQuestionAnswering,
				    RemBertForSequenceClassification,
				    RemBertForTokenClassification,
				    RemBertModel,
				)
				from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST





class   SCREAMING_SNAKE_CASE :





							"""simple docstring"""

							def __init__(     self:      Any						,   __A:      int						,   __A:      Tuple=13						,   __A:      Dict=7						,   __A:      Tuple=True						,   __A:      Union[str, Any]=True						,   __A:      str=True						,   __A:      str=True						,   __A:      Union[str, Any]=99						,   __A:      Union[str, Any]=32						,   __A:      Dict=18						,   __A:      int=43						,   __A:      Optional[Any]=5						,   __A:      Tuple=4						,   __A:      Union[str, Any]=37						,   __A:      Any="gelu"						,   __A:      List[str]=0.1						,   __A:      Union[str, Any]=0.1						,   __A:      Union[str, Any]=5_12						,   __A:      int=16						,   __A:      Dict=2						,   __A:      Optional[Any]=0.02						,   __A:      List[Any]=3						,   __A:      Tuple=4						,   __A:      List[Any]=None						,   )			-> Dict:
												_A									=       parent
												_A									=       batch_size
												_A									=       seq_length
												_A									=       is_training
												_A									=       use_input_mask
												_A									=       use_token_type_ids
												_A									=       use_labels
												_A									=       vocab_size
												_A									=       hidden_size
												_A									=       input_embedding_size
												_A									=       output_embedding_size
												_A									=       num_hidden_layers
												_A									=       num_attention_heads
												_A									=       intermediate_size
												_A									=       hidden_act
												_A									=       hidden_dropout_prob
												_A									=       attention_probs_dropout_prob
												_A									=       max_position_embeddings
												_A									=       type_vocab_size
												_A									=       type_sequence_label_size
												_A									=       initializer_range
												_A									=       num_labels
												_A									=       num_choices
												_A									=       scope

							def 					__A       (     self:      List[Any]      )			-> Union[str, Any]:
												_A									=       ids_tensor([self.batch_size, self.seq_length]						,   self.vocab_size      )

												_A									=       None
												if self.use_input_mask:
																	_A									=       random_attention_mask([self.batch_size, self.seq_length]      )

												_A									=       None
												if self.use_token_type_ids:
																	_A									=       ids_tensor([self.batch_size, self.seq_length]						,   self.type_vocab_size      )

												_A									=       None
												_A									=       None
												_A									=       None
												if self.use_labels:
																	_A									=       ids_tensor([self.batch_size]						,   self.type_sequence_label_size      )
																	_A									=       ids_tensor([self.batch_size, self.seq_length]						,   self.num_labels      )
																	_A									=       ids_tensor([self.batch_size]						,   self.num_choices      )

												_A									=       RemBertConfig(
												    vocab_size=self.vocab_size						,   hidden_size=self.hidden_size						,   input_embedding_size=self.input_embedding_size						,   output_embedding_size=self.output_embedding_size						,   num_hidden_layers=self.num_hidden_layers						,   num_attention_heads=self.num_attention_heads						,   intermediate_size=self.intermediate_size						,   hidden_act=self.hidden_act						,   hidden_dropout_prob=self.hidden_dropout_prob						,   attention_probs_dropout_prob=self.attention_probs_dropout_prob						,   max_position_embeddings=self.max_position_embeddings						,   type_vocab_size=self.type_vocab_size						,   is_decoder=__A						,   initializer_range=self.initializer_range						,   )

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

							def 					__A       (     self:      int      )			-> Optional[Any]:
												(
												    (
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,
												)									=       self.prepare_config_and_inputs()

												_A									=       True
												_A									=       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]      )
												_A									=       ids_tensor([self.batch_size, self.seq_length]						,   vocab_size=2      )

												return (
												    config,
												    input_ids,
												    token_type_ids,
												    input_mask,
												    sequence_labels,
												    token_labels,
												    choice_labels,
												    encoder_hidden_states,
												    encoder_attention_mask,
												)

							def 					__A       (     self:      int						,   __A:      Union[str, Any]						,   __A:      Tuple						,   __A:      int						,   __A:      Union[str, Any]						,   __A:      Any						,   __A:      str						,   __A:      List[Any]      )			-> Optional[Any]:
												_A									=       RemBertModel(config=__A      )
												model.to(__A      )
												model.eval()
												_A									=       model(__A						,   attention_mask=__A						,   token_type_ids=__A      )
												_A									=       model(__A						,   token_type_ids=__A      )
												_A									=       model(__A      )
												self.parent.assertEqual(result.last_hidden_state.shape						,   (self.batch_size, self.seq_length, self.hidden_size)      )

							def 					__A       (     self:      str						,   __A:      List[Any]						,   __A:      Tuple						,   __A:      Any						,   __A:      List[Any]						,   __A:      Optional[int]						,   __A:      List[str]						,   __A:      Optional[Any]						,   __A:      str						,   __A:      Tuple						,   )			-> Dict:
												_A									=       True
												_A									=       RemBertModel(__A      )
												model.to(__A      )
												model.eval()
												_A									=       model(
												    __A						,   attention_mask=__A						,   token_type_ids=__A						,   encoder_hidden_states=__A						,   encoder_attention_mask=__A						,   )
												_A									=       model(
												    __A						,   attention_mask=__A						,   token_type_ids=__A						,   encoder_hidden_states=__A						,   )
												_A									=       model(__A						,   attention_mask=__A						,   token_type_ids=__A      )
												self.parent.assertEqual(result.last_hidden_state.shape						,   (self.batch_size, self.seq_length, self.hidden_size)      )

							def 					__A       (     self:      Dict						,   __A:      Any						,   __A:      Optional[int]						,   __A:      int						,   __A:      Any						,   __A:      str						,   __A:      Optional[Any]						,   __A:      List[Any]						,   __A:      Dict						,   __A:      List[Any]						,   )			-> Union[str, Any]:
												_A									=       RemBertForCausalLM(config=__A      )
												model.to(__A      )
												model.eval()
												_A									=       model(__A						,   attention_mask=__A						,   token_type_ids=__A						,   labels=__A      )
												self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.seq_length, self.vocab_size)      )

							def 					__A       (     self:      List[str]						,   __A:      str						,   __A:      Dict						,   __A:      Tuple						,   __A:      Optional[int]						,   __A:      Optional[Any]						,   __A:      List[Any]						,   __A:      Optional[Any]      )			-> List[str]:
												_A									=       RemBertForMaskedLM(config=__A      )
												model.to(__A      )
												model.eval()
												_A									=       model(__A						,   attention_mask=__A						,   token_type_ids=__A						,   labels=__A      )
												self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.seq_length, self.vocab_size)      )

							def 					__A       (     self:      Union[str, Any]						,   __A:      Any						,   __A:      List[str]						,   __A:      Dict						,   __A:      Any						,   __A:      Tuple						,   __A:      Union[str, Any]						,   __A:      Union[str, Any]						,   __A:      Dict						,   __A:      List[str]						,   )			-> Union[str, Any]:
												_A									=       True
												_A									=       True
												_A									=       RemBertForCausalLM(config=__A      )
												model.to(__A      )
												model.eval()

												# first forward pass
												_A									=       model(
												    __A						,   attention_mask=__A						,   encoder_hidden_states=__A						,   encoder_attention_mask=__A						,   use_cache=__A						,   )
												_A									=       outputs.past_key_values

												# create hypothetical multiple next token and extent to next_input_ids
												_A									=       ids_tensor((self.batch_size, 3)						,   config.vocab_size      )
												_A									=       ids_tensor((self.batch_size, 3)						,   vocab_size=2      )

												# append to next input_ids and
												_A									=       torch.cat([input_ids, next_tokens]						,   dim=-1      )
												_A									=       torch.cat([input_mask, next_mask]						,   dim=-1      )

												_A									=       model(
												    __A						,   attention_mask=__A						,   encoder_hidden_states=__A						,   encoder_attention_mask=__A						,   output_hidden_states=__A						,   )['''hidden_states'''][0]
												_A									=       model(
												    __A						,   attention_mask=__A						,   encoder_hidden_states=__A						,   encoder_attention_mask=__A						,   past_key_values=__A						,   output_hidden_states=__A						,   )['''hidden_states'''][0]

												# select random slice
												_A									=       ids_tensor((1,)						,   output_from_past.shape[-1]      ).item()
												_A									=       output_from_no_past[:, -3:, random_slice_idx].detach()
												_A									=       output_from_past[:, :, random_slice_idx].detach()

												self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]      )

												# test that outputs are equal for slice
												self.parent.assertTrue(torch.allclose(__A						,   __A						,   atol=1e-3      )      )

							def 					__A       (     self:      Union[str, Any]						,   __A:      Union[str, Any]						,   __A:      Dict						,   __A:      Any						,   __A:      Dict						,   __A:      Optional[int]						,   __A:      Optional[int]						,   __A:      Tuple      )			-> Tuple:
												_A									=       RemBertForQuestionAnswering(config=__A      )
												model.to(__A      )
												model.eval()
												_A									=       model(
												    __A						,   attention_mask=__A						,   token_type_ids=__A						,   start_positions=__A						,   end_positions=__A						,   )
												self.parent.assertEqual(result.start_logits.shape						,   (self.batch_size, self.seq_length)      )
												self.parent.assertEqual(result.end_logits.shape						,   (self.batch_size, self.seq_length)      )

							def 					__A       (     self:      Any						,   __A:      Any						,   __A:      List[str]						,   __A:      List[Any]						,   __A:      Optional[Any]						,   __A:      Any						,   __A:      Union[str, Any]						,   __A:      Optional[int]      )			-> Optional[int]:
												_A									=       self.num_labels
												_A									=       RemBertForSequenceClassification(__A      )
												model.to(__A      )
												model.eval()
												_A									=       model(__A						,   attention_mask=__A						,   token_type_ids=__A						,   labels=__A      )
												self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.num_labels)      )

							def 					__A       (     self:      Union[str, Any]						,   __A:      str						,   __A:      Union[str, Any]						,   __A:      Any						,   __A:      Tuple						,   __A:      Optional[Any]						,   __A:      Optional[int]						,   __A:      Tuple      )			-> Optional[Any]:
												_A									=       self.num_labels
												_A									=       RemBertForTokenClassification(config=__A      )
												model.to(__A      )
												model.eval()
												_A									=       model(__A						,   attention_mask=__A						,   token_type_ids=__A						,   labels=__A      )
												self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.seq_length, self.num_labels)      )

							def 					__A       (     self:      Tuple						,   __A:      str						,   __A:      Any						,   __A:      Union[str, Any]						,   __A:      Optional[Any]						,   __A:      List[Any]						,   __A:      Dict						,   __A:      Tuple      )			-> int:
												_A									=       self.num_choices
												_A									=       RemBertForMultipleChoice(config=__A      )
												model.to(__A      )
												model.eval()
												_A									=       input_ids.unsqueeze(1      ).expand(-1						,   self.num_choices						,   -1      ).contiguous()
												_A									=       token_type_ids.unsqueeze(1      ).expand(-1						,   self.num_choices						,   -1      ).contiguous()
												_A									=       input_mask.unsqueeze(1      ).expand(-1						,   self.num_choices						,   -1      ).contiguous()
												_A									=       model(
												    __A						,   attention_mask=__A						,   token_type_ids=__A						,   labels=__A						,   )
												self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.num_choices)      )



							def 					__A       (     self:      int      )			-> Any:
												_A									=       self.prepare_config_and_inputs()
												(
												    (
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,
												)									=       config_and_inputs
												_A									=       {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
												return config, inputs_dict





@require_torch
class   SCREAMING_SNAKE_CASE (       snake_case     ,		snake_case     ,		unittest.TestCase    ):





							"""simple docstring"""

							A_       =    (
							    (
							        RemBertModel,
							        RemBertForMaskedLM,
							        RemBertForCausalLM,
							        RemBertForMultipleChoice,
							        RemBertForQuestionAnswering,
							        RemBertForSequenceClassification,
							        RemBertForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							A_       =    (RemBertForCausalLM,) if is_torch_available() else ()
							A_       =    (
							    {
							        "feature-extraction": RemBertModel,
							        "fill-mask": RemBertForMaskedLM,
							        "question-answering": RemBertForQuestionAnswering,
							        "text-classification": RemBertForSequenceClassification,
							        "text-generation": RemBertForCausalLM,
							        "token-classification": RemBertForTokenClassification,
							        "zero-shot": RemBertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)

							def 					__A       (     self:      Optional[Any]      )			-> int:
												_A									=       RemBertModelTester(self      )
												_A									=       ConfigTester(self						,   config_class=__A						,   hidden_size=37      )

							def 					__A       (     self:      Optional[int]      )			-> Any:
												self.config_tester.run_common_tests()

							def 					__A       (     self:      Union[str, Any]      )			-> str:
												_A									=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*__A      )

							def 					__A       (     self:      Any      )			-> Optional[int]:
												_A									=       self.model_tester.prepare_config_and_inputs()
												for type in ["absolute", "relative_key", "relative_key_query"]:
																	_A									=       type
																	self.model_tester.create_and_check_model(*__A      )

							def 					__A       (     self:      List[Any]      )			-> Any:
												_A									=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*__A      )

							def 					__A       (     self:      List[str]      )			-> Tuple:
												_A									=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*__A      )

							def 					__A       (     self:      Optional[Any]      )			-> Optional[int]:
												_A									=       self.model_tester.prepare_config_and_inputs_for_decoder()
												self.model_tester.create_and_check_decoder_model_past_large_inputs(*__A      )

							def 					__A       (     self:      Union[str, Any]      )			-> Optional[Any]:
												_A									=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*__A      )

							def 					__A       (     self:      List[Any]      )			-> Dict:
												_A									=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*__A      )

							def 					__A       (     self:      List[Any]      )			-> Dict:
												_A									=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*__A      )

							def 					__A       (     self:      Optional[int]      )			-> Dict:
												_A									=       self.model_tester.prepare_config_and_inputs_for_decoder()
												self.model_tester.create_and_check_model_as_decoder(*__A      )

							def 					__A       (     self:      Tuple      )			-> str:
												# This regression test was failing with PyTorch < 1.3
												(
												    (
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,
												)									=       self.model_tester.prepare_config_and_inputs_for_decoder()

												_A									=       None

												self.model_tester.create_and_check_model_as_decoder(
												    __A						,   __A						,   __A						,   __A						,   __A						,   __A						,   __A						,   __A						,   __A						,   )



							@slow
							def 					__A       (     self:      Union[str, Any]      )			-> List[Any]:
												for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	_A									=       RemBertModel.from_pretrained(__A      )
																	self.assertIsNotNone(__A      )





@require_torch
class   SCREAMING_SNAKE_CASE (       unittest.TestCase    ):





							"""simple docstring"""

							@slow
							def 					__A       (     self:      str      )			-> str:
												# Test exact values at the last hidden layer
												_A									=       RemBertModel.from_pretrained('''google/rembert'''      )
												_A									=       torch.tensor([[3_12, 5_64_98, 3_13, 21_25, 3_13]]      )
												_A									=       torch.tensor([[0, 0, 0, 1, 1]]      )
												with torch.no_grad():
																	_A									=       model(__A						,   token_type_ids=__A						,   output_hidden_states=__A      )

												_A									=       11_52

												_A									=       torch.Size((1, 5, hidden_size)      )
												self.assertEqual(output['''last_hidden_state'''].shape						,   __A      )

												_A									=       torch.tensor(
												    [
												        [
												            [0.0_754, -0.2_022, 0.1_904],
												            [-0.3_354, -0.3_692, -0.4_791],
												            [-0.2_314, -0.6_729, -0.0_749],
												            [-0.0_396, -0.3_105, -0.4_234],
												            [-0.1_571, -0.0_525, 0.5_353],
												        ]
												    ]      )

												# Running on the original tf implementation gives slightly different results here.
												# Not clear why this variations is present
												# TODO: Find reason for discrepancy
												# expected_original_implementation = [[
												#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
												#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
												#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
												#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
												#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
												# ]]

												self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3]						,   __A						,   atol=1e-4      )      )







"""simple docstring"""


import warnings

from ...utils import logging
from .image_processing_mobilenet_va import MobileNetVaImageProcessor


a_											=			logging.get_logger(__name__)
class snake_case					(     _UpperCamelCase):







		def __init__(    self					:				str			,      *a__					:				Union[str, Any]			,      **a__					:				Optional[Any]						)				->  None:





						'''simple docstring'''







						warnings.warn(
						    "The class MobileNetV2FeatureExtractor is deprecated and will be removed in version 5 of Transformers."
						    " Please use MobileNetV2ImageProcessor instead."			,      a__			,      )
						super().__init__(*a__			,      **a__						)