

import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    FunnelBaseModel,
					    FunnelForMaskedLM,
					    FunnelForMultipleChoice,
					    FunnelForPreTraining,
					    FunnelForQuestionAnswering,
					    FunnelForSequenceClassification,
					    FunnelForTokenClassification,
					    FunnelModel,
					)

class  a     :


						def __init__(				self  ,    A_  ,    A_=13  ,    A_=7  ,    A_=True  ,    A_=True  ,    A_=True  ,    A_=True  ,    A_=99  ,    A_=[1, 1, 2]  ,    A_=1  ,    A_=32  ,    A_=4  ,    A_=8  ,    A_=37  ,    A_="gelu_new"  ,    A_=0.1  ,    A_=0.1  ,    A_=0.0  ,    A_=512  ,    A_=3  ,    A_=0.02  ,    A_=3  ,    A_=4  ,    A_=None  ,    A_=False  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						List[str]       						=			parent
										_UpperCAmelCase				:						Tuple       						=			batch_size
										_UpperCAmelCase				:						Any       						=			seq_length
										_UpperCAmelCase				:						Optional[int]       						=			is_training
										_UpperCAmelCase				:						Any       						=			use_input_mask
										_UpperCAmelCase				:						str       						=			use_token_type_ids
										_UpperCAmelCase				:						List[str]       						=			use_labels
										_UpperCAmelCase				:						List[str]       						=			vocab_size
										_UpperCAmelCase				:						str       						=			block_sizes
										_UpperCAmelCase				:						int       						=			num_decoder_layers
										_UpperCAmelCase				:						Optional[Any]       						=			d_model
										_UpperCAmelCase				:						Union[str, Any]       						=			n_head
										_UpperCAmelCase				:						Any       						=			d_head
										_UpperCAmelCase				:						List[Any]       						=			d_inner
										_UpperCAmelCase				:						Union[str, Any]       						=			hidden_act
										_UpperCAmelCase				:						Optional[int]       						=			hidden_dropout
										_UpperCAmelCase				:						Dict       						=			attention_dropout
										_UpperCAmelCase				:						Any       						=			activation_dropout
										_UpperCAmelCase				:						Tuple       						=			max_position_embeddings
										_UpperCAmelCase				:						int       						=			type_vocab_size
										_UpperCAmelCase				:						Dict       						=			2
										_UpperCAmelCase				:						Union[str, Any]       						=			num_labels
										_UpperCAmelCase				:						str       						=			num_choices
										_UpperCAmelCase				:						Optional[int]       						=			scope
										_UpperCAmelCase				:						str       						=			initializer_std

										# Used in the tests to check the size of the first attention layer
										_UpperCAmelCase				:						Dict       						=			n_head
										# Used in the tests to check the size of the first hidden state
										_UpperCAmelCase				:						Optional[Any]       						=			self.d_model
										# Used in the tests to check the number of output hidden states/attentions
										_UpperCAmelCase				:						Tuple       						=			sum(self.block_sizes		) + (0 if base else self.num_decoder_layers)
										# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
										# the last hidden state of the first block (which is the first hidden state of the decoder).
										if not base:
														_UpperCAmelCase				:						List[str]       						=			self.num_hidden_layers + 2


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Any       						=			ids_tensor([self.batch_size, self.seq_length]  ,    self.vocab_size		)

										_UpperCAmelCase				:						Union[str, Any]       						=			None
										if self.use_input_mask:
														_UpperCAmelCase				:						Tuple       						=			ids_tensor([self.batch_size, self.seq_length]  ,    vocab_size=2		)

										_UpperCAmelCase				:						Optional[int]       						=			None
										if self.use_token_type_ids:
														_UpperCAmelCase				:						Optional[Any]       						=			ids_tensor([self.batch_size, self.seq_length]  ,    self.type_vocab_size		)

										_UpperCAmelCase				:						Tuple       						=			None
										_UpperCAmelCase				:						str       						=			None
										_UpperCAmelCase				:						Tuple       						=			None
										if self.use_labels:
														_UpperCAmelCase				:						Dict       						=			ids_tensor([self.batch_size]  ,    self.type_sequence_label_size		)
														_UpperCAmelCase				:						str       						=			ids_tensor([self.batch_size, self.seq_length]  ,    self.num_labels		)
														_UpperCAmelCase				:						Union[str, Any]       						=			ids_tensor([self.batch_size]  ,    self.num_choices		)
														_UpperCAmelCase				:						List[Any]       						=			ids_tensor([self.batch_size, self.seq_length]  ,    1		)

										_UpperCAmelCase				:						Tuple       						=			self.get_config()

										return (
										    config,
										    input_ids,
										    token_type_ids,
										    input_mask,
										    sequence_labels,
										    token_labels,
										    choice_labels,
										    fake_token_labels,
										)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										return FunnelConfig(
										    vocab_size=self.vocab_size  ,    block_sizes=self.block_sizes  ,    num_decoder_layers=self.num_decoder_layers  ,    d_model=self.d_model  ,    n_head=self.n_head  ,    d_head=self.d_head  ,    d_inner=self.d_inner  ,    hidden_act=self.hidden_act  ,    hidden_dropout=self.hidden_dropout  ,    attention_dropout=self.attention_dropout  ,    activation_dropout=self.activation_dropout  ,    max_position_embeddings=self.max_position_embeddings  ,    type_vocab_size=self.type_vocab_size  ,    initializer_std=self.initializer_std  ,    )


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						str       						=			FunnelModel(config=lowerCamelCase__		)
										model.to(lowerCamelCase__		)
										model.eval()
										_UpperCAmelCase				:						List[Any]       						=			model(lowerCamelCase__  ,    attention_mask=lowerCamelCase__  ,    token_type_ids=lowerCamelCase__		)
										_UpperCAmelCase				:						str       						=			model(lowerCamelCase__  ,    token_type_ids=lowerCamelCase__		)
										_UpperCAmelCase				:						Tuple       						=			model(lowerCamelCase__		)
										self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.d_model)		)

										_UpperCAmelCase				:						List[Any]       						=			False
										_UpperCAmelCase				:						Any       						=			model(lowerCamelCase__		)
										self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.d_model)		)

										_UpperCAmelCase				:						int       						=			False
										_UpperCAmelCase				:						Optional[Any]       						=			model(lowerCamelCase__		)
										self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.d_model)		)


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						List[Any]       						=			FunnelBaseModel(config=lowerCamelCase__		)
										model.to(lowerCamelCase__		)
										model.eval()
										_UpperCAmelCase				:						str       						=			model(lowerCamelCase__  ,    attention_mask=lowerCamelCase__  ,    token_type_ids=lowerCamelCase__		)
										_UpperCAmelCase				:						str       						=			model(lowerCamelCase__  ,    token_type_ids=lowerCamelCase__		)
										_UpperCAmelCase				:						Dict       						=			model(lowerCamelCase__		)
										self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, 2, self.d_model)		)

										_UpperCAmelCase				:						Optional[int]       						=			False
										_UpperCAmelCase				:						str       						=			model(lowerCamelCase__		)
										self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, 3, self.d_model)		)

										_UpperCAmelCase				:						Any       						=			False
										_UpperCAmelCase				:						List[Any]       						=			model(lowerCamelCase__		)
										self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, 2, self.d_model)		)


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						Optional[int]       						=			self.num_labels
										_UpperCAmelCase				:						List[str]       						=			FunnelForPreTraining(config=lowerCamelCase__		)
										model.to(lowerCamelCase__		)
										model.eval()
										_UpperCAmelCase				:						str       						=			model(lowerCamelCase__  ,    attention_mask=lowerCamelCase__  ,    token_type_ids=lowerCamelCase__  ,    labels=lowerCamelCase__		)
										self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length)		)


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						List[str]       						=			FunnelForMaskedLM(config=lowerCamelCase__		)
										model.to(lowerCamelCase__		)
										model.eval()
										_UpperCAmelCase				:						Dict       						=			model(lowerCamelCase__  ,    attention_mask=lowerCamelCase__  ,    token_type_ids=lowerCamelCase__  ,    labels=lowerCamelCase__		)
										self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.vocab_size)		)


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						Tuple       						=			self.num_labels
										_UpperCAmelCase				:						str       						=			FunnelForSequenceClassification(lowerCamelCase__		)
										model.to(lowerCamelCase__		)
										model.eval()
										_UpperCAmelCase				:						Union[str, Any]       						=			model(lowerCamelCase__  ,    attention_mask=lowerCamelCase__  ,    token_type_ids=lowerCamelCase__  ,    labels=lowerCamelCase__		)
										self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.num_labels)		)


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						str       						=			self.num_choices
										_UpperCAmelCase				:						List[str]       						=			FunnelForMultipleChoice(config=lowerCamelCase__		)
										model.to(lowerCamelCase__		)
										model.eval()
										_UpperCAmelCase				:						Union[str, Any]       						=			input_ids.unsqueeze(1		).expand(-1  ,    self.num_choices  ,    -1		).contiguous()
										_UpperCAmelCase				:						Tuple       						=			token_type_ids.unsqueeze(1		).expand(-1  ,    self.num_choices  ,    -1		).contiguous()
										_UpperCAmelCase				:						Any       						=			input_mask.unsqueeze(1		).expand(-1  ,    self.num_choices  ,    -1		).contiguous()
										_UpperCAmelCase				:						List[Any]       						=			model(
										    lowerCamelCase__  ,    attention_mask=lowerCamelCase__  ,    token_type_ids=lowerCamelCase__  ,    labels=lowerCamelCase__  ,    )
										self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.num_choices)		)


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						Dict       						=			self.num_labels
										_UpperCAmelCase				:						str       						=			FunnelForTokenClassification(config=lowerCamelCase__		)
										model.to(lowerCamelCase__		)
										model.eval()
										_UpperCAmelCase				:						Union[str, Any]       						=			model(lowerCamelCase__  ,    attention_mask=lowerCamelCase__  ,    token_type_ids=lowerCamelCase__  ,    labels=lowerCamelCase__		)
										self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.num_labels)		)


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    A_  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						Optional[Any]       						=			FunnelForQuestionAnswering(config=lowerCamelCase__		)
										model.to(lowerCamelCase__		)
										model.eval()
										_UpperCAmelCase				:						Optional[Any]       						=			model(
										    lowerCamelCase__  ,    attention_mask=lowerCamelCase__  ,    token_type_ids=lowerCamelCase__  ,    start_positions=lowerCamelCase__  ,    end_positions=lowerCamelCase__  ,    )
										self.parent.assertEqual(result.start_logits.shape  ,    (self.batch_size, self.seq_length)		)
										self.parent.assertEqual(result.end_logits.shape  ,    (self.batch_size, self.seq_length)		)





						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Dict       						=			self.prepare_config_and_inputs()
										(
										    (
										    _UpperCAmelCase   
										)						,							(
										    _UpperCAmelCase   
										)						,							(
										    _UpperCAmelCase   
										)						,							(
										    _UpperCAmelCase   
										)						,							(
										    _UpperCAmelCase   
										)						,							(
										    _UpperCAmelCase   
										)						,							(
										    _UpperCAmelCase   
										)						,							(
										    _UpperCAmelCase   
										)						,							   
										)				:						int       						=			config_and_inputs
										_UpperCAmelCase				:						Optional[int]       						=			{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
										return config, inputs_dict







@require_torch
class  a     (		A__							,       A__							,       unittest.TestCase ):
						_lowercase			  =	False
						_lowercase			  =	False
						_lowercase			  =	(
						    (
						        FunnelModel,
						        FunnelForMaskedLM,
						        FunnelForPreTraining,
						        FunnelForQuestionAnswering,
						        FunnelForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						_lowercase			  =	(
						    {
						        'feature-extraction': (FunnelBaseModel, FunnelModel),
						        'fill-mask': FunnelForMaskedLM,
						        'question-answering': FunnelForQuestionAnswering,
						        'text-classification': FunnelForSequenceClassification,
						        'token-classification': FunnelForTokenClassification,
						        'zero-shot': FunnelForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_=False		):







										'''simple docstring'''


										_UpperCAmelCase				:						int       						=			super()._prepare_for_class(lowerCamelCase__  ,    lowerCamelCase__  ,    return_labels=lowerCamelCase__		)

										if return_labels:
														if model_class in get_values(lowerCamelCase__		):
																		_UpperCAmelCase				:						Optional[Any]       						=			torch.zeros(
																		    (self.model_tester.batch_size, self.model_tester.seq_length)  ,    dtype=torch.long  ,    device=lowerCamelCase__		)
										return inputs_dict


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Union[str, Any]       						=			FunnelModelTester(self		)
										_UpperCAmelCase				:						Any       						=			ConfigTester(self  ,    config_class=lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										self.config_tester.run_common_tests()


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						int       						=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[str]       						=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_pretraining(*lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Union[str, Any]       						=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Any       						=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[str]       						=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*lowerCamelCase__		)





						def 				_UpperCAmelCase (				self  ,    A_		):







										'''simple docstring'''


										if hasattr(lowerCamelCase__  ,    "weight"		) and module.weight is not None:
														module.weight.data.fill_(3		)
										if hasattr(lowerCamelCase__  ,    "bias"		) and module.bias is not None:
														module.bias.data.fill_(3		)

										for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
														if hasattr(lowerCamelCase__  ,    lowerCamelCase__		) and getattr(lowerCamelCase__  ,    lowerCamelCase__		) is not None:
																		_UpperCAmelCase				:						Tuple       						=			getattr(lowerCamelCase__  ,    lowerCamelCase__		)
																		weight.data.fill_(3		)







@require_torch
class  a     (		A__							,       unittest.TestCase ):
						_lowercase			  =	False
						_lowercase			  =	False
						_lowercase			  =	(
						    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
						)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Union[str, Any]       						=			FunnelModelTester(self  ,    base=lowerCamelCase__		)
										_UpperCAmelCase				:						Optional[Any]       						=			ConfigTester(self  ,    config_class=lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										self.config_tester.run_common_tests()


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						str       						=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_base_model(*lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[Any]       						=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						str       						=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase						,							_UpperCAmelCase				:						str       						=			self.model_tester.prepare_config_and_inputs_for_common()
										_UpperCAmelCase				:						Dict       						=			True

										for model_class in self.all_model_classes:
														if model_class.__name__ == "FunnelBaseModel":
																		continue
														_UpperCAmelCase				:						int       						=			model_class(lowerCamelCase__		)
														model.to(lowerCamelCase__		)
														model.train()
														_UpperCAmelCase				:						Optional[int]       						=			self._prepare_for_class(lowerCamelCase__  ,    lowerCamelCase__  ,    return_labels=lowerCamelCase__		)
														_UpperCAmelCase				:						List[Any]       						=			model(**lowerCamelCase__		).loss
														loss.backward()





						def 				_UpperCAmelCase (				self  ,    A_		):







										'''simple docstring'''


										if hasattr(lowerCamelCase__  ,    "weight"		) and module.weight is not None:
														module.weight.data.fill_(3		)
										if hasattr(lowerCamelCase__  ,    "bias"		) and module.bias is not None:
														module.bias.data.fill_(3		)

										for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
														if hasattr(lowerCamelCase__  ,    lowerCamelCase__		) and getattr(lowerCamelCase__  ,    lowerCamelCase__		) is not None:
																		_UpperCAmelCase				:						Any       						=			getattr(lowerCamelCase__  ,    lowerCamelCase__		)
																		weight.data.fill_(3		)







@require_torch
@require_sentencepiece
@require_tokenizers
class  a     (		unittest.TestCase ):


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Optional[int]       						=			13
										_UpperCAmelCase				:						List[Any]       						=			7
										_UpperCAmelCase				:						Any       						=			torch.arange(0  ,    batch_size * sequence_length		).long().reshape(lowerCamelCase__  ,    lowerCamelCase__		)
										_UpperCAmelCase				:						List[str]       						=			[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
										_UpperCAmelCase				:						List[Any]       						=			torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]		)

										_UpperCAmelCase				:						Union[str, Any]       						=			FunnelModel.from_pretrained("sgugger/funnel-random-tiny"		)
										_UpperCAmelCase				:						Dict       						=			model(lowerCamelCase__  ,    token_type_ids=lowerCamelCase__		)[0].abs()

										_UpperCAmelCase				:						List[str]       						=			torch.tensor(2344.8352		)
										_UpperCAmelCase				:						Dict       						=			torch.tensor(0.80_52		)
										self.assertTrue(torch.allclose(output.sum()  ,    lowerCamelCase__  ,    atol=1e-4		)		)
										self.assertTrue(torch.allclose(output.mean()  ,    lowerCamelCase__  ,    atol=1e-4		)		)

										_UpperCAmelCase				:						str       						=			torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]		)
										_UpperCAmelCase				:						int       						=			model(lowerCamelCase__  ,    attention_mask=lowerCamelCase__  ,    token_type_ids=lowerCamelCase__		)[0].abs()

										_UpperCAmelCase				:						Optional[int]       						=			torch.tensor(2343.8425		)
										_UpperCAmelCase				:						str       						=			torch.tensor(0.80_49		)
										self.assertTrue(torch.allclose(output.sum()  ,    lowerCamelCase__  ,    atol=1e-4		)		)
										self.assertTrue(torch.allclose(output.mean()  ,    lowerCamelCase__  ,    atol=1e-4		)		)





						@slow
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[Any]       						=			FunnelTokenizer.from_pretrained("huggingface/funnel-small"		)
										_UpperCAmelCase				:						Optional[Any]       						=			FunnelModel.from_pretrained("huggingface/funnel-small"		)
										_UpperCAmelCase				:						int       						=			tokenizer("Hello! I am the Funnel Transformer model."  ,    return_tensors="pt"		)
										_UpperCAmelCase				:						str       						=			model(**lowerCamelCase__		)[0]

										_UpperCAmelCase				:						Dict       						=			torch.tensor(235.7246		)
										_UpperCAmelCase				:						List[str]       						=			torch.tensor(0.02_56		)
										self.assertTrue(torch.allclose(output.sum()  ,    lowerCamelCase__  ,    atol=1e-4		)		)
										self.assertTrue(torch.allclose(output.mean()  ,    lowerCamelCase__  ,    atol=1e-4		)		)







"""simple docstring"""
import unittest

from transformers import ErnieMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    ErnieMForInformationExtraction,
			    ErnieMForMultipleChoice,
			    ErnieMForQuestionAnswering,
			    ErnieMForSequenceClassification,
			    ErnieMForTokenClassification,
			    ErnieMModel,
			)
			from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST


class 							lowerCamelCase_:





					'''simple docstring'''



					def __init__(			self ,				lowerCamelCase__ ,				lowerCamelCase__=1_3 ,				lowerCamelCase__=7 ,				lowerCamelCase__=True ,				lowerCamelCase__=True ,				lowerCamelCase__=True ,				lowerCamelCase__=9_9 ,				lowerCamelCase__=3_2 ,				lowerCamelCase__=5 ,				lowerCamelCase__=4 ,				lowerCamelCase__=3_7 ,				lowerCamelCase__="gelu" ,				lowerCamelCase__=0.1 ,				lowerCamelCase__=0.1 ,				lowerCamelCase__=5_1_2 ,				lowerCamelCase__=1_6 ,				lowerCamelCase__=2 ,				lowerCamelCase__=0.0_2 ,				lowerCamelCase__=3 ,				lowerCamelCase__=4 ,				lowerCamelCase__=None ,				):
									_lowerCamelCase					     = parent
									_lowerCamelCase					     = batch_size
									_lowerCamelCase					     = seq_length
									_lowerCamelCase					     = is_training
									_lowerCamelCase					     = use_input_mask
									_lowerCamelCase					     = use_labels
									_lowerCamelCase					     = vocab_size
									_lowerCamelCase					     = hidden_size
									_lowerCamelCase					     = num_hidden_layers
									_lowerCamelCase					     = num_attention_heads
									_lowerCamelCase					     = intermediate_size
									_lowerCamelCase					     = hidden_act
									_lowerCamelCase					     = hidden_dropout_prob
									_lowerCamelCase					     = attention_probs_dropout_prob
									_lowerCamelCase					     = max_position_embeddings
									_lowerCamelCase					     = type_vocab_size
									_lowerCamelCase					     = type_sequence_label_size
									_lowerCamelCase					     = initializer_range
									_lowerCamelCase					     = num_labels
									_lowerCamelCase					     = num_choices
									_lowerCamelCase					     = scope

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = ids_tensor([self.batch_size, self.seq_length] ,				self.vocab_size      )

									_lowerCamelCase					     = None
									if self.use_input_mask:
													_lowerCamelCase					     = random_attention_mask([self.batch_size, self.seq_length]      )

									_lowerCamelCase					     = None
									_lowerCamelCase					     = None
									_lowerCamelCase					     = None
									if self.use_labels:
													_lowerCamelCase					     = ids_tensor([self.batch_size] ,				self.type_sequence_label_size      )
													_lowerCamelCase					     = ids_tensor([self.batch_size, self.seq_length] ,				self.num_labels      )
													_lowerCamelCase					     = ids_tensor([self.batch_size] ,				self.num_choices      )

									_lowerCamelCase					     = self.get_config()

									return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = ids_tensor([self.batch_size, self.seq_length] ,				self.vocab_size      )

									_lowerCamelCase					     = None
									if self.use_input_mask:
													_lowerCamelCase					     = random_attention_mask([self.batch_size, self.seq_length]      )
									_lowerCamelCase					     = self.get_config()

									return config, input_ids, input_mask

					def 	snake_case__					(			self      ):
									return ErnieMConfig(
									    vocab_size=self.vocab_size ,				hidden_size=self.hidden_size ,				num_hidden_layers=self.num_hidden_layers ,				num_attention_heads=self.num_attention_heads ,				intermediate_size=self.intermediate_size ,				hidden_act=self.hidden_act ,				hidden_dropout_prob=self.hidden_dropout_prob ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob ,				max_position_embeddings=self.max_position_embeddings ,				type_vocab_size=self.type_vocab_size ,				initializer_range=self.initializer_range ,				)

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      ):
									_lowerCamelCase					     = ErnieMModel(config=lowerCamelCase__      )
									model.to(lowerCamelCase__      )
									model.eval()
									_lowerCamelCase					     = model(lowerCamelCase__ ,				return_dict=lowerCamelCase__      )
									self.parent.assertEqual(result.last_hidden_state.shape ,				(self.batch_size, self.seq_length, self.hidden_size)      )

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      ):
									_lowerCamelCase					     = ErnieMForQuestionAnswering(config=lowerCamelCase__      )
									model.to(lowerCamelCase__      )
									model.eval()
									_lowerCamelCase					     = model(
									    lowerCamelCase__ ,				attention_mask=lowerCamelCase__ ,				start_positions=lowerCamelCase__ ,				end_positions=lowerCamelCase__ ,				)
									self.parent.assertEqual(result.start_logits.shape ,				(self.batch_size, self.seq_length)      )
									self.parent.assertEqual(result.end_logits.shape ,				(self.batch_size, self.seq_length)      )

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      ):
									_lowerCamelCase					     = ErnieMForInformationExtraction(config=lowerCamelCase__      )
									model.to(lowerCamelCase__      )
									model.eval()
									_lowerCamelCase					     = torch.ones_like(lowerCamelCase__ ,				dtype=torch.floataa      )
									_lowerCamelCase					     = model(
									    lowerCamelCase__ ,				attention_mask=lowerCamelCase__ ,				start_positions=lowerCamelCase__ ,				end_positions=lowerCamelCase__ ,				)
									self.parent.assertEqual(result.start_logits.shape ,				(self.batch_size, self.seq_length)      )
									self.parent.assertEqual(result.end_logits.shape ,				(self.batch_size, self.seq_length)      )

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      ):
									_lowerCamelCase					     = self.num_labels
									_lowerCamelCase					     = ErnieMForSequenceClassification(lowerCamelCase__      )
									model.to(lowerCamelCase__      )
									model.eval()
									_lowerCamelCase					     = model(lowerCamelCase__ ,				attention_mask=lowerCamelCase__ ,				labels=lowerCamelCase__      )
									self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.num_labels)      )

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      ):
									_lowerCamelCase					     = self.num_labels
									_lowerCamelCase					     = ErnieMForTokenClassification(config=lowerCamelCase__      )
									model.to(lowerCamelCase__      )
									model.eval()
									input_ids.to(lowerCamelCase__      )
									input_mask.to(lowerCamelCase__      )
									token_labels.to(lowerCamelCase__      )

									_lowerCamelCase					     = model(lowerCamelCase__ ,				attention_mask=lowerCamelCase__ ,				labels=lowerCamelCase__      )

									self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.seq_length, self.num_labels)      )

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      ):
									_lowerCamelCase					     = self.num_choices
									_lowerCamelCase					     = ErnieMForMultipleChoice(config=lowerCamelCase__      )
									model.to(lowerCamelCase__      )
									model.eval()
									_lowerCamelCase					     = input_ids.unsqueeze(1      ).expand(-1 ,				self.num_choices ,				-1      ).contiguous()
									_lowerCamelCase					     = input_mask.unsqueeze(1      ).expand(-1 ,				self.num_choices ,				-1      ).contiguous()
									_lowerCamelCase					     = model(
									    lowerCamelCase__ ,				attention_mask=lowerCamelCase__ ,				labels=lowerCamelCase__ ,				)
									self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.num_choices)      )





					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.prepare_config_and_inputs()
									(
									    (
									    _lowerCamelCase
									)     ,   (
									    _lowerCamelCase
									)     ,   (
									    _lowerCamelCase
									)     ,   (
									    _lowerCamelCase
									)     ,   (
									    _lowerCamelCase
									)     ,   (
									    _lowerCamelCase
									)     ,   
									)					     = config_and_inputs
									_lowerCamelCase					     = {'''input_ids''': input_ids, '''attention_mask''': input_mask}
									return config, inputs_dict


@require_torch
class 							lowerCamelCase_(      A__, A__, unittest.TestCase			):





					'''simple docstring'''



					lowercase__      :				Optional[Any]        =       (
					    (
					        ErnieMModel,
					        ErnieMForMultipleChoice,
					        ErnieMForQuestionAnswering,
					        ErnieMForSequenceClassification,
					        ErnieMForTokenClassification,
					    )
					    if is_torch_available()
					    else ()
					)
					lowercase__      :				List[Any]        =       ()
					lowercase__      :				Union[str, Any]        =       (
					    {
					        'feature-extraction': ErnieMModel,
					        'question-answering': ErnieMForQuestionAnswering,
					        'text-classification': ErnieMForSequenceClassification,
					        'token-classification': ErnieMForTokenClassification,
					        'zero-shot': ErnieMForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					lowercase__      :				Optional[int]        =       False

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      ):
									if pipeline_test_casse_name == "QAPipelineTests":
													return True

									return False

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = ErnieMModelTester(self      )
									_lowerCamelCase					     = ConfigTester(self ,				config_class=lowerCamelCase__ ,				hidden_size=3_7      )

					def 	snake_case__					(			self      ):
									self.config_tester.run_common_tests()

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*lowerCamelCase__      )

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
													_lowerCamelCase					     = type
													self.model_tester.create_and_check_model(*lowerCamelCase__      )

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase__      )

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*lowerCamelCase__      )

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase__      )

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_information_extraction(*lowerCamelCase__      )

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*lowerCamelCase__      )





					@slow
					def 	snake_case__					(			self      ):
									for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													_lowerCamelCase					     = ErnieMModel.from_pretrained(lowerCamelCase__      )
													self.assertIsNotNone(lowerCamelCase__      )


@require_torch
class 							lowerCamelCase_(      unittest.TestCase			):





					'''simple docstring'''



					@slow
					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = ErnieMModel.from_pretrained('''susnato/ernie-m-base_pytorch'''      )
									model.eval()
									_lowerCamelCase					     = torch.tensor([[0, 1, 2, 3, 4, 5]]      )
									_lowerCamelCase					     = model(lowerCamelCase__      )[0]

									# TODO Replace vocab size
									_lowerCamelCase					     = 7_6_8

									_lowerCamelCase					     = torch.Size((1, 6, hidden_size)      )
									self.assertEqual(output.shape ,				lowerCamelCase__      )

									_lowerCamelCase					     = torch.tensor(
									    [[[-0.0_0_1_2, 0.1_2_4_5, -0.0_2_1_4], [-0.0_7_4_2, 0.0_2_4_4, -0.0_7_7_1], [-0.0_3_3_3, 0.1_1_6_4, -0.1_5_5_4]]]      )

									self.assertTrue(torch.allclose(output[:, :3, :3] ,				lowerCamelCase__ ,				atol=1e-3      )      )
