






import unittest

from transformers import DebertaVaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    DebertaVaForMaskedLM,
			    DebertaVaForMultipleChoice,
			    DebertaVaForQuestionAnswering,
			    DebertaVaForSequenceClassification,
			    DebertaVaForTokenClassification,
			    DebertaVaModel,
			)
			from transformers.models.deberta_va.modeling_deberta_va import DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST
class  lowerCAmelCase_       (				snake_case__					):




							def __init__(   self     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   List[str]=13 ,							SCREAMING_SNAKE_CASE_     :   Tuple=7 ,							SCREAMING_SNAKE_CASE_     :   List[Any]=True ,							SCREAMING_SNAKE_CASE_     :   List[Any]=True ,							SCREAMING_SNAKE_CASE_     :   int=True ,							SCREAMING_SNAKE_CASE_     :   Tuple=True ,							SCREAMING_SNAKE_CASE_     :   List[Any]=99 ,							SCREAMING_SNAKE_CASE_     :   Any=32 ,							SCREAMING_SNAKE_CASE_     :   Dict=5 ,							SCREAMING_SNAKE_CASE_     :   Tuple=4 ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=37 ,							SCREAMING_SNAKE_CASE_     :   List[Any]="gelu" ,							SCREAMING_SNAKE_CASE_     :   List[str]=0.1 ,							SCREAMING_SNAKE_CASE_     :   Dict=0.1 ,							SCREAMING_SNAKE_CASE_     :   str=512 ,							SCREAMING_SNAKE_CASE_     :   Tuple=16 ,							SCREAMING_SNAKE_CASE_     :   int=2 ,							SCREAMING_SNAKE_CASE_     :   List[str]=0.02 ,							SCREAMING_SNAKE_CASE_     :   str=False ,							SCREAMING_SNAKE_CASE_     :   List[Any]=True ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]="None" ,							SCREAMING_SNAKE_CASE_     :   Optional[int]=3 ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=4 ,							SCREAMING_SNAKE_CASE_     :   int=None ,							):
											lowerCAmelCase__      			=						parent
											lowerCAmelCase__      			=						batch_size
											lowerCAmelCase__      			=						seq_length
											lowerCAmelCase__      			=						is_training
											lowerCAmelCase__      			=						use_input_mask
											lowerCAmelCase__      			=						use_token_type_ids
											lowerCAmelCase__      			=						use_labels
											lowerCAmelCase__      			=						vocab_size
											lowerCAmelCase__      			=						hidden_size
											lowerCAmelCase__      			=						num_hidden_layers
											lowerCAmelCase__      			=						num_attention_heads
											lowerCAmelCase__      			=						intermediate_size
											lowerCAmelCase__      			=						hidden_act
											lowerCAmelCase__      			=						hidden_dropout_prob
											lowerCAmelCase__      			=						attention_probs_dropout_prob
											lowerCAmelCase__      			=						max_position_embeddings
											lowerCAmelCase__      			=						type_vocab_size
											lowerCAmelCase__      			=						type_sequence_label_size
											lowerCAmelCase__      			=						initializer_range
											lowerCAmelCase__      			=						num_labels
											lowerCAmelCase__      			=						num_choices
											lowerCAmelCase__      			=						relative_attention
											lowerCAmelCase__      			=						position_biased_input
											lowerCAmelCase__      			=						pos_att_type
											lowerCAmelCase__      			=						scope




							def 						__snake_case					(   self     :   Optional[Any]	):
											lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.vocab_size	)

											lowerCAmelCase__      			=						None
											if self.use_input_mask:
															lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							vocab_size=2	)

											lowerCAmelCase__      			=						None
											if self.use_token_type_ids:
															lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.type_vocab_size	)

											lowerCAmelCase__      			=						None
											lowerCAmelCase__      			=						None
											lowerCAmelCase__      			=						None
											if self.use_labels:
															lowerCAmelCase__      			=						ids_tensor([self.batch_size] ,							self.type_sequence_label_size	)
															lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.num_labels	)
															lowerCAmelCase__      			=						ids_tensor([self.batch_size] ,							self.num_choices	)

											lowerCAmelCase__      			=						self.get_config()

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




							def 						__snake_case					(   self     :   Tuple	):
											return DebertaVaConfig(
											    vocab_size=self.vocab_size ,							hidden_size=self.hidden_size ,							num_hidden_layers=self.num_hidden_layers ,							num_attention_heads=self.num_attention_heads ,							intermediate_size=self.intermediate_size ,							hidden_act=self.hidden_act ,							hidden_dropout_prob=self.hidden_dropout_prob ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob ,							max_position_embeddings=self.max_position_embeddings ,							type_vocab_size=self.type_vocab_size ,							initializer_range=self.initializer_range ,							relative_attention=self.relative_attention ,							position_biased_input=self.position_biased_input ,							pos_att_type=self.pos_att_type ,							)




							def 						__snake_case					(   self     :   Any ,							SCREAMING_SNAKE_CASE_     :   str	):
											self.parent.assertListEqual(list(result.loss.size()	) ,							[]	)




							def 						__snake_case					(   self     :   str ,							SCREAMING_SNAKE_CASE_     :   Dict ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   str	):
											lowerCAmelCase__      			=						DebertaVaModel(config=SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							token_type_ids=SCREAMING_SNAKE_CASE_	)[0]
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							token_type_ids=SCREAMING_SNAKE_CASE_	)[0]
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)[0]

											self.parent.assertListEqual(list(sequence_output.size()	) ,							[self.batch_size, self.seq_length, self.hidden_size]	)




							def 						__snake_case					(   self     :   int ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Dict ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Any	):
											lowerCAmelCase__      			=						DebertaVaForMaskedLM(config=SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							token_type_ids=SCREAMING_SNAKE_CASE_ ,							labels=SCREAMING_SNAKE_CASE_	)

											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.seq_length, self.vocab_size)	)




							def 						__snake_case					(   self     :   int ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Any	):
											lowerCAmelCase__      			=						self.num_labels
											lowerCAmelCase__      			=						DebertaVaForSequenceClassification(SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							token_type_ids=SCREAMING_SNAKE_CASE_ ,							labels=SCREAMING_SNAKE_CASE_	)
											self.parent.assertListEqual(list(result.logits.size()	) ,							[self.batch_size, self.num_labels]	)
											self.check_loss_output(SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   List[str]	):
											lowerCAmelCase__      			=						self.num_labels
											lowerCAmelCase__      			=						DebertaVaForTokenClassification(config=SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							token_type_ids=SCREAMING_SNAKE_CASE_ ,							labels=SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.seq_length, self.num_labels)	)




							def 						__snake_case					(   self     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   Dict ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   List[Any]	):
											lowerCAmelCase__      			=						DebertaVaForQuestionAnswering(config=SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							token_type_ids=SCREAMING_SNAKE_CASE_ ,							start_positions=SCREAMING_SNAKE_CASE_ ,							end_positions=SCREAMING_SNAKE_CASE_ ,							)
											self.parent.assertEqual(result.start_logits.shape ,							(self.batch_size, self.seq_length)	)
											self.parent.assertEqual(result.end_logits.shape ,							(self.batch_size, self.seq_length)	)




							def 						__snake_case					(   self     :   str ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   Dict ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Dict ,							SCREAMING_SNAKE_CASE_     :   List[Any]	):
											lowerCAmelCase__      			=						DebertaVaForMultipleChoice(config=SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						input_ids.unsqueeze(1	).expand(-1 ,							self.num_choices ,							-1	).contiguous()
											lowerCAmelCase__      			=						token_type_ids.unsqueeze(1	).expand(-1 ,							self.num_choices ,							-1	).contiguous()
											lowerCAmelCase__      			=						input_mask.unsqueeze(1	).expand(-1 ,							self.num_choices ,							-1	).contiguous()
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							token_type_ids=SCREAMING_SNAKE_CASE_ ,							labels=SCREAMING_SNAKE_CASE_ ,							)
											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.num_choices)	)




							def 						__snake_case					(   self     :   Dict	):
											lowerCAmelCase__      			=						self.prepare_config_and_inputs()
											(
											    (
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		
											)      			=						config_and_inputs
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
											return config, inputs_dict



@require_torch
class  lowerCAmelCase_       (				snake_case__     ,    snake_case__     ,    unittest.TestCase					):
							UpperCamelCase_		:Dict		      =    (
							    (
							        DebertaVaModel,
							        DebertaVaForMaskedLM,
							        DebertaVaForSequenceClassification,
							        DebertaVaForTokenClassification,
							        DebertaVaForQuestionAnswering,
							        DebertaVaForMultipleChoice,
							    )
							    if is_torch_available()
							    else ()
							)
							UpperCamelCase_		:Tuple		      =    (
							    {
							        'feature-extraction': DebertaVaModel,
							        'fill-mask': DebertaVaForMaskedLM,
							        'question-answering': DebertaVaForQuestionAnswering,
							        'text-classification': DebertaVaForSequenceClassification,
							        'token-classification': DebertaVaForTokenClassification,
							        'zero-shot': DebertaVaForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)

							UpperCamelCase_		:str		      =    True
							UpperCamelCase_		:List[Any]		      =    False
							UpperCamelCase_		:Tuple		      =    False
							UpperCamelCase_		:Any		      =    False
							UpperCamelCase_		:Optional[Any]		      =    False




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__      			=						DebertaVaModelTester(self	)
											lowerCAmelCase__      			=						ConfigTester(self ,							config_class=SCREAMING_SNAKE_CASE_ ,							hidden_size=37	)




							def 						__snake_case					(   self     :   List[str]	):
											self.config_tester.run_common_tests()




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_model(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_sequence_classification(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Optional[int]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_masked_lm(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Dict	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_question_answering(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_token_classification(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Optional[int]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_multiple_choice(*SCREAMING_SNAKE_CASE_	)




							@slow
							def 						__snake_case					(   self     :   Optional[Any]	):
											for model_name in DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															lowerCAmelCase__      			=						DebertaVaModel.from_pretrained(SCREAMING_SNAKE_CASE_	)
															self.assertIsNotNone(SCREAMING_SNAKE_CASE_	)



@require_torch
@require_sentencepiece
@require_tokenizers
class  lowerCAmelCase_       (				unittest.TestCase					):




							@unittest.skip(reason='''Model not available yet'''	)
							def 						__snake_case					(   self     :   int	):
											pass




							@slow
							def 						__snake_case					(   self     :   List[str]	):
											lowerCAmelCase__      			=						DebertaVaModel.from_pretrained('''microsoft/deberta-v2-xlarge'''	)

											lowerCAmelCase__      			=						torch.tensor([[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]]	)
											lowerCAmelCase__      			=						torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
											with torch.no_grad():
															lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_	)[0]
											# compare the actual values for a slice.
											lowerCAmelCase__      			=						torch.tensor(
											    [[[0.2_356, 0.1_948, 0.0_369], [-0.1_063, 0.3_586, -0.5_152], [-0.6_399, -0.0_259, -0.2_525]]]	)
											self.assertTrue(torch.allclose(output[:, 1:4, 1:4] ,							SCREAMING_SNAKE_CASE_ ,							atol=1e-4	) ,							f'{output[:, 1:4, 1:4]}'	)













from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


_UpperCAmelCase			:		Tuple   				=    logging.get_logger(__name__)


_UpperCAmelCase			:		int   				=    {
    "Salesforce/codegen-350M-nl": "https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json",
    "Salesforce/codegen-350M-multi": "https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json",
    "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json",
    "Salesforce/codegen-2B-nl": "https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json",
    "Salesforce/codegen-2B-multi": "https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json",
    "Salesforce/codegen-2B-mono": "https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json",
    "Salesforce/codegen-6B-nl": "https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json",
    "Salesforce/codegen-6B-multi": "https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json",
    "Salesforce/codegen-6B-mono": "https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json",
    "Salesforce/codegen-16B-nl": "https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json",
    "Salesforce/codegen-16B-multi": "https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json",
    "Salesforce/codegen-16B-mono": "https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json",
}
class  lowerCAmelCase_       (				snake_case__					):
							UpperCamelCase_		:int		      =    'codegen'
							UpperCamelCase_		:int		      =    {
							    'max_position_embeddings': 'n_positions',
							    'hidden_size': 'n_embd',
							    'num_attention_heads': 'n_head',
							    'num_hidden_layers': 'n_layer',
							}




							def __init__(   self     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   str=50_400 ,							SCREAMING_SNAKE_CASE_     :   str=2_048 ,							SCREAMING_SNAKE_CASE_     :   int=2_048 ,							SCREAMING_SNAKE_CASE_     :   Any=4_096 ,							SCREAMING_SNAKE_CASE_     :   List[Any]=28 ,							SCREAMING_SNAKE_CASE_     :   str=16 ,							SCREAMING_SNAKE_CASE_     :   str=64 ,							SCREAMING_SNAKE_CASE_     :   Tuple=None ,							SCREAMING_SNAKE_CASE_     :   Dict="gelu_new" ,							SCREAMING_SNAKE_CASE_     :   str=0.0 ,							SCREAMING_SNAKE_CASE_     :   Any=0.0 ,							SCREAMING_SNAKE_CASE_     :   Any=0.0 ,							SCREAMING_SNAKE_CASE_     :   Any=1e-5 ,							SCREAMING_SNAKE_CASE_     :   Any=0.02 ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=True ,							SCREAMING_SNAKE_CASE_     :   Optional[int]=50_256 ,							SCREAMING_SNAKE_CASE_     :   Any=50_256 ,							SCREAMING_SNAKE_CASE_     :   List[str]=False ,							**SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							):
											lowerCAmelCase__      			=						vocab_size
											lowerCAmelCase__      			=						n_ctx
											lowerCAmelCase__      			=						n_positions
											lowerCAmelCase__      			=						n_embd
											lowerCAmelCase__      			=						n_layer
											lowerCAmelCase__      			=						n_head
											lowerCAmelCase__      			=						n_inner
											lowerCAmelCase__      			=						rotary_dim
											lowerCAmelCase__      			=						activation_function
											lowerCAmelCase__      			=						resid_pdrop
											lowerCAmelCase__      			=						embd_pdrop
											lowerCAmelCase__      			=						attn_pdrop
											lowerCAmelCase__      			=						layer_norm_epsilon
											lowerCAmelCase__      			=						initializer_range
											lowerCAmelCase__      			=						use_cache

											lowerCAmelCase__      			=						bos_token_id
											lowerCAmelCase__      			=						eos_token_id

											super().__init__(
											    bos_token_id=SCREAMING_SNAKE_CASE_ ,							eos_token_id=SCREAMING_SNAKE_CASE_ ,							tie_word_embeddings=SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)



class  lowerCAmelCase_       (				snake_case__					):




							def __init__(   self     :   str ,							SCREAMING_SNAKE_CASE_     :   PretrainedConfig ,							SCREAMING_SNAKE_CASE_     :   str = "default" ,							SCREAMING_SNAKE_CASE_     :   List[PatchingSpec] = None ,							SCREAMING_SNAKE_CASE_     :   bool = False ,							):
											super().__init__(SCREAMING_SNAKE_CASE_ ,							task=SCREAMING_SNAKE_CASE_ ,							patching_specs=SCREAMING_SNAKE_CASE_ ,							use_past=SCREAMING_SNAKE_CASE_	)
											if not getattr(self._config ,							'''pad_token_id''' ,							SCREAMING_SNAKE_CASE_	):
															# TODO: how to do that better?
															lowerCAmelCase__      			=						0




							@property
							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__      			=						OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}	)
											if self.use_past:
															self.fill_with_past_key_values_(SCREAMING_SNAKE_CASE_ ,							direction='''inputs'''	)
															lowerCAmelCase__      			=						{0: '''batch''', 1: '''past_sequence + sequence'''}
											else:
															lowerCAmelCase__      			=						{0: '''batch''', 1: '''sequence'''}

											return common_inputs




							@property
							def 						__snake_case					(   self     :   Dict	):
											return self._config.n_layer




							@property
							def 						__snake_case					(   self     :   Union[str, Any]	):
											return self._config.n_head




							def 						__snake_case					(   self     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   PreTrainedTokenizer ,							SCREAMING_SNAKE_CASE_     :   int = -1 ,							SCREAMING_SNAKE_CASE_     :   int = -1 ,							SCREAMING_SNAKE_CASE_     :   bool = False ,							SCREAMING_SNAKE_CASE_     :   Optional[TensorType] = None ,							):
											lowerCAmelCase__      			=						super(SCREAMING_SNAKE_CASE_ ,							self	).generate_dummy_inputs(
											    SCREAMING_SNAKE_CASE_ ,							batch_size=SCREAMING_SNAKE_CASE_ ,							seq_length=SCREAMING_SNAKE_CASE_ ,							is_pair=SCREAMING_SNAKE_CASE_ ,							framework=SCREAMING_SNAKE_CASE_	)

											# We need to order the input in the way they appears in the forward()
											lowerCAmelCase__      			=						OrderedDict({'''input_ids''': common_inputs['''input_ids''']}	)

											# Need to add the past_keys
											if self.use_past:
															if not is_torch_available():
																			raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
															else:
																			import torch

																			lowerCAmelCase__			,		lowerCAmelCase__      			=						common_inputs['''input_ids'''].shape
																			# Not using the same length for past_key_values
																			lowerCAmelCase__      			=						seqlen + 2
																			lowerCAmelCase__      			=						(
																			    batch,
																			    self.num_attention_heads,
																			    past_key_values_length,
																			    self._config.hidden_size // self.num_attention_heads,
																			)
																			lowerCAmelCase__      			=						[
																			    (torch.zeros(SCREAMING_SNAKE_CASE_	), torch.zeros(SCREAMING_SNAKE_CASE_	)) for _ in range(self.num_layers	)
																			]

											lowerCAmelCase__      			=						common_inputs['''attention_mask''']
											if self.use_past:
															lowerCAmelCase__      			=						ordered_inputs['''attention_mask'''].dtype
															lowerCAmelCase__      			=						torch.cat(
															    [ordered_inputs['''attention_mask'''], torch.ones(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							dtype=SCREAMING_SNAKE_CASE_	)] ,							dim=1	)

											return ordered_inputs




							@property
							def 						__snake_case					(   self     :   Optional[int]	):
											return 13





