




"""simple docstring"""




from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TFFunnelBaseModel,
			    TFFunnelForMaskedLM,
			    TFFunnelForMultipleChoice,
			    TFFunnelForPreTraining,
			    TFFunnelForQuestionAnswering,
			    TFFunnelForSequenceClassification,
			    TFFunnelForTokenClassification,
			    TFFunnelModel,
			)

class   UpperCAmelCase_    :
		def __init__(						self				,						UpperCamelCase_				,						UpperCamelCase_=13				,						UpperCamelCase_=7				,						UpperCamelCase_=True				,						UpperCamelCase_=True				,						UpperCamelCase_=True				,						UpperCamelCase_=True				,						UpperCamelCase_=99				,						UpperCamelCase_=[1, 1, 2]				,						UpperCamelCase_=1				,						UpperCamelCase_=32				,						UpperCamelCase_=4				,						UpperCamelCase_=8				,						UpperCamelCase_=37				,						UpperCamelCase_="gelu_new"				,						UpperCamelCase_=0.1				,						UpperCamelCase_=0.1				,						UpperCamelCase_=0.0				,						UpperCamelCase_=5_12				,						UpperCamelCase_=3				,						UpperCamelCase_=0.0_2				,						UpperCamelCase_=3				,						UpperCamelCase_=4				,						UpperCamelCase_=None				,						UpperCamelCase_=False				,						)						->       Tuple:
					__lowercase			:  int								=       parent
					__lowercase			:  Optional[int]								=       batch_size
					__lowercase			:  Dict								=       seq_length
					__lowercase			:  int								=       is_training
					__lowercase			:  int								=       use_input_mask
					__lowercase			:  str								=       use_token_type_ids
					__lowercase			:  List[Any]								=       use_labels
					__lowercase			:  Union[str, Any]								=       vocab_size
					__lowercase			:  Any								=       block_sizes
					__lowercase			:  Optional[Any]								=       num_decoder_layers
					__lowercase			:  Union[str, Any]								=       d_model
					__lowercase			:  str								=       n_head
					__lowercase			:  List[str]								=       d_head
					__lowercase			:  str								=       d_inner
					__lowercase			:  Tuple								=       hidden_act
					__lowercase			:  str								=       hidden_dropout
					__lowercase			:  Optional[Any]								=       attention_dropout
					__lowercase			:  Dict								=       activation_dropout
					__lowercase			:  Dict								=       max_position_embeddings
					__lowercase			:  int								=       type_vocab_size
					__lowercase			:  List[Any]								=       2
					__lowercase			:  Optional[int]								=       num_labels
					__lowercase			:  int								=       num_choices
					__lowercase			:  List[Any]								=       scope
					__lowercase			:  Optional[Any]								=       initializer_std

					# Used in the tests to check the size of the first attention layer
					__lowercase			:  List[str]								=       n_head
					# Used in the tests to check the size of the first hidden state
					__lowercase			:  Optional[int]								=       self.d_model
					# Used in the tests to check the number of output hidden states/attentions
					__lowercase			:  Any								=       sum(self.block_sizes       ) + (0 if base else self.num_decoder_layers)
					# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
					# the last hidden state of the first block (which is the first hidden state of the decoder).
					if not base:
								__lowercase			:  Dict								=       self.num_hidden_layers + 2
		def 	_lowerCamelCase							(						self       )						->       Optional[Any]:
					__lowercase			:  Optional[int]								=       ids_tensor([self.batch_size, self.seq_length]				,						self.vocab_size       )

					__lowercase			:  Optional[int]								=       None
					if self.use_input_mask:
								__lowercase			:  Union[str, Any]								=       random_attention_mask([self.batch_size, self.seq_length]       )

					__lowercase			:  Optional[int]								=       None
					if self.use_token_type_ids:
								__lowercase			:  Optional[Any]								=       ids_tensor([self.batch_size, self.seq_length]				,						self.type_vocab_size       )

					__lowercase			:  List[Any]								=       None
					__lowercase			:  str								=       None
					__lowercase			:  Any								=       None
					if self.use_labels:
								__lowercase			:  Optional[int]								=       ids_tensor([self.batch_size]				,						self.type_sequence_label_size       )
								__lowercase			:  Any								=       ids_tensor([self.batch_size, self.seq_length]				,						self.num_labels       )
								__lowercase			:  List[Any]								=       ids_tensor([self.batch_size]				,						self.num_choices       )

					__lowercase			:  Optional[int]								=       FunnelConfig(
					    vocab_size=self.vocab_size				,						block_sizes=self.block_sizes				,						num_decoder_layers=self.num_decoder_layers				,						d_model=self.d_model				,						n_head=self.n_head				,						d_head=self.d_head				,						d_inner=self.d_inner				,						hidden_act=self.hidden_act				,						hidden_dropout=self.hidden_dropout				,						attention_dropout=self.attention_dropout				,						activation_dropout=self.activation_dropout				,						max_position_embeddings=self.max_position_embeddings				,						type_vocab_size=self.type_vocab_size				,						initializer_std=self.initializer_std				,						)

					return (
					    config,
					    input_ids,
					    token_type_ids,
					    input_mask,
					    sequence_labels,
					    token_labels,
					    choice_labels,
					)
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						)						->       Dict:
					__lowercase			:  Any								=       TFFunnelModel(config=UpperCamelCase_       )
					__lowercase			:  Tuple								=       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
					__lowercase			:  str								=       model(UpperCamelCase_       )

					__lowercase			:  int								=       [input_ids, input_mask]
					__lowercase			:  Optional[int]								=       model(UpperCamelCase_       )

					__lowercase			:  Any								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.d_model)       )

					__lowercase			:  int								=       False
					__lowercase			:  List[str]								=       TFFunnelModel(config=UpperCamelCase_       )
					__lowercase			:  Optional[int]								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.d_model)       )

					__lowercase			:  Tuple								=       False
					__lowercase			:  Any								=       TFFunnelModel(config=UpperCamelCase_       )
					__lowercase			:  str								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.d_model)       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						)						->       Union[str, Any]:
					__lowercase			:  Optional[int]								=       TFFunnelBaseModel(config=UpperCamelCase_       )
					__lowercase			:  Tuple								=       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
					__lowercase			:  Any								=       model(UpperCamelCase_       )

					__lowercase			:  List[Any]								=       [input_ids, input_mask]
					__lowercase			:  Any								=       model(UpperCamelCase_       )

					__lowercase			:  Any								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, 2, self.d_model)       )

					__lowercase			:  Any								=       False
					__lowercase			:  Optional[Any]								=       TFFunnelBaseModel(config=UpperCamelCase_       )
					__lowercase			:  Optional[int]								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, 3, self.d_model)       )

					__lowercase			:  Any								=       False
					__lowercase			:  str								=       TFFunnelBaseModel(config=UpperCamelCase_       )
					__lowercase			:  Tuple								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, 2, self.d_model)       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						)						->       List[Any]:
					__lowercase			:  List[str]								=       TFFunnelForPreTraining(config=UpperCamelCase_       )
					__lowercase			:  Tuple								=       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
					__lowercase			:  List[Any]								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length)       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						)						->       Any:
					__lowercase			:  str								=       TFFunnelForMaskedLM(config=UpperCamelCase_       )
					__lowercase			:  str								=       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
					__lowercase			:  Any								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length, self.vocab_size)       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						)						->       Tuple:
					__lowercase			:  int								=       self.num_labels
					__lowercase			:  List[str]								=       TFFunnelForSequenceClassification(config=UpperCamelCase_       )
					__lowercase			:  Any								=       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
					__lowercase			:  Tuple								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.num_labels)       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						)						->       Dict:
					__lowercase			:  Optional[int]								=       self.num_choices
					__lowercase			:  List[Any]								=       TFFunnelForMultipleChoice(config=UpperCamelCase_       )
					__lowercase			:  Union[str, Any]								=       tf.tile(tf.expand_dims(UpperCamelCase_				,						1       )				,						(1, self.num_choices, 1)       )
					__lowercase			:  Tuple								=       tf.tile(tf.expand_dims(UpperCamelCase_				,						1       )				,						(1, self.num_choices, 1)       )
					__lowercase			:  str								=       tf.tile(tf.expand_dims(UpperCamelCase_				,						1       )				,						(1, self.num_choices, 1)       )
					__lowercase			:  Optional[Any]								=       {
					    '''input_ids''': multiple_choice_inputs_ids,
					    '''attention_mask''': multiple_choice_input_mask,
					    '''token_type_ids''': multiple_choice_token_type_ids,
					}
					__lowercase			:  List[str]								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.num_choices)       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						)						->       Union[str, Any]:
					__lowercase			:  str								=       self.num_labels
					__lowercase			:  Optional[Any]								=       TFFunnelForTokenClassification(config=UpperCamelCase_       )
					__lowercase			:  Dict								=       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
					__lowercase			:  str								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length, self.num_labels)       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						)						->       Optional[Any]:
					__lowercase			:  Any								=       TFFunnelForQuestionAnswering(config=UpperCamelCase_       )
					__lowercase			:  str								=       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
					__lowercase			:  List[Any]								=       model(UpperCamelCase_       )
					self.parent.assertEqual(result.start_logits.shape				,						(self.batch_size, self.seq_length)       )
					self.parent.assertEqual(result.end_logits.shape				,						(self.batch_size, self.seq_length)       )
		def 	_lowerCamelCase							(						self       )						->       Optional[Any]:
					__lowercase			:  Tuple								=       self.prepare_config_and_inputs()
					(
					    (
					    __lowercase
					)			,(
					    __lowercase
					)			,(
					    __lowercase
					)			,(
					    __lowercase
					)			,(
					    __lowercase
					)			,(
					    __lowercase
					)			,(
					    __lowercase
					)			,
					)			:  str								=       config_and_inputs
					__lowercase			:  List[Any]								=       {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
					return config, inputs_dict


@require_tf
class   UpperCAmelCase_    (			snake_case   ,					snake_case   ,					unittest.TestCase		):
		UpperCamelCase							=(
		    (
		        TFFunnelModel,
		        TFFunnelForMaskedLM,
		        TFFunnelForPreTraining,
		        TFFunnelForQuestionAnswering,
		        TFFunnelForTokenClassification,
		    )
		    if is_tf_available()
		    else ()
		)
		UpperCamelCase							=(
		    {
		        "feature-extraction": (TFFunnelBaseModel, TFFunnelModel),
		        "fill-mask": TFFunnelForMaskedLM,
		        "question-answering": TFFunnelForQuestionAnswering,
		        "text-classification": TFFunnelForSequenceClassification,
		        "token-classification": TFFunnelForTokenClassification,
		        "zero-shot": TFFunnelForSequenceClassification,
		    }
		    if is_tf_available()
		    else {}
		)
		UpperCamelCase							=False
		UpperCamelCase							=False
		def 	_lowerCamelCase							(						self       )						->       int:
					__lowercase			:  Optional[int]								=       TFFunnelModelTester(self       )
					__lowercase			:  Any								=       ConfigTester(self				,						config_class=UpperCamelCase_       )
		def 	_lowerCamelCase							(						self       )						->       Optional[Any]:
					self.config_tester.run_common_tests()
		def 	_lowerCamelCase							(						self       )						->       Any:
					__lowercase			:  List[str]								=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(*UpperCamelCase_       )
		def 	_lowerCamelCase							(						self       )						->       Dict:
					__lowercase			:  Any								=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_pretraining(*UpperCamelCase_       )
		def 	_lowerCamelCase							(						self       )						->       List[str]:
					__lowercase			:  Dict								=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_masked_lm(*UpperCamelCase_       )
		def 	_lowerCamelCase							(						self       )						->       Optional[int]:
					__lowercase			:  Union[str, Any]								=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_token_classification(*UpperCamelCase_       )
		def 	_lowerCamelCase							(						self       )						->       List[Any]:
					__lowercase			:  List[Any]								=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_question_answering(*UpperCamelCase_       )


@require_tf
class   UpperCAmelCase_    (			snake_case   ,					unittest.TestCase		):
		UpperCamelCase							=(
		    (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
		)
		UpperCamelCase							=False
		UpperCamelCase							=False
		def 	_lowerCamelCase							(						self       )						->       str:
					__lowercase			:  List[str]								=       TFFunnelModelTester(self				,						base=UpperCamelCase_       )
					__lowercase			:  Optional[Any]								=       ConfigTester(self				,						config_class=UpperCamelCase_       )
		def 	_lowerCamelCase							(						self       )						->       List[str]:
					self.config_tester.run_common_tests()
		def 	_lowerCamelCase							(						self       )						->       Optional[Any]:
					__lowercase			:  List[Any]								=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_base_model(*UpperCamelCase_       )
		def 	_lowerCamelCase							(						self       )						->       List[str]:
					__lowercase			:  int								=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase_       )
		def 	_lowerCamelCase							(						self       )						->       Tuple:
					__lowercase			:  str								=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_multiple_choice(*UpperCamelCase_       )





"""simple docstring"""






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


_lowercase							=				logging.get_logger(__name__)


_lowercase							=				{
    '''Salesforce/codegen-350M-nl''': '''https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json''',
    '''Salesforce/codegen-350M-multi''': '''https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json''',
    '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json''',
    '''Salesforce/codegen-2B-nl''': '''https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-2B-multi''': '''https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-2B-mono''': '''https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-6B-nl''': '''https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-6B-multi''': '''https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-6B-mono''': '''https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-16B-nl''': '''https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-16B-multi''': '''https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-16B-mono''': '''https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json''',
}

class 					__a     ( __a       ):



							'''simple docstring'''







							_lowerCamelCase					:			List[Any]          =   """codegen"""
							_lowerCamelCase					:			List[str]          =   {
							    """max_position_embeddings""": """n_positions""",
							    """hidden_size""": """n_embd""",
							    """num_attention_heads""": """n_head""",
							    """num_hidden_layers""": """n_layer""",
							}
							def __init__(     self  ,    _lowerCamelCase=50_400  ,    _lowerCamelCase=2_048  ,    _lowerCamelCase=2_048  ,    _lowerCamelCase=4_096  ,    _lowerCamelCase=28  ,    _lowerCamelCase=16  ,    _lowerCamelCase=64  ,    _lowerCamelCase=None  ,    _lowerCamelCase="gelu_new"  ,    _lowerCamelCase=0.0  ,    _lowerCamelCase=0.0  ,    _lowerCamelCase=0.0  ,    _lowerCamelCase=1e-5  ,    _lowerCamelCase=0.02  ,    _lowerCamelCase=True  ,    _lowerCamelCase=50_256  ,    _lowerCamelCase=50_256  ,    _lowerCamelCase=False  ,    **_lowerCamelCase  ,    )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       vocab_size
										__lowercase       =       n_ctx
										__lowercase       =       n_positions
										__lowercase       =       n_embd
										__lowercase       =       n_layer
										__lowercase       =       n_head
										__lowercase       =       n_inner
										__lowercase       =       rotary_dim
										__lowercase       =       activation_function
										__lowercase       =       resid_pdrop
										__lowercase       =       embd_pdrop
										__lowercase       =       attn_pdrop
										__lowercase       =       layer_norm_epsilon
										__lowercase       =       initializer_range
										__lowercase       =       use_cache

										__lowercase       =       bos_token_id
										__lowercase       =       eos_token_id

										super().__init__(
										    bos_token_id=_lowerCamelCase  ,    eos_token_id=_lowerCamelCase  ,    tie_word_embeddings=_lowerCamelCase  ,    **_lowerCamelCase       )





class 					__a     ( __a       ):



							'''simple docstring'''







							def __init__(     self  ,    _lowerCamelCase  ,    _lowerCamelCase = "default"  ,    _lowerCamelCase = None  ,    _lowerCamelCase = False  ,    )			->							str:




										'''simple docstring'''
										super().__init__(_lowerCamelCase  ,    task=_lowerCamelCase  ,    patching_specs=_lowerCamelCase  ,    use_past=_lowerCamelCase       )
										if not getattr(self._config  ,    "pad_token_id"  ,    _lowerCamelCase       ):
													# TODO: how to do that better?
													__lowercase       =       0
							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Mapping[str, Mapping[int, str]]:




										'''simple docstring'''
										__lowercase       =       OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}       )
										if self.use_past:
													self.fill_with_past_key_values_(_lowerCamelCase  ,    direction="inputs"       )
													__lowercase       =       {0: "batch", 1: "past_sequence + sequence"}
										else:
													__lowercase       =       {0: "batch", 1: "sequence"}

										return common_inputs
							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										return self._config.n_layer
							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										return self._config.n_head
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase = -1  ,    _lowerCamelCase = -1  ,    _lowerCamelCase = False  ,    _lowerCamelCase = None  ,    )			->							Mapping[str, Any]:




										'''simple docstring'''
										__lowercase       =       super(_lowerCamelCase  ,    self       ).generate_dummy_inputs(
										    _lowerCamelCase  ,    batch_size=_lowerCamelCase  ,    seq_length=_lowerCamelCase  ,    is_pair=_lowerCamelCase  ,    framework=_lowerCamelCase       )

										# We need to order the input in the way they appears in the forward()
										__lowercase       =       OrderedDict({"input_ids": common_inputs["input_ids"]}       )

										# Need to add the past_keys
										if self.use_past:
													if not is_torch_available():
																raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."       )
													else:
																import torch

																__lowercase   ,					__lowercase       =       common_inputs["input_ids"].shape
																# Not using the same length for past_key_values
																__lowercase       =       seqlen + 2
																__lowercase       =       (
																    batch,
																    self.num_attention_heads,
																    past_key_values_length,
																    self._config.hidden_size // self.num_attention_heads,
																)
																__lowercase       =       [
																    (torch.zeros(_lowerCamelCase       ), torch.zeros(_lowerCamelCase       )) for _ in range(self.num_layers       )
																]

										__lowercase       =       common_inputs["attention_mask"]
										if self.use_past:
													__lowercase       =       ordered_inputs["attention_mask"].dtype
													__lowercase       =       torch.cat(
													    [ordered_inputs["attention_mask"], torch.ones(_lowerCamelCase  ,    _lowerCamelCase  ,    dtype=_lowerCamelCase       )]  ,    dim=1       )

										return ordered_inputs





							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										return 13





