
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    OpenAIGPTConfig,
						    OpenAIGPTDoubleHeadsModel,
						    OpenAIGPTForSequenceClassification,
						    OpenAIGPTLMHeadModel,
						    OpenAIGPTModel,
						)
class 					lowercase__:



	"""simple docstring"""




	def __init__(			self    :     Optional[int]			,				SCREAMING_SNAKE_CASE_    :     str			,				SCREAMING_SNAKE_CASE_    :     Tuple=1_3			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]=7			,				SCREAMING_SNAKE_CASE_    :     Dict=True			,				SCREAMING_SNAKE_CASE_    :     str=True			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]=True			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]=9_9			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]=3_2			,				SCREAMING_SNAKE_CASE_    :     Dict=5			,				SCREAMING_SNAKE_CASE_    :     int=4			,				SCREAMING_SNAKE_CASE_    :     Any=3_7			,				SCREAMING_SNAKE_CASE_    :     Any="gelu"			,				SCREAMING_SNAKE_CASE_    :     Tuple=0.1			,				SCREAMING_SNAKE_CASE_    :     Dict=0.1			,				SCREAMING_SNAKE_CASE_    :     Tuple=5_1_2			,				SCREAMING_SNAKE_CASE_    :     List[Any]=1_6			,				SCREAMING_SNAKE_CASE_    :     Dict=2			,				SCREAMING_SNAKE_CASE_    :     int=0.02			,				SCREAMING_SNAKE_CASE_    :     Tuple=3			,				SCREAMING_SNAKE_CASE_    :     str=4			,				SCREAMING_SNAKE_CASE_    :     Tuple=None			,				)			->	List[str]:
					lowercase_	    =   parent
					lowercase_	    =   batch_size
					lowercase_	    =   seq_length
					lowercase_	    =   is_training
					lowercase_	    =   use_token_type_ids
					lowercase_	    =   use_labels
					lowercase_	    =   vocab_size
					lowercase_	    =   hidden_size
					lowercase_	    =   num_hidden_layers
					lowercase_	    =   num_attention_heads
					lowercase_	    =   intermediate_size
					lowercase_	    =   hidden_act
					lowercase_	    =   hidden_dropout_prob
					lowercase_	    =   attention_probs_dropout_prob
					lowercase_	    =   max_position_embeddings
					lowercase_	    =   type_vocab_size
					lowercase_	    =   type_sequence_label_size
					lowercase_	    =   initializer_range
					lowercase_	    =   num_labels
					lowercase_	    =   num_choices
					lowercase_	    =   scope
					lowercase_	    =   self.vocab_size - 1
	def 		_lowercase    (			self    :     str			)			->	List[str]:
					lowercase_	    =   ids_tensor([self.batch_size, self.seq_length]			,				self.vocab_size			)

					lowercase_	    =   None
					if self.use_token_type_ids:
									lowercase_	    =   ids_tensor([self.batch_size, self.seq_length]			,				self.type_vocab_size			)

					lowercase_	    =   None
					lowercase_	    =   None
					lowercase_	    =   None
					if self.use_labels:
									lowercase_	    =   ids_tensor([self.batch_size]			,				self.type_sequence_label_size			)
									lowercase_	    =   ids_tensor([self.batch_size, self.seq_length]			,				self.num_labels			)
									lowercase_	    =   ids_tensor([self.batch_size]			,				self.num_choices			)

					lowercase_	    =   OpenAIGPTConfig(
					    vocab_size=self.vocab_size			,				n_embd=self.hidden_size			,				n_layer=self.num_hidden_layers			,				n_head=self.num_attention_heads			,				n_positions=self.max_position_embeddings			,				pad_token_id=self.pad_token_id			,				)

					lowercase_	    =   ids_tensor([self.num_hidden_layers, self.num_attention_heads]			,				2			)

					return (
					    config,
					    input_ids,
					    head_mask,
					    token_type_ids,
					    sequence_labels,
					    token_labels,
					    choice_labels,
					)
	def 		_lowercase    (			self    :     str			,				SCREAMING_SNAKE_CASE_    :     str			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]			,				SCREAMING_SNAKE_CASE_    :     List[Any]			,				SCREAMING_SNAKE_CASE_    :     int			,				*SCREAMING_SNAKE_CASE_    :     Optional[Any]			)			->	Union[str, Any]:
					lowercase_	    =   OpenAIGPTModel(config=SCREAMING_SNAKE_CASE_			)
					model.to(SCREAMING_SNAKE_CASE_			)
					model.eval()

					lowercase_	    =   model(SCREAMING_SNAKE_CASE_			,				token_type_ids=SCREAMING_SNAKE_CASE_			,				head_mask=SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   model(SCREAMING_SNAKE_CASE_			,				token_type_ids=SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   model(SCREAMING_SNAKE_CASE_			)

					self.parent.assertEqual(result.last_hidden_state.shape			,				(self.batch_size, self.seq_length, self.hidden_size)			)
	def 		_lowercase    (			self    :     int			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]			,				SCREAMING_SNAKE_CASE_    :     Dict			,				SCREAMING_SNAKE_CASE_    :     Tuple			,				SCREAMING_SNAKE_CASE_    :     str			,				*SCREAMING_SNAKE_CASE_    :     List[Any]			)			->	List[Any]:
					lowercase_	    =   OpenAIGPTLMHeadModel(SCREAMING_SNAKE_CASE_			)
					model.to(SCREAMING_SNAKE_CASE_			)
					model.eval()

					lowercase_	    =   model(SCREAMING_SNAKE_CASE_			,				token_type_ids=SCREAMING_SNAKE_CASE_			,				labels=SCREAMING_SNAKE_CASE_			)
					self.parent.assertEqual(result.loss.shape			,				()			)
					self.parent.assertEqual(result.logits.shape			,				(self.batch_size, self.seq_length, self.vocab_size)			)
	def 		_lowercase    (			self    :     Optional[Any]			,				SCREAMING_SNAKE_CASE_    :     List[str]			,				SCREAMING_SNAKE_CASE_    :     str			,				SCREAMING_SNAKE_CASE_    :     Optional[int]			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]			,				*SCREAMING_SNAKE_CASE_    :     List[Any]			)			->	Optional[int]:
					lowercase_	    =   OpenAIGPTDoubleHeadsModel(SCREAMING_SNAKE_CASE_			)
					model.to(SCREAMING_SNAKE_CASE_			)
					model.eval()

					lowercase_	    =   model(SCREAMING_SNAKE_CASE_			,				token_type_ids=SCREAMING_SNAKE_CASE_			,				labels=SCREAMING_SNAKE_CASE_			)
					self.parent.assertEqual(result.loss.shape			,				()			)
					self.parent.assertEqual(result.logits.shape			,				(self.batch_size, self.seq_length, self.vocab_size)			)
	def 		_lowercase    (			self    :     Any			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]			,				SCREAMING_SNAKE_CASE_    :     List[Any]			,				SCREAMING_SNAKE_CASE_    :     List[str]			,				SCREAMING_SNAKE_CASE_    :     Optional[int]			,				*SCREAMING_SNAKE_CASE_    :     Tuple			)			->	List[str]:
					lowercase_	    =   self.num_labels
					lowercase_	    =   OpenAIGPTForSequenceClassification(SCREAMING_SNAKE_CASE_			)
					model.to(SCREAMING_SNAKE_CASE_			)
					model.eval()

					lowercase_	    =   ids_tensor([self.batch_size]			,				self.type_sequence_label_size			)
					lowercase_	    =   model(SCREAMING_SNAKE_CASE_			,				token_type_ids=SCREAMING_SNAKE_CASE_			,				labels=SCREAMING_SNAKE_CASE_			)
					self.parent.assertEqual(result.logits.shape			,				(self.batch_size, self.num_labels)			)




	def 		_lowercase    (			self    :     Optional[Any]			)			->	Optional[Any]:
					lowercase_	    =   self.prepare_config_and_inputs()
					(
					    (
					    lowercase_
					)    ,      (
					    lowercase_
					)    ,      (
					    lowercase_
					)    ,      (
					    lowercase_
					)    ,      (
					    lowercase_
					)    ,      (
					    lowercase_
					)    ,      (
					    lowercase_
					)    ,      
					)	    =   config_and_inputs
					lowercase_	    =   {
					    '''input_ids''': input_ids,
					    '''token_type_ids''': token_type_ids,
					    '''head_mask''': head_mask,
					}

					return config, inputs_dict

@require_torch
class 					lowercase__(							UpperCAmelCase					,		UpperCAmelCase					,		UpperCAmelCase					,		unittest.TestCase				):



	"""simple docstring"""




	a     :Dict       	=      (
	    (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
	    if is_torch_available()
	    else ()
	)
	a     :Optional[Any]       	=      (
	    (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
	)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
	a     :int       	=      (
	    {
	        'feature-extraction': OpenAIGPTModel,
	        'text-classification': OpenAIGPTForSequenceClassification,
	        'text-generation': OpenAIGPTLMHeadModel,
	        'zero-shot': OpenAIGPTForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	def 		_lowercase    (			self    :     str			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]			,				SCREAMING_SNAKE_CASE_    :     int			,				SCREAMING_SNAKE_CASE_    :     Tuple			,				SCREAMING_SNAKE_CASE_    :     List[str]			,				SCREAMING_SNAKE_CASE_    :     Dict			)			->	int:
					if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
									# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
									# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
									# tiny config could not be created.
									return True

					return False
	def 		_lowercase    (			self    :     Optional[int]			,				SCREAMING_SNAKE_CASE_    :     Tuple			,				SCREAMING_SNAKE_CASE_    :     int			,				SCREAMING_SNAKE_CASE_    :     List[Any]=False			)			->	List[str]:
					lowercase_	    =   super()._prepare_for_class(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				return_labels=SCREAMING_SNAKE_CASE_			)

					if return_labels:
									if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
													lowercase_	    =   torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)			,				dtype=torch.long			,				device=SCREAMING_SNAKE_CASE_			,				)
													lowercase_	    =   inputs_dict['''labels''']
													lowercase_	    =   inputs_dict['''labels''']
													lowercase_	    =   torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.num_choices)			,				dtype=torch.long			,				device=SCREAMING_SNAKE_CASE_			,				)
													lowercase_	    =   torch.zeros(
													    self.model_tester.batch_size			,				dtype=torch.long			,				device=SCREAMING_SNAKE_CASE_			)
					return inputs_dict
	def 		_lowercase    (			self    :     Optional[Any]			)			->	int:
					lowercase_	    =   OpenAIGPTModelTester(self			)
					lowercase_	    =   ConfigTester(self			,				config_class=SCREAMING_SNAKE_CASE_			,				n_embd=3_7			)
	def 		_lowercase    (			self    :     List[str]			)			->	Any:
					self.config_tester.run_common_tests()
	def 		_lowercase    (			self    :     str			)			->	Dict:
					lowercase_	    =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_openai_gpt_model(*SCREAMING_SNAKE_CASE_			)
	def 		_lowercase    (			self    :     List[str]			)			->	Tuple:
					lowercase_	    =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_lm_head_model(*SCREAMING_SNAKE_CASE_			)
	def 		_lowercase    (			self    :     Optional[int]			)			->	Optional[Any]:
					lowercase_	    =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_double_lm_head_model(*SCREAMING_SNAKE_CASE_			)
	def 		_lowercase    (			self    :     Tuple			)			->	Union[str, Any]:
					lowercase_	    =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*SCREAMING_SNAKE_CASE_			)




	@slow
	def 		_lowercase    (			self    :     Any			)			->	List[Any]:
					for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowercase_	    =   OpenAIGPTModel.from_pretrained(SCREAMING_SNAKE_CASE_			)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE_			)

@require_torch
class 					lowercase__(							unittest.TestCase				):



	"""simple docstring"""




	@slow
	def 		_lowercase    (			self    :     List[Any]			)			->	str:
					lowercase_	    =   OpenAIGPTLMHeadModel.from_pretrained('''openai-gpt'''			)
					model.to(SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   torch.tensor([[4_8_1, 4_7_3_5, 5_4_4]]			,				dtype=torch.long			,				device=SCREAMING_SNAKE_CASE_			)  # the president is
					lowercase_	    =   [
					    4_8_1,
					    4_7_3_5,
					    5_4_4,
					    2_4_6,
					    9_6_3,
					    8_7_0,
					    7_6_2,
					    2_3_9,
					    2_4_4,
					    4_0_4_7_7,
					    2_4_4,
					    2_4_9,
					    7_1_9,
					    8_8_1,
					    4_8_7,
					    5_4_4,
					    2_4_0,
					    2_4_4,
					    6_0_3,
					    4_8_1,
					]  # the president is a very good man. " \n " i\'m sure he is, " said the

					lowercase_	    =   model.generate(SCREAMING_SNAKE_CASE_			,				do_sample=SCREAMING_SNAKE_CASE_			)
					self.assertListEqual(output_ids[0].tolist()			,				SCREAMING_SNAKE_CASE_			)










from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
   from ...onnx.config import PatchingSpec
   from ...tokenization_utils_base import PreTrainedTokenizerBase


lowerCamelCase__					:  Optional[int]      							=				logging.get_logger(__name__)

lowerCamelCase__					:  Dict      							=				{
    'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json',
    'allenai/longformer-large-4096': 'https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json',
    'allenai/longformer-large-4096-finetuned-triviaqa': (
        'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json'
    ),
    'allenai/longformer-base-4096-extra.pos.embd.only': (
        'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json'
    ),
    'allenai/longformer-large-4096-extra.pos.embd.only': (
        'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json'
    ),
}





class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE   ):


 '''simple docstring'''







 lowercase_							       =       "longformer"



 def __init__(				self    :						Union[str, Any]							,							_lowerCAmelCase    :						Union[List[int], int] = 512							,							_lowerCAmelCase    :						int = 2							,							_lowerCAmelCase    :						int = 1							,							_lowerCAmelCase    :						int = 0							,							_lowerCAmelCase    :						int = 2							,							_lowerCAmelCase    :						int = 30_522							,							_lowerCAmelCase    :						int = 768							,							_lowerCAmelCase    :						int = 12							,							_lowerCAmelCase    :						int = 12							,							_lowerCAmelCase    :						int = 3_072							,							_lowerCAmelCase    :						str = "gelu"							,							_lowerCAmelCase    :						float = 0.1							,							_lowerCAmelCase    :						float = 0.1							,							_lowerCAmelCase    :						int = 512							,							_lowerCAmelCase    :						int = 2							,							_lowerCAmelCase    :						float = 0.02							,							_lowerCAmelCase    :						float = 1E-12							,							_lowerCAmelCase    :						bool = False							,							**_lowerCAmelCase    :						Union[str, Any]							,							):
       super().__init__(pad_token_id=_lowerCAmelCase							,							**_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     attention_window
       SCREAMING_SNAKE_CASE_      		=     sep_token_id
       SCREAMING_SNAKE_CASE_      		=     bos_token_id
       SCREAMING_SNAKE_CASE_      		=     eos_token_id
       SCREAMING_SNAKE_CASE_      		=     vocab_size
       SCREAMING_SNAKE_CASE_      		=     hidden_size
       SCREAMING_SNAKE_CASE_      		=     num_hidden_layers
       SCREAMING_SNAKE_CASE_      		=     num_attention_heads
       SCREAMING_SNAKE_CASE_      		=     hidden_act
       SCREAMING_SNAKE_CASE_      		=     intermediate_size
       SCREAMING_SNAKE_CASE_      		=     hidden_dropout_prob
       SCREAMING_SNAKE_CASE_      		=     attention_probs_dropout_prob
       SCREAMING_SNAKE_CASE_      		=     max_position_embeddings
       SCREAMING_SNAKE_CASE_      		=     type_vocab_size
       SCREAMING_SNAKE_CASE_      		=     initializer_range
       SCREAMING_SNAKE_CASE_      		=     layer_norm_eps
       SCREAMING_SNAKE_CASE_      		=     onnx_export





class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE   ):


 '''simple docstring'''







 def __init__(				self    :						Optional[Any]							,							_lowerCAmelCase    :						"PretrainedConfig"							,							_lowerCAmelCase    :						str = "default"							,							_lowerCAmelCase    :						"List[PatchingSpec]" = None		):
       super().__init__(_lowerCAmelCase							,							_lowerCAmelCase							,							_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     True



 @property
 def  lowerCAmelCase_    (				self    :						Any		):
       if self.task == "multiple-choice":
             SCREAMING_SNAKE_CASE_      		=     {0: 'batch', 1: 'choice', 2: 'sequence'}
       else:
             SCREAMING_SNAKE_CASE_      		=     {0: 'batch', 1: 'sequence'}
       return OrderedDict(
           [
               ('input_ids', dynamic_axis),
               ('attention_mask', dynamic_axis),
               ('global_attention_mask', dynamic_axis),
           ]		)



 @property
 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       SCREAMING_SNAKE_CASE_      		=     super().outputs
       if self.task == "default":
             SCREAMING_SNAKE_CASE_      		=     {0: 'batch'}
       return outputs



 @property
 def  lowerCAmelCase_    (				self    :						str		):
       return 1E-4



 @property
 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       # needs to be >= 14 to support tril operator
       return max(super().default_onnx_opset							,							14		)




 def  lowerCAmelCase_    (				self    :						str							,							_lowerCAmelCase    :						"PreTrainedTokenizerBase"							,							_lowerCAmelCase    :						int = -1							,							_lowerCAmelCase    :						int = -1							,							_lowerCAmelCase    :						bool = False							,							_lowerCAmelCase    :						Optional[TensorType] = None							,							):
       SCREAMING_SNAKE_CASE_      		=     super().generate_dummy_inputs(
           preprocessor=_lowerCAmelCase							,							batch_size=_lowerCAmelCase							,							seq_length=_lowerCAmelCase							,							is_pair=_lowerCAmelCase							,							framework=_lowerCAmelCase		)
       import torch

       # for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
       # makes the export fail randomly
       SCREAMING_SNAKE_CASE_      		=     torch.zeros_like(inputs['input_ids']		)
       # make every second token global
       SCREAMING_SNAKE_CASE_      		=     1

       return inputs