



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    OpenAIGPTConfig,
				    OpenAIGPTDoubleHeadsModel,
				    OpenAIGPTForSequenceClassification,
				    OpenAIGPTLMHeadModel,
				    OpenAIGPTModel,
				)



class        lowerCAmelCase__					:


				def __init__(							self      :  Optional[Any]						, __UpperCamelCase      :  Tuple						, __UpperCamelCase      :  Tuple=13						, __UpperCamelCase      :  int=7						, __UpperCamelCase      :  Any=True						, __UpperCamelCase      :  List[str]=True						, __UpperCamelCase      :  Tuple=True						, __UpperCamelCase      :  int=99						, __UpperCamelCase      :  Optional[Any]=32						, __UpperCamelCase      :  str=5						, __UpperCamelCase      :  Optional[Any]=4						, __UpperCamelCase      :  Any=37						, __UpperCamelCase      :  Optional[int]="gelu"						, __UpperCamelCase      :  List[str]=0.1						, __UpperCamelCase      :  Any=0.1						, __UpperCamelCase      :  Tuple=512						, __UpperCamelCase      :  Dict=16						, __UpperCamelCase      :  str=2						, __UpperCamelCase      :  List[str]=0.0_2						, __UpperCamelCase      :  List[Any]=3						, __UpperCamelCase      :  int=4						, __UpperCamelCase      :  List[Any]=None						, )		->     List[str]:
											A      						=    parent
											A      						=    batch_size
											A      						=    seq_length
											A      						=    is_training
											A      						=    use_token_type_ids
											A      						=    use_labels
											A      						=    vocab_size
											A      						=    hidden_size
											A      						=    num_hidden_layers
											A      						=    num_attention_heads
											A      						=    intermediate_size
											A      						=    hidden_act
											A      						=    hidden_dropout_prob
											A      						=    attention_probs_dropout_prob
											A      						=    max_position_embeddings
											A      						=    type_vocab_size
											A      						=    type_sequence_label_size
											A      						=    initializer_range
											A      						=    num_labels
											A      						=    num_choices
											A      						=    scope
											A      						=    self.vocab_size - 1


				def __UpperCamelCase						(							self      :  List[str]       )		->     Optional[int]:
											A      						=    ids_tensor([self.batch_size, self.seq_length]						, self.vocab_size       )

											A      						=    None
											if self.use_token_type_ids:
																		A      						=    ids_tensor([self.batch_size, self.seq_length]						, self.type_vocab_size       )

											A      						=    None
											A      						=    None
											A      						=    None
											if self.use_labels:
																		A      						=    ids_tensor([self.batch_size]						, self.type_sequence_label_size       )
																		A      						=    ids_tensor([self.batch_size, self.seq_length]						, self.num_labels       )
																		A      						=    ids_tensor([self.batch_size]						, self.num_choices       )

											A      						=    OpenAIGPTConfig(
											    vocab_size=self.vocab_size						, n_embd=self.hidden_size						, n_layer=self.num_hidden_layers						, n_head=self.num_attention_heads						, n_positions=self.max_position_embeddings						, pad_token_id=self.pad_token_id						, )

											A      						=    ids_tensor([self.num_hidden_layers, self.num_attention_heads]						, 2       )

											return (
											    config,
											    input_ids,
											    head_mask,
											    token_type_ids,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											)


				def __UpperCamelCase						(							self      :  Tuple						, __UpperCamelCase      :  Union[str, Any]						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  Dict						, *__UpperCamelCase      :  Optional[Any]       )		->     str:
											A      						=    OpenAIGPTModel(config=__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()

											A      						=    model(__UpperCamelCase						, token_type_ids=__UpperCamelCase						, head_mask=__UpperCamelCase       )
											A      						=    model(__UpperCamelCase						, token_type_ids=__UpperCamelCase       )
											A      						=    model(__UpperCamelCase       )

											self.parent.assertEqual(result.last_hidden_state.shape						, (self.batch_size, self.seq_length, self.hidden_size)       )


				def __UpperCamelCase						(							self      :  Dict						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  Union[str, Any]						, __UpperCamelCase      :  Tuple						, __UpperCamelCase      :  Dict						, *__UpperCamelCase      :  Tuple       )		->     List[str]:
											A      						=    OpenAIGPTLMHeadModel(__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()

											A      						=    model(__UpperCamelCase						, token_type_ids=__UpperCamelCase						, labels=__UpperCamelCase       )
											self.parent.assertEqual(result.loss.shape						, ()       )
											self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length, self.vocab_size)       )


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  List[Any]						, __UpperCamelCase      :  Tuple						, *__UpperCamelCase      :  Tuple       )		->     Optional[Any]:
											A      						=    OpenAIGPTDoubleHeadsModel(__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()

											A      						=    model(__UpperCamelCase						, token_type_ids=__UpperCamelCase						, labels=__UpperCamelCase       )
											self.parent.assertEqual(result.loss.shape						, ()       )
											self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length, self.vocab_size)       )


				def __UpperCamelCase						(							self      :  Dict						, __UpperCamelCase      :  Optional[Any]						, __UpperCamelCase      :  Tuple						, __UpperCamelCase      :  Any						, __UpperCamelCase      :  Union[str, Any]						, *__UpperCamelCase      :  str       )		->     str:
											A      						=    self.num_labels
											A      						=    OpenAIGPTForSequenceClassification(__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()

											A      						=    ids_tensor([self.batch_size]						, self.type_sequence_label_size       )
											A      						=    model(__UpperCamelCase						, token_type_ids=__UpperCamelCase						, labels=__UpperCamelCase       )
											self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.num_labels)       )


				def __UpperCamelCase						(							self      :  Dict       )		->     Optional[Any]:
											A      						=    self.prepare_config_and_inputs()
											(
											    (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   
											)      						=    config_and_inputs
											A      						=    {
											    'input_ids': input_ids,
											    'token_type_ids': token_type_ids,
											    'head_mask': head_mask,
											}

											return config, inputs_dict





@require_torch
class        lowerCAmelCase__					(  _lowerCamelCase ,				_lowerCamelCase ,				_lowerCamelCase ,				unittest.TestCase       ):
				A_   :     List[str]								=     (
				    (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
				    if is_torch_available()
				    else ()
				)
				A_   :     List[Any]								=     (
				    (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
				)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
				A_   :     Any								=     (
				    {
				        'feature-extraction': OpenAIGPTModel,
				        'text-classification': OpenAIGPTForSequenceClassification,
				        'text-generation': OpenAIGPTLMHeadModel,
				        'zero-shot': OpenAIGPTForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)


				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  Any						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  str						, __UpperCamelCase      :  List[Any]       )		->     int:
											if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
																		# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
																		# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
																		# tiny config could not be created.
																		return True

											return False


				def __UpperCamelCase						(							self      :  List[Any]						, __UpperCamelCase      :  List[Any]						, __UpperCamelCase      :  List[Any]						, __UpperCamelCase      :  Optional[Any]=False       )		->     int:
											A      						=    super()._prepare_for_class(__UpperCamelCase						, __UpperCamelCase						, return_labels=__UpperCamelCase       )

											if return_labels:
																		if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
																									A      						=    torch.zeros(
																									    (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)						, dtype=torch.long						, device=__UpperCamelCase						, )
																									A      						=    inputs_dict['labels']
																									A      						=    inputs_dict['labels']
																									A      						=    torch.zeros(
																									    (self.model_tester.batch_size, self.model_tester.num_choices)						, dtype=torch.long						, device=__UpperCamelCase						, )
																									A      						=    torch.zeros(
																									    self.model_tester.batch_size						, dtype=torch.long						, device=__UpperCamelCase       )
											return inputs_dict


				def __UpperCamelCase						(							self      :  Any       )		->     List[str]:
											A      						=    OpenAIGPTModelTester(self       )
											A      						=    ConfigTester(self						, config_class=__UpperCamelCase						, n_embd=37       )


				def __UpperCamelCase						(							self      :  str       )		->     Union[str, Any]:
											self.config_tester.run_common_tests()


				def __UpperCamelCase						(							self      :  List[str]       )		->     Union[str, Any]:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_openai_gpt_model(*__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     Optional[int]:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_lm_head_model(*__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     Optional[int]:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_double_lm_head_model(*__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     List[Any]:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*__UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Dict       )		->     Optional[Any]:
											for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		A      						=    OpenAIGPTModel.from_pretrained(__UpperCamelCase       )
																		self.assertIsNotNone(__UpperCamelCase       )





@require_torch
class        lowerCAmelCase__					(  unittest.TestCase       ):


				@slow
				def __UpperCamelCase						(							self      :  Optional[int]       )		->     Optional[Any]:
											A      						=    OpenAIGPTLMHeadModel.from_pretrained('openai-gpt'       )
											model.to(__UpperCamelCase       )
											A      						=    torch.tensor([[481, 4_735, 544]]						, dtype=torch.long						, device=__UpperCamelCase       )  # the president is
											A      						=    [
											    481,
											    4_735,
											    544,
											    246,
											    963,
											    870,
											    762,
											    239,
											    244,
											    40_477,
											    244,
											    249,
											    719,
											    881,
											    487,
											    544,
											    240,
											    244,
											    603,
											    481,
											]  # the president is a very good man. " \n " i\'m sure he is, " said the

											A      						=    model.generate(__UpperCamelCase						, do_sample=__UpperCamelCase       )
											self.assertListEqual(output_ids[0].tolist()						, __UpperCamelCase       )




from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case     :Optional[Any]			       =logging.get_logger(__name__)

__snake_case     :Tuple			       ={
    'transfo-xl-wt103': 'https://huggingface.co/transfo-xl-wt103/resolve/main/config.json',
}



class        lowerCAmelCase__					(  _lowerCamelCase       ):
				A_   :     Any								=     'transfo-xl'
				A_   :     Dict								=     ['mems']
				A_   :     int								=     {
				    'n_token': 'vocab_size',
				    'hidden_size': 'd_model',
				    'num_attention_heads': 'n_head',
				    'num_hidden_layers': 'n_layer',
				}


				def __init__(							self      :  Optional[Any]						, __UpperCamelCase      :  str=267_735						, __UpperCamelCase      :  int=[20_000, 40_000, 200_000]						, __UpperCamelCase      :  List[str]=1_024						, __UpperCamelCase      :  Union[str, Any]=1_024						, __UpperCamelCase      :  Optional[Any]=16						, __UpperCamelCase      :  Dict=64						, __UpperCamelCase      :  Optional[Any]=4_096						, __UpperCamelCase      :  int=4						, __UpperCamelCase      :  List[str]=False						, __UpperCamelCase      :  Optional[int]=18						, __UpperCamelCase      :  int=1_600						, __UpperCamelCase      :  Dict=1_000						, __UpperCamelCase      :  Optional[Any]=True						, __UpperCamelCase      :  Optional[Any]=True						, __UpperCamelCase      :  Optional[Any]=0						, __UpperCamelCase      :  Any=-1						, __UpperCamelCase      :  Any=True						, __UpperCamelCase      :  List[Any]=0.1						, __UpperCamelCase      :  str=0.0						, __UpperCamelCase      :  Dict=True						, __UpperCamelCase      :  int="normal"						, __UpperCamelCase      :  List[Any]=0.0_1						, __UpperCamelCase      :  str=0.0_1						, __UpperCamelCase      :  Any=0.0_2						, __UpperCamelCase      :  str=1e-5						, __UpperCamelCase      :  Optional[int]=0						, **__UpperCamelCase      :  Any						, )		->     int:
											A      						=    vocab_size
											A      						=    []
											self.cutoffs.extend(__UpperCamelCase       )
											if proj_share_all_but_first:
																		A      						=    [False] + [True] * len(self.cutoffs       )
											else:
																		A      						=    [False] + [False] * len(self.cutoffs       )
											A      						=    d_model
											A      						=    d_embed
											A      						=    d_head
											A      						=    d_inner
											A      						=    div_val
											A      						=    pre_lnorm
											A      						=    n_layer
											A      						=    n_head
											A      						=    mem_len
											A      						=    same_length
											A      						=    attn_type
											A      						=    clamp_len
											A      						=    sample_softmax
											A      						=    adaptive
											A      						=    dropout
											A      						=    dropatt
											A      						=    untie_r
											A      						=    init
											A      						=    init_range
											A      						=    proj_init_std
											A      						=    init_std
											A      						=    layer_norm_epsilon
											super().__init__(eos_token_id=__UpperCamelCase						, **__UpperCamelCase       )


				@property
				def __UpperCamelCase						(							self      :  Any       )		->     Optional[int]:
											# Message copied from Transformer-XL documentation
											logger.info(f'''The model {self.model_type} is one of the few models that has no sequence length limit.'''       )
											return -1


				@max_position_embeddings.setter
				def __UpperCamelCase						(							self      :  List[Any]						, __UpperCamelCase      :  List[Any]       )		->     List[Any]:
											# Message copied from Transformer-XL documentation
											raise NotImplementedError(
											    f'''The model {self.model_type} is one of the few models that has no sequence length limit.'''       )