


"""simple docstring"""




import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         GPTNeoXForCausalLM,
         GPTNeoXForQuestionAnswering,
         GPTNeoXForSequenceClassification,
         GPTNeoXForTokenClassification,
         GPTNeoXModel,
     )







class 		_snake_case  :




 def __init__(		self ,  _lowerCamelCase ,  _lowerCamelCase=13 ,  _lowerCamelCase=7 ,  _lowerCamelCase=True ,  _lowerCamelCase=True ,  _lowerCamelCase=True ,  _lowerCamelCase=True ,  _lowerCamelCase=99 ,  _lowerCamelCase=64 ,  _lowerCamelCase=5 ,  _lowerCamelCase=4 ,  _lowerCamelCase=37 ,  _lowerCamelCase="gelu" ,  _lowerCamelCase=0.1 ,  _lowerCamelCase=0.1 ,  _lowerCamelCase=512 ,  _lowerCamelCase=16 ,  _lowerCamelCase=2 ,  _lowerCamelCase=0.02 ,  _lowerCamelCase=3 ,  _lowerCamelCase=4 ,  _lowerCamelCase=None ,  ):
       a						:str										=	parent
       a						:Tuple										=	batch_size
       a						:Union[str, Any]										=	seq_length
       a						:str										=	is_training
       a						:Dict										=	use_input_mask
       a						:Any										=	use_token_type_ids
       a						:Union[str, Any]										=	use_labels
       a						:Union[str, Any]										=	vocab_size
       a						:Dict										=	hidden_size
       a						:Optional[int]										=	num_hidden_layers
       a						:Any										=	num_attention_heads
       a						:Tuple										=	intermediate_size
       a						:List[Any]										=	hidden_act
       a						:Any										=	hidden_dropout_prob
       a						:Optional[Any]										=	attention_probs_dropout_prob
       a						:str										=	max_position_embeddings
       a						:Optional[int]										=	type_vocab_size
       a						:Optional[Any]										=	type_sequence_label_size
       a						:Dict										=	initializer_range
       a						:List[str]										=	num_labels
       a						:Dict										=	num_choices
       a						:Union[str, Any]										=	scope
       a						:Optional[int]										=	vocab_size - 1




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Dict										=	ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size   )

       a						:str										=	None
       if self.use_input_mask:
             a						:List[str]										=	random_attention_mask([self.batch_size, self.seq_length]   )

       a						:Optional[int]										=	None
       if self.use_labels:
             a						:Any										=	ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels   )

       a						:Optional[int]										=	self.get_config()

       return config, input_ids, input_mask, token_labels




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       return GPTNeoXConfig(
           vocab_size=self.vocab_size ,  hidden_size=self.hidden_size ,  num_hidden_layers=self.num_hidden_layers ,  num_attention_heads=self.num_attention_heads ,  intermediate_size=self.intermediate_size ,  hidden_act=self.hidden_act ,  hidden_dropout_prob=self.hidden_dropout_prob ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob ,  max_position_embeddings=self.max_position_embeddings ,  type_vocab_size=self.type_vocab_size ,  is_decoder=_UpperCAmelCase ,  initializer_range=self.initializer_range ,  pad_token_id=self.pad_token_id ,  )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Dict										=	self.prepare_config_and_inputs()

       a						:Any										=	True

       return config, input_ids, input_mask, token_labels




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:List[Any]										=	GPTNeoXModel(config=_UpperCAmelCase   )
       model.to(_UpperCAmelCase   )
       model.eval()
       a						:Dict										=	model(_UpperCAmelCase ,  attention_mask=_UpperCAmelCase   )
       a						:Dict										=	model(_UpperCAmelCase   )
       self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:Any										=	True
       a						:List[str]										=	GPTNeoXModel(_UpperCAmelCase   )
       model.to(_UpperCAmelCase   )
       model.eval()
       a						:Tuple										=	model(_UpperCAmelCase ,  attention_mask=_UpperCAmelCase   )
       self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:Optional[Any]										=	GPTNeoXForCausalLM(config=_UpperCAmelCase   )
       model.to(_UpperCAmelCase   )
       model.eval()
       a						:Optional[Any]										=	model(_UpperCAmelCase ,  attention_mask=_UpperCAmelCase ,  labels=_UpperCAmelCase   )
       self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:Optional[int]										=	self.num_labels
       a						:Optional[int]										=	GPTNeoXForQuestionAnswering(_UpperCAmelCase   )
       model.to(_UpperCAmelCase   )
       model.eval()
       a						:List[Any]										=	model(_UpperCAmelCase ,  attention_mask=_UpperCAmelCase   )
       self.parent.assertEqual(result.start_logits.shape ,  (self.batch_size, self.seq_length)   )
       self.parent.assertEqual(result.end_logits.shape ,  (self.batch_size, self.seq_length)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:Tuple										=	self.num_labels
       a						:Any										=	GPTNeoXForSequenceClassification(_UpperCAmelCase   )
       model.to(_UpperCAmelCase   )
       model.eval()
       a						:Optional[Any]										=	ids_tensor([self.batch_size] ,  self.type_sequence_label_size   )
       a						:List[str]										=	model(_UpperCAmelCase ,  attention_mask=_UpperCAmelCase ,  labels=_UpperCAmelCase   )
       self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_labels)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:Any										=	self.num_labels
       a						:Any										=	GPTNeoXForTokenClassification(_UpperCAmelCase   )
       model.to(_UpperCAmelCase   )
       model.eval()
       a						:List[Any]										=	model(_UpperCAmelCase ,  attention_mask=_UpperCAmelCase ,  labels=_UpperCAmelCase   )
       self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.num_labels)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:List[Any]										=	True
       a						:Union[str, Any]										=	GPTNeoXForCausalLM(config=_UpperCAmelCase   )
       model.to(_UpperCAmelCase   )
       model.eval()

       # first forward pass
       a						:Any										=	model(_UpperCAmelCase ,  attention_mask=_UpperCAmelCase ,  use_cache=_UpperCAmelCase   )
       a						:Union[str, Any]										=	outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       a						:int										=	ids_tensor((self.batch_size, 3) ,  config.vocab_size   )
       a						:Any										=	ids_tensor((self.batch_size, 3) ,  vocab_size=2   )

       # append to next input_ids and
       a						:int										=	torch.cat([input_ids, next_tokens] ,  dim=-1   )
       a						:Optional[int]										=	torch.cat([input_mask, next_mask] ,  dim=-1   )

       a						:Any										=	model(_UpperCAmelCase ,  attention_mask=_UpperCAmelCase ,  output_hidden_states=_UpperCAmelCase   )
       a						:List[str]										=	output_from_no_past['hidden_states'][0]
       a						:str										=	model(
           _UpperCAmelCase ,  attention_mask=_UpperCAmelCase ,  past_key_values=_UpperCAmelCase ,  output_hidden_states=_UpperCAmelCase ,  )['hidden_states'][0]

       # select random slice
       a						:int										=	ids_tensor((1,) ,  output_from_past.shape[-1]   ).item()
       a						:List[Any]										=	output_from_no_past[:, -3:, random_slice_idx].detach()
       a						:Union[str, Any]										=	output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(_UpperCAmelCase ,  _UpperCAmelCase ,  atol=1e-3   )   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Union[str, Any]										=	self.prepare_config_and_inputs()
       a						:Dict										=	config_and_inputs
       a						:List[Any]										=	{'input_ids': input_ids, 'attention_mask': input_mask}
       return config, inputs_dict







@require_torch
class 		_snake_case  (				_UpperCamelCase						,   _UpperCamelCase						,   _UpperCamelCase						,   unittest.TestCase							):
 SCREAMING_SNAKE_CASE__				=	(
     (
         GPTNeoXModel,
         GPTNeoXForCausalLM,
         GPTNeoXForQuestionAnswering,
         GPTNeoXForSequenceClassification,
         GPTNeoXForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 SCREAMING_SNAKE_CASE__				=	(GPTNeoXForCausalLM,) if is_torch_available() else ()
 SCREAMING_SNAKE_CASE__				=	(
     {
         'feature-extraction': GPTNeoXModel,
         'question-answering': GPTNeoXForQuestionAnswering,
         'text-classification': GPTNeoXForSequenceClassification,
         'text-generation': GPTNeoXForCausalLM,
         'token-classification': GPTNeoXForTokenClassification,
         'zero-shot': GPTNeoXForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 SCREAMING_SNAKE_CASE__				=	False
 SCREAMING_SNAKE_CASE__				=	False
 SCREAMING_SNAKE_CASE__				=	False
 SCREAMING_SNAKE_CASE__				=	False




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:int										=	GPTNeoXModelTester(self   )
       a						:Tuple										=	ConfigTester(self ,  config_class=_UpperCAmelCase ,  hidden_size=64 ,  num_attention_heads=8   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       self.config_tester.run_common_tests()




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Optional[int]										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(_UpperCAmelCase ,  _UpperCAmelCase ,  _UpperCAmelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:List[Any]										=	self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_model_as_decoder(_UpperCAmelCase ,  _UpperCAmelCase ,  _UpperCAmelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       # This regression test was failing with PyTorch < 1.3
       a						:str										=	self.model_tester.prepare_config_and_inputs_for_decoder()

       a						:Union[str, Any]										=	None

       self.model_tester.create_and_check_model_as_decoder(_UpperCAmelCase ,  _UpperCAmelCase ,  _UpperCAmelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Optional[int]										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_decoder_model_past_large_inputs(_UpperCAmelCase ,  _UpperCAmelCase ,  _UpperCAmelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:str										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_causal_lm(*_UpperCAmelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Tuple										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:str										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:List[str]										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase   )




 @unittest.skip(reason='''Feed forward chunking is not implemented'''   )
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       pass




 @parameterized.expand([('''linear''',), ('''dynamic''',)]   )
 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       a						:Tuple										=	self.model_tester.prepare_config_and_inputs_for_common()
       a						:Optional[Any]										=	ids_tensor([1, 10] ,  config.vocab_size   )
       a						:Any										=	ids_tensor([1, int(config.max_position_embeddings * 1.5   )] ,  config.vocab_size   )

       set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
       a						:Dict										=	GPTNeoXModel(_UpperCAmelCase   )
       original_model.to(_UpperCAmelCase   )
       original_model.eval()
       a						:Optional[int]										=	original_model(_UpperCAmelCase   ).last_hidden_state
       a						:str										=	original_model(_UpperCAmelCase   ).last_hidden_state

       set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
       a						:Dict										=	{'type': scaling_type, 'factor': 10.0}
       a						:int										=	GPTNeoXModel(_UpperCAmelCase   )
       scaled_model.to(_UpperCAmelCase   )
       scaled_model.eval()
       a						:Dict										=	scaled_model(_UpperCAmelCase   ).last_hidden_state
       a						:Tuple										=	scaled_model(_UpperCAmelCase   ).last_hidden_state

       # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
       # maximum sequence length, so the outputs for the short input should match.
       if scaling_type == "dynamic":
             self.assertTrue(torch.allclose(_UpperCAmelCase ,  _UpperCAmelCase ,  atol=1e-5   )   )
       else:
             self.assertFalse(torch.allclose(_UpperCAmelCase ,  _UpperCAmelCase ,  atol=1e-5   )   )

       # The output should be different for long inputs
       self.assertFalse(torch.allclose(_UpperCAmelCase ,  _UpperCAmelCase ,  atol=1e-5   )   )







@require_torch
class 		_snake_case  (				unittest.TestCase							):




 @slow
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Tuple										=	AutoTokenizer.from_pretrained('''EleutherAI/pythia-410m-deduped'''   )
       for checkpointing in [True, False]:
             a						:List[str]										=	GPTNeoXForCausalLM.from_pretrained('''EleutherAI/pythia-410m-deduped'''   )

             if checkpointing:
                   model.gradient_checkpointing_enable()
             else:
                   model.gradient_checkpointing_disable()
             model.to(_UpperCAmelCase   )

             a						:str										=	tokenizer('''My favorite food is''' ,  return_tensors='''pt'''   ).to(_UpperCAmelCase   )
             # The hub repo. is updated on 2023-04-04, resulting in poor outputs.
             # See: https://github.com/huggingface/transformers/pull/24193
             a						:Dict										=	'My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI\'m not sure'

             a						:int										=	model.generate(**_UpperCAmelCase ,  do_sample=_UpperCAmelCase ,  max_new_tokens=20   )
             a						:int										=	tokenizer.batch_decode(_UpperCAmelCase   )[0]

             self.assertEqual(_UpperCAmelCase ,  _UpperCAmelCase   )









from . import __version__

# Backward compatibility imports, to make sure all those objects can be found in file_utils
from .utils import (
    CLOUDFRONT_DISTRIB_PREFIX,
    CONFIG_NAME,
    DISABLE_TELEMETRY,
    DUMMY_INPUTS,
    DUMMY_MASK,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    FEATURE_EXTRACTOR_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    MODEL_CARD_NAME,
    MULTIPLE_CHOICE_DUMMY_INPUTS,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    SENTENCEPIECE_UNDERLINE,
    SPIECE_UNDERLINE,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    TORCH_FX_REQUIRED_VERSION,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
    ContextManagers,
    DummyObject,
    EntryNotFoundError,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    TensorType,
    _LazyModule,
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    cached_property,
    copy_func,
    default_cache_path,
    define_sagemaker_information,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    get_torch_version,
    has_file,
    http_user_agent,
    is_apex_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_datasets_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_librosa_available,
    is_offline_mode,
    is_onnx_available,
    is_pandas_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_tensor,
    is_tensorflow_probability_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    replace_return_docstrings,
    requires_backends,
    to_numpy,
    to_py_obj,
    torch_only_method,
)





