


import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         GPTNeoXForCausalLM,
         GPTNeoXForQuestionAnswering,
         GPTNeoXForSequenceClassification,
         GPTNeoXForTokenClassification,
         GPTNeoXModel,
     )







class 		_snake_case  :




 def __init__(		self ,  _lowerCamelCase ,  _lowerCamelCase=13 ,  _lowerCamelCase=7 ,  _lowerCamelCase=True ,  _lowerCamelCase=True ,  _lowerCamelCase=True ,  _lowerCamelCase=True ,  _lowerCamelCase=99 ,  _lowerCamelCase=64 ,  _lowerCamelCase=5 ,  _lowerCamelCase=4 ,  _lowerCamelCase=37 ,  _lowerCamelCase="gelu" ,  _lowerCamelCase=0.1 ,  _lowerCamelCase=0.1 ,  _lowerCamelCase=512 ,  _lowerCamelCase=16 ,  _lowerCamelCase=2 ,  _lowerCamelCase=0.02 ,  _lowerCamelCase=3 ,  _lowerCamelCase=4 ,  _lowerCamelCase=None ,  ):
       a						:Optional[Any]										=	parent
       a						:Any										=	batch_size
       a						:Any										=	seq_length
       a						:Tuple										=	is_training
       a						:Dict										=	use_input_mask
       a						:Dict										=	use_token_type_ids
       a						:int										=	use_labels
       a						:str										=	vocab_size
       a						:Optional[Any]										=	hidden_size
       a						:str										=	num_hidden_layers
       a						:int										=	num_attention_heads
       a						:Tuple										=	intermediate_size
       a						:Union[str, Any]										=	hidden_act
       a						:List[str]										=	hidden_dropout_prob
       a						:List[Any]										=	attention_probs_dropout_prob
       a						:Union[str, Any]										=	max_position_embeddings
       a						:Dict										=	type_vocab_size
       a						:Optional[int]										=	type_sequence_label_size
       a						:List[str]										=	initializer_range
       a						:Union[str, Any]										=	num_labels
       a						:Union[str, Any]										=	num_choices
       a						:List[str]										=	scope
       a						:Any										=	vocab_size - 1




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Tuple										=	ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size   )

       a						:List[Any]										=	None
       if self.use_input_mask:
             a						:str										=	random_attention_mask([self.batch_size, self.seq_length]   )

       a						:Tuple										=	None
       if self.use_labels:
             a						:Union[str, Any]										=	ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels   )

       a						:Union[str, Any]										=	self.get_config()

       return config, input_ids, input_mask, token_labels




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       return GPTNeoXConfig(
           vocab_size=self.vocab_size ,  hidden_size=self.hidden_size ,  num_hidden_layers=self.num_hidden_layers ,  num_attention_heads=self.num_attention_heads ,  intermediate_size=self.intermediate_size ,  hidden_act=self.hidden_act ,  hidden_dropout_prob=self.hidden_dropout_prob ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob ,  max_position_embeddings=self.max_position_embeddings ,  type_vocab_size=self.type_vocab_size ,  is_decoder=_lowerCamelCase ,  initializer_range=self.initializer_range ,  pad_token_id=self.pad_token_id ,  )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Union[str, Any]										=	self.prepare_config_and_inputs()

       a						:Optional[Any]										=	True

       return config, input_ids, input_mask, token_labels




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:Dict										=	GPTNeoXModel(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       a						:Dict										=	model(_lowerCamelCase ,  attention_mask=_lowerCamelCase   )
       a						:Any										=	model(_lowerCamelCase   )
       self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:str										=	True
       a						:Dict										=	GPTNeoXModel(_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       a						:int										=	model(_lowerCamelCase ,  attention_mask=_lowerCamelCase   )
       self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:Optional[Any]										=	GPTNeoXForCausalLM(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       a						:int										=	model(_lowerCamelCase ,  attention_mask=_lowerCamelCase ,  labels=_lowerCamelCase   )
       self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:List[Any]										=	self.num_labels
       a						:Union[str, Any]										=	GPTNeoXForQuestionAnswering(_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       a						:Any										=	model(_lowerCamelCase ,  attention_mask=_lowerCamelCase   )
       self.parent.assertEqual(result.start_logits.shape ,  (self.batch_size, self.seq_length)   )
       self.parent.assertEqual(result.end_logits.shape ,  (self.batch_size, self.seq_length)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:Any										=	self.num_labels
       a						:Optional[Any]										=	GPTNeoXForSequenceClassification(_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       a						:Dict										=	ids_tensor([self.batch_size] ,  self.type_sequence_label_size   )
       a						:Dict										=	model(_lowerCamelCase ,  attention_mask=_lowerCamelCase ,  labels=_lowerCamelCase   )
       self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_labels)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:Optional[int]										=	self.num_labels
       a						:int										=	GPTNeoXForTokenClassification(_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       a						:Optional[Any]										=	model(_lowerCamelCase ,  attention_mask=_lowerCamelCase ,  labels=_lowerCamelCase   )
       self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.num_labels)   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   ):
       a						:List[str]										=	True
       a						:Dict										=	GPTNeoXForCausalLM(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()

       # first forward pass
       a						:Optional[int]										=	model(_lowerCamelCase ,  attention_mask=_lowerCamelCase ,  use_cache=_lowerCamelCase   )
       a						:int										=	outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       a						:int										=	ids_tensor((self.batch_size, 3) ,  config.vocab_size   )
       a						:Any										=	ids_tensor((self.batch_size, 3) ,  vocab_size=2   )

       # append to next input_ids and
       a						:Union[str, Any]										=	torch.cat([input_ids, next_tokens] ,  dim=-1   )
       a						:Optional[Any]										=	torch.cat([input_mask, next_mask] ,  dim=-1   )

       a						:Union[str, Any]										=	model(_lowerCamelCase ,  attention_mask=_lowerCamelCase ,  output_hidden_states=_lowerCamelCase   )
       a						:Optional[int]										=	output_from_no_past["hidden_states"][0]
       a						:List[str]										=	model(
           _lowerCamelCase ,  attention_mask=_lowerCamelCase ,  past_key_values=_lowerCamelCase ,  output_hidden_states=_lowerCamelCase ,  )["hidden_states"][0]

       # select random slice
       a						:Dict										=	ids_tensor((1,) ,  output_from_past.shape[-1]   ).item()
       a						:str										=	output_from_no_past[:, -3:, random_slice_idx].detach()
       a						:List[Any]										=	output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(_lowerCamelCase ,  _lowerCamelCase ,  atol=1e-3   )   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:List[str]										=	self.prepare_config_and_inputs()
       a						:Optional[Any]										=	config_and_inputs
       a						:List[Any]										=	{"input_ids": input_ids, "attention_mask": input_mask}
       return config, inputs_dict







@require_torch
class 		_snake_case  (				a__						,   a__						,   a__						,   unittest.TestCase							):
 SCREAMING_SNAKE_CASE__				=	(
     (
         GPTNeoXModel,
         GPTNeoXForCausalLM,
         GPTNeoXForQuestionAnswering,
         GPTNeoXForSequenceClassification,
         GPTNeoXForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 SCREAMING_SNAKE_CASE__				=	(GPTNeoXForCausalLM,) if is_torch_available() else ()
 SCREAMING_SNAKE_CASE__				=	(
     {
         'feature-extraction': GPTNeoXModel,
         'question-answering': GPTNeoXForQuestionAnswering,
         'text-classification': GPTNeoXForSequenceClassification,
         'text-generation': GPTNeoXForCausalLM,
         'token-classification': GPTNeoXForTokenClassification,
         'zero-shot': GPTNeoXForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 SCREAMING_SNAKE_CASE__				=	False
 SCREAMING_SNAKE_CASE__				=	False
 SCREAMING_SNAKE_CASE__				=	False
 SCREAMING_SNAKE_CASE__				=	False




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:int										=	GPTNeoXModelTester(self   )
       a						:Tuple										=	ConfigTester(self ,  config_class=_lowerCamelCase ,  hidden_size=64 ,  num_attention_heads=8   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       self.config_tester.run_common_tests()




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:List[str]										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(_lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Dict										=	self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_model_as_decoder(_lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       # This regression test was failing with PyTorch < 1.3
       a						:Dict										=	self.model_tester.prepare_config_and_inputs_for_decoder()

       a						:int										=	None

       self.model_tester.create_and_check_model_as_decoder(_lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:List[Any]										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_decoder_model_past_large_inputs(_lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Dict										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_causal_lm(*_lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Union[str, Any]										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Any										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:List[Any]										=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase   )




 @unittest.skip(reason='''Feed forward chunking is not implemented'''   )
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       pass




 @parameterized.expand([('''linear''',), ('''dynamic''',)]   )
 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       a						:str										=	self.model_tester.prepare_config_and_inputs_for_common()
       a						:int										=	ids_tensor([1, 10] ,  config.vocab_size   )
       a						:Union[str, Any]										=	ids_tensor([1, int(config.max_position_embeddings * 1.5   )] ,  config.vocab_size   )

       set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
       a						:Any										=	GPTNeoXModel(_lowerCamelCase   )
       original_model.to(_lowerCamelCase   )
       original_model.eval()
       a						:Dict										=	original_model(_lowerCamelCase   ).last_hidden_state
       a						:Optional[int]										=	original_model(_lowerCamelCase   ).last_hidden_state

       set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
       a						:str										=	{"type": scaling_type, "factor": 10.0}
       a						:Optional[int]										=	GPTNeoXModel(_lowerCamelCase   )
       scaled_model.to(_lowerCamelCase   )
       scaled_model.eval()
       a						:List[Any]										=	scaled_model(_lowerCamelCase   ).last_hidden_state
       a						:Tuple										=	scaled_model(_lowerCamelCase   ).last_hidden_state

       # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
       # maximum sequence length, so the outputs for the short input should match.
       if scaling_type == "dynamic":
             self.assertTrue(torch.allclose(_lowerCamelCase ,  _lowerCamelCase ,  atol=1e-5   )   )
       else:
             self.assertFalse(torch.allclose(_lowerCamelCase ,  _lowerCamelCase ,  atol=1e-5   )   )

       # The output should be different for long inputs
       self.assertFalse(torch.allclose(_lowerCamelCase ,  _lowerCamelCase ,  atol=1e-5   )   )







@require_torch
class 		_snake_case  (				unittest.TestCase							):




 @slow
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:List[str]										=	AutoTokenizer.from_pretrained('''EleutherAI/pythia-410m-deduped'''   )
       for checkpointing in [True, False]:
             a						:Any										=	GPTNeoXForCausalLM.from_pretrained('''EleutherAI/pythia-410m-deduped'''   )

             if checkpointing:
                   model.gradient_checkpointing_enable()
             else:
                   model.gradient_checkpointing_disable()
             model.to(_lowerCamelCase   )

             a						:Tuple										=	tokenizer('''My favorite food is''' ,  return_tensors='''pt'''   ).to(_lowerCamelCase   )
             # The hub repo. is updated on 2023-04-04, resulting in poor outputs.
             # See: https://github.com/huggingface/transformers/pull/24193
             a						:Optional[int]										=	"My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI'm not sure"

             a						:Dict										=	model.generate(**_lowerCamelCase ,  do_sample=_lowerCamelCase ,  max_new_tokens=20   )
             a						:int										=	tokenizer.batch_decode(_lowerCamelCase   )[0]

             self.assertEqual(_lowerCamelCase ,  _lowerCamelCase   )









import math
def     __lowerCamelCase						(			UpperCAmelCase_						:		float			,  UpperCAmelCase_						:		float						):




      """simple docstring"""





      return math.pow(UpperCAmelCase_			,  2						) - a
def     __lowerCamelCase						(			UpperCAmelCase_						:		float						):




      """simple docstring"""





      return 2 * x
def     __lowerCamelCase						(			UpperCAmelCase_						:		float						):




      """simple docstring"""





      a						:int										=	2.0

      while start <= a:
            a						:int										=	math.pow(UpperCAmelCase_			,  2						)

      return start
def     __lowerCamelCase						(			UpperCAmelCase_						:		float			,  UpperCAmelCase_						:		int = 9999			,  UpperCAmelCase_						:		float = 0.00000000000001						):




      """simple docstring"""





      if a < 0:
            raise ValueError('''math domain error'''						)

      a						:List[Any]										=	get_initial_point(UpperCAmelCase_						)

      for _ in range(UpperCAmelCase_						):
            a						:Optional[int]										=	value
            a						:int										=	value - fx(UpperCAmelCase_			,  UpperCAmelCase_						) / fx_derivative(UpperCAmelCase_						)
            if abs(prev_value - value						) < tolerance:
                  return value

      return value


if __name__ == "__main__":
     from doctest import testmod

     testmod()





