





import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      GPTNeoXForCausalLM,
      GPTNeoXForQuestionAnswering,
      GPTNeoXForSequenceClassification,
      GPTNeoXForTokenClassification,
      GPTNeoXModel,
  )







class 	__lowerCAmelCase      :





   def __init__(			self       ,      snake_case       ,      snake_case=13       ,      snake_case=7       ,      snake_case=True       ,      snake_case=True       ,      snake_case=True       ,      snake_case=True       ,      snake_case=99       ,      snake_case=64       ,      snake_case=5       ,      snake_case=4       ,      snake_case=37       ,      snake_case="gelu"       ,      snake_case=0.1       ,      snake_case=0.1       ,      snake_case=512       ,      snake_case=16       ,      snake_case=2       ,      snake_case=0.02       ,      snake_case=3       ,      snake_case=4       ,      snake_case=None       ,      )					->	Any:
      """simple docstring"""





      a__						:    str   		=	parent
      a__						:    List[Any]   		=	batch_size
      a__						:    List[Any]   		=	seq_length
      a__						:    List[Any]   		=	is_training
      a__						:    Optional[int]   		=	use_input_mask
      a__						:    Any   		=	use_token_type_ids
      a__						:    Tuple   		=	use_labels
      a__						:    Dict   		=	vocab_size
      a__						:    List[str]   		=	hidden_size
      a__						:    Tuple   		=	num_hidden_layers
      a__						:    List[Any]   		=	num_attention_heads
      a__						:    Optional[int]   		=	intermediate_size
      a__						:    List[Any]   		=	hidden_act
      a__						:    List[str]   		=	hidden_dropout_prob
      a__						:    List[str]   		=	attention_probs_dropout_prob
      a__						:    Any   		=	max_position_embeddings
      a__						:    Any   		=	type_vocab_size
      a__						:    str   		=	type_sequence_label_size
      a__						:    Tuple   		=	initializer_range
      a__						:    int   		=	num_labels
      a__						:    Any   		=	num_choices
      a__						:    Optional[Any]   		=	scope
      a__						:    List[str]   		=	vocab_size - 1





   def 						_snake_case			(			self   )					->	List[str]:
      """simple docstring"""





      a__						:    List[str]   		=	ids_tensor([self.batch_size, self.seq_length]       ,      self.vocab_size   )

      a__						:    Any   		=	None
      if self.use_input_mask:
         a__						:    Any   		=	random_attention_mask([self.batch_size, self.seq_length]   )

      a__						:    str   		=	None
      if self.use_labels:
         a__						:    Tuple   		=	ids_tensor([self.batch_size, self.seq_length]       ,      self.num_labels   )

      a__						:    Tuple   		=	self.get_config()

      return config, input_ids, input_mask, token_labels





   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      return GPTNeoXConfig(
          vocab_size=self.vocab_size       ,      hidden_size=self.hidden_size       ,      num_hidden_layers=self.num_hidden_layers       ,      num_attention_heads=self.num_attention_heads       ,      intermediate_size=self.intermediate_size       ,      hidden_act=self.hidden_act       ,      hidden_dropout_prob=self.hidden_dropout_prob       ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,      max_position_embeddings=self.max_position_embeddings       ,      type_vocab_size=self.type_vocab_size       ,      is_decoder=snake_case       ,      initializer_range=self.initializer_range       ,      pad_token_id=self.pad_token_id       ,      )





   def 						_snake_case			(			self   )					->	Tuple:
      """simple docstring"""





      a__		,  a__		,  a__		,  a__						:    int   		=	self.prepare_config_and_inputs()

      a__						:    Dict   		=	True

      return config, input_ids, input_mask, token_labels





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case       ,      snake_case   )					->	Optional[Any]:
      """simple docstring"""





      a__						:    Any   		=	GPTNeoXModel(config=snake_case   )
      model.to(snake_case   )
      model.eval()
      a__						:    Tuple   		=	model(snake_case       ,      attention_mask=snake_case   )
      a__						:    Dict   		=	model(snake_case   )
      self.parent.assertEqual(result.last_hidden_state.shape       ,      (self.batch_size, self.seq_length, self.hidden_size)   )





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case       ,      snake_case   )					->	List[str]:
      """simple docstring"""





      a__						:    List[str]   		=	True
      a__						:    Optional[Any]   		=	GPTNeoXModel(snake_case   )
      model.to(snake_case   )
      model.eval()
      a__						:    Optional[int]   		=	model(snake_case       ,      attention_mask=snake_case   )
      self.parent.assertEqual(result.last_hidden_state.shape       ,      (self.batch_size, self.seq_length, self.hidden_size)   )





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case       ,      snake_case       ,      snake_case   )					->	str:
      """simple docstring"""





      a__						:    List[Any]   		=	GPTNeoXForCausalLM(config=snake_case   )
      model.to(snake_case   )
      model.eval()
      a__						:    str   		=	model(snake_case       ,      attention_mask=snake_case       ,      labels=snake_case   )
      self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.seq_length, self.vocab_size)   )





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case       ,      snake_case       ,      snake_case   )					->	int:
      """simple docstring"""





      a__						:    Union[str, Any]   		=	self.num_labels
      a__						:    int   		=	GPTNeoXForQuestionAnswering(snake_case   )
      model.to(snake_case   )
      model.eval()
      a__						:    Any   		=	model(snake_case       ,      attention_mask=snake_case   )
      self.parent.assertEqual(result.start_logits.shape       ,      (self.batch_size, self.seq_length)   )
      self.parent.assertEqual(result.end_logits.shape       ,      (self.batch_size, self.seq_length)   )





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case       ,      snake_case       ,      snake_case   )					->	Dict:
      """simple docstring"""





      a__						:    Optional[int]   		=	self.num_labels
      a__						:    List[str]   		=	GPTNeoXForSequenceClassification(snake_case   )
      model.to(snake_case   )
      model.eval()
      a__						:    Dict   		=	ids_tensor([self.batch_size]       ,      self.type_sequence_label_size   )
      a__						:    Tuple   		=	model(snake_case       ,      attention_mask=snake_case       ,      labels=snake_case   )
      self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.num_labels)   )





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case       ,      snake_case       ,      snake_case   )					->	Optional[Any]:
      """simple docstring"""





      a__						:    str   		=	self.num_labels
      a__						:    List[str]   		=	GPTNeoXForTokenClassification(snake_case   )
      model.to(snake_case   )
      model.eval()
      a__						:    List[Any]   		=	model(snake_case       ,      attention_mask=snake_case       ,      labels=snake_case   )
      self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.seq_length, self.num_labels)   )





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case       ,      snake_case   )					->	Union[str, Any]:
      """simple docstring"""





      a__						:    Tuple   		=	True
      a__						:    Tuple   		=	GPTNeoXForCausalLM(config=snake_case   )
      model.to(snake_case   )
      model.eval()

      # first forward pass
      a__						:    List[Any]   		=	model(snake_case       ,      attention_mask=snake_case       ,      use_cache=snake_case   )
      a__						:    Dict   		=	outputs.past_key_values

      # create hypothetical multiple next token and extent to next_input_ids
      a__						:    Optional[int]   		=	ids_tensor((self.batch_size, 3)       ,      config.vocab_size   )
      a__						:    Any   		=	ids_tensor((self.batch_size, 3)       ,      vocab_size=2   )

      # append to next input_ids and
      a__						:    Optional[Any]   		=	torch.cat([input_ids, next_tokens]       ,      dim=-1   )
      a__						:    Optional[int]   		=	torch.cat([input_mask, next_mask]       ,      dim=-1   )

      a__						:    Any   		=	model(snake_case       ,      attention_mask=snake_case       ,      output_hidden_states=snake_case   )
      a__						:    int   		=	output_from_no_past["hidden_states"][0]
      a__						:    Optional[int]   		=	model(
          snake_case       ,      attention_mask=snake_case       ,      past_key_values=snake_case       ,      output_hidden_states=snake_case       ,      )["hidden_states"][0]

      # select random slice
      a__						:    str   		=	ids_tensor((1,)       ,      output_from_past.shape[-1]   ).item()
      a__						:    Tuple   		=	output_from_no_past[:, -3:, random_slice_idx].detach()
      a__						:    Tuple   		=	output_from_past[:, :, random_slice_idx].detach()

      self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

      # test that outputs are equal for slice
      self.parent.assertTrue(torch.allclose(snake_case       ,      snake_case       ,      atol=1E-3   )   )





   def 						_snake_case			(			self   )					->	Optional[Any]:
      """simple docstring"""





      a__						:    Optional[int]   		=	self.prepare_config_and_inputs()
      a__		,  a__		,  a__		,  a__						:    List[str]   		=	config_and_inputs
      a__						:    int   		=	{"input_ids": input_ids, "attention_mask": input_mask}
      return config, inputs_dict







@require_torch
class 	__lowerCAmelCase      (       _UpperCamelCase		,_UpperCamelCase		,_UpperCamelCase		,unittest.TestCase	):
   _UpperCamelCase			:    Dict  		=					(
       (
           GPTNeoXModel,
           GPTNeoXForCausalLM,
           GPTNeoXForQuestionAnswering,
           GPTNeoXForSequenceClassification,
           GPTNeoXForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   _UpperCamelCase			:    int  		=					(GPTNeoXForCausalLM,) if is_torch_available() else ()
   _UpperCamelCase			:    Union[str, Any]  		=					(
       {
           """feature-extraction""": GPTNeoXModel,
           """question-answering""": GPTNeoXForQuestionAnswering,
           """text-classification""": GPTNeoXForSequenceClassification,
           """text-generation""": GPTNeoXForCausalLM,
           """token-classification""": GPTNeoXForTokenClassification,
           """zero-shot""": GPTNeoXForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   _UpperCamelCase			:    List[str]  		=					False
   _UpperCamelCase			:    List[str]  		=					False
   _UpperCamelCase			:    Optional[int]  		=					False
   _UpperCamelCase			:    Any  		=					False





   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      a__						:    Optional[int]   		=	GPTNeoXModelTester(self   )
      a__						:    Optional[Any]   		=	ConfigTester(self       ,      config_class=snake_case       ,      hidden_size=64       ,      num_attention_heads=8   )





   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      self.config_tester.run_common_tests()





   def 						_snake_case			(			self   )					->	Union[str, Any]:
      """simple docstring"""





      a__		,  a__		,  a__		,  a__						:    Union[str, Any]   		=	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(snake_case       ,      snake_case       ,      snake_case   )





   def 						_snake_case			(			self   )					->	Dict:
      """simple docstring"""





      a__		,  a__		,  a__		,  a__						:    Dict   		=	self.model_tester.prepare_config_and_inputs_for_decoder()
      self.model_tester.create_and_check_model_as_decoder(snake_case       ,      snake_case       ,      snake_case   )





   def 						_snake_case			(			self   )					->	Tuple:
      """simple docstring"""





      a__		,  a__		,  a__		,  a__						:    Optional[int]   		=	self.model_tester.prepare_config_and_inputs_for_decoder()

      a__						:    List[str]   		=	None

      self.model_tester.create_and_check_model_as_decoder(snake_case       ,      snake_case       ,      snake_case   )





   def 						_snake_case			(			self   )					->	List[Any]:
      """simple docstring"""





      a__		,  a__		,  a__		,  a__						:    str   		=	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_decoder_model_past_large_inputs(snake_case       ,      snake_case       ,      snake_case   )





   def 						_snake_case			(			self   )					->	List[Any]:
      """simple docstring"""





      a__						:    int   		=	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_causal_lm(*snake_case   )





   def 						_snake_case			(			self   )					->	Dict:
      """simple docstring"""





      a__						:    Dict   		=	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*snake_case   )





   def 						_snake_case			(			self   )					->	str:
      """simple docstring"""





      a__						:    List[str]   		=	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*snake_case   )





   def 						_snake_case			(			self   )					->	Dict:
      """simple docstring"""





      a__						:    Optional[int]   		=	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*snake_case   )





   @unittest.skip(reason="Feed forward chunking is not implemented"   )
   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      pass





   @parameterized.expand([("linear",), ("dynamic",)]   )
   def 						_snake_case			(			self       ,      snake_case   )					->	Dict:
      """simple docstring"""





      a__		,  a__						:    Tuple   		=	self.model_tester.prepare_config_and_inputs_for_common()
      a__						:    Optional[Any]   		=	ids_tensor([1, 10]       ,      config.vocab_size   )
      a__						:    Union[str, Any]   		=	ids_tensor([1, int(config.max_position_embeddings * 1.5   )]       ,      config.vocab_size   )

      set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
      a__						:    List[str]   		=	GPTNeoXModel(snake_case   )
      original_model.to(snake_case   )
      original_model.eval()
      a__						:    List[str]   		=	original_model(snake_case   ).last_hidden_state
      a__						:    str   		=	original_model(snake_case   ).last_hidden_state

      set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
      a__						:    str   		=	{"type": scaling_type, "factor": 10.0}
      a__						:    Any   		=	GPTNeoXModel(snake_case   )
      scaled_model.to(snake_case   )
      scaled_model.eval()
      a__						:    Optional[int]   		=	scaled_model(snake_case   ).last_hidden_state
      a__						:    Tuple   		=	scaled_model(snake_case   ).last_hidden_state

      # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
      # maximum sequence length, so the outputs for the short input should match.
      if scaling_type == "dynamic":
         self.assertTrue(torch.allclose(snake_case       ,      snake_case       ,      atol=1E-5   )   )
      else:
         self.assertFalse(torch.allclose(snake_case       ,      snake_case       ,      atol=1E-5   )   )

      # The output should be different for long inputs
      self.assertFalse(torch.allclose(snake_case       ,      snake_case       ,      atol=1E-5   )   )







@require_torch
class 	__lowerCAmelCase      (       unittest.TestCase	):





   @slow
   def 						_snake_case			(			self   )					->	List[Any]:
      """simple docstring"""





      a__						:    Tuple   		=	AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped"   )
      for checkpointing in [True, False]:
         a__						:    int   		=	GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped"   )

         if checkpointing:
            model.gradient_checkpointing_enable()
         else:
            model.gradient_checkpointing_disable()
         model.to(snake_case   )

         a__						:    Optional[int]   		=	tokenizer("My favorite food is"       ,      return_tensors="pt"   ).to(snake_case   )
         # The hub repo. is updated on 2023-04-04, resulting in poor outputs.
         # See: https://github.com/huggingface/transformers/pull/24193
         a__						:    Optional[Any]   		=	"My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI'm not sure"

         a__						:    str   		=	model.generate(**snake_case       ,      do_sample=snake_case       ,      max_new_tokens=20   )
         a__						:    int   		=	tokenizer.batch_decode(snake_case   )[0]

         self.assertEqual(snake_case       ,      snake_case   )







import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class 						__lowerCAmelCase      (      __magic_name__ ,			unittest.TestCase					):
							UpperCamelCase__   	=    TransfoXLTokenizer
							UpperCamelCase__   	=    False
							UpperCamelCase__   	=    False
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								super().setUp()

								a									=				[
								    """<unk>""",
								    """[CLS]""",
								    """[SEP]""",
								    """want""",
								    """unwanted""",
								    """wa""",
								    """un""",
								    """running""",
								    """,""",
								    """low""",
								    """l""",
								]
								a									=				os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["""vocab_file"""]							)
								with open(self.vocab_file						, """w"""						, encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)
							def  lowerCamelCase__				(		self		:List[Any]						, **__magic_name__		:Optional[int]							):







								'''simple docstring'''







								a									=				True
								return TransfoXLTokenizer.from_pretrained(self.tmpdirname						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:Union[str, Any]						, __magic_name__		:int							):







								'''simple docstring'''







								a									=				"""<unk> UNwanted , running"""
								a									=				"""<unk> unwanted, running"""
								return input_text, output_text
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(vocab_file=self.vocab_file						, lower_case=__magic_name__							)

								a									=				tokenizer.tokenize("""<unk> UNwanted , running"""							)
								self.assertListEqual(__magic_name__						, ["""<unk>""", """unwanted""", """,""", """running"""]							)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__							)						, [0, 4, 8, 7]							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)						, ["""hello""", """!""", """how""", """are""", """you""", """?"""]							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)						, ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)
								a									=				"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
								a									=				[
								    """Hello""",
								    """(""",
								    """bracket""",
								    """)""",
								    """and""",
								    """side""",
								    """@-@""",
								    """scrolled""",
								    """[""",
								    """and""",
								    """]""",
								    """Henry""",
								    """'s""",
								    """$""",
								    """5""",
								    """@,@""",
								    """000""",
								    """with""",
								    """3""",
								    """@.@""",
								    """34""",
								    """m""",
								    """.""",
								    """What""",
								    """'s""",
								    """up""",
								    """!""",
								    """?""",
								]

								self.assertListEqual(tokenizer.tokenize(__magic_name__							)						, __magic_name__							)

								self.assertEqual(tokenizer.convert_tokens_to_string(__magic_name__							)						, __magic_name__							)





							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				self.get_tokenizer()
								a									=				len(__magic_name__							)

								tokenizer.add_tokens(["""new1""", """new2"""]							)
								tokenizer.move_added_token("""new1"""						, 1							)

								# Check that moved token is not copied (duplicate)
								self.assertEqual(len(__magic_name__							)						, original_len + 2							)
								# Check that token is moved to specified id
								self.assertEqual(tokenizer.encode("""new1"""							)						, [1]							)
								self.assertEqual(tokenizer.decode([1]							)						, """new1"""							)

