




import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    GPTNeoXForCausalLM,
		    GPTNeoXForQuestionAnswering,
		    GPTNeoXForSequenceClassification,
		    GPTNeoXForTokenClassification,
		    GPTNeoXModel,
		)

class 						__lowerCAmelCase      :
							def __init__(		self		:List[Any]						, __magic_name__		:Tuple						, __magic_name__		:Any=13						, __magic_name__		:Any=7						, __magic_name__		:int=True						, __magic_name__		:Tuple=True						, __magic_name__		:Union[str, Any]=True						, __magic_name__		:Any=True						, __magic_name__		:Optional[int]=99						, __magic_name__		:Union[str, Any]=64						, __magic_name__		:Optional[int]=5						, __magic_name__		:List[str]=4						, __magic_name__		:str=37						, __magic_name__		:Optional[int]="gelu"						, __magic_name__		:List[Any]=0.1						, __magic_name__		:Dict=0.1						, __magic_name__		:int=512						, __magic_name__		:Optional[int]=16						, __magic_name__		:Any=2						, __magic_name__		:Tuple=0.02						, __magic_name__		:str=3						, __magic_name__		:int=4						, __magic_name__		:int=None						, ):







								'''simple docstring'''







								a									=				parent
								a									=				batch_size
								a									=				seq_length
								a									=				is_training
								a									=				use_input_mask
								a									=				use_token_type_ids
								a									=				use_labels
								a									=				vocab_size
								a									=				hidden_size
								a									=				num_hidden_layers
								a									=				num_attention_heads
								a									=				intermediate_size
								a									=				hidden_act
								a									=				hidden_dropout_prob
								a									=				attention_probs_dropout_prob
								a									=				max_position_embeddings
								a									=				type_vocab_size
								a									=				type_sequence_label_size
								a									=				initializer_range
								a									=				num_labels
								a									=				num_choices
								a									=				scope
								a									=				vocab_size - 1
							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				ids_tensor([self.batch_size, self.seq_length]						, self.vocab_size							)

								a									=				None
								if self.use_input_mask:
									a									=				random_attention_mask([self.batch_size, self.seq_length]							)

								a									=				None
								if self.use_labels:
									a									=				ids_tensor([self.batch_size, self.seq_length]						, self.num_labels							)

								a									=				self.get_config()

								return config, input_ids, input_mask, token_labels
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								return GPTNeoXConfig(
								    vocab_size=self.vocab_size						, hidden_size=self.hidden_size						, num_hidden_layers=self.num_hidden_layers						, num_attention_heads=self.num_attention_heads						, intermediate_size=self.intermediate_size						, hidden_act=self.hidden_act						, hidden_dropout_prob=self.hidden_dropout_prob						, attention_probs_dropout_prob=self.attention_probs_dropout_prob						, max_position_embeddings=self.max_position_embeddings						, type_vocab_size=self.type_vocab_size						, is_decoder=__magic_name__						, initializer_range=self.initializer_range						, pad_token_id=self.pad_token_id						, )
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a							,					a							,					a							,					a									=				self.prepare_config_and_inputs()

								a									=				True

								return config, input_ids, input_mask, token_labels
							def  lowerCamelCase__				(		self		:str						, __magic_name__		:int						, __magic_name__		:Union[str, Any]						, __magic_name__		:Optional[Any]							):







								'''simple docstring'''







								a									=				GPTNeoXModel(config=__magic_name__							)
								model.to(__magic_name__							)
								model.eval()
								a									=				model(__magic_name__						, attention_mask=__magic_name__							)
								a									=				model(__magic_name__							)
								self.parent.assertEqual(result.last_hidden_state.shape						, (self.batch_size, self.seq_length, self.hidden_size)							)
							def  lowerCamelCase__				(		self		:Tuple						, __magic_name__		:Optional[Any]						, __magic_name__		:Union[str, Any]						, __magic_name__		:Optional[Any]							):







								'''simple docstring'''







								a									=				True
								a									=				GPTNeoXModel(__magic_name__							)
								model.to(__magic_name__							)
								model.eval()
								a									=				model(__magic_name__						, attention_mask=__magic_name__							)
								self.parent.assertEqual(result.last_hidden_state.shape						, (self.batch_size, self.seq_length, self.hidden_size)							)
							def  lowerCamelCase__				(		self		:int						, __magic_name__		:int						, __magic_name__		:List[str]						, __magic_name__		:List[Any]						, __magic_name__		:Union[str, Any]							):







								'''simple docstring'''







								a									=				GPTNeoXForCausalLM(config=__magic_name__							)
								model.to(__magic_name__							)
								model.eval()
								a									=				model(__magic_name__						, attention_mask=__magic_name__						, labels=__magic_name__							)
								self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length, self.vocab_size)							)
							def  lowerCamelCase__				(		self		:Any						, __magic_name__		:Union[str, Any]						, __magic_name__		:str						, __magic_name__		:List[Any]						, __magic_name__		:List[Any]							):







								'''simple docstring'''







								a									=				self.num_labels
								a									=				GPTNeoXForQuestionAnswering(__magic_name__							)
								model.to(__magic_name__							)
								model.eval()
								a									=				model(__magic_name__						, attention_mask=__magic_name__							)
								self.parent.assertEqual(result.start_logits.shape						, (self.batch_size, self.seq_length)							)
								self.parent.assertEqual(result.end_logits.shape						, (self.batch_size, self.seq_length)							)
							def  lowerCamelCase__				(		self		:Tuple						, __magic_name__		:List[str]						, __magic_name__		:int						, __magic_name__		:Union[str, Any]						, __magic_name__		:str							):







								'''simple docstring'''







								a									=				self.num_labels
								a									=				GPTNeoXForSequenceClassification(__magic_name__							)
								model.to(__magic_name__							)
								model.eval()
								a									=				ids_tensor([self.batch_size]						, self.type_sequence_label_size							)
								a									=				model(__magic_name__						, attention_mask=__magic_name__						, labels=__magic_name__							)
								self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.num_labels)							)
							def  lowerCamelCase__				(		self		:Any						, __magic_name__		:Optional[Any]						, __magic_name__		:Any						, __magic_name__		:str						, __magic_name__		:Optional[Any]							):







								'''simple docstring'''







								a									=				self.num_labels
								a									=				GPTNeoXForTokenClassification(__magic_name__							)
								model.to(__magic_name__							)
								model.eval()
								a									=				model(__magic_name__						, attention_mask=__magic_name__						, labels=__magic_name__							)
								self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length, self.num_labels)							)
							def  lowerCamelCase__				(		self		:int						, __magic_name__		:Optional[Any]						, __magic_name__		:int						, __magic_name__		:List[str]							):







								'''simple docstring'''







								a									=				True
								a									=				GPTNeoXForCausalLM(config=__magic_name__							)
								model.to(__magic_name__							)
								model.eval()

								# first forward pass
								a									=				model(__magic_name__						, attention_mask=__magic_name__						, use_cache=__magic_name__							)
								a									=				outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								a									=				ids_tensor((self.batch_size, 3)						, config.vocab_size							)
								a									=				ids_tensor((self.batch_size, 3)						, vocab_size=2							)

								# append to next input_ids and
								a									=				torch.cat([input_ids, next_tokens]						, dim=-1							)
								a									=				torch.cat([input_mask, next_mask]						, dim=-1							)

								a									=				model(__magic_name__						, attention_mask=__magic_name__						, output_hidden_states=__magic_name__							)
								a									=				output_from_no_past["""hidden_states"""][0]
								a									=				model(
								    __magic_name__						, attention_mask=__magic_name__						, past_key_values=__magic_name__						, output_hidden_states=__magic_name__						, )["""hidden_states"""][0]

								# select random slice
								a									=				ids_tensor((1,)						, output_from_past.shape[-1]							).item()
								a									=				output_from_no_past[:, -3:, random_slice_idx].detach()
								a									=				output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]							)

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(__magic_name__						, __magic_name__						, atol=1E-3							)							)





							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				self.prepare_config_and_inputs()
								a							,					a							,					a							,					a									=				config_and_inputs
								a									=				{"""input_ids""": input_ids, """attention_mask""": input_mask}
								return config, inputs_dict







@require_torch
class 						__lowerCAmelCase      (      __magic_name__ ,			__magic_name__ ,			__magic_name__ ,			unittest.TestCase					):
							UpperCamelCase__   	=    (
							    (
							        GPTNeoXModel,
							        GPTNeoXForCausalLM,
							        GPTNeoXForQuestionAnswering,
							        GPTNeoXForSequenceClassification,
							        GPTNeoXForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							UpperCamelCase__   	=    (GPTNeoXForCausalLM,) if is_torch_available() else ()
							UpperCamelCase__   	=    (
							    {
							        '''feature-extraction''': GPTNeoXModel,
							        '''question-answering''': GPTNeoXForQuestionAnswering,
							        '''text-classification''': GPTNeoXForSequenceClassification,
							        '''text-generation''': GPTNeoXForCausalLM,
							        '''token-classification''': GPTNeoXForTokenClassification,
							        '''zero-shot''': GPTNeoXForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							UpperCamelCase__   	=    False
							UpperCamelCase__   	=    False
							UpperCamelCase__   	=    False
							UpperCamelCase__   	=    False
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				GPTNeoXModelTester(self							)
								a									=				ConfigTester(self						, config_class=__magic_name__						, hidden_size=64						, num_attention_heads=8							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								self.config_tester.run_common_tests()
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								a							,					a							,					a							,					a									=				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(__magic_name__						, __magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a							,					a							,					a							,					a									=				self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(__magic_name__						, __magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								a							,					a							,					a							,					a									=				self.model_tester.prepare_config_and_inputs_for_decoder()

								a									=				None

								self.model_tester.create_and_check_model_as_decoder(__magic_name__						, __magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a							,					a							,					a							,					a									=				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_decoder_model_past_large_inputs(__magic_name__						, __magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_causal_lm(*__magic_name__							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*__magic_name__							)
							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*__magic_name__							)
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*__magic_name__							)
							@unittest.skip(reason="""Feed forward chunking is not implemented"""							)
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								pass





							@parameterized.expand([("""linear""",), ("""dynamic""",)]							)
							def  lowerCamelCase__				(		self		:Optional[Any]						, __magic_name__		:Dict							):







								'''simple docstring'''







								a							,					a									=				self.model_tester.prepare_config_and_inputs_for_common()
								a									=				ids_tensor([1, 10]						, config.vocab_size							)
								a									=				ids_tensor([1, int(config.max_position_embeddings * 1.5							)]						, config.vocab_size							)

								set_seed(42							)  # Fixed seed at init time so the two models get the same random weights
								a									=				GPTNeoXModel(__magic_name__							)
								original_model.to(__magic_name__							)
								original_model.eval()
								a									=				original_model(__magic_name__							).last_hidden_state
								a									=				original_model(__magic_name__							).last_hidden_state

								set_seed(42							)  # Fixed seed at init time so the two models get the same random weights
								a									=				{"""type""": scaling_type, """factor""": 10.0}
								a									=				GPTNeoXModel(__magic_name__							)
								scaled_model.to(__magic_name__							)
								scaled_model.eval()
								a									=				scaled_model(__magic_name__							).last_hidden_state
								a									=				scaled_model(__magic_name__							).last_hidden_state

								# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
								# maximum sequence length, so the outputs for the short input should match.
								if scaling_type == "dynamic":
									self.assertTrue(torch.allclose(__magic_name__						, __magic_name__						, atol=1E-5							)							)
								else:
									self.assertFalse(torch.allclose(__magic_name__						, __magic_name__						, atol=1E-5							)							)

								# The output should be different for long inputs
								self.assertFalse(torch.allclose(__magic_name__						, __magic_name__						, atol=1E-5							)							)







@require_torch
class 						__lowerCAmelCase      (      unittest.TestCase					):
							@slow
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""EleutherAI/pythia-410m-deduped"""							)
								for checkpointing in [True, False]:
									a									=				GPTNeoXForCausalLM.from_pretrained("""EleutherAI/pythia-410m-deduped"""							)

									if checkpointing:
										model.gradient_checkpointing_enable()
									else:
										model.gradient_checkpointing_disable()
									model.to(__magic_name__							)

									a									=				tokenizer("""My favorite food is"""						, return_tensors="""pt"""							).to(__magic_name__							)
									# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
									# See: https://github.com/huggingface/transformers/pull/24193
									a									=				"""My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI'm not sure"""

									a									=				model.generate(**__magic_name__						, do_sample=__magic_name__						, max_new_tokens=20							)
									a									=				tokenizer.batch_decode(__magic_name__							)[0]

									self.assertEqual(__magic_name__						, __magic_name__							)







import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class 						__lowerCAmelCase      (      __magic_name__ ,			unittest.TestCase					):
							UpperCamelCase__   	=    TransfoXLTokenizer
							UpperCamelCase__   	=    False
							UpperCamelCase__   	=    False
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								super().setUp()

								a									=				[
								    """<unk>""",
								    """[CLS]""",
								    """[SEP]""",
								    """want""",
								    """unwanted""",
								    """wa""",
								    """un""",
								    """running""",
								    """,""",
								    """low""",
								    """l""",
								]
								a									=				os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["""vocab_file"""]							)
								with open(self.vocab_file						, """w"""						, encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)
							def  lowerCamelCase__				(		self		:List[Any]						, **__magic_name__		:Optional[int]							):







								'''simple docstring'''







								a									=				True
								return TransfoXLTokenizer.from_pretrained(self.tmpdirname						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:Union[str, Any]						, __magic_name__		:int							):







								'''simple docstring'''







								a									=				"""<unk> UNwanted , running"""
								a									=				"""<unk> unwanted, running"""
								return input_text, output_text
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(vocab_file=self.vocab_file						, lower_case=__magic_name__							)

								a									=				tokenizer.tokenize("""<unk> UNwanted , running"""							)
								self.assertListEqual(__magic_name__						, ["""<unk>""", """unwanted""", """,""", """running"""]							)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__							)						, [0, 4, 8, 7]							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)						, ["""hello""", """!""", """how""", """are""", """you""", """?"""]							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)						, ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)
								a									=				"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
								a									=				[
								    """Hello""",
								    """(""",
								    """bracket""",
								    """)""",
								    """and""",
								    """side""",
								    """@-@""",
								    """scrolled""",
								    """[""",
								    """and""",
								    """]""",
								    """Henry""",
								    """'s""",
								    """$""",
								    """5""",
								    """@,@""",
								    """000""",
								    """with""",
								    """3""",
								    """@.@""",
								    """34""",
								    """m""",
								    """.""",
								    """What""",
								    """'s""",
								    """up""",
								    """!""",
								    """?""",
								]

								self.assertListEqual(tokenizer.tokenize(__magic_name__							)						, __magic_name__							)

								self.assertEqual(tokenizer.convert_tokens_to_string(__magic_name__							)						, __magic_name__							)





							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				self.get_tokenizer()
								a									=				len(__magic_name__							)

								tokenizer.add_tokens(["""new1""", """new2"""]							)
								tokenizer.move_added_token("""new1"""						, 1							)

								# Check that moved token is not copied (duplicate)
								self.assertEqual(len(__magic_name__							)						, original_len + 2							)
								# Check that token is moved to specified id
								self.assertEqual(tokenizer.encode("""new1"""							)						, [1]							)
								self.assertEqual(tokenizer.decode([1]							)						, """new1"""							)

