


'''simple docstring'''



import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    GPTNeoXForCausalLM,
		    GPTNeoXForQuestionAnswering,
		    GPTNeoXForSequenceClassification,
		    GPTNeoXForTokenClassification,
		    GPTNeoXModel,
		)


class 		_snake_case					:




					def __init__(      self							,							_lowerCamelCase							,							_lowerCamelCase=13							,							_lowerCamelCase=7							,							_lowerCamelCase=True							,							_lowerCamelCase=True							,							_lowerCamelCase=True							,							_lowerCamelCase=True							,							_lowerCamelCase=99							,							_lowerCamelCase=64							,							_lowerCamelCase=5							,							_lowerCamelCase=4							,							_lowerCamelCase=37							,							_lowerCamelCase="gelu"							,							_lowerCamelCase=0.1							,							_lowerCamelCase=0.1							,							_lowerCamelCase=512							,							_lowerCamelCase=16							,							_lowerCamelCase=2							,							_lowerCamelCase=0.02							,							_lowerCamelCase=3							,							_lowerCamelCase=4							,							_lowerCamelCase=None							,							):
								UpperCAmelCase__  :					Any        =     parent
								UpperCAmelCase__  :					List[str]        =     batch_size
								UpperCAmelCase__  :					Any        =     seq_length
								UpperCAmelCase__  :					int        =     is_training
								UpperCAmelCase__  :					List[str]        =     use_input_mask
								UpperCAmelCase__  :					Dict        =     use_token_type_ids
								UpperCAmelCase__  :					List[str]        =     use_labels
								UpperCAmelCase__  :					Dict        =     vocab_size
								UpperCAmelCase__  :					List[str]        =     hidden_size
								UpperCAmelCase__  :					Dict        =     num_hidden_layers
								UpperCAmelCase__  :					Optional[int]        =     num_attention_heads
								UpperCAmelCase__  :					List[Any]        =     intermediate_size
								UpperCAmelCase__  :					Union[str, Any]        =     hidden_act
								UpperCAmelCase__  :					List[Any]        =     hidden_dropout_prob
								UpperCAmelCase__  :					Union[str, Any]        =     attention_probs_dropout_prob
								UpperCAmelCase__  :					int        =     max_position_embeddings
								UpperCAmelCase__  :					Any        =     type_vocab_size
								UpperCAmelCase__  :					Optional[Any]        =     type_sequence_label_size
								UpperCAmelCase__  :					int        =     initializer_range
								UpperCAmelCase__  :					Optional[Any]        =     num_labels
								UpperCAmelCase__  :					str        =     num_choices
								UpperCAmelCase__  :					Optional[int]        =     scope
								UpperCAmelCase__  :					List[Any]        =     vocab_size - 1




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[Any]        =     ids_tensor([self.batch_size, self.seq_length]							,							self.vocab_size)

								UpperCAmelCase__  :					int        =     None
								if self.use_input_mask:
											UpperCAmelCase__  :					Dict        =     random_attention_mask([self.batch_size, self.seq_length])

								UpperCAmelCase__  :					List[Any]        =     None
								if self.use_labels:
											UpperCAmelCase__  :					Any        =     ids_tensor([self.batch_size, self.seq_length]							,							self.num_labels)

								UpperCAmelCase__  :					Union[str, Any]        =     self.get_config()

								return config, input_ids, input_mask, token_labels




					def 				snake_case__						(      self):
								return GPTNeoXConfig(
								    vocab_size=self.vocab_size							,							hidden_size=self.hidden_size							,							num_hidden_layers=self.num_hidden_layers							,							num_attention_heads=self.num_attention_heads							,							intermediate_size=self.intermediate_size							,							hidden_act=self.hidden_act							,							hidden_dropout_prob=self.hidden_dropout_prob							,							attention_probs_dropout_prob=self.attention_probs_dropout_prob							,							max_position_embeddings=self.max_position_embeddings							,							type_vocab_size=self.type_vocab_size							,							is_decoder=_lowerCamelCase							,							initializer_range=self.initializer_range							,							pad_token_id=self.pad_token_id							,							)




					def 				snake_case__						(      self):
								UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					str        =     self.prepare_config_and_inputs()

								UpperCAmelCase__  :					Dict        =     True

								return config, input_ids, input_mask, token_labels




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
								UpperCAmelCase__  :					Tuple        =     GPTNeoXModel(config=_lowerCamelCase)
								model.to(_lowerCamelCase)
								model.eval()
								UpperCAmelCase__  :					Optional[int]        =     model(_lowerCamelCase							,							attention_mask=_lowerCamelCase)
								UpperCAmelCase__  :					List[str]        =     model(_lowerCamelCase)
								self.parent.assertEqual(result.last_hidden_state.shape							,							(self.batch_size, self.seq_length, self.hidden_size))




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
								UpperCAmelCase__  :					int        =     True
								UpperCAmelCase__  :					List[str]        =     GPTNeoXModel(_lowerCamelCase)
								model.to(_lowerCamelCase)
								model.eval()
								UpperCAmelCase__  :					int        =     model(_lowerCamelCase							,							attention_mask=_lowerCamelCase)
								self.parent.assertEqual(result.last_hidden_state.shape							,							(self.batch_size, self.seq_length, self.hidden_size))




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
								UpperCAmelCase__  :					int        =     GPTNeoXForCausalLM(config=_lowerCamelCase)
								model.to(_lowerCamelCase)
								model.eval()
								UpperCAmelCase__  :					str        =     model(_lowerCamelCase							,							attention_mask=_lowerCamelCase							,							labels=_lowerCamelCase)
								self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size))




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
								UpperCAmelCase__  :					int        =     self.num_labels
								UpperCAmelCase__  :					int        =     GPTNeoXForQuestionAnswering(_lowerCamelCase)
								model.to(_lowerCamelCase)
								model.eval()
								UpperCAmelCase__  :					List[str]        =     model(_lowerCamelCase							,							attention_mask=_lowerCamelCase)
								self.parent.assertEqual(result.start_logits.shape							,							(self.batch_size, self.seq_length))
								self.parent.assertEqual(result.end_logits.shape							,							(self.batch_size, self.seq_length))




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
								UpperCAmelCase__  :					Dict        =     self.num_labels
								UpperCAmelCase__  :					Any        =     GPTNeoXForSequenceClassification(_lowerCamelCase)
								model.to(_lowerCamelCase)
								model.eval()
								UpperCAmelCase__  :					int        =     ids_tensor([self.batch_size]							,							self.type_sequence_label_size)
								UpperCAmelCase__  :					Union[str, Any]        =     model(_lowerCamelCase							,							attention_mask=_lowerCamelCase							,							labels=_lowerCamelCase)
								self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.num_labels))




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
								UpperCAmelCase__  :					List[Any]        =     self.num_labels
								UpperCAmelCase__  :					Tuple        =     GPTNeoXForTokenClassification(_lowerCamelCase)
								model.to(_lowerCamelCase)
								model.eval()
								UpperCAmelCase__  :					Tuple        =     model(_lowerCamelCase							,							attention_mask=_lowerCamelCase							,							labels=_lowerCamelCase)
								self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.num_labels))




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
								UpperCAmelCase__  :					Tuple        =     True
								UpperCAmelCase__  :					str        =     GPTNeoXForCausalLM(config=_lowerCamelCase)
								model.to(_lowerCamelCase)
								model.eval()

								# first forward pass
								UpperCAmelCase__  :					Dict        =     model(_lowerCamelCase							,							attention_mask=_lowerCamelCase							,							use_cache=_lowerCamelCase)
								UpperCAmelCase__  :					str        =     outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								UpperCAmelCase__  :					int        =     ids_tensor((self.batch_size, 3)							,							config.vocab_size)
								UpperCAmelCase__  :					Optional[Any]        =     ids_tensor((self.batch_size, 3)							,							vocab_size=2)

								# append to next input_ids and
								UpperCAmelCase__  :					str        =     torch.cat([input_ids, next_tokens]							,							dim=-1)
								UpperCAmelCase__  :					List[str]        =     torch.cat([input_mask, next_mask]							,							dim=-1)

								UpperCAmelCase__  :					Optional[int]        =     model(_lowerCamelCase							,							attention_mask=_lowerCamelCase							,							output_hidden_states=_lowerCamelCase)
								UpperCAmelCase__  :					List[Any]        =     output_from_no_past["""hidden_states"""][0]
								UpperCAmelCase__  :					Optional[Any]        =     model(
								    _lowerCamelCase							,							attention_mask=_lowerCamelCase							,							past_key_values=_lowerCamelCase							,							output_hidden_states=_lowerCamelCase							,							)["""hidden_states"""][0]

								# select random slice
								UpperCAmelCase__  :					Optional[int]        =     ids_tensor((1,)							,							output_from_past.shape[-1]).item()
								UpperCAmelCase__  :					Union[str, Any]        =     output_from_no_past[:, -3:, random_slice_idx].detach()
								UpperCAmelCase__  :					List[str]        =     output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(_lowerCamelCase							,							_lowerCamelCase							,							atol=1e-3))




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Tuple        =     self.prepare_config_and_inputs()
								UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					str        =     config_and_inputs
								UpperCAmelCase__  :					Optional[Any]        =     {"""input_ids""": input_ids, """attention_mask""": input_mask}
								return config, inputs_dict


@require_torch
class 		_snake_case					(     a__       ,   a__       ,   a__       ,   unittest.TestCase	):
					lowerCAmelCase      :Dict   =					(
					    (
					        GPTNeoXModel,
					        GPTNeoXForCausalLM,
					        GPTNeoXForQuestionAnswering,
					        GPTNeoXForSequenceClassification,
					        GPTNeoXForTokenClassification,
					    )
					    if is_torch_available()
					    else ()
					)
					lowerCAmelCase      :Tuple   =					(GPTNeoXForCausalLM,) if is_torch_available() else ()
					lowerCAmelCase      :List[str]   =					(
					    {
					        '''feature-extraction''': GPTNeoXModel,
					        '''question-answering''': GPTNeoXForQuestionAnswering,
					        '''text-classification''': GPTNeoXForSequenceClassification,
					        '''text-generation''': GPTNeoXForCausalLM,
					        '''token-classification''': GPTNeoXForTokenClassification,
					        '''zero-shot''': GPTNeoXForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					lowerCAmelCase      :List[str]   =					False
					lowerCAmelCase      :str   =					False
					lowerCAmelCase      :Dict   =					False
					lowerCAmelCase      :Optional[int]   =					False




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Optional[int]        =     GPTNeoXModelTester(self)
								UpperCAmelCase__  :					Optional[Any]        =     ConfigTester(self							,							config_class=_lowerCamelCase							,							hidden_size=64							,							num_attention_heads=8)




					def 				snake_case__						(      self):
								self.config_tester.run_common_tests()




					def 				snake_case__						(      self):
								UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					int        =     self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase)




					def 				snake_case__						(      self):
								UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					Optional[Any]        =     self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase)




					def 				snake_case__						(      self):
								# This regression test was failing with PyTorch < 1.3
								UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					int        =     self.model_tester.prepare_config_and_inputs_for_decoder()

								UpperCAmelCase__  :					Union[str, Any]        =     None

								self.model_tester.create_and_check_model_as_decoder(_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase)




					def 				snake_case__						(      self):
								UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					int        =     self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_decoder_model_past_large_inputs(_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Any        =     self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_causal_lm(*_lowerCamelCase)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Tuple        =     self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					int        =     self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					int        =     self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase)




					@unittest.skip(reason="""Feed forward chunking is not implemented""")
					def 				snake_case__						(      self):
								pass




					@parameterized.expand([("""linear""",), ("""dynamic""",)])
					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__   ,       UpperCAmelCase__  :					str        =     self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase__  :					Any        =     ids_tensor([1, 10]							,							config.vocab_size)
								UpperCAmelCase__  :					List[str]        =     ids_tensor([1, int(config.max_position_embeddings * 1.5)]							,							config.vocab_size)

								set_seed(42)  # Fixed seed at init time so the two models get the same random weights
								UpperCAmelCase__  :					Optional[int]        =     GPTNeoXModel(_lowerCamelCase)
								original_model.to(_lowerCamelCase)
								original_model.eval()
								UpperCAmelCase__  :					Any        =     original_model(_lowerCamelCase).last_hidden_state
								UpperCAmelCase__  :					Optional[int]        =     original_model(_lowerCamelCase).last_hidden_state

								set_seed(42)  # Fixed seed at init time so the two models get the same random weights
								UpperCAmelCase__  :					Dict        =     {"""type""": scaling_type, """factor""": 10.0}
								UpperCAmelCase__  :					List[Any]        =     GPTNeoXModel(_lowerCamelCase)
								scaled_model.to(_lowerCamelCase)
								scaled_model.eval()
								UpperCAmelCase__  :					str        =     scaled_model(_lowerCamelCase).last_hidden_state
								UpperCAmelCase__  :					Any        =     scaled_model(_lowerCamelCase).last_hidden_state

								# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
								# maximum sequence length, so the outputs for the short input should match.
								if scaling_type == "dynamic":
											self.assertTrue(torch.allclose(_lowerCamelCase							,							_lowerCamelCase							,							atol=1e-5))
								else:
											self.assertFalse(torch.allclose(_lowerCamelCase							,							_lowerCamelCase							,							atol=1e-5))

								# The output should be different for long inputs
								self.assertFalse(torch.allclose(_lowerCamelCase							,							_lowerCamelCase							,							atol=1e-5))


@require_torch
class 		_snake_case					(     unittest.TestCase	):




					@slow
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Any        =     AutoTokenizer.from_pretrained("""EleutherAI/pythia-410m-deduped""")
								for checkpointing in [True, False]:
											UpperCAmelCase__  :					Dict        =     GPTNeoXForCausalLM.from_pretrained("""EleutherAI/pythia-410m-deduped""")

											if checkpointing:
														model.gradient_checkpointing_enable()
											else:
														model.gradient_checkpointing_disable()
											model.to(_lowerCamelCase)

											UpperCAmelCase__  :					Any        =     tokenizer("""My favorite food is"""							,							return_tensors="""pt""").to(_lowerCamelCase)
											# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
											# See: https://github.com/huggingface/transformers/pull/24193
											UpperCAmelCase__  :					List[Any]        =     """My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI'm not sure"""

											UpperCAmelCase__  :					Union[str, Any]        =     model.generate(**_lowerCamelCase							,							do_sample=_lowerCamelCase							,							max_new_tokens=20)
											UpperCAmelCase__  :					int        =     tokenizer.batch_decode(_lowerCamelCase)[0]

											self.assertEqual(_lowerCamelCase							,							_lowerCamelCase)



'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


__A	={
    'configuration_encodec': [
        'ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'EncodecConfig',
    ],
    'feature_extraction_encodec': ['EncodecFeatureExtractor'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=[
		    'ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'EncodecModel',
		    'EncodecPreTrainedModel',
		]

if TYPE_CHECKING:
		from .configuration_encodec import (
		    ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    EncodecConfig,
		)
		from .feature_extraction_encodec import EncodecFeatureExtractor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_encodec import (
				    ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST,
				    EncodecModel,
				    EncodecPreTrainedModel,
				)

else:
		import sys

		__A	=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)