


import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel







class        lowerCamelCase__	:






			"""simple docstring"""






			def __init__(							self      ,						snake_case      ,						snake_case=13      ,						snake_case=7      ,						snake_case=True      ,						snake_case=True      ,						snake_case=False      ,						snake_case=True      ,						snake_case=99      ,						snake_case=32      ,						snake_case=5      ,						snake_case=4      ,						snake_case=37      ,						snake_case="gelu"      ,						snake_case=0.1      ,						snake_case=0.1      ,						snake_case=512      ,						snake_case=16      ,						snake_case=2      ,						snake_case=0.02      ,						snake_case=3      ,						snake_case=4      ,						snake_case=None      ,						):

							'''simple docstring'''


							UpperCamelCase__					=       parent
							UpperCamelCase__					=       batch_size
							UpperCamelCase__					=       seq_length
							UpperCamelCase__					=       is_training
							UpperCamelCase__					=       use_input_mask
							UpperCamelCase__					=       use_token_type_ids
							UpperCamelCase__					=       use_labels
							UpperCamelCase__					=       vocab_size
							UpperCamelCase__					=       hidden_size
							UpperCamelCase__					=       num_hidden_layers
							UpperCamelCase__					=       num_attention_heads
							UpperCamelCase__					=       intermediate_size
							UpperCamelCase__					=       hidden_act
							UpperCamelCase__					=       hidden_dropout_prob
							UpperCamelCase__					=       attention_probs_dropout_prob
							UpperCamelCase__					=       max_position_embeddings
							UpperCamelCase__					=       type_vocab_size
							UpperCamelCase__					=       type_sequence_label_size
							UpperCamelCase__					=       initializer_range
							UpperCamelCase__					=       num_labels
							UpperCamelCase__					=       num_choices
							UpperCamelCase__					=       scope




			def 				snake_case__				(							self     ):

							'''simple docstring'''


							UpperCamelCase__					=       ids_tensor([self.batch_size, self.seq_length]      ,						self.vocab_size     )

							UpperCamelCase__					=       None
							if self.use_input_mask:
											UpperCamelCase__					=       random_attention_mask([self.batch_size, self.seq_length]     )

							UpperCamelCase__					=       None
							if self.use_token_type_ids:
											UpperCamelCase__					=       ids_tensor([self.batch_size, self.seq_length]      ,						self.type_vocab_size     )

							UpperCamelCase__					=       None
							UpperCamelCase__					=       None
							UpperCamelCase__					=       None
							if self.use_labels:
											UpperCamelCase__					=       ids_tensor([self.batch_size]      ,						self.type_sequence_label_size     )
											UpperCamelCase__					=       ids_tensor([self.batch_size, self.seq_length]      ,						self.num_labels     )
											UpperCamelCase__					=       ids_tensor([self.batch_size]      ,						self.num_choices     )

							UpperCamelCase__					=       self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




			def 				snake_case__				(							self     ):

							'''simple docstring'''


							return OpenLlamaConfig(
							    vocab_size=self.vocab_size      ,						hidden_size=self.hidden_size      ,						num_hidden_layers=self.num_hidden_layers      ,						num_attention_heads=self.num_attention_heads      ,						intermediate_size=self.intermediate_size      ,						hidden_act=self.hidden_act      ,						hidden_dropout_prob=self.hidden_dropout_prob      ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,						max_position_embeddings=self.max_position_embeddings      ,						type_vocab_size=self.type_vocab_size      ,						is_decoder=__UpperCamelCase      ,						initializer_range=self.initializer_range      ,						use_stable_embedding=__UpperCamelCase      ,						)




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case     ):

							'''simple docstring'''


							UpperCamelCase__					=       OpenLlamaModel(config=__UpperCamelCase     )
							model.to(__UpperCamelCase     )
							model.eval()
							UpperCamelCase__					=       model(__UpperCamelCase      ,						attention_mask=__UpperCamelCase     )
							UpperCamelCase__					=       model(__UpperCamelCase     )
							self.parent.assertEqual(result.last_hidden_state.shape      ,						(self.batch_size, self.seq_length, self.hidden_size)     )




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						):

							'''simple docstring'''


							UpperCamelCase__					=       True
							UpperCamelCase__					=       OpenLlamaModel(__UpperCamelCase     )
							model.to(__UpperCamelCase     )
							model.eval()
							UpperCamelCase__					=       model(
							    __UpperCamelCase      ,						attention_mask=__UpperCamelCase      ,						encoder_hidden_states=__UpperCamelCase      ,						encoder_attention_mask=__UpperCamelCase      ,						)
							UpperCamelCase__					=       model(
							    __UpperCamelCase      ,						attention_mask=__UpperCamelCase      ,						encoder_hidden_states=__UpperCamelCase      ,						)
							UpperCamelCase__					=       model(__UpperCamelCase      ,						attention_mask=__UpperCamelCase     )
							self.parent.assertEqual(result.last_hidden_state.shape      ,						(self.batch_size, self.seq_length, self.hidden_size)     )




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						):

							'''simple docstring'''


							UpperCamelCase__					=       OpenLlamaForCausalLM(config=__UpperCamelCase     )
							model.to(__UpperCamelCase     )
							model.eval()
							UpperCamelCase__					=       model(__UpperCamelCase      ,						attention_mask=__UpperCamelCase      ,						labels=__UpperCamelCase     )
							self.parent.assertEqual(result.logits.shape      ,						(self.batch_size, self.seq_length, self.vocab_size)     )




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						):

							'''simple docstring'''


							UpperCamelCase__					=       True
							UpperCamelCase__					=       True
							UpperCamelCase__					=       OpenLlamaForCausalLM(config=__UpperCamelCase     )
							model.to(__UpperCamelCase     )
							model.eval()

							# first forward pass
							UpperCamelCase__					=       model(
							    __UpperCamelCase      ,						attention_mask=__UpperCamelCase      ,						encoder_hidden_states=__UpperCamelCase      ,						encoder_attention_mask=__UpperCamelCase      ,						use_cache=__UpperCamelCase      ,						)
							UpperCamelCase__					=       outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							UpperCamelCase__					=       ids_tensor((self.batch_size, 3)      ,						config.vocab_size     )
							UpperCamelCase__					=       ids_tensor((self.batch_size, 3)      ,						vocab_size=2     )

							# append to next input_ids and
							UpperCamelCase__					=       torch.cat([input_ids, next_tokens]      ,						dim=-1     )
							UpperCamelCase__					=       torch.cat([input_mask, next_mask]      ,						dim=-1     )

							UpperCamelCase__					=       model(
							    __UpperCamelCase      ,						attention_mask=__UpperCamelCase      ,						encoder_hidden_states=__UpperCamelCase      ,						encoder_attention_mask=__UpperCamelCase      ,						output_hidden_states=__UpperCamelCase      ,						)["hidden_states"][0]
							UpperCamelCase__					=       model(
							    __UpperCamelCase      ,						attention_mask=__UpperCamelCase      ,						encoder_hidden_states=__UpperCamelCase      ,						encoder_attention_mask=__UpperCamelCase      ,						past_key_values=__UpperCamelCase      ,						output_hidden_states=__UpperCamelCase      ,						)["hidden_states"][0]

							# select random slice
							UpperCamelCase__					=       ids_tensor((1,)      ,						output_from_past.shape[-1]     ).item()
							UpperCamelCase__					=       output_from_no_past[:, -3:, random_slice_idx].detach()
							UpperCamelCase__					=       output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(__UpperCamelCase      ,						__UpperCamelCase      ,						atol=1E-3     )     )







			def 				snake_case__				(							self     ):

							'''simple docstring'''


							UpperCamelCase__					=       self.prepare_config_and_inputs()
							(
							    (
							    UpperCamelCase__						
							),      (
							    UpperCamelCase__						
							),      (
							    UpperCamelCase__						
							),      (
							    UpperCamelCase__						
							),      (
							    UpperCamelCase__						
							),      (
							    UpperCamelCase__						
							),      (
							    UpperCamelCase__						
							),      						
							)					=       config_and_inputs
							UpperCamelCase__					=       {"input_ids": input_ids, "attention_mask": input_mask}
							return config, inputs_dict







@require_torch
class        lowerCamelCase__	(							UpperCAmelCase					,	UpperCAmelCase					,	UpperCAmelCase					,	unittest.TestCase      ):






			"""simple docstring"""






			_UpperCamelCase		:					List[Any]    =		(
			    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
			)
			_UpperCamelCase		:					str    =		(OpenLlamaForCausalLM,) if is_torch_available() else ()
			_UpperCamelCase		:					List[Any]    =		(
			    {
			        'feature-extraction': OpenLlamaModel,
			        'text-classification': OpenLlamaForSequenceClassification,
			        'text-generation': OpenLlamaForCausalLM,
			        'zero-shot': OpenLlamaForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			_UpperCamelCase		:					str    =		False
			_UpperCamelCase		:					Optional[Any]    =		False




			def 				snake_case__				(							self     ):

							'''simple docstring'''


							UpperCamelCase__					=       OpenLlamaModelTester(self     )
							UpperCamelCase__					=       ConfigTester(self      ,						config_class=__UpperCamelCase      ,						hidden_size=37     )




			def 				snake_case__				(							self     ):

							'''simple docstring'''


							self.config_tester.run_common_tests()




			def 				snake_case__				(							self     ):

							'''simple docstring'''


							UpperCamelCase__					=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__UpperCamelCase     )




			def 				snake_case__				(							self     ):

							'''simple docstring'''


							UpperCamelCase__					=       self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
											UpperCamelCase__					=       type
											self.model_tester.create_and_check_model(*__UpperCamelCase     )




			def 				snake_case__				(							self     ):

							'''simple docstring'''


							UpperCamelCase__,      UpperCamelCase__					=       self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase__					=       3
							UpperCamelCase__					=       input_dict["input_ids"]
							UpperCamelCase__					=       input_ids.ne(1     ).to(__UpperCamelCase     )
							UpperCamelCase__					=       ids_tensor([self.model_tester.batch_size]      ,						self.model_tester.type_sequence_label_size     )
							UpperCamelCase__					=       OpenLlamaForSequenceClassification(__UpperCamelCase     )
							model.to(__UpperCamelCase     )
							model.eval()
							UpperCamelCase__					=       model(__UpperCamelCase      ,						attention_mask=__UpperCamelCase      ,						labels=__UpperCamelCase     )
							self.assertEqual(result.logits.shape      ,						(self.model_tester.batch_size, self.model_tester.num_labels)     )




			def 				snake_case__				(							self     ):

							'''simple docstring'''


							UpperCamelCase__,      UpperCamelCase__					=       self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase__					=       3
							UpperCamelCase__					=       "single_label_classification"
							UpperCamelCase__					=       input_dict["input_ids"]
							UpperCamelCase__					=       input_ids.ne(1     ).to(__UpperCamelCase     )
							UpperCamelCase__					=       ids_tensor([self.model_tester.batch_size]      ,						self.model_tester.type_sequence_label_size     )
							UpperCamelCase__					=       OpenLlamaForSequenceClassification(__UpperCamelCase     )
							model.to(__UpperCamelCase     )
							model.eval()
							UpperCamelCase__					=       model(__UpperCamelCase      ,						attention_mask=__UpperCamelCase      ,						labels=__UpperCamelCase     )
							self.assertEqual(result.logits.shape      ,						(self.model_tester.batch_size, self.model_tester.num_labels)     )




			def 				snake_case__				(							self     ):

							'''simple docstring'''


							UpperCamelCase__,      UpperCamelCase__					=       self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase__					=       3
							UpperCamelCase__					=       "multi_label_classification"
							UpperCamelCase__					=       input_dict["input_ids"]
							UpperCamelCase__					=       input_ids.ne(1     ).to(__UpperCamelCase     )
							UpperCamelCase__					=       ids_tensor(
							    [self.model_tester.batch_size, config.num_labels]      ,						self.model_tester.type_sequence_label_size     ).to(torch.float     )
							UpperCamelCase__					=       OpenLlamaForSequenceClassification(__UpperCamelCase     )
							model.to(__UpperCamelCase     )
							model.eval()
							UpperCamelCase__					=       model(__UpperCamelCase      ,						attention_mask=__UpperCamelCase      ,						labels=__UpperCamelCase     )
							self.assertEqual(result.logits.shape      ,						(self.model_tester.batch_size, self.model_tester.num_labels)     )




			@unittest.skip("Open-Llama buffers include complex numbers, which breaks this test"     )
			def 				snake_case__				(							self     ):

							'''simple docstring'''


							pass







			@parameterized.expand([("linear",), ("dynamic",)]     )
			def 				snake_case__				(							self      ,						snake_case     ):

							'''simple docstring'''


							UpperCamelCase__,      UpperCamelCase__					=       self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase__					=       ids_tensor([1, 10]      ,						config.vocab_size     )
							UpperCamelCase__					=       ids_tensor([1, int(config.max_position_embeddings * 1.5     )]      ,						config.vocab_size     )

							set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
							UpperCamelCase__					=       OpenLlamaModel(__UpperCamelCase     )
							original_model.to(__UpperCamelCase     )
							original_model.eval()
							UpperCamelCase__					=       original_model(__UpperCamelCase     ).last_hidden_state
							UpperCamelCase__					=       original_model(__UpperCamelCase     ).last_hidden_state

							set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
							UpperCamelCase__					=       {"type": scaling_type, "factor": 10.0}
							UpperCamelCase__					=       OpenLlamaModel(__UpperCamelCase     )
							scaled_model.to(__UpperCamelCase     )
							scaled_model.eval()
							UpperCamelCase__					=       scaled_model(__UpperCamelCase     ).last_hidden_state
							UpperCamelCase__					=       scaled_model(__UpperCamelCase     ).last_hidden_state

							# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
							# maximum sequence length, so the outputs for the short input should match.
							if scaling_type == "dynamic":
											self.assertTrue(torch.allclose(__UpperCamelCase      ,						__UpperCamelCase      ,						atol=1E-5     )     )
							else:
											self.assertFalse(torch.allclose(__UpperCamelCase      ,						__UpperCamelCase      ,						atol=1E-5     )     )

							# The output should be different for long inputs
							self.assertFalse(torch.allclose(__UpperCamelCase      ,						__UpperCamelCase      ,						atol=1E-5     )     )






from __future__ import annotations

import numpy as np
def        UpperCamelCase_(				_A   :list[float]  )->		Union[str, Any]:
			return np.maximum(0				,     _A  )


if __name__ == "__main__":
						print(np.array(relu([-1, 0, 5])))  # --> [0, 0, 5]


