





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel
class        snake_case_		:







		def __init__(		self			,     __lowercase			,     __lowercase=1_3			,     __lowercase=7			,     __lowercase=True			,     __lowercase=True			,     __lowercase=False			,     __lowercase=True			,     __lowercase=9_9			,     __lowercase=3_2			,     __lowercase=5			,     __lowercase=4			,     __lowercase=3_7			,     __lowercase="gelu"			,     __lowercase=0.1			,     __lowercase=0.1			,     __lowercase=5_1_2			,     __lowercase=1_6			,     __lowercase=2			,     __lowercase=0.0_2			,     __lowercase=3			,     __lowercase=4			,     __lowercase=None			,     )				->   Dict:
				lowerCamelCase   : List[Any]								=parent
				lowerCamelCase   : Any								=batch_size
				lowerCamelCase   : Tuple								=seq_length
				lowerCamelCase   : Dict								=is_training
				lowerCamelCase   : List[str]								=use_input_mask
				lowerCamelCase   : Union[str, Any]								=use_token_type_ids
				lowerCamelCase   : Optional[Any]								=use_labels
				lowerCamelCase   : List[Any]								=vocab_size
				lowerCamelCase   : Optional[int]								=hidden_size
				lowerCamelCase   : Optional[Any]								=num_hidden_layers
				lowerCamelCase   : Optional[Any]								=num_attention_heads
				lowerCamelCase   : str								=intermediate_size
				lowerCamelCase   : Any								=hidden_act
				lowerCamelCase   : Dict								=hidden_dropout_prob
				lowerCamelCase   : Dict								=attention_probs_dropout_prob
				lowerCamelCase   : int								=max_position_embeddings
				lowerCamelCase   : Optional[Any]								=type_vocab_size
				lowerCamelCase   : str								=type_sequence_label_size
				lowerCamelCase   : int								=initializer_range
				lowerCamelCase   : Union[str, Any]								=num_labels
				lowerCamelCase   : Union[str, Any]								=num_choices
				lowerCamelCase   : str								=scope







		def 			__lowercase   (		self   )				->   List[Any]:
				lowerCamelCase   : Union[str, Any]								=ids_tensor([self.batch_size, self.seq_length]			,     self.vocab_size   )

				lowerCamelCase   : List[str]								=None
				if self.use_input_mask:
						lowerCamelCase   : int								=random_attention_mask([self.batch_size, self.seq_length]   )

				lowerCamelCase   : List[str]								=None
				if self.use_token_type_ids:
						lowerCamelCase   : int								=ids_tensor([self.batch_size, self.seq_length]			,     self.type_vocab_size   )

				lowerCamelCase   : str								=None
				lowerCamelCase   : List[str]								=None
				lowerCamelCase   : List[Any]								=None
				if self.use_labels:
						lowerCamelCase   : Tuple								=ids_tensor([self.batch_size]			,     self.type_sequence_label_size   )
						lowerCamelCase   : List[Any]								=ids_tensor([self.batch_size, self.seq_length]			,     self.num_labels   )
						lowerCamelCase   : List[str]								=ids_tensor([self.batch_size]			,     self.num_choices   )

				lowerCamelCase   : List[str]								=self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







		def 			__lowercase   (		self   )				->   List[str]:
				return OpenLlamaConfig(
				    vocab_size=self.vocab_size			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     max_position_embeddings=self.max_position_embeddings			,     type_vocab_size=self.type_vocab_size			,     is_decoder=__lowercase			,     initializer_range=self.initializer_range			,     use_stable_embedding=__lowercase			,     )







		def 			__lowercase   (		self			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase   )				->   List[str]:
				lowerCamelCase   : str								=OpenLlamaModel(config=__lowercase   )
				model.to(__lowercase   )
				model.eval()
				lowerCamelCase   : Optional[Any]								=model(__lowercase			,     attention_mask=__lowercase   )
				lowerCamelCase   : Dict								=model(__lowercase   )
				self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)   )







		def 			__lowercase   (		self			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     )				->   Union[str, Any]:
				lowerCamelCase   : Optional[Any]								=True
				lowerCamelCase   : int								=OpenLlamaModel(__lowercase   )
				model.to(__lowercase   )
				model.eval()
				lowerCamelCase   : Union[str, Any]								=model(
				    __lowercase			,     attention_mask=__lowercase			,     encoder_hidden_states=__lowercase			,     encoder_attention_mask=__lowercase			,     )
				lowerCamelCase   : Optional[int]								=model(
				    __lowercase			,     attention_mask=__lowercase			,     encoder_hidden_states=__lowercase			,     )
				lowerCamelCase   : Optional[int]								=model(__lowercase			,     attention_mask=__lowercase   )
				self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)   )







		def 			__lowercase   (		self			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     )				->   str:
				lowerCamelCase   : List[Any]								=OpenLlamaForCausalLM(config=__lowercase   )
				model.to(__lowercase   )
				model.eval()
				lowerCamelCase   : str								=model(__lowercase			,     attention_mask=__lowercase			,     labels=__lowercase   )
				self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.seq_length, self.vocab_size)   )







		def 			__lowercase   (		self			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     )				->   Dict:
				lowerCamelCase   : Optional[Any]								=True
				lowerCamelCase   : Union[str, Any]								=True
				lowerCamelCase   : Optional[Any]								=OpenLlamaForCausalLM(config=__lowercase   )
				model.to(__lowercase   )
				model.eval()

				# first forward pass
				lowerCamelCase   : List[Any]								=model(
				    __lowercase			,     attention_mask=__lowercase			,     encoder_hidden_states=__lowercase			,     encoder_attention_mask=__lowercase			,     use_cache=__lowercase			,     )
				lowerCamelCase   : Union[str, Any]								=outputs.past_key_values

				# create hypothetical multiple next token and extent to next_input_ids
				lowerCamelCase   : Any								=ids_tensor((self.batch_size, 3)			,     config.vocab_size   )
				lowerCamelCase   : Any								=ids_tensor((self.batch_size, 3)			,     vocab_size=2   )

				# append to next input_ids and
				lowerCamelCase   : Optional[int]								=torch.cat([input_ids, next_tokens]			,     dim=-1   )
				lowerCamelCase   : Optional[int]								=torch.cat([input_mask, next_mask]			,     dim=-1   )

				lowerCamelCase   : Optional[Any]								=model(
				    __lowercase			,     attention_mask=__lowercase			,     encoder_hidden_states=__lowercase			,     encoder_attention_mask=__lowercase			,     output_hidden_states=__lowercase			,     )['''hidden_states'''][0]
				lowerCamelCase   : Optional[int]								=model(
				    __lowercase			,     attention_mask=__lowercase			,     encoder_hidden_states=__lowercase			,     encoder_attention_mask=__lowercase			,     past_key_values=__lowercase			,     output_hidden_states=__lowercase			,     )['''hidden_states'''][0]

				# select random slice
				lowerCamelCase   : Optional[int]								=ids_tensor((1,)			,     output_from_past.shape[-1]   ).item()
				lowerCamelCase   : Union[str, Any]								=output_from_no_past[:, -3:, random_slice_idx].detach()
				lowerCamelCase   : str								=output_from_past[:, :, random_slice_idx].detach()

				self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

				# test that outputs are equal for slice
				self.parent.assertTrue(torch.allclose(__lowercase			,     __lowercase			,     atol=1e-3   )   )







		def 			__lowercase   (		self   )				->   Optional[Any]:
				lowerCamelCase   : Tuple								=self.prepare_config_and_inputs()
				(
				    lowerCamelCase   
				)   : List[Any]								=config_and_inputs
				lowerCamelCase   : Tuple								={'''input_ids''': input_ids, '''attention_mask''': input_mask}
				return config, inputs_dict

@require_torch
class        snake_case_		(			_A						,					_A						,					_A						,					unittest.TestCase):
		lowerCamelCase    :Any						=     (
		    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
		)
		lowerCamelCase    :Union[str, Any]						=     (OpenLlamaForCausalLM,) if is_torch_available() else ()
		lowerCamelCase    :Dict						=     (
		    {
		        "feature-extraction": OpenLlamaModel,
		        "text-classification": OpenLlamaForSequenceClassification,
		        "text-generation": OpenLlamaForCausalLM,
		        "zero-shot": OpenLlamaForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		lowerCamelCase    :int						=     False
		lowerCamelCase    :Optional[Any]						=     False







		def 			__lowercase   (		self   )				->   Any:
				lowerCamelCase   : Any								=OpenLlamaModelTester(self   )
				lowerCamelCase   : Tuple								=ConfigTester(self			,     config_class=__lowercase			,     hidden_size=3_7   )







		def 			__lowercase   (		self   )				->   Optional[int]:
				self.config_tester.run_common_tests()







		def 			__lowercase   (		self   )				->   Any:
				lowerCamelCase   : Any								=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*__lowercase   )







		def 			__lowercase   (		self   )				->   List[str]:
				lowerCamelCase   : Tuple								=self.model_tester.prepare_config_and_inputs()
				for type in ["absolute", "relative_key", "relative_key_query"]:
						lowerCamelCase   : str								=type
						self.model_tester.create_and_check_model(*__lowercase   )







		def 			__lowercase   (		self   )				->   Optional[Any]:
				lowerCamelCase   : int								=self.model_tester.prepare_config_and_inputs_for_common()
				lowerCamelCase   : List[str]								=3
				lowerCamelCase   : Tuple								=input_dict['''input_ids''']
				lowerCamelCase   : Tuple								=input_ids.ne(1   ).to(__lowercase   )
				lowerCamelCase   : Optional[Any]								=ids_tensor([self.model_tester.batch_size]			,     self.model_tester.type_sequence_label_size   )
				lowerCamelCase   : str								=OpenLlamaForSequenceClassification(__lowercase   )
				model.to(__lowercase   )
				model.eval()
				lowerCamelCase   : Optional[int]								=model(__lowercase			,     attention_mask=__lowercase			,     labels=__lowercase   )
				self.assertEqual(result.logits.shape			,     (self.model_tester.batch_size, self.model_tester.num_labels)   )







		def 			__lowercase   (		self   )				->   Dict:
				lowerCamelCase   : Union[str, Any]								=self.model_tester.prepare_config_and_inputs_for_common()
				lowerCamelCase   : Optional[Any]								=3
				lowerCamelCase   : Union[str, Any]								='''single_label_classification'''
				lowerCamelCase   : List[str]								=input_dict['''input_ids''']
				lowerCamelCase   : Optional[int]								=input_ids.ne(1   ).to(__lowercase   )
				lowerCamelCase   : int								=ids_tensor([self.model_tester.batch_size]			,     self.model_tester.type_sequence_label_size   )
				lowerCamelCase   : Optional[Any]								=OpenLlamaForSequenceClassification(__lowercase   )
				model.to(__lowercase   )
				model.eval()
				lowerCamelCase   : List[Any]								=model(__lowercase			,     attention_mask=__lowercase			,     labels=__lowercase   )
				self.assertEqual(result.logits.shape			,     (self.model_tester.batch_size, self.model_tester.num_labels)   )







		def 			__lowercase   (		self   )				->   List[Any]:
				lowerCamelCase   : List[Any]								=self.model_tester.prepare_config_and_inputs_for_common()
				lowerCamelCase   : Union[str, Any]								=3
				lowerCamelCase   : Optional[int]								='''multi_label_classification'''
				lowerCamelCase   : Union[str, Any]								=input_dict['''input_ids''']
				lowerCamelCase   : Any								=input_ids.ne(1   ).to(__lowercase   )
				lowerCamelCase   : Dict								=ids_tensor(
				    [self.model_tester.batch_size, config.num_labels]			,     self.model_tester.type_sequence_label_size   ).to(torch.float   )
				lowerCamelCase   : int								=OpenLlamaForSequenceClassification(__lowercase   )
				model.to(__lowercase   )
				model.eval()
				lowerCamelCase   : List[str]								=model(__lowercase			,     attention_mask=__lowercase			,     labels=__lowercase   )
				self.assertEqual(result.logits.shape			,     (self.model_tester.batch_size, self.model_tester.num_labels)   )







		@unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test'''   )
		def 			__lowercase   (		self   )				->   Tuple:
				pass







		@parameterized.expand([('''linear''',), ('''dynamic''',)]   )
		def 			__lowercase   (		self			,     __lowercase   )				->   int:
				lowerCamelCase   : str								=self.model_tester.prepare_config_and_inputs_for_common()
				lowerCamelCase   : List[str]								=ids_tensor([1, 1_0]			,     config.vocab_size   )
				lowerCamelCase   : List[Any]								=ids_tensor([1, int(config.max_position_embeddings * 1.5   )]			,     config.vocab_size   )

				set_seed(4_2   )  # Fixed seed at init time so the two models get the same random weights
				lowerCamelCase   : Dict								=OpenLlamaModel(__lowercase   )
				original_model.to(__lowercase   )
				original_model.eval()
				lowerCamelCase   : Any								=original_model(__lowercase   ).last_hidden_state
				lowerCamelCase   : Optional[int]								=original_model(__lowercase   ).last_hidden_state

				set_seed(4_2   )  # Fixed seed at init time so the two models get the same random weights
				lowerCamelCase   : Tuple								={'''type''': scaling_type, '''factor''': 1_0.0}
				lowerCamelCase   : List[Any]								=OpenLlamaModel(__lowercase   )
				scaled_model.to(__lowercase   )
				scaled_model.eval()
				lowerCamelCase   : List[Any]								=scaled_model(__lowercase   ).last_hidden_state
				lowerCamelCase   : List[Any]								=scaled_model(__lowercase   ).last_hidden_state

				# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
				# maximum sequence length, so the outputs for the short input should match.
				if scaling_type == "dynamic":
						self.assertTrue(torch.allclose(__lowercase			,     __lowercase			,     atol=1e-5   )   )
				else:
						self.assertFalse(torch.allclose(__lowercase			,     __lowercase			,     atol=1e-5   )   )

				# The output should be different for long inputs
				self.assertFalse(torch.allclose(__lowercase			,     __lowercase			,     atol=1e-5   )   )








import numpy as np
from sklearn.datasets import fetch_california_housing
from sklearn.metrics import mean_absolute_error, mean_squared_error
from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor


def 					A__	(       SCREAMING_SNAKE_CASE_		)   ->     tuple:
		return (data["data"], data["target"])


def 					A__	(       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		)   ->     np.ndarray:
		lowerCamelCase   : Dict								=XGBRegressor(verbosity=0							,       random_state=4_2		)
		xgb.fit(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		)
		# Predict target for test data
		lowerCamelCase   : List[str]								=xgb.predict(SCREAMING_SNAKE_CASE_		)
		lowerCamelCase   : List[Any]								=predictions.reshape(len(SCREAMING_SNAKE_CASE_		)							,       1		)
		return predictions


def 					A__	(       )   ->     None:
		lowerCamelCase   : Union[str, Any]								=fetch_california_housing()
		lowerCamelCase		,   lowerCamelCase   : Optional[Any]								=data_handling(SCREAMING_SNAKE_CASE_		)
		lowerCamelCase		,   lowerCamelCase		,   lowerCamelCase		,   lowerCamelCase   : str								=train_test_split(
		    SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       test_size=0.2_5							,       random_state=1		)
		lowerCamelCase   : str								=xgboost(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		)
		# Error printing
		print(F"Mean Absolute Error : {mean_absolute_error(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		)}"		)
		print(F"Mean Square Error  : {mean_squared_error(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		)}"		)


if __name__ == "__main__":
				import doctest

				doctest.testmod(verbose=True)
				main()

