
'''simple docstring'''
import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel


class 			SCREAMING_SNAKE_CASE__      :







				def __init__(       self						,				A_						,				A_=13						,				A_=7						,				A_=True						,				A_=True						,				A_=False						,				A_=True						,				A_=99						,				A_=32						,				A_=5						,				A_=4						,				A_=37						,				A_="gelu"						,				A_=0.1						,				A_=0.1						,				A_=512						,				A_=16						,				A_=2						,				A_=0.02						,				A_=3						,				A_=4						,				A_=None						,				)->		Optional[Any]:







											'''simple docstring'''






											UpperCamelCase   			=     parent
											UpperCamelCase   			=     batch_size
											UpperCamelCase   			=     seq_length
											UpperCamelCase   			=     is_training
											UpperCamelCase   			=     use_input_mask
											UpperCamelCase   			=     use_token_type_ids
											UpperCamelCase   			=     use_labels
											UpperCamelCase   			=     vocab_size
											UpperCamelCase   			=     hidden_size
											UpperCamelCase   			=     num_hidden_layers
											UpperCamelCase   			=     num_attention_heads
											UpperCamelCase   			=     intermediate_size
											UpperCamelCase   			=     hidden_act
											UpperCamelCase   			=     hidden_dropout_prob
											UpperCamelCase   			=     attention_probs_dropout_prob
											UpperCamelCase   			=     max_position_embeddings
											UpperCamelCase   			=     type_vocab_size
											UpperCamelCase   			=     type_sequence_label_size
											UpperCamelCase   			=     initializer_range
											UpperCamelCase   			=     num_labels
											UpperCamelCase   			=     num_choices
											UpperCamelCase   			=     scope







				def 						UpperCAmelCase_    (       self     )->		int:







											'''simple docstring'''






											UpperCamelCase   			=     ids_tensor([self.batch_size, self.seq_length]						,				self.vocab_size     )

											UpperCamelCase   			=     None
											if self.use_input_mask:
																		UpperCamelCase   			=     random_attention_mask([self.batch_size, self.seq_length]     )

											UpperCamelCase   			=     None
											if self.use_token_type_ids:
																		UpperCamelCase   			=     ids_tensor([self.batch_size, self.seq_length]						,				self.type_vocab_size     )

											UpperCamelCase   			=     None
											UpperCamelCase   			=     None
											UpperCamelCase   			=     None
											if self.use_labels:
																		UpperCamelCase   			=     ids_tensor([self.batch_size]						,				self.type_sequence_label_size     )
																		UpperCamelCase   			=     ids_tensor([self.batch_size, self.seq_length]						,				self.num_labels     )
																		UpperCamelCase   			=     ids_tensor([self.batch_size]						,				self.num_choices     )

											UpperCamelCase   			=     self.get_config()

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







				def 						UpperCAmelCase_    (       self     )->		Dict:







											'''simple docstring'''






											return OpenLlamaConfig(
											    vocab_size=self.vocab_size						,				hidden_size=self.hidden_size						,				num_hidden_layers=self.num_hidden_layers						,				num_attention_heads=self.num_attention_heads						,				intermediate_size=self.intermediate_size						,				hidden_act=self.hidden_act						,				hidden_dropout_prob=self.hidden_dropout_prob						,				attention_probs_dropout_prob=self.attention_probs_dropout_prob						,				max_position_embeddings=self.max_position_embeddings						,				type_vocab_size=self.type_vocab_size						,				is_decoder=A_						,				initializer_range=self.initializer_range						,				use_stable_embedding=A_						,				)







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_     )->		str:







											'''simple docstring'''






											UpperCamelCase   			=     OpenLlamaModel(config=A_     )
											model.to(A_     )
											model.eval()
											UpperCamelCase   			=     model(A_						,				attention_mask=A_     )
											UpperCamelCase   			=     model(A_     )
											self.parent.assertEqual(result.last_hidden_state.shape						,				(self.batch_size, self.seq_length, self.hidden_size)     )







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				)->		Tuple:







											'''simple docstring'''






											UpperCamelCase   			=     True
											UpperCamelCase   			=     OpenLlamaModel(A_     )
											model.to(A_     )
											model.eval()
											UpperCamelCase   			=     model(
											    A_						,				attention_mask=A_						,				encoder_hidden_states=A_						,				encoder_attention_mask=A_						,				)
											UpperCamelCase   			=     model(
											    A_						,				attention_mask=A_						,				encoder_hidden_states=A_						,				)
											UpperCamelCase   			=     model(A_						,				attention_mask=A_     )
											self.parent.assertEqual(result.last_hidden_state.shape						,				(self.batch_size, self.seq_length, self.hidden_size)     )







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				)->		str:







											'''simple docstring'''






											UpperCamelCase   			=     OpenLlamaForCausalLM(config=A_     )
											model.to(A_     )
											model.eval()
											UpperCamelCase   			=     model(A_						,				attention_mask=A_						,				labels=A_     )
											self.parent.assertEqual(result.logits.shape						,				(self.batch_size, self.seq_length, self.vocab_size)     )







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				A_						,				)->		Any:







											'''simple docstring'''






											UpperCamelCase   			=     True
											UpperCamelCase   			=     True
											UpperCamelCase   			=     OpenLlamaForCausalLM(config=A_     )
											model.to(A_     )
											model.eval()

											# first forward pass
											UpperCamelCase   			=     model(
											    A_						,				attention_mask=A_						,				encoder_hidden_states=A_						,				encoder_attention_mask=A_						,				use_cache=A_						,				)
											UpperCamelCase   			=     outputs.past_key_values

											# create hypothetical multiple next token and extent to next_input_ids
											UpperCamelCase   			=     ids_tensor((self.batch_size, 3)						,				config.vocab_size     )
											UpperCamelCase   			=     ids_tensor((self.batch_size, 3)						,				vocab_size=2     )

											# append to next input_ids and
											UpperCamelCase   			=     torch.cat([input_ids, next_tokens]						,				dim=-1     )
											UpperCamelCase   			=     torch.cat([input_mask, next_mask]						,				dim=-1     )

											UpperCamelCase   			=     model(
											    A_						,				attention_mask=A_						,				encoder_hidden_states=A_						,				encoder_attention_mask=A_						,				output_hidden_states=A_						,				)['hidden_states'][0]
											UpperCamelCase   			=     model(
											    A_						,				attention_mask=A_						,				encoder_hidden_states=A_						,				encoder_attention_mask=A_						,				past_key_values=A_						,				output_hidden_states=A_						,				)['hidden_states'][0]

											# select random slice
											UpperCamelCase   			=     ids_tensor((1,)						,				output_from_past.shape[-1]     ).item()
											UpperCamelCase   			=     output_from_no_past[:, -3:, random_slice_idx].detach()
											UpperCamelCase   			=     output_from_past[:, :, random_slice_idx].detach()

											self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

											# test that outputs are equal for slice
											self.parent.assertTrue(torch.allclose(A_						,				A_						,				atol=1e-3     )     )







				def 						UpperCAmelCase_    (       self     )->		Optional[int]:







											'''simple docstring'''






											UpperCamelCase   			=     self.prepare_config_and_inputs()
											(
											    (
											    UpperCamelCase
											)			,	(
											    UpperCamelCase
											)			,	(
											    UpperCamelCase
											)			,	(
											    UpperCamelCase
											)			,	(
											    UpperCamelCase
											)			,	(
											    UpperCamelCase
											)			,	(
											    UpperCamelCase
											)			,	
											)   			=     config_and_inputs
											UpperCamelCase   			=     {'input_ids': input_ids, 'attention_mask': input_mask}
											return config, inputs_dict


@require_torch
class 			SCREAMING_SNAKE_CASE__      (     snake_case_							,       snake_case_							,       snake_case_							,       unittest.TestCase):
				lowerCAmelCase_     			=							(
				    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
				)
				lowerCAmelCase_     			=							(OpenLlamaForCausalLM,) if is_torch_available() else ()
				lowerCAmelCase_     			=							(
				    {
				        """feature-extraction""": OpenLlamaModel,
				        """text-classification""": OpenLlamaForSequenceClassification,
				        """text-generation""": OpenLlamaForCausalLM,
				        """zero-shot""": OpenLlamaForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				lowerCAmelCase_     			=							False
				lowerCAmelCase_     			=							False







				def 						UpperCAmelCase_    (       self     )->		Union[str, Any]:







											'''simple docstring'''






											UpperCamelCase   			=     OpenLlamaModelTester(self     )
											UpperCamelCase   			=     ConfigTester(self						,				config_class=A_						,				hidden_size=37     )







				def 						UpperCAmelCase_    (       self     )->		Tuple:







											'''simple docstring'''






											self.config_tester.run_common_tests()







				def 						UpperCAmelCase_    (       self     )->		List[Any]:







											'''simple docstring'''






											UpperCamelCase   			=     self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*A_     )







				def 						UpperCAmelCase_    (       self     )->		Optional[int]:







											'''simple docstring'''






											UpperCamelCase   			=     self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
																		UpperCamelCase   			=     type
																		self.model_tester.create_and_check_model(*A_     )







				def 						UpperCAmelCase_    (       self     )->		Tuple:







											'''simple docstring'''






											UpperCamelCase			,	UpperCamelCase   			=     self.model_tester.prepare_config_and_inputs_for_common()
											UpperCamelCase   			=     3
											UpperCamelCase   			=     input_dict['input_ids']
											UpperCamelCase   			=     input_ids.ne(1     ).to(A_     )
											UpperCamelCase   			=     ids_tensor([self.model_tester.batch_size]						,				self.model_tester.type_sequence_label_size     )
											UpperCamelCase   			=     OpenLlamaForSequenceClassification(A_     )
											model.to(A_     )
											model.eval()
											UpperCamelCase   			=     model(A_						,				attention_mask=A_						,				labels=A_     )
											self.assertEqual(result.logits.shape						,				(self.model_tester.batch_size, self.model_tester.num_labels)     )







				def 						UpperCAmelCase_    (       self     )->		Tuple:







											'''simple docstring'''






											UpperCamelCase			,	UpperCamelCase   			=     self.model_tester.prepare_config_and_inputs_for_common()
											UpperCamelCase   			=     3
											UpperCamelCase   			=     'single_label_classification'
											UpperCamelCase   			=     input_dict['input_ids']
											UpperCamelCase   			=     input_ids.ne(1     ).to(A_     )
											UpperCamelCase   			=     ids_tensor([self.model_tester.batch_size]						,				self.model_tester.type_sequence_label_size     )
											UpperCamelCase   			=     OpenLlamaForSequenceClassification(A_     )
											model.to(A_     )
											model.eval()
											UpperCamelCase   			=     model(A_						,				attention_mask=A_						,				labels=A_     )
											self.assertEqual(result.logits.shape						,				(self.model_tester.batch_size, self.model_tester.num_labels)     )







				def 						UpperCAmelCase_    (       self     )->		Union[str, Any]:







											'''simple docstring'''






											UpperCamelCase			,	UpperCamelCase   			=     self.model_tester.prepare_config_and_inputs_for_common()
											UpperCamelCase   			=     3
											UpperCamelCase   			=     'multi_label_classification'
											UpperCamelCase   			=     input_dict['input_ids']
											UpperCamelCase   			=     input_ids.ne(1     ).to(A_     )
											UpperCamelCase   			=     ids_tensor(
											    [self.model_tester.batch_size, config.num_labels]						,				self.model_tester.type_sequence_label_size     ).to(torch.float     )
											UpperCamelCase   			=     OpenLlamaForSequenceClassification(A_     )
											model.to(A_     )
											model.eval()
											UpperCamelCase   			=     model(A_						,				attention_mask=A_						,				labels=A_     )
											self.assertEqual(result.logits.shape						,				(self.model_tester.batch_size, self.model_tester.num_labels)     )







				@unittest.skip('Open-Llama buffers include complex numbers, which breaks this test'     )
				def 						UpperCAmelCase_    (       self     )->		Any:







											'''simple docstring'''






											pass







				@parameterized.expand([('linear',), ('dynamic',)]     )
				def 						UpperCAmelCase_    (       self						,				A_     )->		Union[str, Any]:







											'''simple docstring'''






											UpperCamelCase			,	UpperCamelCase   			=     self.model_tester.prepare_config_and_inputs_for_common()
											UpperCamelCase   			=     ids_tensor([1, 10]						,				config.vocab_size     )
											UpperCamelCase   			=     ids_tensor([1, int(config.max_position_embeddings * 1.5     )]						,				config.vocab_size     )

											set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
											UpperCamelCase   			=     OpenLlamaModel(A_     )
											original_model.to(A_     )
											original_model.eval()
											UpperCamelCase   			=     original_model(A_     ).last_hidden_state
											UpperCamelCase   			=     original_model(A_     ).last_hidden_state

											set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
											UpperCamelCase   			=     {'type': scaling_type, 'factor': 10.0}
											UpperCamelCase   			=     OpenLlamaModel(A_     )
											scaled_model.to(A_     )
											scaled_model.eval()
											UpperCamelCase   			=     scaled_model(A_     ).last_hidden_state
											UpperCamelCase   			=     scaled_model(A_     ).last_hidden_state

											# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
											# maximum sequence length, so the outputs for the short input should match.
											if scaling_type == "dynamic":
																		self.assertTrue(torch.allclose(A_						,				A_						,				atol=1e-5     )     )
											else:
																		self.assertFalse(torch.allclose(A_						,				A_						,				atol=1e-5     )     )

											# The output should be different for long inputs
											self.assertFalse(torch.allclose(A_						,				A_						,				atol=1e-5     )     )









from typing import Dict

from .base import GenericTensor, Pipeline


class 	__snake_case   ( SCREAMING_SNAKE_CASE ):
		def 						SCREAMING_SNAKE_CASE_					(							self    ,a_=None    ,a_=None    ,a_=None    ,**a_						):







						"""simple docstring"""
						if tokenize_kwargs is None:
										lowerCAmelCase__		  =  {}

						if truncation is not None:
										if "truncation" in tokenize_kwargs:
														raise ValueError(
														    'truncation parameter defined twice (given as keyword argument as well as in tokenize_kwargs)'						)
										lowerCAmelCase__		  =  truncation

						lowerCAmelCase__		  =  tokenize_kwargs

						lowerCAmelCase__		  =  {}
						if return_tensors is not None:
										lowerCAmelCase__		  =  return_tensors

						return preprocess_params, {}, postprocess_params
		def 						SCREAMING_SNAKE_CASE_					(							self    ,a_    ,**a_						):







						"""simple docstring"""
						lowerCAmelCase__		  =  self.framework
						lowerCAmelCase__		  =  self.tokenizer(a_    ,return_tensors=a_    ,**a_						)
						return model_inputs
		def 						SCREAMING_SNAKE_CASE_					(							self    ,a_						):







						"""simple docstring"""
						lowerCAmelCase__		  =  self.model(**a_						)
						return model_outputs
		def 						SCREAMING_SNAKE_CASE_					(							self    ,a_    ,a_=False						):







						"""simple docstring"""
						# [0] is the first available tensor, logits or last_hidden_state.
						if return_tensors:
										return model_outputs[0]
						if self.framework == "pt":
										return model_outputs[0].tolist()
						elif self.framework == "tf":
										return model_outputs[0].numpy().tolist()



		def __call__(							self    ,*a_    ,**a_						):







						"""simple docstring"""
						return super().__call__(*a_    ,**a_						)
