
"""simple docstring"""





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel
class 		SCREAMING_SNAKE_CASE_					:




		"""simple docstring"""





		def __init__(	self				,    lowerCAmelCase__				,    lowerCAmelCase__=1_3				,    lowerCAmelCase__=7				,    lowerCAmelCase__=True				,    lowerCAmelCase__=True				,    lowerCAmelCase__=False				,    lowerCAmelCase__=True				,    lowerCAmelCase__=9_9				,    lowerCAmelCase__=3_2				,    lowerCAmelCase__=5				,    lowerCAmelCase__=4				,    lowerCAmelCase__=3_7				,    lowerCAmelCase__="gelu"				,    lowerCAmelCase__=0.1				,    lowerCAmelCase__=0.1				,    lowerCAmelCase__=5_1_2				,    lowerCAmelCase__=1_6				,    lowerCAmelCase__=2				,    lowerCAmelCase__=0.02				,    lowerCAmelCase__=3				,    lowerCAmelCase__=4				,    lowerCAmelCase__=None				,    ):
				__SCREAMING_SNAKE_CASE				=					parent
				__SCREAMING_SNAKE_CASE				=					batch_size
				__SCREAMING_SNAKE_CASE				=					seq_length
				__SCREAMING_SNAKE_CASE				=					is_training
				__SCREAMING_SNAKE_CASE				=					use_input_mask
				__SCREAMING_SNAKE_CASE				=					use_token_type_ids
				__SCREAMING_SNAKE_CASE				=					use_labels
				__SCREAMING_SNAKE_CASE				=					vocab_size
				__SCREAMING_SNAKE_CASE				=					hidden_size
				__SCREAMING_SNAKE_CASE				=					num_hidden_layers
				__SCREAMING_SNAKE_CASE				=					num_attention_heads
				__SCREAMING_SNAKE_CASE				=					intermediate_size
				__SCREAMING_SNAKE_CASE				=					hidden_act
				__SCREAMING_SNAKE_CASE				=					hidden_dropout_prob
				__SCREAMING_SNAKE_CASE				=					attention_probs_dropout_prob
				__SCREAMING_SNAKE_CASE				=					max_position_embeddings
				__SCREAMING_SNAKE_CASE				=					type_vocab_size
				__SCREAMING_SNAKE_CASE				=					type_sequence_label_size
				__SCREAMING_SNAKE_CASE				=					initializer_range
				__SCREAMING_SNAKE_CASE				=					num_labels
				__SCREAMING_SNAKE_CASE				=					num_choices
				__SCREAMING_SNAKE_CASE				=					scope




		def       snake_case_	(	self):
				__SCREAMING_SNAKE_CASE				=					ids_tensor([self.batch_size, self.seq_length]				,    self.vocab_size)

				__SCREAMING_SNAKE_CASE				=					None
				if self.use_input_mask:
						__SCREAMING_SNAKE_CASE				=					random_attention_mask([self.batch_size, self.seq_length])

				__SCREAMING_SNAKE_CASE				=					None
				if self.use_token_type_ids:
						__SCREAMING_SNAKE_CASE				=					ids_tensor([self.batch_size, self.seq_length]				,    self.type_vocab_size)

				__SCREAMING_SNAKE_CASE				=					None
				__SCREAMING_SNAKE_CASE				=					None
				__SCREAMING_SNAKE_CASE				=					None
				if self.use_labels:
						__SCREAMING_SNAKE_CASE				=					ids_tensor([self.batch_size]				,    self.type_sequence_label_size)
						__SCREAMING_SNAKE_CASE				=					ids_tensor([self.batch_size, self.seq_length]				,    self.num_labels)
						__SCREAMING_SNAKE_CASE				=					ids_tensor([self.batch_size]				,    self.num_choices)

				__SCREAMING_SNAKE_CASE				=					self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




		def       snake_case_	(	self):
				return OpenLlamaConfig(
				    vocab_size=self.vocab_size				,    hidden_size=self.hidden_size				,    num_hidden_layers=self.num_hidden_layers				,    num_attention_heads=self.num_attention_heads				,    intermediate_size=self.intermediate_size				,    hidden_act=self.hidden_act				,    hidden_dropout_prob=self.hidden_dropout_prob				,    attention_probs_dropout_prob=self.attention_probs_dropout_prob				,    max_position_embeddings=self.max_position_embeddings				,    type_vocab_size=self.type_vocab_size				,    is_decoder=lowerCAmelCase_				,    initializer_range=self.initializer_range				,    use_stable_embedding=lowerCAmelCase_				,    )




		def       snake_case_	(	self				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE				=					OpenLlamaModel(config=lowerCAmelCase_)
				model.to(lowerCAmelCase_)
				model.eval()
				__SCREAMING_SNAKE_CASE				=					model(lowerCAmelCase_				,    attention_mask=lowerCAmelCase_)
				__SCREAMING_SNAKE_CASE				=					model(lowerCAmelCase_)
				self.parent.assertEqual(result.last_hidden_state.shape				,    (self.batch_size, self.seq_length, self.hidden_size))




		def       snake_case_	(	self				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    ):
				__SCREAMING_SNAKE_CASE				=					True
				__SCREAMING_SNAKE_CASE				=					OpenLlamaModel(lowerCAmelCase_)
				model.to(lowerCAmelCase_)
				model.eval()
				__SCREAMING_SNAKE_CASE				=					model(
				    lowerCAmelCase_				,    attention_mask=lowerCAmelCase_				,    encoder_hidden_states=lowerCAmelCase_				,    encoder_attention_mask=lowerCAmelCase_				,    )
				__SCREAMING_SNAKE_CASE				=					model(
				    lowerCAmelCase_				,    attention_mask=lowerCAmelCase_				,    encoder_hidden_states=lowerCAmelCase_				,    )
				__SCREAMING_SNAKE_CASE				=					model(lowerCAmelCase_				,    attention_mask=lowerCAmelCase_)
				self.parent.assertEqual(result.last_hidden_state.shape				,    (self.batch_size, self.seq_length, self.hidden_size))




		def       snake_case_	(	self				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    ):
				__SCREAMING_SNAKE_CASE				=					OpenLlamaForCausalLM(config=lowerCAmelCase_)
				model.to(lowerCAmelCase_)
				model.eval()
				__SCREAMING_SNAKE_CASE				=					model(lowerCAmelCase_				,    attention_mask=lowerCAmelCase_				,    labels=lowerCAmelCase_)
				self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.seq_length, self.vocab_size))




		def       snake_case_	(	self				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    ):
				__SCREAMING_SNAKE_CASE				=					True
				__SCREAMING_SNAKE_CASE				=					True
				__SCREAMING_SNAKE_CASE				=					OpenLlamaForCausalLM(config=lowerCAmelCase_)
				model.to(lowerCAmelCase_)
				model.eval()

				# first forward pass
				__SCREAMING_SNAKE_CASE				=					model(
				    lowerCAmelCase_				,    attention_mask=lowerCAmelCase_				,    encoder_hidden_states=lowerCAmelCase_				,    encoder_attention_mask=lowerCAmelCase_				,    use_cache=lowerCAmelCase_				,    )
				__SCREAMING_SNAKE_CASE				=					outputs.past_key_values

				# create hypothetical multiple next token and extent to next_input_ids
				__SCREAMING_SNAKE_CASE				=					ids_tensor((self.batch_size, 3)				,    config.vocab_size)
				__SCREAMING_SNAKE_CASE				=					ids_tensor((self.batch_size, 3)				,    vocab_size=2)

				# append to next input_ids and
				__SCREAMING_SNAKE_CASE				=					torch.cat([input_ids, next_tokens]				,    dim=-1)
				__SCREAMING_SNAKE_CASE				=					torch.cat([input_mask, next_mask]				,    dim=-1)

				__SCREAMING_SNAKE_CASE				=					model(
				    lowerCAmelCase_				,    attention_mask=lowerCAmelCase_				,    encoder_hidden_states=lowerCAmelCase_				,    encoder_attention_mask=lowerCAmelCase_				,    output_hidden_states=lowerCAmelCase_				,    )["""hidden_states"""][0]
				__SCREAMING_SNAKE_CASE				=					model(
				    lowerCAmelCase_				,    attention_mask=lowerCAmelCase_				,    encoder_hidden_states=lowerCAmelCase_				,    encoder_attention_mask=lowerCAmelCase_				,    past_key_values=lowerCAmelCase_				,    output_hidden_states=lowerCAmelCase_				,    )["""hidden_states"""][0]

				# select random slice
				__SCREAMING_SNAKE_CASE				=					ids_tensor((1,)				,    output_from_past.shape[-1]).item()
				__SCREAMING_SNAKE_CASE				=					output_from_no_past[:, -3:, random_slice_idx].detach()
				__SCREAMING_SNAKE_CASE				=					output_from_past[:, :, random_slice_idx].detach()

				self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

				# test that outputs are equal for slice
				self.parent.assertTrue(torch.allclose(lowerCAmelCase_				,    lowerCAmelCase_				,    atol=1E-3))




		def       snake_case_	(	self):
				__SCREAMING_SNAKE_CASE				=					self.prepare_config_and_inputs()
				(
				    __SCREAMING_SNAKE_CASE  
				)				=					config_and_inputs
				__SCREAMING_SNAKE_CASE				=					{"""input_ids""": input_ids, """attention_mask""": input_mask}
				return config, inputs_dict



@require_torch
class 		SCREAMING_SNAKE_CASE_					(		_SCREAMING_SNAKE_CASE	,							_SCREAMING_SNAKE_CASE	,							_SCREAMING_SNAKE_CASE	,							unittest.TestCase					):




		"""simple docstring"""





		__lowercase				:    Union[str, Any]		  =						(
		    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
		)
		__lowercase				:    Tuple		  =						(OpenLlamaForCausalLM,) if is_torch_available() else ()
		__lowercase				:    Dict		  =						(
		    {
		        """feature-extraction""": OpenLlamaModel,
		        """text-classification""": OpenLlamaForSequenceClassification,
		        """text-generation""": OpenLlamaForCausalLM,
		        """zero-shot""": OpenLlamaForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		__lowercase				:    Optional[Any]		  =						False
		__lowercase				:    Any		  =						False




		def       snake_case_	(	self):
				__SCREAMING_SNAKE_CASE				=					OpenLlamaModelTester(self)
				__SCREAMING_SNAKE_CASE				=					ConfigTester(self				,    config_class=lowerCAmelCase_				,    hidden_size=3_7)




		def       snake_case_	(	self):
				self.config_tester.run_common_tests()




		def       snake_case_	(	self):
				__SCREAMING_SNAKE_CASE				=					self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*lowerCAmelCase_)




		def       snake_case_	(	self):
				__SCREAMING_SNAKE_CASE				=					self.model_tester.prepare_config_and_inputs()
				for type in ["absolute", "relative_key", "relative_key_query"]:
						__SCREAMING_SNAKE_CASE				=					type
						self.model_tester.create_and_check_model(*lowerCAmelCase_)




		def       snake_case_	(	self):
				__SCREAMING_SNAKE_CASE				=					self.model_tester.prepare_config_and_inputs_for_common()
				__SCREAMING_SNAKE_CASE				=					3
				__SCREAMING_SNAKE_CASE				=					input_dict["""input_ids"""]
				__SCREAMING_SNAKE_CASE				=					input_ids.ne(1).to(lowerCAmelCase_)
				__SCREAMING_SNAKE_CASE				=					ids_tensor([self.model_tester.batch_size]				,    self.model_tester.type_sequence_label_size)
				__SCREAMING_SNAKE_CASE				=					OpenLlamaForSequenceClassification(lowerCAmelCase_)
				model.to(lowerCAmelCase_)
				model.eval()
				__SCREAMING_SNAKE_CASE				=					model(lowerCAmelCase_				,    attention_mask=lowerCAmelCase_				,    labels=lowerCAmelCase_)
				self.assertEqual(result.logits.shape				,    (self.model_tester.batch_size, self.model_tester.num_labels))




		def       snake_case_	(	self):
				__SCREAMING_SNAKE_CASE				=					self.model_tester.prepare_config_and_inputs_for_common()
				__SCREAMING_SNAKE_CASE				=					3
				__SCREAMING_SNAKE_CASE				=					"""single_label_classification"""
				__SCREAMING_SNAKE_CASE				=					input_dict["""input_ids"""]
				__SCREAMING_SNAKE_CASE				=					input_ids.ne(1).to(lowerCAmelCase_)
				__SCREAMING_SNAKE_CASE				=					ids_tensor([self.model_tester.batch_size]				,    self.model_tester.type_sequence_label_size)
				__SCREAMING_SNAKE_CASE				=					OpenLlamaForSequenceClassification(lowerCAmelCase_)
				model.to(lowerCAmelCase_)
				model.eval()
				__SCREAMING_SNAKE_CASE				=					model(lowerCAmelCase_				,    attention_mask=lowerCAmelCase_				,    labels=lowerCAmelCase_)
				self.assertEqual(result.logits.shape				,    (self.model_tester.batch_size, self.model_tester.num_labels))




		def       snake_case_	(	self):
				__SCREAMING_SNAKE_CASE				=					self.model_tester.prepare_config_and_inputs_for_common()
				__SCREAMING_SNAKE_CASE				=					3
				__SCREAMING_SNAKE_CASE				=					"""multi_label_classification"""
				__SCREAMING_SNAKE_CASE				=					input_dict["""input_ids"""]
				__SCREAMING_SNAKE_CASE				=					input_ids.ne(1).to(lowerCAmelCase_)
				__SCREAMING_SNAKE_CASE				=					ids_tensor(
				    [self.model_tester.batch_size, config.num_labels]				,    self.model_tester.type_sequence_label_size).to(torch.float)
				__SCREAMING_SNAKE_CASE				=					OpenLlamaForSequenceClassification(lowerCAmelCase_)
				model.to(lowerCAmelCase_)
				model.eval()
				__SCREAMING_SNAKE_CASE				=					model(lowerCAmelCase_				,    attention_mask=lowerCAmelCase_				,    labels=lowerCAmelCase_)
				self.assertEqual(result.logits.shape				,    (self.model_tester.batch_size, self.model_tester.num_labels))




		@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test""")
		def       snake_case_	(	self):
				pass




		@parameterized.expand([("""linear""",), ("""dynamic""",)])
		def       snake_case_	(	self				,    lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE				=					self.model_tester.prepare_config_and_inputs_for_common()
				__SCREAMING_SNAKE_CASE				=					ids_tensor([1, 1_0]				,    config.vocab_size)
				__SCREAMING_SNAKE_CASE				=					ids_tensor([1, int(config.max_position_embeddings * 1.5)]				,    config.vocab_size)

				set_seed(4_2)  # Fixed seed at init time so the two models get the same random weights
				__SCREAMING_SNAKE_CASE				=					OpenLlamaModel(lowerCAmelCase_)
				original_model.to(lowerCAmelCase_)
				original_model.eval()
				__SCREAMING_SNAKE_CASE				=					original_model(lowerCAmelCase_).last_hidden_state
				__SCREAMING_SNAKE_CASE				=					original_model(lowerCAmelCase_).last_hidden_state

				set_seed(4_2)  # Fixed seed at init time so the two models get the same random weights
				__SCREAMING_SNAKE_CASE				=					{"""type""": scaling_type, """factor""": 10.0}
				__SCREAMING_SNAKE_CASE				=					OpenLlamaModel(lowerCAmelCase_)
				scaled_model.to(lowerCAmelCase_)
				scaled_model.eval()
				__SCREAMING_SNAKE_CASE				=					scaled_model(lowerCAmelCase_).last_hidden_state
				__SCREAMING_SNAKE_CASE				=					scaled_model(lowerCAmelCase_).last_hidden_state

				# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
				# maximum sequence length, so the outputs for the short input should match.
				if scaling_type == "dynamic":
						self.assertTrue(torch.allclose(lowerCAmelCase_				,    lowerCAmelCase_				,    atol=1E-5))
				else:
						self.assertFalse(torch.allclose(lowerCAmelCase_				,    lowerCAmelCase_				,    atol=1E-5))

				# The output should be different for long inputs
				self.assertFalse(torch.allclose(lowerCAmelCase_				,    lowerCAmelCase_				,    atol=1E-5))












"""simple docstring"""
_lowerCAmelCase       =				[
    "Audio",
    "Array2D",
    "Array3D",
    "Array4D",
    "Array5D",
    "ClassLabel",
    "Features",
    "Sequence",
    "Value",
    "Image",
    "Translation",
    "TranslationVariableLanguages",
]
from .audio import Audio
from .features import ArrayaD, ArrayaD, ArrayaD, ArrayaD, ClassLabel, Features, Sequence, Value
from .image import Image
from .translation import Translation, TranslationVariableLanguages

