






import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel


class     __SCREAMING_SNAKE_CASE					:






						def __init__(	self     ,						_UpperCamelCase     ,						_UpperCamelCase=13     ,						_UpperCamelCase=7     ,						_UpperCamelCase=True     ,						_UpperCamelCase=True     ,						_UpperCamelCase=False     ,						_UpperCamelCase=True     ,						_UpperCamelCase=99     ,						_UpperCamelCase=32     ,						_UpperCamelCase=5     ,						_UpperCamelCase=4     ,						_UpperCamelCase=37     ,						_UpperCamelCase="gelu"     ,						_UpperCamelCase=0.1     ,						_UpperCamelCase=0.1     ,						_UpperCamelCase=5_12     ,						_UpperCamelCase=16     ,						_UpperCamelCase=2     ,						_UpperCamelCase=0.02     ,						_UpperCamelCase=3     ,						_UpperCamelCase=4     ,						_UpperCamelCase=None     ,						):






												"""simple docstring"""
												lowerCAmelCase__    						=						parent
												lowerCAmelCase__    						=						batch_size
												lowerCAmelCase__    						=						seq_length
												lowerCAmelCase__    						=						is_training
												lowerCAmelCase__    						=						use_input_mask
												lowerCAmelCase__    						=						use_token_type_ids
												lowerCAmelCase__    						=						use_labels
												lowerCAmelCase__    						=						vocab_size
												lowerCAmelCase__    						=						hidden_size
												lowerCAmelCase__    						=						num_hidden_layers
												lowerCAmelCase__    						=						num_attention_heads
												lowerCAmelCase__    						=						intermediate_size
												lowerCAmelCase__    						=						hidden_act
												lowerCAmelCase__    						=						hidden_dropout_prob
												lowerCAmelCase__    						=						attention_probs_dropout_prob
												lowerCAmelCase__    						=						max_position_embeddings
												lowerCAmelCase__    						=						type_vocab_size
												lowerCAmelCase__    						=						type_sequence_label_size
												lowerCAmelCase__    						=						initializer_range
												lowerCAmelCase__    						=						num_labels
												lowerCAmelCase__    						=						num_choices
												lowerCAmelCase__    						=						scope






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						ids_tensor([self.batch_size, self.seq_length]     ,						self.vocab_size   )

												lowerCAmelCase__    						=						None
												if self.use_input_mask:
																		lowerCAmelCase__    						=						random_attention_mask([self.batch_size, self.seq_length]   )

												lowerCAmelCase__    						=						None
												if self.use_token_type_ids:
																		lowerCAmelCase__    						=						ids_tensor([self.batch_size, self.seq_length]     ,						self.type_vocab_size   )

												lowerCAmelCase__    						=						None
												lowerCAmelCase__    						=						None
												lowerCAmelCase__    						=						None
												if self.use_labels:
																		lowerCAmelCase__    						=						ids_tensor([self.batch_size]     ,						self.type_sequence_label_size   )
																		lowerCAmelCase__    						=						ids_tensor([self.batch_size, self.seq_length]     ,						self.num_labels   )
																		lowerCAmelCase__    						=						ids_tensor([self.batch_size]     ,						self.num_choices   )

												lowerCAmelCase__    						=						self.get_config()

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												return OpenLlamaConfig(
												    vocab_size=self.vocab_size     ,						hidden_size=self.hidden_size     ,						num_hidden_layers=self.num_hidden_layers     ,						num_attention_heads=self.num_attention_heads     ,						intermediate_size=self.intermediate_size     ,						hidden_act=self.hidden_act     ,						hidden_dropout_prob=self.hidden_dropout_prob     ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,						max_position_embeddings=self.max_position_embeddings     ,						type_vocab_size=self.type_vocab_size     ,						is_decoder=UpperCamelCase_     ,						initializer_range=self.initializer_range     ,						use_stable_embedding=UpperCamelCase_     ,						)






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						OpenLlamaModel(config=UpperCamelCase_   )
												model.to(UpperCamelCase_   )
												model.eval()
												lowerCAmelCase__    						=						model(UpperCamelCase_     ,						attention_mask=UpperCamelCase_   )
												lowerCAmelCase__    						=						model(UpperCamelCase_   )
												self.parent.assertEqual(result.last_hidden_state.shape     ,						(self.batch_size, self.seq_length, self.hidden_size)   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						):






												"""simple docstring"""
												lowerCAmelCase__    						=						True
												lowerCAmelCase__    						=						OpenLlamaModel(UpperCamelCase_   )
												model.to(UpperCamelCase_   )
												model.eval()
												lowerCAmelCase__    						=						model(
												    UpperCamelCase_     ,						attention_mask=UpperCamelCase_     ,						encoder_hidden_states=UpperCamelCase_     ,						encoder_attention_mask=UpperCamelCase_     ,						)
												lowerCAmelCase__    						=						model(
												    UpperCamelCase_     ,						attention_mask=UpperCamelCase_     ,						encoder_hidden_states=UpperCamelCase_     ,						)
												lowerCAmelCase__    						=						model(UpperCamelCase_     ,						attention_mask=UpperCamelCase_   )
												self.parent.assertEqual(result.last_hidden_state.shape     ,						(self.batch_size, self.seq_length, self.hidden_size)   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						):






												"""simple docstring"""
												lowerCAmelCase__    						=						OpenLlamaForCausalLM(config=UpperCamelCase_   )
												model.to(UpperCamelCase_   )
												model.eval()
												lowerCAmelCase__    						=						model(UpperCamelCase_     ,						attention_mask=UpperCamelCase_     ,						labels=UpperCamelCase_   )
												self.parent.assertEqual(result.logits.shape     ,						(self.batch_size, self.seq_length, self.vocab_size)   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						):






												"""simple docstring"""
												lowerCAmelCase__    						=						True
												lowerCAmelCase__    						=						True
												lowerCAmelCase__    						=						OpenLlamaForCausalLM(config=UpperCamelCase_   )
												model.to(UpperCamelCase_   )
												model.eval()

												# first forward pass
												lowerCAmelCase__    						=						model(
												    UpperCamelCase_     ,						attention_mask=UpperCamelCase_     ,						encoder_hidden_states=UpperCamelCase_     ,						encoder_attention_mask=UpperCamelCase_     ,						use_cache=UpperCamelCase_     ,						)
												lowerCAmelCase__    						=						outputs.past_key_values

												# create hypothetical multiple next token and extent to next_input_ids
												lowerCAmelCase__    						=						ids_tensor((self.batch_size, 3)     ,						config.vocab_size   )
												lowerCAmelCase__    						=						ids_tensor((self.batch_size, 3)     ,						vocab_size=2   )

												# append to next input_ids and
												lowerCAmelCase__    						=						torch.cat([input_ids, next_tokens]     ,						dim=-1   )
												lowerCAmelCase__    						=						torch.cat([input_mask, next_mask]     ,						dim=-1   )

												lowerCAmelCase__    						=						model(
												    UpperCamelCase_     ,						attention_mask=UpperCamelCase_     ,						encoder_hidden_states=UpperCamelCase_     ,						encoder_attention_mask=UpperCamelCase_     ,						output_hidden_states=UpperCamelCase_     ,						)['hidden_states'][0]
												lowerCAmelCase__    						=						model(
												    UpperCamelCase_     ,						attention_mask=UpperCamelCase_     ,						encoder_hidden_states=UpperCamelCase_     ,						encoder_attention_mask=UpperCamelCase_     ,						past_key_values=UpperCamelCase_     ,						output_hidden_states=UpperCamelCase_     ,						)['hidden_states'][0]

												# select random slice
												lowerCAmelCase__    						=						ids_tensor((1,)     ,						output_from_past.shape[-1]   ).item()
												lowerCAmelCase__    						=						output_from_no_past[:, -3:, random_slice_idx].detach()
												lowerCAmelCase__    						=						output_from_past[:, :, random_slice_idx].detach()

												self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

												# test that outputs are equal for slice
												self.parent.assertTrue(torch.allclose(UpperCamelCase_     ,						UpperCamelCase_     ,						atol=1E-3   )   )







						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.prepare_config_and_inputs()
												(
												    lowerCAmelCase__    
												)    						=						config_and_inputs
												lowerCAmelCase__    						=						{'input_ids': input_ids, 'attention_mask': input_mask}
												return config, inputs_dict




@require_torch
class     __SCREAMING_SNAKE_CASE					(   __lowercase		,					__lowercase		,					__lowercase		,					unittest.TestCase):
						_SCREAMING_SNAKE_CASE	:       Any   =					(
						    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
						)
						_SCREAMING_SNAKE_CASE	:       Any   =					(OpenLlamaForCausalLM,) if is_torch_available() else ()
						_SCREAMING_SNAKE_CASE	:       List[str]   =					(
						    {
						        '''feature-extraction''': OpenLlamaModel,
						        '''text-classification''': OpenLlamaForSequenceClassification,
						        '''text-generation''': OpenLlamaForCausalLM,
						        '''zero-shot''': OpenLlamaForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						_SCREAMING_SNAKE_CASE	:       List[Any]   =					False
						_SCREAMING_SNAKE_CASE	:       Tuple   =					False






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						OpenLlamaModelTester(self   )
												lowerCAmelCase__    						=						ConfigTester(self     ,						config_class=UpperCamelCase_     ,						hidden_size=37   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												self.config_tester.run_common_tests()






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*UpperCamelCase_   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.model_tester.prepare_config_and_inputs()
												for type in ["absolute", "relative_key", "relative_key_query"]:
																		lowerCAmelCase__    						=						type
																		self.model_tester.create_and_check_model(*UpperCamelCase_   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.model_tester.prepare_config_and_inputs_for_common()
												lowerCAmelCase__    						=						3
												lowerCAmelCase__    						=						input_dict['input_ids']
												lowerCAmelCase__    						=						input_ids.ne(1   ).to(UpperCamelCase_   )
												lowerCAmelCase__    						=						ids_tensor([self.model_tester.batch_size]     ,						self.model_tester.type_sequence_label_size   )
												lowerCAmelCase__    						=						OpenLlamaForSequenceClassification(UpperCamelCase_   )
												model.to(UpperCamelCase_   )
												model.eval()
												lowerCAmelCase__    						=						model(UpperCamelCase_     ,						attention_mask=UpperCamelCase_     ,						labels=UpperCamelCase_   )
												self.assertEqual(result.logits.shape     ,						(self.model_tester.batch_size, self.model_tester.num_labels)   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.model_tester.prepare_config_and_inputs_for_common()
												lowerCAmelCase__    						=						3
												lowerCAmelCase__    						=						'single_label_classification'
												lowerCAmelCase__    						=						input_dict['input_ids']
												lowerCAmelCase__    						=						input_ids.ne(1   ).to(UpperCamelCase_   )
												lowerCAmelCase__    						=						ids_tensor([self.model_tester.batch_size]     ,						self.model_tester.type_sequence_label_size   )
												lowerCAmelCase__    						=						OpenLlamaForSequenceClassification(UpperCamelCase_   )
												model.to(UpperCamelCase_   )
												model.eval()
												lowerCAmelCase__    						=						model(UpperCamelCase_     ,						attention_mask=UpperCamelCase_     ,						labels=UpperCamelCase_   )
												self.assertEqual(result.logits.shape     ,						(self.model_tester.batch_size, self.model_tester.num_labels)   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.model_tester.prepare_config_and_inputs_for_common()
												lowerCAmelCase__    						=						3
												lowerCAmelCase__    						=						'multi_label_classification'
												lowerCAmelCase__    						=						input_dict['input_ids']
												lowerCAmelCase__    						=						input_ids.ne(1   ).to(UpperCamelCase_   )
												lowerCAmelCase__    						=						ids_tensor(
												    [self.model_tester.batch_size, config.num_labels]     ,						self.model_tester.type_sequence_label_size   ).to(torch.float   )
												lowerCAmelCase__    						=						OpenLlamaForSequenceClassification(UpperCamelCase_   )
												model.to(UpperCamelCase_   )
												model.eval()
												lowerCAmelCase__    						=						model(UpperCamelCase_     ,						attention_mask=UpperCamelCase_     ,						labels=UpperCamelCase_   )
												self.assertEqual(result.logits.shape     ,						(self.model_tester.batch_size, self.model_tester.num_labels)   )






						@unittest.skip('Open-Llama buffers include complex numbers, which breaks this test'   )
						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												pass







						@parameterized.expand([('linear',), ('dynamic',)]   )
						def       UpperCamelCase__      (	self     ,						_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.model_tester.prepare_config_and_inputs_for_common()
												lowerCAmelCase__    						=						ids_tensor([1, 10]     ,						config.vocab_size   )
												lowerCAmelCase__    						=						ids_tensor([1, int(config.max_position_embeddings * 1.5   )]     ,						config.vocab_size   )

												set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
												lowerCAmelCase__    						=						OpenLlamaModel(UpperCamelCase_   )
												original_model.to(UpperCamelCase_   )
												original_model.eval()
												lowerCAmelCase__    						=						original_model(UpperCamelCase_   ).last_hidden_state
												lowerCAmelCase__    						=						original_model(UpperCamelCase_   ).last_hidden_state

												set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
												lowerCAmelCase__    						=						{'type': scaling_type, 'factor': 10.0}
												lowerCAmelCase__    						=						OpenLlamaModel(UpperCamelCase_   )
												scaled_model.to(UpperCamelCase_   )
												scaled_model.eval()
												lowerCAmelCase__    						=						scaled_model(UpperCamelCase_   ).last_hidden_state
												lowerCAmelCase__    						=						scaled_model(UpperCamelCase_   ).last_hidden_state

												# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
												# maximum sequence length, so the outputs for the short input should match.
												if scaling_type == "dynamic":
																		self.assertTrue(torch.allclose(UpperCamelCase_     ,						UpperCamelCase_     ,						atol=1E-5   )   )
												else:
																		self.assertFalse(torch.allclose(UpperCamelCase_     ,						UpperCamelCase_     ,						atol=1E-5   )   )

												# The output should be different for long inputs
												self.assertFalse(torch.allclose(UpperCamelCase_     ,						UpperCamelCase_     ,						atol=1E-5   )   )













def 	_UpperCamelCase   (    UpperCamelCase_       :					int		)  ->   int:

						"""simple docstring"""



						if not isinstance(UpperCamelCase_    ,       UpperCamelCase_		):
												raise ValueError('multiplicative_persistence() only accepts integral values'		)
						if num < 0:
												raise ValueError('multiplicative_persistence() does not accept negative values'		)

						lowerCAmelCase__    						=						0
						lowerCAmelCase__    						=						str(UpperCamelCase_		)

						while len(UpperCamelCase_		) != 1:
												lowerCAmelCase__    						=						[int(UpperCamelCase_		) for i in num_string]

												lowerCAmelCase__    						=						1
												for i in range(0    ,       len(UpperCamelCase_		)		):
																		total *= numbers[i]

												lowerCAmelCase__    						=						str(UpperCamelCase_		)

												steps += 1
						return steps







def 	_UpperCamelCase   (    UpperCamelCase_       :					int		)  ->   int:

						"""simple docstring"""



						if not isinstance(UpperCamelCase_    ,       UpperCamelCase_		):
												raise ValueError('additive_persistence() only accepts integral values'		)
						if num < 0:
												raise ValueError('additive_persistence() does not accept negative values'		)

						lowerCAmelCase__    						=						0
						lowerCAmelCase__    						=						str(UpperCamelCase_		)

						while len(UpperCamelCase_		) != 1:
												lowerCAmelCase__    						=						[int(UpperCamelCase_		) for i in num_string]

												lowerCAmelCase__    						=						0
												for i in range(0    ,       len(UpperCamelCase_		)		):
																		total += numbers[i]

												lowerCAmelCase__    						=						str(UpperCamelCase_		)

												steps += 1
						return steps


if __name__ == "__main__":
					import doctest

					doctest.testmod()





