

'''simple docstring'''





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel





class 		__SCREAMING_SNAKE_CASE							:







				def __init__(       self     :		str		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		List[str]=13		,					UpperCAmelCase__     :		Optional[Any]=7		,					UpperCAmelCase__     :		Dict=True		,					UpperCAmelCase__     :		Any=True		,					UpperCAmelCase__     :		Tuple=False		,					UpperCAmelCase__     :		Tuple=True		,					UpperCAmelCase__     :		Optional[Any]=99		,					UpperCAmelCase__     :		Any=32		,					UpperCAmelCase__     :		Optional[int]=5		,					UpperCAmelCase__     :		Optional[Any]=4		,					UpperCAmelCase__     :		Dict=37		,					UpperCAmelCase__     :		str="gelu"		,					UpperCAmelCase__     :		Tuple=0.1		,					UpperCAmelCase__     :		Dict=0.1		,					UpperCAmelCase__     :		Optional[Any]=512		,					UpperCAmelCase__     :		Union[str, Any]=16		,					UpperCAmelCase__     :		Optional[Any]=2		,					UpperCAmelCase__     :		List[str]=0.02		,					UpperCAmelCase__     :		List[Any]=3		,					UpperCAmelCase__     :		Tuple=4		,					UpperCAmelCase__     :		List[str]=None		,					):
										'''simple docstring'''
										lowercase :							List[str]    		=parent
										lowercase :							Dict    		=batch_size
										lowercase :							List[str]    		=seq_length
										lowercase :							List[str]    		=is_training
										lowercase :							Union[str, Any]    		=use_input_mask
										lowercase :							str    		=use_token_type_ids
										lowercase :							List[Any]    		=use_labels
										lowercase :							List[Any]    		=vocab_size
										lowercase :							Tuple    		=hidden_size
										lowercase :							Union[str, Any]    		=num_hidden_layers
										lowercase :							Dict    		=num_attention_heads
										lowercase :							List[str]    		=intermediate_size
										lowercase :							str    		=hidden_act
										lowercase :							Any    		=hidden_dropout_prob
										lowercase :							str    		=attention_probs_dropout_prob
										lowercase :							str    		=max_position_embeddings
										lowercase :							Dict    		=type_vocab_size
										lowercase :							List[str]    		=type_sequence_label_size
										lowercase :							List[Any]    		=initializer_range
										lowercase :							Optional[Any]    		=num_labels
										lowercase :							Any    		=num_choices
										lowercase :							Optional[int]    		=scope







				def  lowerCamelCase_	(       self     :		Optional[int]   ):
										'''simple docstring'''
										lowercase :							List[str]    		=ids_tensor([self.batch_size, self.seq_length]		,					self.vocab_size   )

										lowercase :							Optional[Any]    		=None
										if self.use_input_mask:
																lowercase :							Tuple    		=random_attention_mask([self.batch_size, self.seq_length]   )

										lowercase :							Union[str, Any]    		=None
										if self.use_token_type_ids:
																lowercase :							List[str]    		=ids_tensor([self.batch_size, self.seq_length]		,					self.type_vocab_size   )

										lowercase :							str    		=None
										lowercase :							int    		=None
										lowercase :							Any    		=None
										if self.use_labels:
																lowercase :							Any    		=ids_tensor([self.batch_size]		,					self.type_sequence_label_size   )
																lowercase :							List[Any]    		=ids_tensor([self.batch_size, self.seq_length]		,					self.num_labels   )
																lowercase :							List[str]    		=ids_tensor([self.batch_size]		,					self.num_choices   )

										lowercase :							List[str]    		=self.get_config()

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







				def  lowerCamelCase_	(       self     :		Any   ):
										'''simple docstring'''
										return OpenLlamaConfig(
										    vocab_size=self.vocab_size		,					hidden_size=self.hidden_size		,					num_hidden_layers=self.num_hidden_layers		,					num_attention_heads=self.num_attention_heads		,					intermediate_size=self.intermediate_size		,					hidden_act=self.hidden_act		,					hidden_dropout_prob=self.hidden_dropout_prob		,					attention_probs_dropout_prob=self.attention_probs_dropout_prob		,					max_position_embeddings=self.max_position_embeddings		,					type_vocab_size=self.type_vocab_size		,					is_decoder=UpperCAmelCase__		,					initializer_range=self.initializer_range		,					use_stable_embedding=UpperCAmelCase__		,					)







				def  lowerCamelCase_	(       self     :		Dict		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		Optional[Any]		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		List[str]   ):
										'''simple docstring'''
										lowercase :							Union[str, Any]    		=OpenLlamaModel(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							str    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__   )
										lowercase :							Tuple    		=model(UpperCAmelCase__   )
										self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.hidden_size)   )







				def  lowerCamelCase_	(       self     :		Optional[Any]		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		Any		,					):
										'''simple docstring'''
										lowercase :							Optional[Any]    		=True
										lowercase :							Optional[Any]    		=OpenLlamaModel(UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							List[str]    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					encoder_hidden_states=UpperCAmelCase__		,					encoder_attention_mask=UpperCAmelCase__		,					)
										lowercase :							int    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					encoder_hidden_states=UpperCAmelCase__		,					)
										lowercase :							Optional[Any]    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__   )
										self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.hidden_size)   )







				def  lowerCamelCase_	(       self     :		str		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Optional[Any]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Union[str, Any]		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		Any		,					):
										'''simple docstring'''
										lowercase :							List[str]    		=OpenLlamaForCausalLM(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							Union[str, Any]    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.vocab_size)   )







				def  lowerCamelCase_	(       self     :		str		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		Optional[Any]		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		List[Any]		,					):
										'''simple docstring'''
										lowercase :							int    		=True
										lowercase :							List[str]    		=True
										lowercase :							Tuple    		=OpenLlamaForCausalLM(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()

										# first forward pass
										lowercase :							Optional[int]    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					encoder_hidden_states=UpperCAmelCase__		,					encoder_attention_mask=UpperCAmelCase__		,					use_cache=UpperCAmelCase__		,					)
										lowercase :							Union[str, Any]    		=outputs.past_key_values

										# create hypothetical multiple next token and extent to next_input_ids
										lowercase :							Any    		=ids_tensor((self.batch_size, 3)		,					config.vocab_size   )
										lowercase :							Tuple    		=ids_tensor((self.batch_size, 3)		,					vocab_size=2   )

										# append to next input_ids and
										lowercase :							Any    		=torch.cat([input_ids, next_tokens]		,					dim=-1   )
										lowercase :							int    		=torch.cat([input_mask, next_mask]		,					dim=-1   )

										lowercase :							Dict    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					encoder_hidden_states=UpperCAmelCase__		,					encoder_attention_mask=UpperCAmelCase__		,					output_hidden_states=UpperCAmelCase__		,					)['''hidden_states'''][0]
										lowercase :							Optional[int]    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					encoder_hidden_states=UpperCAmelCase__		,					encoder_attention_mask=UpperCAmelCase__		,					past_key_values=UpperCAmelCase__		,					output_hidden_states=UpperCAmelCase__		,					)['''hidden_states'''][0]

										# select random slice
										lowercase :							List[Any]    		=ids_tensor((1,)		,					output_from_past.shape[-1]   ).item()
										lowercase :							Any    		=output_from_no_past[:, -3:, random_slice_idx].detach()
										lowercase :							Tuple    		=output_from_past[:, :, random_slice_idx].detach()

										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

										# test that outputs are equal for slice
										self.parent.assertTrue(torch.allclose(UpperCAmelCase__		,					UpperCAmelCase__		,					atol=1E-3   )   )







				def  lowerCamelCase_	(       self     :		Any   ):
										'''simple docstring'''
										lowercase :							Tuple    		=self.prepare_config_and_inputs()
										(
										    (
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	
										) :							str    		=config_and_inputs
										lowercase :							List[Any]    		={'''input_ids''': input_ids, '''attention_mask''': input_mask}
										return config, inputs_dict





@require_torch
class 		__SCREAMING_SNAKE_CASE							(			lowercase__						,							lowercase__						,							lowercase__						,							unittest.TestCase      ):
				lowerCamelCase_		       =    (
				    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
				)
				lowerCamelCase_		       =    (OpenLlamaForCausalLM,) if is_torch_available() else ()
				lowerCamelCase_		       =    (
				    {
				        'feature-extraction': OpenLlamaModel,
				        'text-classification': OpenLlamaForSequenceClassification,
				        'text-generation': OpenLlamaForCausalLM,
				        'zero-shot': OpenLlamaForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				lowerCamelCase_		       =    False
				lowerCamelCase_		       =    False







				def  lowerCamelCase_	(       self     :		Any   ):
										'''simple docstring'''
										lowercase :							Any    		=OpenLlamaModelTester(self   )
										lowercase :							Dict    		=ConfigTester(self		,					config_class=UpperCAmelCase__		,					hidden_size=37   )







				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										self.config_tester.run_common_tests()







				def  lowerCamelCase_	(       self     :		List[str]   ):
										'''simple docstring'''
										lowercase :							Tuple    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		Optional[Any]   ):
										'''simple docstring'''
										lowercase :							Optional[int]    		=self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
																lowercase :							Dict    		=type
																self.model_tester.create_and_check_model(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										lowercase		,	lowercase :							Dict    		=self.model_tester.prepare_config_and_inputs_for_common()
										lowercase :							Optional[int]    		=3
										lowercase :							List[Any]    		=input_dict['''input_ids''']
										lowercase :							Any    		=input_ids.ne(1   ).to(UpperCAmelCase__   )
										lowercase :							Optional[Any]    		=ids_tensor([self.model_tester.batch_size]		,					self.model_tester.type_sequence_label_size   )
										lowercase :							Optional[Any]    		=OpenLlamaForSequenceClassification(UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							List[str]    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.assertEqual(result.logits.shape		,					(self.model_tester.batch_size, self.model_tester.num_labels)   )







				def  lowerCamelCase_	(       self     :		Any   ):
										'''simple docstring'''
										lowercase		,	lowercase :							List[Any]    		=self.model_tester.prepare_config_and_inputs_for_common()
										lowercase :							Dict    		=3
										lowercase :							List[str]    		='''single_label_classification'''
										lowercase :							Tuple    		=input_dict['''input_ids''']
										lowercase :							Dict    		=input_ids.ne(1   ).to(UpperCAmelCase__   )
										lowercase :							List[str]    		=ids_tensor([self.model_tester.batch_size]		,					self.model_tester.type_sequence_label_size   )
										lowercase :							Dict    		=OpenLlamaForSequenceClassification(UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							str    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.assertEqual(result.logits.shape		,					(self.model_tester.batch_size, self.model_tester.num_labels)   )







				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										lowercase		,	lowercase :							str    		=self.model_tester.prepare_config_and_inputs_for_common()
										lowercase :							Optional[Any]    		=3
										lowercase :							Optional[int]    		='''multi_label_classification'''
										lowercase :							Dict    		=input_dict['''input_ids''']
										lowercase :							Optional[int]    		=input_ids.ne(1   ).to(UpperCAmelCase__   )
										lowercase :							Dict    		=ids_tensor(
										    [self.model_tester.batch_size, config.num_labels]		,					self.model_tester.type_sequence_label_size   ).to(torch.float   )
										lowercase :							str    		=OpenLlamaForSequenceClassification(UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							List[str]    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.assertEqual(result.logits.shape		,					(self.model_tester.batch_size, self.model_tester.num_labels)   )







				@unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test'''   )
				def  lowerCamelCase_	(       self     :		Optional[Any]   ):
										'''simple docstring'''
										pass







				@parameterized.expand([('''linear''',), ('''dynamic''',)]   )
				def  lowerCamelCase_	(       self     :		Optional[int]		,					UpperCAmelCase__     :		List[Any]   ):
										'''simple docstring'''
										lowercase		,	lowercase :							Tuple    		=self.model_tester.prepare_config_and_inputs_for_common()
										lowercase :							List[str]    		=ids_tensor([1, 10]		,					config.vocab_size   )
										lowercase :							Union[str, Any]    		=ids_tensor([1, int(config.max_position_embeddings * 1.5   )]		,					config.vocab_size   )

										set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
										lowercase :							Dict    		=OpenLlamaModel(UpperCAmelCase__   )
										original_model.to(UpperCAmelCase__   )
										original_model.eval()
										lowercase :							Optional[Any]    		=original_model(UpperCAmelCase__   ).last_hidden_state
										lowercase :							Tuple    		=original_model(UpperCAmelCase__   ).last_hidden_state

										set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
										lowercase :							Optional[int]    		={'''type''': scaling_type, '''factor''': 10.0}
										lowercase :							Tuple    		=OpenLlamaModel(UpperCAmelCase__   )
										scaled_model.to(UpperCAmelCase__   )
										scaled_model.eval()
										lowercase :							Optional[Any]    		=scaled_model(UpperCAmelCase__   ).last_hidden_state
										lowercase :							Optional[int]    		=scaled_model(UpperCAmelCase__   ).last_hidden_state

										# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
										# maximum sequence length, so the outputs for the short input should match.
										if scaling_type == "dynamic":
																self.assertTrue(torch.allclose(UpperCAmelCase__		,					UpperCAmelCase__		,					atol=1E-5   )   )
										else:
																self.assertFalse(torch.allclose(UpperCAmelCase__		,					UpperCAmelCase__		,					atol=1E-5   )   )

										# The output should be different for long inputs
										self.assertFalse(torch.allclose(UpperCAmelCase__		,					UpperCAmelCase__		,					atol=1E-5   )   )











'''simple docstring'''



import inspect
import warnings
from typing import Any, Dict, Optional, Union

from packaging import version




def   lowercase__(			*__UpperCamelCase:					Union[str, Any]      ,__UpperCamelCase:					Optional[Union[Dict, Any]] = None      ,__UpperCamelCase:					Dict=True      ,__UpperCamelCase:					List[Any]=2    ):

      """simple docstring"""






      from .. import __version__

      SCREAMING_SNAKE_CASE   :      int    					=						take_from
      SCREAMING_SNAKE_CASE   :      Optional[int]    					=						()
      if not isinstance(args[0]      ,__UpperCamelCase    ):
            SCREAMING_SNAKE_CASE   :      List[str]    					=						(args,)

      for attribute, version_name, message in args:
            if version.parse(version.parse(__UpperCamelCase    ).base_version    ) >= version.parse(__UpperCamelCase    ):
                  raise ValueError(
                      f"The deprecation tuple {(attribute, version_name, message)} should be removed since diffusers'"
                      f" version {__version__} is >= {version_name}"    )

            SCREAMING_SNAKE_CASE   :      Tuple    					=						None
            if isinstance(__UpperCamelCase      ,__UpperCamelCase    ) and attribute in deprecated_kwargs:
                  values += (deprecated_kwargs.pop(__UpperCamelCase    ),)
                  SCREAMING_SNAKE_CASE   :      Dict    					=						f"The `{attribute}` argument is deprecated and will be removed in version {version_name}."
            elif hasattr(__UpperCamelCase      ,__UpperCamelCase    ):
                  values += (getattr(__UpperCamelCase      ,__UpperCamelCase    ),)
                  SCREAMING_SNAKE_CASE   :      Optional[int]    					=						f"The `{attribute}` attribute is deprecated and will be removed in version {version_name}."
            elif deprecated_kwargs is None:
                  SCREAMING_SNAKE_CASE   :      Dict    					=						f"`{attribute}` is deprecated and will be removed in version {version_name}."

            if warning is not None:
                  SCREAMING_SNAKE_CASE   :      Dict    					=						warning + ' ' if standard_warn else ''
                  warnings.warn(warning + message      ,__UpperCamelCase      ,stacklevel=__UpperCamelCase    )

      if isinstance(__UpperCamelCase      ,__UpperCamelCase    ) and len(__UpperCamelCase    ) > 0:
            SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						inspect.getouterframes(inspect.currentframe()    )[1]
            SCREAMING_SNAKE_CASE   :      Any    					=						call_frame.filename
            SCREAMING_SNAKE_CASE   :      Tuple    					=						call_frame.lineno
            SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						call_frame.function
            SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE   :      Tuple    					=						next(iter(deprecated_kwargs.items()    )    )
            raise TypeError(f"{function} in {filename} line {line_number-1} got an unexpected keyword argument `{key}`"    )

      if len(__UpperCamelCase    ) == 0:
            return
      elif len(__UpperCamelCase    ) == 1:
            return values[0]
      return values


