

import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel



class        _a		:


					'''simple docstring'''






					def __init__(					self       ,			A__       ,			A__=13       ,			A__=7       ,			A__=True       ,			A__=True       ,			A__=False       ,			A__=True       ,			A__=99       ,			A__=32       ,			A__=5       ,			A__=4       ,			A__=37       ,			A__="gelu"       ,			A__=0.1       ,			A__=0.1       ,			A__=512       ,			A__=16       ,			A__=2       ,			A__=0.0_2       ,			A__=3       ,			A__=4       ,			A__=None       ,			):
						A__				:       Tuple	  =						parent
						A__				:       Optional[Any]	  =						batch_size
						A__				:       Optional[Any]	  =						seq_length
						A__				:       str	  =						is_training
						A__				:       int	  =						use_input_mask
						A__				:       int	  =						use_token_type_ids
						A__				:       int	  =						use_labels
						A__				:       Tuple	  =						vocab_size
						A__				:       str	  =						hidden_size
						A__				:       int	  =						num_hidden_layers
						A__				:       Any	  =						num_attention_heads
						A__				:       str	  =						intermediate_size
						A__				:       List[Any]	  =						hidden_act
						A__				:       List[str]	  =						hidden_dropout_prob
						A__				:       List[Any]	  =						attention_probs_dropout_prob
						A__				:       List[str]	  =						max_position_embeddings
						A__				:       Tuple	  =						type_vocab_size
						A__				:       List[Any]	  =						type_sequence_label_size
						A__				:       Dict	  =						initializer_range
						A__				:       Optional[int]	  =						num_labels
						A__				:       List[Any]	  =						num_choices
						A__				:       Tuple	  =						scope






					def     __A     (					self     ):
						A__				:       Tuple	  =						ids_tensor([self.batch_size, self.seq_length]       ,			self.vocab_size     )

						A__				:       List[str]	  =						None
						if self.use_input_mask:
							A__				:       Optional[Any]	  =						random_attention_mask([self.batch_size, self.seq_length]     )

						A__				:       Union[str, Any]	  =						None
						if self.use_token_type_ids:
							A__				:       Optional[Any]	  =						ids_tensor([self.batch_size, self.seq_length]       ,			self.type_vocab_size     )

						A__				:       List[Any]	  =						None
						A__				:       List[str]	  =						None
						A__				:       Optional[Any]	  =						None
						if self.use_labels:
							A__				:       Union[str, Any]	  =						ids_tensor([self.batch_size]       ,			self.type_sequence_label_size     )
							A__				:       Dict	  =						ids_tensor([self.batch_size, self.seq_length]       ,			self.num_labels     )
							A__				:       str	  =						ids_tensor([self.batch_size]       ,			self.num_choices     )

						A__				:       Union[str, Any]	  =						self.get_config()

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






					def     __A     (					self     ):
						return OpenLlamaConfig(
						    vocab_size=self.vocab_size       ,			hidden_size=self.hidden_size       ,			num_hidden_layers=self.num_hidden_layers       ,			num_attention_heads=self.num_attention_heads       ,			intermediate_size=self.intermediate_size       ,			hidden_act=self.hidden_act       ,			hidden_dropout_prob=self.hidden_dropout_prob       ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,			max_position_embeddings=self.max_position_embeddings       ,			type_vocab_size=self.type_vocab_size       ,			is_decoder=A__       ,			initializer_range=self.initializer_range       ,			use_stable_embedding=A__       ,			)






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__     ):
						A__				:       List[Any]	  =						OpenLlamaModel(config=A__     )
						model.to(A__     )
						model.eval()
						A__				:       str	  =						model(A__       ,			attention_mask=A__     )
						A__				:       str	  =						model(A__     )
						self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)     )






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			):
						A__				:       Union[str, Any]	  =						True
						A__				:       List[Any]	  =						OpenLlamaModel(A__     )
						model.to(A__     )
						model.eval()
						A__				:       List[str]	  =						model(
						    A__       ,			attention_mask=A__       ,			encoder_hidden_states=A__       ,			encoder_attention_mask=A__       ,			)
						A__				:       str	  =						model(
						    A__       ,			attention_mask=A__       ,			encoder_hidden_states=A__       ,			)
						A__				:       Any	  =						model(A__       ,			attention_mask=A__     )
						self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)     )






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			):
						A__				:       Any	  =						OpenLlamaForCausalLM(config=A__     )
						model.to(A__     )
						model.eval()
						A__				:       Tuple	  =						model(A__       ,			attention_mask=A__       ,			labels=A__     )
						self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.seq_length, self.vocab_size)     )






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			):
						A__				:       Union[str, Any]	  =						True
						A__				:       Tuple	  =						True
						A__				:       Optional[Any]	  =						OpenLlamaForCausalLM(config=A__     )
						model.to(A__     )
						model.eval()

						# first forward pass
						A__				:       Union[str, Any]	  =						model(
						    A__       ,			attention_mask=A__       ,			encoder_hidden_states=A__       ,			encoder_attention_mask=A__       ,			use_cache=A__       ,			)
						A__				:       Dict	  =						outputs.past_key_values

						# create hypothetical multiple next token and extent to next_input_ids
						A__				:       List[Any]	  =						ids_tensor((self.batch_size, 3)       ,			config.vocab_size     )
						A__				:       int	  =						ids_tensor((self.batch_size, 3)       ,			vocab_size=2     )

						# append to next input_ids and
						A__				:       Optional[Any]	  =						torch.cat([input_ids, next_tokens]       ,			dim=-1     )
						A__				:       Any	  =						torch.cat([input_mask, next_mask]       ,			dim=-1     )

						A__				:       Any	  =						model(
						    A__       ,			attention_mask=A__       ,			encoder_hidden_states=A__       ,			encoder_attention_mask=A__       ,			output_hidden_states=A__       ,			)["""hidden_states"""][0]
						A__				:       Optional[Any]	  =						model(
						    A__       ,			attention_mask=A__       ,			encoder_hidden_states=A__       ,			encoder_attention_mask=A__       ,			past_key_values=A__       ,			output_hidden_states=A__       ,			)["""hidden_states"""][0]

						# select random slice
						A__				:       Tuple	  =						ids_tensor((1,)       ,			output_from_past.shape[-1]     ).item()
						A__				:       Dict	  =						output_from_no_past[:, -3:, random_slice_idx].detach()
						A__				:       List[Any]	  =						output_from_past[:, :, random_slice_idx].detach()

						self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

						# test that outputs are equal for slice
						self.parent.assertTrue(torch.allclose(A__       ,			A__       ,			atol=1e-3     )     )






					def     __A     (					self     ):
						A__				:       List[str]	  =						self.prepare_config_and_inputs()
						(
						    (
						    A__
						)  ,  (
						    A__
						)  ,  (
						    A__
						)  ,  (
						    A__
						)  ,  (
						    A__
						)  ,  (
						    A__
						)  ,  (
						    A__
						)  ,  
						)				:       List[Any]	  =						config_and_inputs
						A__				:       Union[str, Any]	  =						{"""input_ids""": input_ids, """attention_mask""": input_mask}
						return config, inputs_dict



@require_torch
class        _a		(__magic_name__						,  __magic_name__						,  __magic_name__						,  unittest.TestCase							):


					'''simple docstring'''






					UpperCAmelCase__:  Optional[Any]    				=			(
					    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
					)
					UpperCAmelCase__:  Any    				=			(OpenLlamaForCausalLM,) if is_torch_available() else ()
					UpperCAmelCase__:  str    				=			(
					    {
					        '''feature-extraction''': OpenLlamaModel,
					        '''text-classification''': OpenLlamaForSequenceClassification,
					        '''text-generation''': OpenLlamaForCausalLM,
					        '''zero-shot''': OpenLlamaForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					UpperCAmelCase__:  List[Any]    				=			False
					UpperCAmelCase__:  Dict    				=			False






					def     __A     (					self     ):
						A__				:       List[Any]	  =						OpenLlamaModelTester(self     )
						A__				:       str	  =						ConfigTester(self       ,			config_class=A__       ,			hidden_size=37     )






					def     __A     (					self     ):
						self.config_tester.run_common_tests()






					def     __A     (					self     ):
						A__				:       int	  =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*A__     )






					def     __A     (					self     ):
						A__				:       Optional[int]	  =						self.model_tester.prepare_config_and_inputs()
						for type in ["absolute", "relative_key", "relative_key_query"]:
							A__				:       Tuple	  =						type
							self.model_tester.create_and_check_model(*A__     )






					def     __A     (					self     ):
						A__  ,  A__				:       List[str]	  =						self.model_tester.prepare_config_and_inputs_for_common()
						A__				:       int	  =						3
						A__				:       str	  =						input_dict["""input_ids"""]
						A__				:       Optional[Any]	  =						input_ids.ne(1     ).to(A__     )
						A__				:       str	  =						ids_tensor([self.model_tester.batch_size]       ,			self.model_tester.type_sequence_label_size     )
						A__				:       Tuple	  =						OpenLlamaForSequenceClassification(A__     )
						model.to(A__     )
						model.eval()
						A__				:       str	  =						model(A__       ,			attention_mask=A__       ,			labels=A__     )
						self.assertEqual(result.logits.shape       ,			(self.model_tester.batch_size, self.model_tester.num_labels)     )






					def     __A     (					self     ):
						A__  ,  A__				:       Dict	  =						self.model_tester.prepare_config_and_inputs_for_common()
						A__				:       List[str]	  =						3
						A__				:       Dict	  =						"""single_label_classification"""
						A__				:       str	  =						input_dict["""input_ids"""]
						A__				:       str	  =						input_ids.ne(1     ).to(A__     )
						A__				:       Optional[Any]	  =						ids_tensor([self.model_tester.batch_size]       ,			self.model_tester.type_sequence_label_size     )
						A__				:       Any	  =						OpenLlamaForSequenceClassification(A__     )
						model.to(A__     )
						model.eval()
						A__				:       List[Any]	  =						model(A__       ,			attention_mask=A__       ,			labels=A__     )
						self.assertEqual(result.logits.shape       ,			(self.model_tester.batch_size, self.model_tester.num_labels)     )






					def     __A     (					self     ):
						A__  ,  A__				:       Optional[int]	  =						self.model_tester.prepare_config_and_inputs_for_common()
						A__				:       Dict	  =						3
						A__				:       Optional[Any]	  =						"""multi_label_classification"""
						A__				:       Union[str, Any]	  =						input_dict["""input_ids"""]
						A__				:       int	  =						input_ids.ne(1     ).to(A__     )
						A__				:       Dict	  =						ids_tensor(
						    [self.model_tester.batch_size, config.num_labels]       ,			self.model_tester.type_sequence_label_size     ).to(torch.float     )
						A__				:       List[str]	  =						OpenLlamaForSequenceClassification(A__     )
						model.to(A__     )
						model.eval()
						A__				:       Tuple	  =						model(A__       ,			attention_mask=A__       ,			labels=A__     )
						self.assertEqual(result.logits.shape       ,			(self.model_tester.batch_size, self.model_tester.num_labels)     )






					@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test"""     )
					def     __A     (					self     ):
						pass






					@parameterized.expand([("""linear""",), ("""dynamic""",)]     )
					def     __A     (					self       ,			A__     ):
						A__  ,  A__				:       Optional[int]	  =						self.model_tester.prepare_config_and_inputs_for_common()
						A__				:       Tuple	  =						ids_tensor([1, 10]       ,			config.vocab_size     )
						A__				:       Dict	  =						ids_tensor([1, int(config.max_position_embeddings * 1.5     )]       ,			config.vocab_size     )

						set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
						A__				:       List[Any]	  =						OpenLlamaModel(A__     )
						original_model.to(A__     )
						original_model.eval()
						A__				:       Optional[Any]	  =						original_model(A__     ).last_hidden_state
						A__				:       List[str]	  =						original_model(A__     ).last_hidden_state

						set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
						A__				:       List[str]	  =						{"""type""": scaling_type, """factor""": 1_0.0}
						A__				:       List[str]	  =						OpenLlamaModel(A__     )
						scaled_model.to(A__     )
						scaled_model.eval()
						A__				:       Optional[int]	  =						scaled_model(A__     ).last_hidden_state
						A__				:       int	  =						scaled_model(A__     ).last_hidden_state

						# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
						# maximum sequence length, so the outputs for the short input should match.
						if scaling_type == "dynamic":
							self.assertTrue(torch.allclose(A__       ,			A__       ,			atol=1e-5     )     )
						else:
							self.assertFalse(torch.allclose(A__       ,			A__       ,			atol=1e-5     )     )

						# The output should be different for long inputs
						self.assertFalse(torch.allclose(A__       ,			A__       ,			atol=1e-5     )     )






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


A_			:					Tuple    			=     {
    'configuration_xlm': ['XLM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XLMConfig', 'XLMOnnxConfig'],
    'tokenization_xlm': ['XLMTokenizer'],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A_			:					Dict    			=     [
	    'XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'XLMForMultipleChoice',
	    'XLMForQuestionAnswering',
	    'XLMForQuestionAnsweringSimple',
	    'XLMForSequenceClassification',
	    'XLMForTokenClassification',
	    'XLMModel',
	    'XLMPreTrainedModel',
	    'XLMWithLMHeadModel',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A_			:					str    			=     [
	    'TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'TFXLMForMultipleChoice',
	    'TFXLMForQuestionAnsweringSimple',
	    'TFXLMForSequenceClassification',
	    'TFXLMForTokenClassification',
	    'TFXLMMainLayer',
	    'TFXLMModel',
	    'TFXLMPreTrainedModel',
	    'TFXLMWithLMHeadModel',
	]


if TYPE_CHECKING:
	from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
	from .tokenization_xlm import XLMTokenizer

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_xlm import (
		    XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    XLMForMultipleChoice,
		    XLMForQuestionAnswering,
		    XLMForQuestionAnsweringSimple,
		    XLMForSequenceClassification,
		    XLMForTokenClassification,
		    XLMModel,
		    XLMPreTrainedModel,
		    XLMWithLMHeadModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_xlm import (
		    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFXLMForMultipleChoice,
		    TFXLMForQuestionAnsweringSimple,
		    TFXLMForSequenceClassification,
		    TFXLMForTokenClassification,
		    TFXLMMainLayer,
		    TFXLMModel,
		    TFXLMPreTrainedModel,
		    TFXLMWithLMHeadModel,
		)

else:
	import sys

	A_			:					Tuple    			=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



