
"""simple docstring"""


from __future__ import annotations

import copy
import inspect
import unittest

import numpy as np

from transformers import is_tf_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import (
		    TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
		    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
		    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
		    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
		    LayoutLMvaConfig,
		    TFLayoutLMvaForQuestionAnswering,
		    TFLayoutLMvaForSequenceClassification,
		    TFLayoutLMvaForTokenClassification,
		    TFLayoutLMvaModel,
		)

if is_vision_available():
		from PIL import Image

		from transformers import LayoutLMvaImageProcessor







class 					lowerCAmelCase_					:



						'''simple docstring'''


						def __init__(			self  :   List[str]	,A_  :   Union[str, Any]	,A_  :   str=2	,A_  :   Union[str, Any]=3	,A_  :   Dict=4	,A_  :   Optional[Any]=2	,A_  :   Any=7	,A_  :   Any=True	,A_  :   List[str]=True	,A_  :   Tuple=True	,A_  :   Optional[int]=True	,A_  :   Dict=99	,A_  :   int=36	,A_  :   Dict=2	,A_  :   Tuple=4	,A_  :   List[Any]=37	,A_  :   int="gelu"	,A_  :   Tuple=0.1	,A_  :   Union[str, Any]=0.1	,A_  :   Optional[int]=512	,A_  :   Dict=16	,A_  :   Dict=2	,A_  :   int=0.02	,A_  :   Dict=6	,A_  :   Dict=6	,A_  :   int=3	,A_  :   str=4	,A_  :   List[Any]=None	,A_  :   str=1000	,)  -> Optional[Any]:
							A     = parent
							A     = batch_size
							A     = num_channels
							A     = image_size
							A     = patch_size
							A     = is_training
							A     = use_input_mask
							A     = use_token_type_ids
							A     = use_labels
							A     = vocab_size
							A     = hidden_size
							A     = num_hidden_layers
							A     = num_attention_heads
							A     = intermediate_size
							A     = hidden_act
							A     = hidden_dropout_prob
							A     = attention_probs_dropout_prob
							A     = max_position_embeddings
							A     = type_vocab_size
							A     = type_sequence_label_size
							A     = initializer_range
							A     = coordinate_size
							A     = shape_size
							A     = num_labels
							A     = num_choices
							A     = scope
							A     = range_bbox

							# LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
							A     = text_seq_length
							A     = (image_size // patch_size) ** 2 + 1
							A     = self.text_seq_length + self.image_seq_length


						def     _SCREAMING_SNAKE_CASE   (			self  :   Union[str, Any]   )  -> Any:
							A     = ids_tensor([self.batch_size, self.text_seq_length]	,self.vocab_size   )

							A     = ids_tensor([self.batch_size, self.text_seq_length, 4]	,self.range_bbox   )
							A     = bbox.numpy()
							# Ensure that bbox is legal
							for i in range(bbox.shape[0]   ):
								for j in range(bbox.shape[1]   ):
									if bbox[i, j, 3] < bbox[i, j, 1]:
										A     = bbox[i, j, 3]
										A     = bbox[i, j, 1]
										A     = tmp_coordinate
									if bbox[i, j, 2] < bbox[i, j, 0]:
										A     = bbox[i, j, 2]
										A     = bbox[i, j, 0]
										A     = tmp_coordinate
							A     = tf.constant(A_   )

							A     = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

							A     = None
							if self.use_input_mask:
								A     = random_attention_mask([self.batch_size, self.text_seq_length]   )

							A     = None
							if self.use_token_type_ids:
								A     = ids_tensor([self.batch_size, self.text_seq_length]	,self.type_vocab_size   )

							A     = None
							A     = None
							if self.use_labels:
								A     = ids_tensor([self.batch_size]	,self.type_sequence_label_size   )
								A     = ids_tensor([self.batch_size, self.text_seq_length]	,self.num_labels   )

							A     = LayoutLMvaConfig(
							    vocab_size=self.vocab_size	,hidden_size=self.hidden_size	,num_hidden_layers=self.num_hidden_layers	,num_attention_heads=self.num_attention_heads	,intermediate_size=self.intermediate_size	,hidden_act=self.hidden_act	,hidden_dropout_prob=self.hidden_dropout_prob	,attention_probs_dropout_prob=self.attention_probs_dropout_prob	,max_position_embeddings=self.max_position_embeddings	,type_vocab_size=self.type_vocab_size	,initializer_range=self.initializer_range	,coordinate_size=self.coordinate_size	,shape_size=self.shape_size	,input_size=self.image_size	,patch_size=self.patch_size	,)

							return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]	,A_  :   Optional[int]	,A_  :   Dict	,A_  :   Optional[Any]	,A_  :   Optional[int]	,A_  :   Optional[int]	,A_  :   Tuple   )  -> Union[str, Any]:
							A     = TFLayoutLMvaModel(config=A_   )

							# text + image
							A     = model(A_	,pixel_values=A_	,training=A_   )
							A     = model(
							    A_	,bbox=A_	,pixel_values=A_	,attention_mask=A_	,token_type_ids=A_	,training=A_	,)
							A     = model(A_	,bbox=A_	,pixel_values=A_	,training=A_   )

							self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.hidden_size)   )

							# text only
							A     = model(A_	,training=A_   )
							self.parent.assertEqual(
							    result.last_hidden_state.shape	,(self.batch_size, self.text_seq_length, self.hidden_size)   )

							# image only
							A     = model({'pixel_values': pixel_values}	,training=A_   )
							self.parent.assertEqual(
							    result.last_hidden_state.shape	,(self.batch_size, self.image_seq_length, self.hidden_size)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   int	,A_  :   List[Any]	,A_  :   Any	,A_  :   str	,A_  :   str	,A_  :   List[str]	,A_  :   Optional[Any]	,A_  :   List[str]   )  -> Union[str, Any]:
							A     = self.num_labels
							A     = TFLayoutLMvaForSequenceClassification(config=A_   )
							A     = model(
							    A_	,bbox=A_	,pixel_values=A_	,attention_mask=A_	,token_type_ids=A_	,labels=A_	,training=A_	,)
							self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.num_labels)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]	,A_  :   List[str]	,A_  :   Any	,A_  :   List[Any]	,A_  :   List[str]	,A_  :   List[Any]	,A_  :   Union[str, Any]	,A_  :   List[Any]   )  -> Union[str, Any]:
							A     = self.num_labels
							A     = TFLayoutLMvaForTokenClassification(config=A_   )
							A     = model(
							    A_	,bbox=A_	,pixel_values=A_	,attention_mask=A_	,token_type_ids=A_	,labels=A_	,training=A_	,)
							self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.text_seq_length, self.num_labels)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[Any]	,A_  :   int	,A_  :   List[str]	,A_  :   Tuple	,A_  :   List[str]	,A_  :   Optional[int]	,A_  :   Optional[int]	,A_  :   Optional[Any]   )  -> Optional[Any]:
							A     = 2
							A     = TFLayoutLMvaForQuestionAnswering(config=A_   )
							A     = model(
							    A_	,bbox=A_	,pixel_values=A_	,attention_mask=A_	,token_type_ids=A_	,start_positions=A_	,end_positions=A_	,training=A_	,)
							self.parent.assertEqual(result.start_logits.shape	,(self.batch_size, self.seq_length)   )
							self.parent.assertEqual(result.end_logits.shape	,(self.batch_size, self.seq_length)   )




						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]   )  -> Optional[int]:
							A     = self.prepare_config_and_inputs()
							((A) ,   (A) ,   (A) ,   (A) ,   (A) ,   (A) ,   (A) ,   (A))     = config_and_inputs
							A     = {
							    'input_ids': input_ids,
							    'bbox': bbox,
							    'pixel_values': pixel_values,
							    'token_type_ids': token_type_ids,
							    'attention_mask': input_mask,
							}
							return config, inputs_dict







@require_tf
class 					lowerCAmelCase_					(   _lowercase		,       _lowercase		,       unittest.TestCase     ):



						'''simple docstring'''


						_lowerCamelCase:     str   =     (
						    (
						        TFLayoutLMvaModel,
						        TFLayoutLMvaForQuestionAnswering,
						        TFLayoutLMvaForSequenceClassification,
						        TFLayoutLMvaForTokenClassification,
						    )
						    if is_tf_available()
						    else ()
						)
						_lowerCamelCase:     int   =     (
						    {'''document-question-answering''': TFLayoutLMvaForQuestionAnswering, '''feature-extraction''': TFLayoutLMvaModel}
						    if is_tf_available()
						    else {}
						)

						_lowerCamelCase:     int   =     False
						_lowerCamelCase:     Any   =     False
						_lowerCamelCase:     Any   =     False


						def     _SCREAMING_SNAKE_CASE   (			self  :   Any	,A_  :   Tuple	,A_  :   Any	,A_  :   Optional[int]	,A_  :   Dict	,A_  :   Dict   )  -> List[Any]:
							return True


						def     _SCREAMING_SNAKE_CASE   (			self  :   Union[str, Any]	,A_  :   int	,A_  :   Tuple	,A_  :   Optional[int]=False   )  -> dict:
							A     = copy.deepcopy(A_   )

							if model_class in get_values(A_   ):
								A     = {
								    k: tf.tile(tf.expand_dims(A_	,1   )	,(1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)   )
								    if isinstance(A_	,tf.Tensor   ) and v.ndim > 0
								    else v
								    for k, v in inputs_dict.items()
								}

							if return_labels:
								if model_class in get_values(A_   ):
									A     = tf.ones(self.model_tester.batch_size	,dtype=tf.intaa   )
								elif model_class in get_values(A_   ):
									A     = tf.zeros(self.model_tester.batch_size	,dtype=tf.intaa   )
									A     = tf.zeros(self.model_tester.batch_size	,dtype=tf.intaa   )
								elif model_class in get_values(A_   ):
									A     = tf.zeros(self.model_tester.batch_size	,dtype=tf.intaa   )
								elif model_class in get_values(A_   ):
									A     = tf.zeros(
									    (self.model_tester.batch_size, self.model_tester.text_seq_length)	,dtype=tf.intaa   )

							return inputs_dict


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[Any]   )  -> Dict:
							A     = TFLayoutLMvaModelTester(self   )
							A     = ConfigTester(self	,config_class=A_	,hidden_size=37   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Dict   )  -> Union[str, Any]:
							self.config_tester.run_common_tests()


						def     _SCREAMING_SNAKE_CASE   (			self  :   Any   )  -> str:
							A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()
							for model_class in self.all_model_classes:
								A     = model_class(A_   )
								if getattr(A_	,'hf_compute_loss'	,A_   ):
									# The number of elements in the loss should be the same as the number of elements in the label
									A     = self._prepare_for_class(inputs_dict.copy()	,A_	,return_labels=A_   )
									A     = prepared_for_class[
									    sorted(prepared_for_class.keys() - inputs_dict.keys()	,reverse=A_   )[0]
									]
									A     = added_label.shape.as_list()[:1]

									# Test that model correctly compute the loss with kwargs
									A     = self._prepare_for_class(inputs_dict.copy()	,A_	,return_labels=A_   )
									A     = prepared_for_class.pop('input_ids'   )

									A     = model(A_	,**A_   )[0]
									self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]   )

									# Test that model correctly compute the loss when we mask some positions
									A     = self._prepare_for_class(inputs_dict.copy()	,A_	,return_labels=A_   )
									A     = prepared_for_class.pop('input_ids'   )
									if "labels" in prepared_for_class:
										A     = prepared_for_class['labels'].numpy()
										if len(labels.shape   ) > 1 and labels.shape[1] != 1:
											A     = -100
											A     = tf.convert_to_tensor(A_   )
											A     = model(A_	,**A_   )[0]
											self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]   )
											self.assertTrue(not np.any(np.isnan(loss.numpy()   )   )   )

                # Test that model correctly compute the loss with a dict
									A     = self._prepare_for_class(inputs_dict.copy()	,A_	,return_labels=A_   )
									A     = model(A_   )[0]
									self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]   )

									# Test that model correctly compute the loss with a tuple
									A     = self._prepare_for_class(inputs_dict.copy()	,A_	,return_labels=A_   )

									# Get keys that were added with the _prepare_for_class function
									A     = prepared_for_class.keys() - inputs_dict.keys()
									A     = inspect.signature(model.call   ).parameters
									A     = list(signature.keys()   )

									# Create a dictionary holding the location of the tensors in the tuple
									A     = {0: 'input_ids'}
									for label_key in label_keys:
										A     = signature_names.index(A_   )
										A     = label_key
									A     = sorted(tuple_index_mapping.items()   )
									# Initialize a list with their default values, update the values and convert to a tuple
									A     = []

									for name in signature_names:
										if name != "kwargs":
											list_input.append(signature[name].default   )

									for index, value in sorted_tuple_index_mapping:
										A     = prepared_for_class[value]

									A     = tuple(A_   )

									# Send to model
									A     = model(tuple_input[:-1]   )[0]

									self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]   )  -> List[Any]:
							(
							    (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   
							)     = self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(A_	,A_	,A_	,A_	,A_	,A_   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Optional[Any]   )  -> List[str]:
							(
							    (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   
							)     = self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
								A     = type
								self.model_tester.create_and_check_model(A_	,A_	,A_	,A_	,A_	,A_   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Tuple   )  -> Tuple:
							(
							    (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   
							)     = self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(
							    A_	,A_	,A_	,A_	,A_	,A_	,A_   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Dict   )  -> Any:
							(
							    (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   
							)     = self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(
							    A_	,A_	,A_	,A_	,A_	,A_	,A_   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   str   )  -> Any:
							(
							    (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   
							)     = self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(
							    A_	,A_	,A_	,A_	,A_	,A_	,A_   )




						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   int   )  -> List[Any]:
							for model_name in TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								A     = TFLayoutLMvaModel.from_pretrained(A_   )
								self.assertIsNotNone(A_   )






def    _snake_case					(		):
	A     = Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'  )
	return image







@require_tf
class 					lowerCAmelCase_					(   unittest.TestCase     ):



						'''simple docstring'''


						@cached_property
						def     _SCREAMING_SNAKE_CASE   (			self  :   int   )  -> Any:
							return LayoutLMvaImageProcessor(apply_ocr=A_   ) if is_vision_available() else None




						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   Tuple   )  -> Union[str, Any]:
							A     = TFLayoutLMvaModel.from_pretrained('microsoft/layoutlmv3-base'   )

							A     = self.default_image_processor
							A     = prepare_img()
							A     = image_processor(images=A_	,return_tensors='tf'   ).pixel_values

							A     = tf.constant([[1, 2]]   )
							A     = tf.expand_dims(tf.constant([[1, 2, 3, 4], [5, 6, 7, 8]]   )	,axis=0   )

							# forward pass
							A     = model(input_ids=A_	,bbox=A_	,pixel_values=A_	,training=A_   )

							# verify the logits
							A     = (1, 199, 768)
							self.assertEqual(outputs.last_hidden_state.shape	,A_   )

							A     = tf.constant(
							    [[-0.05_29, 0.36_18, 0.16_32], [-0.15_87, -0.16_67, -0.04_00], [-0.15_57, -0.16_71, -0.05_05]]   )

							self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]	,A_	,atol=1e-4   )   )

"""simple docstring"""


import unittest

from parameterized import parameterized

from transformers import LlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaTokenizer







class 					lowerCAmelCase_					:



						'''simple docstring'''


						def __init__(			self  :   Tuple	,A_  :   Any	,A_  :   int=13	,A_  :   str=7	,A_  :   Tuple=True	,A_  :   str=True	,A_  :   str=False	,A_  :   List[str]=True	,A_  :   str=99	,A_  :   str=32	,A_  :   Optional[int]=5	,A_  :   Optional[Any]=4	,A_  :   str=37	,A_  :   Optional[Any]="gelu"	,A_  :   Union[str, Any]=0.1	,A_  :   Any=0.1	,A_  :   Optional[Any]=512	,A_  :   str=16	,A_  :   int=2	,A_  :   Optional[Any]=0.02	,A_  :   str=3	,A_  :   str=4	,A_  :   List[str]=None	,)  -> str:
							A     = parent
							A     = batch_size
							A     = seq_length
							A     = is_training
							A     = use_input_mask
							A     = use_token_type_ids
							A     = use_labels
							A     = vocab_size
							A     = hidden_size
							A     = num_hidden_layers
							A     = num_attention_heads
							A     = intermediate_size
							A     = hidden_act
							A     = hidden_dropout_prob
							A     = attention_probs_dropout_prob
							A     = max_position_embeddings
							A     = type_vocab_size
							A     = type_sequence_label_size
							A     = initializer_range
							A     = num_labels
							A     = num_choices
							A     = scope


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[Any]   )  -> Any:
							A     = ids_tensor([self.batch_size, self.seq_length]	,self.vocab_size   )

							A     = None
							if self.use_input_mask:
								A     = random_attention_mask([self.batch_size, self.seq_length]   )

							A     = None
							if self.use_token_type_ids:
								A     = ids_tensor([self.batch_size, self.seq_length]	,self.type_vocab_size   )

							A     = None
							A     = None
							A     = None
							if self.use_labels:
								A     = ids_tensor([self.batch_size]	,self.type_sequence_label_size   )
								A     = ids_tensor([self.batch_size, self.seq_length]	,self.num_labels   )
								A     = ids_tensor([self.batch_size]	,self.num_choices   )

							A     = self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


						def     _SCREAMING_SNAKE_CASE   (			self  :   Optional[Any]   )  -> Any:
							return LlamaConfig(
							    vocab_size=self.vocab_size	,hidden_size=self.hidden_size	,num_hidden_layers=self.num_hidden_layers	,num_attention_heads=self.num_attention_heads	,intermediate_size=self.intermediate_size	,hidden_act=self.hidden_act	,hidden_dropout_prob=self.hidden_dropout_prob	,attention_probs_dropout_prob=self.attention_probs_dropout_prob	,max_position_embeddings=self.max_position_embeddings	,type_vocab_size=self.type_vocab_size	,is_decoder=A_	,initializer_range=self.initializer_range	,)


						def     _SCREAMING_SNAKE_CASE   (			self  :   Any	,A_  :   Dict	,A_  :   Optional[int]	,A_  :   Any	,A_  :   Optional[Any]	,A_  :   Any	,A_  :   Union[str, Any]	,A_  :   Tuple   )  -> List[Any]:
							A     = LlamaModel(config=A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_   )
							A     = model(A_   )
							self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.hidden_size)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Tuple	,A_  :   int	,A_  :   Optional[Any]	,A_  :   Optional[int]	,A_  :   Any	,A_  :   Union[str, Any]	,A_  :   Any	,A_  :   Tuple	,A_  :   Union[str, Any]	,A_  :   Dict	,)  -> List[str]:
							A     = True
							A     = LlamaModel(A_   )
							model.to(A_   )
							model.eval()
							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,encoder_attention_mask=A_	,)
							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,)
							A     = model(A_	,attention_mask=A_   )
							self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.hidden_size)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   int	,A_  :   List[str]	,A_  :   Optional[int]	,A_  :   Any	,A_  :   str	,A_  :   Dict	,A_  :   Dict	,A_  :   Tuple	,A_  :   Tuple	,A_  :   Dict	,)  -> Union[str, Any]:
							A     = LlamaForCausalLM(config=A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_	,labels=A_   )
							self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.vocab_size)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   int	,A_  :   Dict	,A_  :   Any	,A_  :   int	,A_  :   List[str]	,A_  :   Tuple	,A_  :   Any	,A_  :   Union[str, Any]	,A_  :   Any	,A_  :   int	,)  -> List[Any]:
							A     = True
							A     = True
							A     = LlamaForCausalLM(config=A_   )
							model.to(A_   )
							model.eval()

							# first forward pass
							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,encoder_attention_mask=A_	,use_cache=A_	,)
							A     = outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							A     = ids_tensor((self.batch_size, 3)	,config.vocab_size   )
							A     = ids_tensor((self.batch_size, 3)	,vocab_size=2   )

							# append to next input_ids and
							A     = torch.cat([input_ids, next_tokens]	,dim=-1   )
							A     = torch.cat([input_mask, next_mask]	,dim=-1   )

							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,encoder_attention_mask=A_	,output_hidden_states=A_	,)['hidden_states'][0]
							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,encoder_attention_mask=A_	,past_key_values=A_	,output_hidden_states=A_	,)['hidden_states'][0]

							# select random slice
							A     = ids_tensor((1,)	,output_from_past.shape[-1]   ).item()
							A     = output_from_no_past[:, -3:, random_slice_idx].detach()
							A     = output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(A_	,A_	,atol=1e-3   )   )




						def     _SCREAMING_SNAKE_CASE   (			self  :   Any   )  -> Optional[int]:
							A     = self.prepare_config_and_inputs()
							(
							    (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   
							)     = config_and_inputs
							A     = {'input_ids': input_ids, 'attention_mask': input_mask}
							return config, inputs_dict







@require_torch
class 					lowerCAmelCase_					(   _lowercase		,       _lowercase		,       _lowercase		,       unittest.TestCase     ):



						'''simple docstring'''


						_lowerCamelCase:     Dict   =     (LlamaModel, LlamaForCausalLM, LlamaForSequenceClassification) if is_torch_available() else ()
						_lowerCamelCase:     List[Any]   =     (LlamaForCausalLM,) if is_torch_available() else ()
						_lowerCamelCase:     Any   =     (
						    {
						        '''feature-extraction''': LlamaModel,
						        '''text-classification''': LlamaForSequenceClassification,
						        '''text-generation''': LlamaForCausalLM,
						        '''zero-shot''': LlamaForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						_lowerCamelCase:     int   =     False
						_lowerCamelCase:     List[str]   =     False


						def     _SCREAMING_SNAKE_CASE   (			self  :   Any   )  -> Dict:
							A     = LlamaModelTester(self   )
							A     = ConfigTester(self	,config_class=A_	,hidden_size=37   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Union[str, Any]   )  -> Any:
							self.config_tester.run_common_tests()


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[Any]   )  -> Tuple:
							A     = self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*A_   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   str   )  -> Optional[int]:
							A     = self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
								A     = type
								self.model_tester.create_and_check_model(*A_   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[Any]   )  -> List[Any]:
							A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()
							A     = 3
							A     = input_dict['input_ids']
							A     = input_ids.ne(1   ).to(A_   )
							A     = ids_tensor([self.model_tester.batch_size]	,self.model_tester.type_sequence_label_size   )
							A     = LlamaForSequenceClassification(A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_	,labels=A_   )
							self.assertEqual(result.logits.shape	,(self.model_tester.batch_size, self.model_tester.num_labels)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   int   )  -> Optional[Any]:
							A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()
							A     = 3
							A     = 'single_label_classification'
							A     = input_dict['input_ids']
							A     = input_ids.ne(1   ).to(A_   )
							A     = ids_tensor([self.model_tester.batch_size]	,self.model_tester.type_sequence_label_size   )
							A     = LlamaForSequenceClassification(A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_	,labels=A_   )
							self.assertEqual(result.logits.shape	,(self.model_tester.batch_size, self.model_tester.num_labels)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   str   )  -> Dict:
							A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()
							A     = 3
							A     = 'multi_label_classification'
							A     = input_dict['input_ids']
							A     = input_ids.ne(1   ).to(A_   )
							A     = ids_tensor(
							    [self.model_tester.batch_size, config.num_labels]	,self.model_tester.type_sequence_label_size   ).to(torch.float   )
							A     = LlamaForSequenceClassification(A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_	,labels=A_   )
							self.assertEqual(result.logits.shape	,(self.model_tester.batch_size, self.model_tester.num_labels)   )


						@unittest.skip('LLaMA buffers include complex numbers, which breaks this test'   )
						def     _SCREAMING_SNAKE_CASE   (			self  :   Dict   )  -> Optional[Any]:
							pass




						@parameterized.expand([('linear',), ('dynamic',)]   )
						def     _SCREAMING_SNAKE_CASE   (			self  :   List[Any]	,A_  :   Any   )  -> str:
							A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()
							A     = ids_tensor([1, 10]	,config.vocab_size   )
							A     = ids_tensor([1, int(config.max_position_embeddings * 1.5   )]	,config.vocab_size   )

							set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
							A     = LlamaModel(A_   )
							original_model.to(A_   )
							original_model.eval()
							A     = original_model(A_   ).last_hidden_state
							A     = original_model(A_   ).last_hidden_state

							set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
							A     = {'type': scaling_type, 'factor': 10.0}
							A     = LlamaModel(A_   )
							scaled_model.to(A_   )
							scaled_model.eval()
							A     = scaled_model(A_   ).last_hidden_state
							A     = scaled_model(A_   ).last_hidden_state

							# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
							# maximum sequence length, so the outputs for the short input should match.
							if scaling_type == "dynamic":
								self.assertTrue(torch.allclose(A_	,A_	,atol=1e-5   )   )
							else:
								self.assertFalse(torch.allclose(A_	,A_	,atol=1e-5   )   )

							# The output should be different for long inputs
							self.assertFalse(torch.allclose(A_	,A_	,atol=1e-5   )   )







@require_torch
class 					lowerCAmelCase_					(   unittest.TestCase     ):



						'''simple docstring'''


						@unittest.skip('Logits are not exactly the same, once we fix the instabalities somehow, will update!'   )
						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]   )  -> Optional[Any]:
							A     = [1, 306, 4658, 278, 6593, 310, 2834, 338]
							A     = LlamaForCausalLM.from_pretrained('meta-llama/Llama-2-7b-hf'	,device_map='auto'   )
							A     = model(torch.tensor([input_ids]   )   )
							# Expected mean on dim = -1
							A     = torch.tensor([[-6.65_50, -4.12_27, -4.98_59, -3.24_06, 0.82_62, -3.00_33, 1.29_64, -3.36_99]]   )
							torch.testing.assert_close(out.mean(-1   )	,A_	,atol=1e-2	,rtol=1e-2   )
							# slicing logits[0, 0, 0:30]
							# fmt: off
							A     = torch.tensor([-12.82_81, -7.44_53, -0.46_39, -8.06_25, -7.25_00, -8.00_00, -6.48_83, -7.76_95, -7.84_38, -7.03_12, -6.21_88, -7.13_28, -1.84_96, 1.99_61, -8.62_50, -6.72_27, -12.82_81, -6.94_92, -7.07_42, -7.78_52, -7.58_20, -7.90_62, -6.93_75, -7.98_05, -8.34_38, -8.15_62, -8.04_69, -7.62_50, -7.74_22, -7.33_98,]   )
							# fmt: on
							torch.testing.assert_close(out[0, 0, :30]	,A_	,atol=1e-5	,rtol=1e-5   )


						@unittest.skip('Logits are not exactly the same, once we fix the instabalities somehow, will update!'   )
						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   Any   )  -> int:
							A     = [1, 306, 4658, 278, 6593, 310, 2834, 338]
							A     = LlamaForCausalLM.from_pretrained('meta-llama/Llama-2-13b-hf'	,device_map='auto'   )
							A     = model(torch.tensor(A_   )   )
							# Expected mean on dim = -1
							A     = torch.tensor([[-2.06_22, -1.27_94, -1.16_38, -0.97_88, -1.46_03, -1.02_38, -1.78_93, -1.44_11]]   )
							torch.testing.assert_close(out.mean(-1   )	,A_	,atol=1e-2	,rtol=1e-2   )
							# slicing logits[0, 0, 0:30]
							# fmt: off
							A     = torch.tensor([-8.14_06, -8.05_47, 2.74_61, -1.23_44, -0.14_48, -1.82_62, -1.00_20, -1.81_54, -1.68_95, -1.85_16, -2.35_74, -0.92_77, 3.75_98, 6.57_42, -1.29_98, -0.11_77, -8.14_06, -2.96_88, -2.91_99, -3.16_99, -3.52_54, -2.35_55, -2.79_88, -3.41_41, -2.82_62, -4.51_95, -3.33_79, -3.31_64, -2.78_32, -3.02_73]   )
							# fmt: on
							torch.testing.assert_close(out[0, 0, :30]	,A_	,atol=1e-5	,rtol=1e-5   )


						@unittest.skip('Logits are not exactly the same, once we fix the instabalities somehow, will update!'   )
						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   int   )  -> str:
							A     = [1, 306, 4658, 278, 6593, 310, 2834, 338]
							A     = LlamaForCausalLM.from_pretrained('meta-llama/Llama-2-13b-chat-hf'	,device_map='auto'   )
							A     = model(torch.tensor(A_   )   )
							# Expected mean on dim = -1
							A     = torch.tensor([[-0.85_62, -1.85_20, -0.75_51, -0.41_62, -1.51_61, -1.20_38, -2.48_23, -2.32_54]]   )
							torch.testing.assert_close(out.mean(-1   )	,A_	,atol=1e-2	,rtol=1e-2   )
							# slicing logits[0, 0, 0:30]
							# fmt: off
							A     = torch.tensor([-2.22_27, 4.88_28, 0.90_23, -0.45_78, -0.78_71, -0.10_33, -0.62_21, -0.57_86, -0.78_03, -1.06_74, -1.29_20, -0.15_70, 0.80_08, 2.07_23, -0.94_97, 0.27_71, -2.22_27, -0.76_12, -1.43_46, -1.20_61, -1.64_26, -0.30_00, -0.71_39, -1.19_34, -1.86_91, -1.69_73, -1.59_47, -1.27_05, -0.35_23, -0.55_13]   )
							# fmt: on
							torch.testing.assert_close(out.mean(-1   )	,A_	,atol=1e-2	,rtol=1e-2   )


						@unittest.skip(
						    'Logits are not exactly the same, once we fix the instabalities somehow, will update! Also it is gonna be a `too_slow` test'   )
						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   Any   )  -> str:
							A     = [1, 306, 4658, 278, 6593, 310, 2834, 338]
							A     = LlamaForCausalLM.from_pretrained('meta-llama/Llama-2-70b-hf'	,device_map='auto'   )
							A     = model(torch.tensor(A_   )   )

							A     = torch.tensor(
							    [[-4.23_27, -3.33_60, -4.66_65, -4.76_31, -1.81_80, -3.41_70, -1.42_11, -3.18_10]]	,dtype=torch.floataa   )
							torch.testing.assert_close(out.mean(-1   )	,A_	,atol=1e-2	,rtol=1e-2   )
							# fmt: off
							A     = torch.tensor([-9.49_22, -3.95_51, 1.79_98, -5.67_58, -5.10_55, -5.89_84, -4.83_20, -6.80_86, -6.53_91, -5.61_72, -5.58_20, -5.53_52, 1.78_81, 3.62_89, -6.51_17, -3.47_85, -9.50_00, -6.03_52, -6.81_25, -6.01_95, -6.68_36, -5.47_27, -6.28_12, -6.03_91, -7.33_98, -7.42_97, -7.48_44, -6.58_20, -5.87_89, -5.53_12]   )
							# fmt: on
							torch.testing.assert_close(out[0, 0, :30]	,A_	,atol=1e-5	,rtol=1e-5   )




						@unittest.skip('Model is curently gated'   )
						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   int   )  -> Tuple:
							A     = 'Simply put, the theory of relativity states that 1) the laws of physics are the same everywhere in the universe and 2) the passage of time and the length of objects can vary depending on the observer\'s frame of reference.\n\nThe first part of the theory, that the laws of physics are the same everywhere, is known as the "princi'
							A     = 'Simply put, the theory of relativity states that '
							A     = LlamaTokenizer.from_pretrained('meta-llama/Llama-2-13b-chat-hf'   )
							A     = tokenizer.encode(A_	,return_tensors='pt'   )
							A     = LlamaForCausalLM.from_pretrained(
							    'meta-llama/Llama-2-13b-chat-hf'	,device_map='sequential'	,use_safetensors=A_   )

							# greedy generation outputs
							A     = model.generate(A_	,max_new_tokens=64	,top_p=A_	,temperature=1	,do_sample=A_   )
							A     = tokenizer.decode(generated_ids[0]	,skip_special_tokens=A_   )
							self.assertEqual(A_	,A_   )