






"""simple docstring"""



import inspect
import unittest
from typing import List

import numpy as np

from transformers import EfficientFormerConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     TFEfficientFormerForImageClassification,
     TFEfficientFormerForImageClassificationWithTeacher,
     TFEfficientFormerModel,
 )
 from transformers.models.efficientformer.modeling_tf_efficientformer import (
     TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
 )


if is_vision_available():
 from PIL import Image

 from transformers import EfficientFormerImageProcessor




class 			lowerCAmelCase__			:
   def __init__(		self      :	Dict							,			snake_case__      :	Union[str, Any]							,			snake_case__      :	int = 1_3							,			snake_case__      :	int = 6_4							,			snake_case__      :	int = 2							,			snake_case__      :	int = 3							,			snake_case__      :	int = 3							,			snake_case__      :	bool = True							,			snake_case__      :	bool = True							,			snake_case__      :	int = 1_2_8							,			snake_case__      :	Optional[Any]=[1_6, 3_2, 6_4, 1_2_8]							,			snake_case__      :	int = 7							,			snake_case__      :	int = 4							,			snake_case__      :	int = 3_7							,			snake_case__      :	str = "gelu"							,			snake_case__      :	float = 0.1							,			snake_case__      :	float = 0.1							,			snake_case__      :	int = 1_0							,			snake_case__      :	float = 0.02							,			snake_case__      :	int = 2							,			snake_case__      :	int = 1							,			snake_case__      :	int = 1_2_8							,			snake_case__      :	List[int] = [2, 2, 2, 2]							,			snake_case__      :	int = 2							,			snake_case__      :	int = 2							,			):


        '''simple docstring'''
        UpperCAmelCase__						:    Dict							=							parent
        UpperCAmelCase__						:    Tuple							=							batch_size
        UpperCAmelCase__						:    Any							=							image_size
        UpperCAmelCase__						:    str							=							patch_size
        UpperCAmelCase__						:    Tuple							=							num_channels
        UpperCAmelCase__						:    List[Any]							=							is_training
        UpperCAmelCase__						:    Union[str, Any]							=							use_labels
        UpperCAmelCase__						:    Optional[int]							=							hidden_size
        UpperCAmelCase__						:    Tuple							=							num_hidden_layers
        UpperCAmelCase__						:    int							=							num_attention_heads
        UpperCAmelCase__						:    Dict							=							intermediate_size
        UpperCAmelCase__						:    int							=							hidden_act
        UpperCAmelCase__						:    Union[str, Any]							=							hidden_dropout_prob
        UpperCAmelCase__						:    Tuple							=							attention_probs_dropout_prob
        UpperCAmelCase__						:    List[Any]							=							type_sequence_label_size
        UpperCAmelCase__						:    Tuple							=							initializer_range
        UpperCAmelCase__						:    Optional[Any]							=							encoder_stride
        UpperCAmelCase__						:    str							=							num_attention_outputs
        UpperCAmelCase__						:    Optional[Any]							=							embed_dim
        UpperCAmelCase__						:    Optional[Any]							=							embed_dim + 1
        UpperCAmelCase__						:    Any							=							resolution
        UpperCAmelCase__						:    Any							=							depths
        UpperCAmelCase__						:    List[str]							=							hidden_sizes
        UpperCAmelCase__						:    int							=							dim
        UpperCAmelCase__						:    Dict							=							mlp_expansion_ratio
   def       __a							(		self      :	List[str]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[int]							=							floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

        UpperCAmelCase__						:    Any							=							None
        if self.use_labels:
             UpperCAmelCase__						:    Optional[int]							=							ids_tensor([self.batch_size]							,			self.type_sequence_label_size							)

        UpperCAmelCase__						:    str							=							self.get_config()

        return config, pixel_values, labels
   def       __a							(		self      :	Optional[Any]							):


        '''simple docstring'''
        return EfficientFormerConfig(
            image_size=self.image_size							,			patch_size=self.patch_size							,			num_channels=self.num_channels							,			hidden_size=self.hidden_size							,			num_hidden_layers=self.num_hidden_layers							,			num_attention_heads=self.num_attention_heads							,			intermediate_size=self.intermediate_size							,			hidden_act=self.hidden_act							,			hidden_dropout_prob=self.hidden_dropout_prob							,			attention_probs_dropout_prob=self.attention_probs_dropout_prob							,			is_decoder=snake_case__							,			initializer_range=self.initializer_range							,			encoder_stride=self.encoder_stride							,			resolution=self.resolution							,			depths=self.depths							,			hidden_sizes=self.hidden_sizes							,			dim=self.dim							,			mlp_expansion_ratio=self.mlp_expansion_ratio							,			)
   def       __a							(		self      :	List[str]							,			snake_case__      :	int							,			snake_case__      :	Optional[int]							,			snake_case__      :	str							):


        '''simple docstring'''
        UpperCAmelCase__						:    Dict							=							TFEfficientFormerModel(config=snake_case__							)
        UpperCAmelCase__						:    Optional[Any]							=							model(snake_case__							,			training=snake_case__							)
        self.parent.assertEqual(result.last_hidden_state.shape							,			(self.batch_size, self.seq_length, self.hidden_size)							)
   def       __a							(		self      :	Optional[int]							,			snake_case__      :	Optional[Any]							,			snake_case__      :	str							,			snake_case__      :	Dict							):


        '''simple docstring'''
        UpperCAmelCase__						:    str							=							self.type_sequence_label_size
        UpperCAmelCase__						:    Optional[Any]							=							TFEfficientFormerForImageClassification(snake_case__							)
        UpperCAmelCase__						:    Dict							=							model(snake_case__							,			labels=snake_case__							,			training=snake_case__							)
        self.parent.assertEqual(result.logits.shape							,			(self.batch_size, self.type_sequence_label_size)							)

        # test greyscale images
        UpperCAmelCase__						:    List[Any]							=							1
        UpperCAmelCase__						:    str							=							TFEfficientFormerForImageClassification(snake_case__							)

        UpperCAmelCase__						:    Optional[Any]							=							floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
        UpperCAmelCase__						:    Tuple							=							model(snake_case__							,			labels=snake_case__							)
        self.parent.assertEqual(result.logits.shape							,			(self.batch_size, self.type_sequence_label_size)							)




   def       __a							(		self      :	Tuple							):


        '''simple docstring'''
        UpperCAmelCase__						:    List[Any]							=							self.prepare_config_and_inputs()
        UpperCAmelCase__							,				UpperCAmelCase__							,				UpperCAmelCase__						:    Union[str, Any]							=							config_and_inputs
        UpperCAmelCase__						:    Optional[int]							=							{"pixel_values": pixel_values}
        return config, inputs_dict




@require_tf
class 			lowerCAmelCase__			(    __magic_name__							,						__magic_name__							,						unittest.TestCase   ):
   SCREAMING_SNAKE_CASE_    =(
       (
           TFEfficientFormerModel,
           TFEfficientFormerForImageClassificationWithTeacher,
           TFEfficientFormerForImageClassification,
       )
       if is_tf_available()
       else ()
   )
   SCREAMING_SNAKE_CASE_    =(
       {
           '''feature-extraction''': TFEfficientFormerModel,
           '''image-classification''': (
               TFEfficientFormerForImageClassification,
               TFEfficientFormerForImageClassificationWithTeacher,
           ),
       }
       if is_tf_available()
       else {}
   )

   SCREAMING_SNAKE_CASE_    =False

   SCREAMING_SNAKE_CASE_    =False
   SCREAMING_SNAKE_CASE_    =False
   SCREAMING_SNAKE_CASE_    =False
   SCREAMING_SNAKE_CASE_    =False
   def       __a							(		self      :	str							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[Any]							=							TFEfficientFormerModelTester(self							)
        UpperCAmelCase__						:    List[Any]							=							ConfigTester(
            self							,			config_class=snake_case__							,			has_text_modality=snake_case__							,			hidden_size=3_7							)
   def       __a							(		self      :	List[str]							):


        '''simple docstring'''
        self.config_tester.run_common_tests()
   @unittest.skip(reason="EfficientFormer does not use inputs_embeds"							)
   def       __a							(		self      :	List[Any]							):


        '''simple docstring'''
        pass
   @unittest.skip(reason="EfficientFormer does not support input and output embeddings"							)
   def       __a							(		self      :	Optional[int]							):


        '''simple docstring'''
        pass
   def       __a							(		self      :	Any							):


        '''simple docstring'''
        UpperCAmelCase__							,				UpperCAmelCase__						:    Union[str, Any]							=							self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
             UpperCAmelCase__						:    int							=							model_class(snake_case__							)
             UpperCAmelCase__						:    Optional[Any]							=							inspect.signature(model.call							)
             # signature.parameters is an OrderedDict => so arg_names order is deterministic
             UpperCAmelCase__						:    Optional[Any]							=							[*signature.parameters.keys()]

             UpperCAmelCase__						:    Union[str, Any]							=							["pixel_values"]
             self.assertListEqual(arg_names[:1]							,			snake_case__							)
   def       __a							(		self      :	List[Any]							):


        '''simple docstring'''
        def check_hidden_states_output(snake_case__      :	Any							,			snake_case__      :	Any							,			snake_case__      :	Tuple							):
             UpperCAmelCase__						:    str							=							model_class(snake_case__							)

             UpperCAmelCase__						:    Dict							=							model(**self._prepare_for_class(snake_case__							,			snake_case__							)							,			training=snake_case__							)
             UpperCAmelCase__						:    Tuple							=							outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

             UpperCAmelCase__						:    List[Any]							=							getattr(
                 self.model_tester							,			"expected_num_hidden_layers"							,			self.model_tester.num_hidden_layers + 1							)
             self.assertEqual(len(snake_case__							)							,			snake_case__							)

             if hasattr(self.model_tester							,			"encoder_seq_length"							):
                  UpperCAmelCase__						:    Tuple							=							self.model_tester.encoder_seq_length
                  if hasattr(self.model_tester							,			"chunk_length"							) and self.model_tester.chunk_length > 1:
                       UpperCAmelCase__						:    Dict							=							seq_length * self.model_tester.chunk_length
             else:
                  UpperCAmelCase__						:    str							=							self.model_tester.seq_length

             self.assertListEqual(
                 list(hidden_states[-1].shape[-2:]							)							,			[seq_length, self.model_tester.hidden_size]							,			)

             if config.is_encoder_decoder:
                  UpperCAmelCase__						:    Dict							=							outputs.decoder_hidden_states

                  self.asseretIsInstance(snake_case__							,			(list, tuple)							)
                  self.assertEqual(len(snake_case__							)							,			snake_case__							)
                  UpperCAmelCase__						:    Any							=							getattr(self.model_tester							,			"seq_length"							,			snake_case__							)
                  UpperCAmelCase__						:    str							=							getattr(self.model_tester							,			"decoder_seq_length"							,			snake_case__							)

                  self.assertListEqual(
                      list(hidden_states[-1].shape[-2:]							)							,			[decoder_seq_length, self.model_tester.hidden_size]							,			)

        UpperCAmelCase__							,				UpperCAmelCase__						:    Tuple							=							self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
             UpperCAmelCase__						:    List[Any]							=							True
             check_hidden_states_output(snake_case__							,			snake_case__							,			snake_case__							)

             # check that output_hidden_states also work using config
             del inputs_dict["output_hidden_states"]
             UpperCAmelCase__						:    List[Any]							=							True

             check_hidden_states_output(snake_case__							,			snake_case__							,			snake_case__							)
   def       __a							(		self      :	Dict							,			snake_case__      :	Any							,			snake_case__      :	Optional[Any]							,			snake_case__      :	Optional[Any]=False							):


        '''simple docstring'''
        UpperCAmelCase__						:    Any							=							super()._prepare_for_class(snake_case__							,			snake_case__							,			return_labels=snake_case__							)

        if return_labels:
             if model_class.__name__ == "TFEfficientFormerForImageClassificationWithTeacher":
                  del inputs_dict["labels"]

        return inputs_dict
   def       __a							(		self      :	str							):


        '''simple docstring'''
        UpperCAmelCase__						:    int							=							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*snake_case__							)
   @unittest.skip(reason="EfficientFormer does not implement masked image modeling yet"							)
   def       __a							(		self      :	str							):


        '''simple docstring'''
        UpperCAmelCase__						:    List[str]							=							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_image_modeling(*snake_case__							)
   def       __a							(		self      :	Union[str, Any]							):


        '''simple docstring'''
        UpperCAmelCase__						:    int							=							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*snake_case__							)
   @slow
   def       __a							(		self      :	str							):


        '''simple docstring'''
        for model_name in TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             UpperCAmelCase__						:    int							=							TFEfficientFormerModel.from_pretrained(snake_case__							)
             self.assertIsNotNone(snake_case__							)
   def       __a							(		self      :	List[str]							):


        '''simple docstring'''
        UpperCAmelCase__							,				UpperCAmelCase__						:    Union[str, Any]							=							self.model_tester.prepare_config_and_inputs_for_common()
        UpperCAmelCase__						:    Dict							=							True

        UpperCAmelCase__						:    Optional[int]							=							getattr(self.model_tester							,			"seq_length"							,			snake_case__							)
        UpperCAmelCase__						:    int							=							getattr(self.model_tester							,			"encoder_seq_length"							,			snake_case__							)
        UpperCAmelCase__						:    List[Any]							=							getattr(self.model_tester							,			"key_length"							,			snake_case__							)
        UpperCAmelCase__						:    Union[str, Any]							=							getattr(self.model_tester							,			"chunk_length"							,			snake_case__							)

        if chunk_length is not None and hasattr(self.model_tester							,			"num_hashes"							):
             UpperCAmelCase__						:    Any							=							encoder_seq_length * self.model_tester.num_hashes

        for model_class in self.all_model_classes:
             UpperCAmelCase__						:    List[str]							=							True
             UpperCAmelCase__						:    Any							=							False
             UpperCAmelCase__						:    int							=							True
             UpperCAmelCase__						:    Optional[Any]							=							model_class(snake_case__							)

             UpperCAmelCase__						:    Tuple							=							model(**self._prepare_for_class(snake_case__							,			snake_case__							)							,			training=snake_case__							)
             UpperCAmelCase__						:    Any							=							outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
             self.assertEqual(len(snake_case__							)							,			self.model_tester.num_attention_outputs							)

             # check that output_attentions also work using config
             del inputs_dict["output_attentions"]
             UpperCAmelCase__						:    Union[str, Any]							=							True
             UpperCAmelCase__						:    Union[str, Any]							=							model_class(snake_case__							)
             UpperCAmelCase__						:    Tuple							=							model(**self._prepare_for_class(snake_case__							,			snake_case__							)							,			training=snake_case__							)

             UpperCAmelCase__						:    int							=							outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
             self.assertEqual(len(snake_case__							)							,			self.model_tester.num_attention_outputs							)

             if chunk_length is not None:
                  self.assertListEqual(
                      list(attentions[0].shape[-4:]							)							,			[self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length]							,			)
             else:
                  self.assertListEqual(
                      list(attentions[0].shape[-3:]							)							,			[self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]							,			)




   def       __a							(		self      :	List[Any]							):


        '''simple docstring'''
        # We use a simplified version of this test for EfficientFormer because it requires training=False
        # and Keras refuses to let us force that during functional construction
        UpperCAmelCase__							,				UpperCAmelCase__						:    Dict							=							self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
             # Prepare our model
             UpperCAmelCase__						:    Dict							=							model_class(snake_case__							)
             # These are maximally general inputs for the model, with multiple None dimensions
             # Hopefully this will catch any conditionals that fail for flexible shapes
             UpperCAmelCase__						:    List[Any]							=							{
                 key: tf.keras.Input(shape=val.shape[1:]							,			dtype=val.dtype							,			name=snake_case__							)
                 for key, val in model.input_signature.items()
                 if key in model.dummy_inputs
             }
             UpperCAmelCase__						:    Optional[int]							=							model(snake_case__							)
             self.assertTrue(outputs_dict is not None							)




def 	SCREAMING_SNAKE_CASE__       (  )->		Tuple:






     '''simple docstring'''



     UpperCAmelCase__						:    Union[str, Any]							=							Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"				)
     return image




@require_tf
@require_vision
class 			lowerCAmelCase__			(    unittest.TestCase   ):
   @cached_property
   def       __a							(		self      :	str							):


        '''simple docstring'''
        return (
            EfficientFormerImageProcessor.from_pretrained("snap-research/efficientformer-l1-300"							)
            if is_vision_available()
            else None
        )
   @slow
   def       __a							(		self      :	List[str]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Union[str, Any]							=							TFEfficientFormerForImageClassification.from_pretrained("snap-research/efficientformer-l1-300"							)
        UpperCAmelCase__						:    str							=							self.default_image_processor
        UpperCAmelCase__						:    Optional[int]							=							prepare_img()
        UpperCAmelCase__						:    Any							=							image_processor(images=snake_case__							,			return_tensors="tf"							)
        # forward pass
        UpperCAmelCase__						:    Optional[int]							=							model(**snake_case__							,			training=snake_case__							)
        # verify the logits
        UpperCAmelCase__						:    Optional[Any]							=							tf.TensorShape((1, 1_0_0_0)							)
        self.assertEqual(outputs.logits.shape							,			snake_case__							)
        UpperCAmelCase__						:    Optional[int]							=							tf.constant([-0.0555, 0.4825, -0.0852]							)
        self.assertTrue(np.allclose(outputs.logits[0, :3]							,			snake_case__							,			atol=1e-4							)							)




   @slow
   def       __a							(		self      :	Tuple							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[Any]							=							TFEfficientFormerForImageClassificationWithTeacher.from_pretrained(
            "snap-research/efficientformer-l1-300"							)
        UpperCAmelCase__						:    str							=							self.default_image_processor
        UpperCAmelCase__						:    List[str]							=							prepare_img()
        UpperCAmelCase__						:    Optional[Any]							=							image_processor(images=snake_case__							,			return_tensors="tf"							)
        # forward pass
        UpperCAmelCase__						:    Optional[int]							=							model(**snake_case__							,			training=snake_case__							)
        # verify the logits
        UpperCAmelCase__						:    Tuple							=							tf.TensorShape((1, 1_0_0_0)							)
        self.assertEqual(outputs.logits.shape							,			snake_case__							)
        UpperCAmelCase__						:    Optional[Any]							=							tf.constant([-0.1312, 0.4353, -1.0499]							)
        self.assertTrue(np.allclose(outputs.logits[0, :3]							,			snake_case__							,			atol=1e-4							)							)










"""simple docstring"""



import unittest

from parameterized import parameterized

from transformers import LlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaTokenizer




class 			lowerCAmelCase__			:
   def __init__(		self      :	Any							,			snake_case__      :	List[str]							,			snake_case__      :	Union[str, Any]=1_3							,			snake_case__      :	Dict=7							,			snake_case__      :	Optional[int]=True							,			snake_case__      :	Optional[int]=True							,			snake_case__      :	int=False							,			snake_case__      :	Optional[Any]=True							,			snake_case__      :	Optional[int]=9_9							,			snake_case__      :	List[Any]=3_2							,			snake_case__      :	Optional[Any]=5							,			snake_case__      :	Union[str, Any]=4							,			snake_case__      :	List[Any]=3_7							,			snake_case__      :	List[str]="gelu"							,			snake_case__      :	List[Any]=0.1							,			snake_case__      :	Tuple=0.1							,			snake_case__      :	Optional[Any]=5_1_2							,			snake_case__      :	Tuple=1_6							,			snake_case__      :	str=2							,			snake_case__      :	Optional[int]=0.02							,			snake_case__      :	int=3							,			snake_case__      :	Union[str, Any]=4							,			snake_case__      :	Optional[int]=None							,			):


        '''simple docstring'''
        UpperCAmelCase__						:    List[str]							=							parent
        UpperCAmelCase__						:    Optional[Any]							=							batch_size
        UpperCAmelCase__						:    Any							=							seq_length
        UpperCAmelCase__						:    int							=							is_training
        UpperCAmelCase__						:    Dict							=							use_input_mask
        UpperCAmelCase__						:    List[Any]							=							use_token_type_ids
        UpperCAmelCase__						:    Union[str, Any]							=							use_labels
        UpperCAmelCase__						:    Union[str, Any]							=							vocab_size
        UpperCAmelCase__						:    Union[str, Any]							=							hidden_size
        UpperCAmelCase__						:    Union[str, Any]							=							num_hidden_layers
        UpperCAmelCase__						:    str							=							num_attention_heads
        UpperCAmelCase__						:    int							=							intermediate_size
        UpperCAmelCase__						:    int							=							hidden_act
        UpperCAmelCase__						:    str							=							hidden_dropout_prob
        UpperCAmelCase__						:    Optional[int]							=							attention_probs_dropout_prob
        UpperCAmelCase__						:    List[Any]							=							max_position_embeddings
        UpperCAmelCase__						:    str							=							type_vocab_size
        UpperCAmelCase__						:    Union[str, Any]							=							type_sequence_label_size
        UpperCAmelCase__						:    str							=							initializer_range
        UpperCAmelCase__						:    Tuple							=							num_labels
        UpperCAmelCase__						:    Union[str, Any]							=							num_choices
        UpperCAmelCase__						:    Tuple							=							scope
   def       __a							(		self      :	Tuple							):


        '''simple docstring'''
        UpperCAmelCase__						:    Union[str, Any]							=							ids_tensor([self.batch_size, self.seq_length]							,			self.vocab_size							)

        UpperCAmelCase__						:    Optional[int]							=							None
        if self.use_input_mask:
             UpperCAmelCase__						:    Union[str, Any]							=							random_attention_mask([self.batch_size, self.seq_length]							)

        UpperCAmelCase__						:    Dict							=							None
        if self.use_token_type_ids:
             UpperCAmelCase__						:    Optional[int]							=							ids_tensor([self.batch_size, self.seq_length]							,			self.type_vocab_size							)

        UpperCAmelCase__						:    str							=							None
        UpperCAmelCase__						:    Optional[Any]							=							None
        UpperCAmelCase__						:    int							=							None
        if self.use_labels:
             UpperCAmelCase__						:    List[Any]							=							ids_tensor([self.batch_size]							,			self.type_sequence_label_size							)
             UpperCAmelCase__						:    Optional[Any]							=							ids_tensor([self.batch_size, self.seq_length]							,			self.num_labels							)
             UpperCAmelCase__						:    Tuple							=							ids_tensor([self.batch_size]							,			self.num_choices							)

        UpperCAmelCase__						:    Dict							=							self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
   def       __a							(		self      :	Tuple							):


        '''simple docstring'''
        return LlamaConfig(
            vocab_size=self.vocab_size							,			hidden_size=self.hidden_size							,			num_hidden_layers=self.num_hidden_layers							,			num_attention_heads=self.num_attention_heads							,			intermediate_size=self.intermediate_size							,			hidden_act=self.hidden_act							,			hidden_dropout_prob=self.hidden_dropout_prob							,			attention_probs_dropout_prob=self.attention_probs_dropout_prob							,			max_position_embeddings=self.max_position_embeddings							,			type_vocab_size=self.type_vocab_size							,			is_decoder=snake_case__							,			initializer_range=self.initializer_range							,			)
   def       __a							(		self      :	Optional[int]							,			snake_case__      :	Optional[int]							,			snake_case__      :	Optional[int]							,			snake_case__      :	Tuple							,			snake_case__      :	Dict							,			snake_case__      :	Tuple							,			snake_case__      :	List[str]							,			snake_case__      :	Union[str, Any]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[Any]							=							LlamaModel(config=snake_case__							)
        model.to(snake_case__							)
        model.eval()
        UpperCAmelCase__						:    List[str]							=							model(snake_case__							,			attention_mask=snake_case__							)
        UpperCAmelCase__						:    Any							=							model(snake_case__							)
        self.parent.assertEqual(result.last_hidden_state.shape							,			(self.batch_size, self.seq_length, self.hidden_size)							)
   def       __a							(		self      :	Any							,			snake_case__      :	Union[str, Any]							,			snake_case__      :	Optional[Any]							,			snake_case__      :	List[Any]							,			snake_case__      :	Dict							,			snake_case__      :	Optional[int]							,			snake_case__      :	Tuple							,			snake_case__      :	Tuple							,			snake_case__      :	str							,			snake_case__      :	int							,			):


        '''simple docstring'''
        UpperCAmelCase__						:    List[Any]							=							True
        UpperCAmelCase__						:    str							=							LlamaModel(snake_case__							)
        model.to(snake_case__							)
        model.eval()
        UpperCAmelCase__						:    Optional[Any]							=							model(
            snake_case__							,			attention_mask=snake_case__							,			encoder_hidden_states=snake_case__							,			encoder_attention_mask=snake_case__							,			)
        UpperCAmelCase__						:    str							=							model(
            snake_case__							,			attention_mask=snake_case__							,			encoder_hidden_states=snake_case__							,			)
        UpperCAmelCase__						:    List[str]							=							model(snake_case__							,			attention_mask=snake_case__							)
        self.parent.assertEqual(result.last_hidden_state.shape							,			(self.batch_size, self.seq_length, self.hidden_size)							)
   def       __a							(		self      :	List[str]							,			snake_case__      :	Dict							,			snake_case__      :	Optional[int]							,			snake_case__      :	int							,			snake_case__      :	Optional[int]							,			snake_case__      :	List[str]							,			snake_case__      :	Optional[int]							,			snake_case__      :	List[Any]							,			snake_case__      :	List[str]							,			snake_case__      :	Union[str, Any]							,			):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[int]							=							LlamaForCausalLM(config=snake_case__							)
        model.to(snake_case__							)
        model.eval()
        UpperCAmelCase__						:    Optional[Any]							=							model(snake_case__							,			attention_mask=snake_case__							,			labels=snake_case__							)
        self.parent.assertEqual(result.logits.shape							,			(self.batch_size, self.seq_length, self.vocab_size)							)
   def       __a							(		self      :	List[str]							,			snake_case__      :	Union[str, Any]							,			snake_case__      :	List[Any]							,			snake_case__      :	int							,			snake_case__      :	Optional[int]							,			snake_case__      :	List[Any]							,			snake_case__      :	int							,			snake_case__      :	Optional[Any]							,			snake_case__      :	Dict							,			snake_case__      :	Optional[int]							,			):


        '''simple docstring'''
        UpperCAmelCase__						:    List[str]							=							True
        UpperCAmelCase__						:    str							=							True
        UpperCAmelCase__						:    Union[str, Any]							=							LlamaForCausalLM(config=snake_case__							)
        model.to(snake_case__							)
        model.eval()

        # first forward pass
        UpperCAmelCase__						:    List[str]							=							model(
            snake_case__							,			attention_mask=snake_case__							,			encoder_hidden_states=snake_case__							,			encoder_attention_mask=snake_case__							,			use_cache=snake_case__							,			)
        UpperCAmelCase__						:    Optional[Any]							=							outputs.past_key_values

        # create hypothetical multiple next token and extent to next_input_ids
        UpperCAmelCase__						:    Any							=							ids_tensor((self.batch_size, 3)							,			config.vocab_size							)
        UpperCAmelCase__						:    Optional[int]							=							ids_tensor((self.batch_size, 3)							,			vocab_size=2							)

        # append to next input_ids and
        UpperCAmelCase__						:    str							=							torch.cat([input_ids, next_tokens]							,			dim=-1							)
        UpperCAmelCase__						:    List[Any]							=							torch.cat([input_mask, next_mask]							,			dim=-1							)

        UpperCAmelCase__						:    str							=							model(
            snake_case__							,			attention_mask=snake_case__							,			encoder_hidden_states=snake_case__							,			encoder_attention_mask=snake_case__							,			output_hidden_states=snake_case__							,			)["hidden_states"][0]
        UpperCAmelCase__						:    str							=							model(
            snake_case__							,			attention_mask=snake_case__							,			encoder_hidden_states=snake_case__							,			encoder_attention_mask=snake_case__							,			past_key_values=snake_case__							,			output_hidden_states=snake_case__							,			)["hidden_states"][0]

        # select random slice
        UpperCAmelCase__						:    Dict							=							ids_tensor((1,)							,			output_from_past.shape[-1]							).item()
        UpperCAmelCase__						:    str							=							output_from_no_past[:, -3:, random_slice_idx].detach()
        UpperCAmelCase__						:    Any							=							output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]							)

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(snake_case__							,			snake_case__							,			atol=1e-3							)							)




   def       __a							(		self      :	Dict							):


        '''simple docstring'''
        UpperCAmelCase__						:    int							=							self.prepare_config_and_inputs()
        (
            (
            UpperCAmelCase__
        )							,				(
            UpperCAmelCase__
        )							,				(
            UpperCAmelCase__
        )							,				(
            UpperCAmelCase__
        )							,				(
            UpperCAmelCase__
        )							,				(
            UpperCAmelCase__
        )							,				(
            UpperCAmelCase__
        )							,				
        )						:    Optional[Any]							=							config_and_inputs
        UpperCAmelCase__						:    Union[str, Any]							=							{"input_ids": input_ids, "attention_mask": input_mask}
        return config, inputs_dict




@require_torch
class 			lowerCAmelCase__			(    __magic_name__							,						__magic_name__							,						__magic_name__							,						unittest.TestCase   ):
   SCREAMING_SNAKE_CASE_    =(LlamaModel, LlamaForCausalLM, LlamaForSequenceClassification) if is_torch_available() else ()
   SCREAMING_SNAKE_CASE_    =(LlamaForCausalLM,) if is_torch_available() else ()
   SCREAMING_SNAKE_CASE_    =(
       {
           '''feature-extraction''': LlamaModel,
           '''text-classification''': LlamaForSequenceClassification,
           '''text-generation''': LlamaForCausalLM,
           '''zero-shot''': LlamaForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   SCREAMING_SNAKE_CASE_    =False
   SCREAMING_SNAKE_CASE_    =False
   def       __a							(		self      :	int							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[int]							=							LlamaModelTester(self							)
        UpperCAmelCase__						:    str							=							ConfigTester(self							,			config_class=snake_case__							,			hidden_size=3_7							)
   def       __a							(		self      :	Union[str, Any]							):


        '''simple docstring'''
        self.config_tester.run_common_tests()
   def       __a							(		self      :	Dict							):


        '''simple docstring'''
        UpperCAmelCase__						:    Union[str, Any]							=							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*snake_case__							)
   def       __a							(		self      :	Optional[Any]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Union[str, Any]							=							self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
             UpperCAmelCase__						:    List[Any]							=							type
             self.model_tester.create_and_check_model(*snake_case__							)
   def       __a							(		self      :	Union[str, Any]							):


        '''simple docstring'''
        UpperCAmelCase__							,				UpperCAmelCase__						:    Tuple							=							self.model_tester.prepare_config_and_inputs_for_common()
        UpperCAmelCase__						:    str							=							3
        UpperCAmelCase__						:    Any							=							input_dict["input_ids"]
        UpperCAmelCase__						:    Any							=							input_ids.ne(1							).to(snake_case__							)
        UpperCAmelCase__						:    str							=							ids_tensor([self.model_tester.batch_size]							,			self.model_tester.type_sequence_label_size							)
        UpperCAmelCase__						:    Optional[Any]							=							LlamaForSequenceClassification(snake_case__							)
        model.to(snake_case__							)
        model.eval()
        UpperCAmelCase__						:    Optional[int]							=							model(snake_case__							,			attention_mask=snake_case__							,			labels=snake_case__							)
        self.assertEqual(result.logits.shape							,			(self.model_tester.batch_size, self.model_tester.num_labels)							)
   def       __a							(		self      :	Optional[int]							):


        '''simple docstring'''
        UpperCAmelCase__							,				UpperCAmelCase__						:    Optional[Any]							=							self.model_tester.prepare_config_and_inputs_for_common()
        UpperCAmelCase__						:    List[Any]							=							3
        UpperCAmelCase__						:    Any							=							"single_label_classification"
        UpperCAmelCase__						:    Tuple							=							input_dict["input_ids"]
        UpperCAmelCase__						:    str							=							input_ids.ne(1							).to(snake_case__							)
        UpperCAmelCase__						:    List[Any]							=							ids_tensor([self.model_tester.batch_size]							,			self.model_tester.type_sequence_label_size							)
        UpperCAmelCase__						:    Any							=							LlamaForSequenceClassification(snake_case__							)
        model.to(snake_case__							)
        model.eval()
        UpperCAmelCase__						:    List[str]							=							model(snake_case__							,			attention_mask=snake_case__							,			labels=snake_case__							)
        self.assertEqual(result.logits.shape							,			(self.model_tester.batch_size, self.model_tester.num_labels)							)
   def       __a							(		self      :	Union[str, Any]							):


        '''simple docstring'''
        UpperCAmelCase__							,				UpperCAmelCase__						:    str							=							self.model_tester.prepare_config_and_inputs_for_common()
        UpperCAmelCase__						:    Dict							=							3
        UpperCAmelCase__						:    Tuple							=							"multi_label_classification"
        UpperCAmelCase__						:    Any							=							input_dict["input_ids"]
        UpperCAmelCase__						:    str							=							input_ids.ne(1							).to(snake_case__							)
        UpperCAmelCase__						:    Any							=							ids_tensor(
            [self.model_tester.batch_size, config.num_labels]							,			self.model_tester.type_sequence_label_size							).to(torch.float							)
        UpperCAmelCase__						:    Dict							=							LlamaForSequenceClassification(snake_case__							)
        model.to(snake_case__							)
        model.eval()
        UpperCAmelCase__						:    int							=							model(snake_case__							,			attention_mask=snake_case__							,			labels=snake_case__							)
        self.assertEqual(result.logits.shape							,			(self.model_tester.batch_size, self.model_tester.num_labels)							)
   @unittest.skip("LLaMA buffers include complex numbers, which breaks this test"							)
   def       __a							(		self      :	str							):


        '''simple docstring'''
        pass




   @parameterized.expand([("linear",), ("dynamic",)]							)
   def       __a							(		self      :	List[str]							,			snake_case__      :	int							):


        '''simple docstring'''
        UpperCAmelCase__							,				UpperCAmelCase__						:    str							=							self.model_tester.prepare_config_and_inputs_for_common()
        UpperCAmelCase__						:    List[str]							=							ids_tensor([1, 1_0]							,			config.vocab_size							)
        UpperCAmelCase__						:    int							=							ids_tensor([1, int(config.max_position_embeddings * 1.5							)]							,			config.vocab_size							)

        set_seed(4_2							)  # Fixed seed at init time so the two models get the same random weights
        UpperCAmelCase__						:    str							=							LlamaModel(snake_case__							)
        original_model.to(snake_case__							)
        original_model.eval()
        UpperCAmelCase__						:    List[Any]							=							original_model(snake_case__							).last_hidden_state
        UpperCAmelCase__						:    Tuple							=							original_model(snake_case__							).last_hidden_state

        set_seed(4_2							)  # Fixed seed at init time so the two models get the same random weights
        UpperCAmelCase__						:    str							=							{"type": scaling_type, "factor": 10.0}
        UpperCAmelCase__						:    Optional[int]							=							LlamaModel(snake_case__							)
        scaled_model.to(snake_case__							)
        scaled_model.eval()
        UpperCAmelCase__						:    Tuple							=							scaled_model(snake_case__							).last_hidden_state
        UpperCAmelCase__						:    Union[str, Any]							=							scaled_model(snake_case__							).last_hidden_state

        # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
        # maximum sequence length, so the outputs for the short input should match.
        if scaling_type == "dynamic":
             self.assertTrue(torch.allclose(snake_case__							,			snake_case__							,			atol=1e-5							)							)
        else:
             self.assertFalse(torch.allclose(snake_case__							,			snake_case__							,			atol=1e-5							)							)

        # The output should be different for long inputs
        self.assertFalse(torch.allclose(snake_case__							,			snake_case__							,			atol=1e-5							)							)




@require_torch
class 			lowerCAmelCase__			(    unittest.TestCase   ):
   @unittest.skip("Logits are not exactly the same, once we fix the instabalities somehow, will update!"							)
   @slow
   def       __a							(		self      :	List[str]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Union[str, Any]							=							[1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
        UpperCAmelCase__						:    Any							=							LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf"							,			device_map="auto"							)
        UpperCAmelCase__						:    Any							=							model(torch.tensor([input_ids]							)							)
        # Expected mean on dim = -1
        UpperCAmelCase__						:    Optional[Any]							=							torch.tensor([[-6.6550, -4.1227, -4.9859, -3.2406, 0.8262, -3.0033, 1.2964, -3.3699]]							)
        torch.testing.assert_close(out.mean(-1							)							,			snake_case__							,			atol=1e-2							,			rtol=1e-2							)
        # slicing logits[0, 0, 0:30]
        # fmt: off
        UpperCAmelCase__						:    Optional[int]							=							torch.tensor([-12.8281, -7.4453, -0.4639, -8.0625, -7.2500, -8.0000, -6.4883, -7.7695, -7.8438, -7.0312, -6.2188, -7.1328, -1.8496, 1.9961, -8.6250, -6.7227, -12.8281, -6.9492, -7.0742, -7.7852, -7.5820, -7.9062, -6.9375, -7.9805, -8.3438, -8.1562, -8.0469, -7.6250, -7.7422, -7.3398,]							)
        # fmt: on
        torch.testing.assert_close(out[0, 0, :3_0]							,			snake_case__							,			atol=1e-5							,			rtol=1e-5							)
   @unittest.skip("Logits are not exactly the same, once we fix the instabalities somehow, will update!"							)
   @slow
   def       __a							(		self      :	Any							):


        '''simple docstring'''
        UpperCAmelCase__						:    List[str]							=							[1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
        UpperCAmelCase__						:    Dict							=							LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-13b-hf"							,			device_map="auto"							)
        UpperCAmelCase__						:    List[str]							=							model(torch.tensor(snake_case__							)							)
        # Expected mean on dim = -1
        UpperCAmelCase__						:    str							=							torch.tensor([[-2.0622, -1.2794, -1.1638, -0.9788, -1.4603, -1.0238, -1.7893, -1.4411]]							)
        torch.testing.assert_close(out.mean(-1							)							,			snake_case__							,			atol=1e-2							,			rtol=1e-2							)
        # slicing logits[0, 0, 0:30]
        # fmt: off
        UpperCAmelCase__						:    str							=							torch.tensor([-8.1406, -8.0547, 2.7461, -1.2344, -0.1448, -1.8262, -1.0020, -1.8154, -1.6895, -1.8516, -2.3574, -0.9277, 3.7598, 6.5742, -1.2998, -0.1177, -8.1406, -2.9688, -2.9199, -3.1699, -3.5254, -2.3555, -2.7988, -3.4141, -2.8262, -4.5195, -3.3379, -3.3164, -2.7832, -3.0273]							)
        # fmt: on
        torch.testing.assert_close(out[0, 0, :3_0]							,			snake_case__							,			atol=1e-5							,			rtol=1e-5							)
   @unittest.skip("Logits are not exactly the same, once we fix the instabalities somehow, will update!"							)
   @slow
   def       __a							(		self      :	int							):


        '''simple docstring'''
        UpperCAmelCase__						:    List[Any]							=							[1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
        UpperCAmelCase__						:    Optional[Any]							=							LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-13b-chat-hf"							,			device_map="auto"							)
        UpperCAmelCase__						:    Dict							=							model(torch.tensor(snake_case__							)							)
        # Expected mean on dim = -1
        UpperCAmelCase__						:    List[str]							=							torch.tensor([[-0.8562, -1.8520, -0.7551, -0.4162, -1.5161, -1.2038, -2.4823, -2.3254]]							)
        torch.testing.assert_close(out.mean(-1							)							,			snake_case__							,			atol=1e-2							,			rtol=1e-2							)
        # slicing logits[0, 0, 0:30]
        # fmt: off
        UpperCAmelCase__						:    str							=							torch.tensor([-2.2227, 4.8828, 0.9023, -0.4578, -0.7871, -0.1033, -0.6221, -0.5786, -0.7803, -1.0674, -1.2920, -0.1570, 0.8008, 2.0723, -0.9497, 0.2771, -2.2227, -0.7612, -1.4346, -1.2061, -1.6426, -0.3000, -0.7139, -1.1934, -1.8691, -1.6973, -1.5947, -1.2705, -0.3523, -0.5513]							)
        # fmt: on
        torch.testing.assert_close(out.mean(-1							)							,			snake_case__							,			atol=1e-2							,			rtol=1e-2							)
   @unittest.skip(
       "Logits are not exactly the same, once we fix the instabalities somehow, will update! Also it is gonna be a `too_slow` test"							)
   @slow
   def       __a							(		self      :	Any							):


        '''simple docstring'''
        UpperCAmelCase__						:    Tuple							=							[1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
        UpperCAmelCase__						:    Optional[int]							=							LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-70b-hf"							,			device_map="auto"							)
        UpperCAmelCase__						:    List[Any]							=							model(torch.tensor(snake_case__							)							)

        UpperCAmelCase__						:    str							=							torch.tensor(
            [[-4.2327, -3.3360, -4.6665, -4.7631, -1.8180, -3.4170, -1.4211, -3.1810]]							,			dtype=torch.floataa							)
        torch.testing.assert_close(out.mean(-1							)							,			snake_case__							,			atol=1e-2							,			rtol=1e-2							)
        # fmt: off
        UpperCAmelCase__						:    Any							=							torch.tensor([-9.4922, -3.9551, 1.7998, -5.6758, -5.1055, -5.8984, -4.8320, -6.8086, -6.5391, -5.6172, -5.5820, -5.5352, 1.7881, 3.6289, -6.5117, -3.4785, -9.5000, -6.0352, -6.8125, -6.0195, -6.6836, -5.4727, -6.2812, -6.0391, -7.3398, -7.4297, -7.4844, -6.5820, -5.8789, -5.5312]							)
        # fmt: on
        torch.testing.assert_close(out[0, 0, :3_0]							,			snake_case__							,			atol=1e-5							,			rtol=1e-5							)




   @unittest.skip("Model is curently gated"							)
   @slow
   def       __a							(		self      :	Dict							):


        '''simple docstring'''
        UpperCAmelCase__						:    List[str]							=							"Simply put, the theory of relativity states that 1) the laws of physics are the same everywhere in the universe and 2) the passage of time and the length of objects can vary depending on the observer\'s frame of reference.\n\nThe first part of the theory, that the laws of physics are the same everywhere, is known as the \"princi"
        UpperCAmelCase__						:    Any							=							"Simply put, the theory of relativity states that "
        UpperCAmelCase__						:    int							=							LlamaTokenizer.from_pretrained("meta-llama/Llama-2-13b-chat-hf"							)
        UpperCAmelCase__						:    List[Any]							=							tokenizer.encode(snake_case__							,			return_tensors="pt"							)
        UpperCAmelCase__						:    Union[str, Any]							=							LlamaForCausalLM.from_pretrained(
            "meta-llama/Llama-2-13b-chat-hf"							,			device_map="sequential"							,			use_safetensors=snake_case__							)

        # greedy generation outputs
        UpperCAmelCase__						:    Optional[int]							=							model.generate(snake_case__							,			max_new_tokens=6_4							,			top_p=snake_case__							,			temperature=1							,			do_sample=snake_case__							)
        UpperCAmelCase__						:    Tuple							=							tokenizer.decode(generated_ids[0]							,			skip_special_tokens=snake_case__							)
        self.assertEqual(snake_case__							,			snake_case__							)


