



"""simple docstring"""







import inspect
import unittest
import warnings
from math import ceil, floor

from transformers import LevitConfig
from transformers.file_utils import cached_property, is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
         MODEL_MAPPING,
         LevitForImageClassification,
         LevitForImageClassificationWithTeacher,
         LevitModel,
     )
     from transformers.models.levit.modeling_levit import LEVIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
     from PIL import Image

     from transformers import LevitImageProcessor







class 				UpperCamelCase__       (     lowercase_  ):




      """simple docstring"""




      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[Any]	):
        lowerCAmelCase_       :    str								=			self.config_class(**self.inputs_dict	)
        self.parent.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,      'hidden_sizes'	)	)
        self.parent.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,      'num_attention_heads'	)	)







class 				UpperCamelCase__       :




      """simple docstring"""




      def __init__(				self    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				List[str]   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=1_3   ,      SCREAMING_SNAKE_CASE_    :				int=6_4   ,      SCREAMING_SNAKE_CASE_    :				Tuple=3   ,      SCREAMING_SNAKE_CASE_    :				Tuple=3   ,      SCREAMING_SNAKE_CASE_    :				Any=2   ,      SCREAMING_SNAKE_CASE_    :				Tuple=1   ,      SCREAMING_SNAKE_CASE_    :				Tuple=1_6   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]=[1_2_8, 2_5_6, 3_8_4]   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=[4, 6, 8]   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=[2, 3, 4]   ,      SCREAMING_SNAKE_CASE_    :				Any=[1_6, 1_6, 1_6]   ,      SCREAMING_SNAKE_CASE_    :				int=0   ,      SCREAMING_SNAKE_CASE_    :				Any=[2, 2, 2]   ,      SCREAMING_SNAKE_CASE_    :				str=[2, 2, 2]   ,      SCREAMING_SNAKE_CASE_    :				Dict=0.02   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=True   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=True   ,      SCREAMING_SNAKE_CASE_    :				Tuple=2   ,      ):
        lowerCAmelCase_       :    Optional[Any]								=			parent
        lowerCAmelCase_       :    Tuple								=			batch_size
        lowerCAmelCase_       :    str								=			image_size
        lowerCAmelCase_       :    str								=			num_channels
        lowerCAmelCase_       :    Dict								=			kernel_size
        lowerCAmelCase_       :    Tuple								=			stride
        lowerCAmelCase_       :    Union[str, Any]								=			padding
        lowerCAmelCase_       :    Union[str, Any]								=			hidden_sizes
        lowerCAmelCase_       :    Any								=			num_attention_heads
        lowerCAmelCase_       :    Optional[int]								=			depths
        lowerCAmelCase_       :    Optional[int]								=			key_dim
        lowerCAmelCase_       :    Dict								=			drop_path_rate
        lowerCAmelCase_       :    List[Any]								=			patch_size
        lowerCAmelCase_       :    int								=			attention_ratio
        lowerCAmelCase_       :    Optional[Any]								=			mlp_ratio
        lowerCAmelCase_       :    Union[str, Any]								=			initializer_range
        lowerCAmelCase_       :    str								=			[
            ['Subsample', key_dim[0], hidden_sizes[0] // key_dim[0], 4, 2, 2],
            ['Subsample', key_dim[0], hidden_sizes[1] // key_dim[0], 4, 2, 2],
        ]
        lowerCAmelCase_       :    List[Any]								=			is_training
        lowerCAmelCase_       :    List[Any]								=			use_labels
        lowerCAmelCase_       :    str								=			num_labels
        lowerCAmelCase_       :    str								=			initializer_range



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[str]	):
        lowerCAmelCase_       :    List[str]								=			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]	)

        lowerCAmelCase_       :    Union[str, Any]								=			None
        if self.use_labels:
          lowerCAmelCase_       :    Dict								=			ids_tensor([self.batch_size]   ,      self.num_labels	)

        lowerCAmelCase_       :    Optional[Any]								=			self.get_config()
        return config, pixel_values, labels



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]	):
        return LevitConfig(
            image_size=self.image_size   ,      num_channels=self.num_channels   ,      kernel_size=self.kernel_size   ,      stride=self.stride   ,      padding=self.padding   ,      patch_size=self.patch_size   ,      hidden_sizes=self.hidden_sizes   ,      num_attention_heads=self.num_attention_heads   ,      depths=self.depths   ,      key_dim=self.key_dim   ,      drop_path_rate=self.drop_path_rate   ,      mlp_ratio=self.mlp_ratio   ,      attention_ratio=self.attention_ratio   ,      initializer_range=self.initializer_range   ,      down_ops=self.down_ops   ,      )



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				str   ,      SCREAMING_SNAKE_CASE_    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]	):
        lowerCAmelCase_       :    Any								=			LevitModel(config=SCREAMING_SNAKE_CASE_	)
        model.to(SCREAMING_SNAKE_CASE_	)
        model.eval()
        lowerCAmelCase_       :    Optional[Any]								=			model(SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    str								=			(self.image_size, self.image_size)
        lowerCAmelCase_						,lowerCAmelCase_       :    Tuple								=			image_size[0], image_size[1]
        for _ in range(4	):
          lowerCAmelCase_       :    str								=			floor(((height + 2 * self.padding - self.kernel_size) / self.stride) + 1	)
          lowerCAmelCase_       :    Optional[int]								=			floor(((width + 2 * self.padding - self.kernel_size) / self.stride) + 1	)
        self.parent.assertEqual(
            result.last_hidden_state.shape   ,      (self.batch_size, ceil(height / 4	) * ceil(width / 4	), self.hidden_sizes[-1])   ,      )



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				List[str]   ,      SCREAMING_SNAKE_CASE_    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				int	):
        lowerCAmelCase_       :    List[str]								=			self.num_labels
        lowerCAmelCase_       :    Optional[Any]								=			LevitForImageClassification(SCREAMING_SNAKE_CASE_	)
        model.to(SCREAMING_SNAKE_CASE_	)
        model.eval()
        lowerCAmelCase_       :    Tuple								=			model(SCREAMING_SNAKE_CASE_   ,      labels=SCREAMING_SNAKE_CASE_	)
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.num_labels)	)





      def 					SCREAMING_SNAKE_CASE__			(				self    :				Any	):
        lowerCAmelCase_       :    List[Any]								=			self.prepare_config_and_inputs()
        lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_       :    Any								=			config_and_inputs
        lowerCAmelCase_       :    Dict								=			{'pixel_values': pixel_values}
        return config, inputs_dict







@require_torch
class 				UpperCamelCase__       (     lowercase_, lowercase_, unittest.TestCase  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE      =					(
          (LevitModel, LevitForImageClassification, LevitForImageClassificationWithTeacher)
          if is_torch_available()
          else ()
      )
      _SCREAMING_SNAKE_CASE      =					(
          {
              """feature-extraction""": LevitModel,
              """image-classification""": (LevitForImageClassification, LevitForImageClassificationWithTeacher),
          }
          if is_torch_available()
          else {}
      )

      _SCREAMING_SNAKE_CASE      =					False
      _SCREAMING_SNAKE_CASE      =					False
      _SCREAMING_SNAKE_CASE      =					False
      _SCREAMING_SNAKE_CASE      =					False
      _SCREAMING_SNAKE_CASE      =					False



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]	):
        lowerCAmelCase_       :    int								=			LevitModelTester(self	)
        lowerCAmelCase_       :    List[str]								=			ConfigTester(self   ,      config_class=SCREAMING_SNAKE_CASE_   ,      has_text_modality=SCREAMING_SNAKE_CASE_   ,      hidden_size=3_7	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]	):
        self.create_and_test_config_common_properties()
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple	):
        return



      @unittest.skip(reason='Levit does not use inputs_embeds'	)
      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict	):
        pass



      @unittest.skip(reason='Levit does not support input and output embeddings'	)
      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict	):
        pass



      @unittest.skip(reason='Levit does not output attentions'	)
      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[int]	):
        pass



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]	):
        lowerCAmelCase_						,lowerCAmelCase_       :    Union[str, Any]								=			self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          lowerCAmelCase_       :    Dict								=			model_class(SCREAMING_SNAKE_CASE_	)
          lowerCAmelCase_       :    Any								=			inspect.signature(model.forward	)
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          lowerCAmelCase_       :    Any								=			[*signature.parameters.keys()]

          lowerCAmelCase_       :    Optional[Any]								=			['pixel_values']
          self.assertListEqual(arg_names[:1]   ,      SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple	):
        def check_hidden_states_output(SCREAMING_SNAKE_CASE_    :				List[str]   ,      SCREAMING_SNAKE_CASE_    :				str   ,      SCREAMING_SNAKE_CASE_    :				str	):
          lowerCAmelCase_       :    int								=			model_class(SCREAMING_SNAKE_CASE_	)
          model.to(SCREAMING_SNAKE_CASE_	)
          model.eval()

          with torch.no_grad():
            lowerCAmelCase_       :    str								=			model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_	)	)

          lowerCAmelCase_       :    List[str]								=			outputs.hidden_states

          lowerCAmelCase_       :    Dict								=			len(self.model_tester.depths	) + 1
          self.assertEqual(len(SCREAMING_SNAKE_CASE_	)   ,      SCREAMING_SNAKE_CASE_	)

          lowerCAmelCase_       :    Optional[Any]								=			(self.model_tester.image_size, self.model_tester.image_size)
          lowerCAmelCase_						,lowerCAmelCase_       :    List[str]								=			image_size[0], image_size[1]
          for _ in range(4	):
            lowerCAmelCase_       :    int								=			floor(
                (
                    (height + 2 * self.model_tester.padding - self.model_tester.kernel_size)
                    / self.model_tester.stride
                )
                + 1	)
            lowerCAmelCase_       :    str								=			floor(
                (
                    (width + 2 * self.model_tester.padding - self.model_tester.kernel_size)
                    / self.model_tester.stride
                )
                + 1	)
          # verify the first hidden states (first block)
          self.assertListEqual(
              list(hidden_states[0].shape[-2:]	)   ,      [
                  height * width,
                  self.model_tester.hidden_sizes[0],
              ]   ,      )

        lowerCAmelCase_						,lowerCAmelCase_       :    Any								=			self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          lowerCAmelCase_       :    str								=			True
          check_hidden_states_output(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_	)

          # check that output_hidden_states also work using config
          del inputs_dict["output_hidden_states"]
          lowerCAmelCase_       :    Dict								=			True

          check_hidden_states_output(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_	)



      @unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.'	)
      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]	):
        pass



      def 					SCREAMING_SNAKE_CASE__			(				self    :				str   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				Any   ,      SCREAMING_SNAKE_CASE_    :				List[str]=False	):
        lowerCAmelCase_       :    Tuple								=			super()._prepare_for_class(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_   ,      return_labels=SCREAMING_SNAKE_CASE_	)

        if return_labels:
          if model_class.__name__ == "LevitForImageClassificationWithTeacher":
            del inputs_dict["labels"]

        return inputs_dict



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]	):
        lowerCAmelCase_       :    Dict								=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[Any]	):
        lowerCAmelCase_       :    List[str]								=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[int]	):
        if not self.model_tester.is_training:
          return

        lowerCAmelCase_						,lowerCAmelCase_       :    Any								=			self.model_tester.prepare_config_and_inputs_for_common()
        lowerCAmelCase_       :    int								=			True

        for model_class in self.all_model_classes:
          # LevitForImageClassificationWithTeacher supports inference-only
          if (
              model_class in get_values(SCREAMING_SNAKE_CASE_	)
              or model_class.__name__ == "LevitForImageClassificationWithTeacher"
          ):
            continue
          lowerCAmelCase_       :    Dict								=			model_class(SCREAMING_SNAKE_CASE_	)
          model.to(SCREAMING_SNAKE_CASE_	)
          model.train()
          lowerCAmelCase_       :    List[str]								=			self._prepare_for_class(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_   ,      return_labels=SCREAMING_SNAKE_CASE_	)
          lowerCAmelCase_       :    str								=			model(**SCREAMING_SNAKE_CASE_	).loss
          loss.backward()



      def 					SCREAMING_SNAKE_CASE__			(				self    :				str	):
        lowerCAmelCase_						,lowerCAmelCase_       :    Dict								=			self.model_tester.prepare_config_and_inputs_for_common()
        if not self.model_tester.is_training:
          return

        lowerCAmelCase_       :    Optional[int]								=			False
        lowerCAmelCase_       :    Dict								=			True

        for model_class in self.all_model_classes:
          if model_class in get_values(SCREAMING_SNAKE_CASE_	) or not model_class.supports_gradient_checkpointing:
            continue
          # LevitForImageClassificationWithTeacher supports inference-only
          if model_class.__name__ == "LevitForImageClassificationWithTeacher":
            continue
          lowerCAmelCase_       :    Dict								=			model_class(SCREAMING_SNAKE_CASE_	)
          model.gradient_checkpointing_enable()
          model.to(SCREAMING_SNAKE_CASE_	)
          model.train()
          lowerCAmelCase_       :    Optional[int]								=			self._prepare_for_class(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_   ,      return_labels=SCREAMING_SNAKE_CASE_	)
          lowerCAmelCase_       :    Union[str, Any]								=			model(**SCREAMING_SNAKE_CASE_	).loss
          loss.backward()



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict	):
        lowerCAmelCase_						,lowerCAmelCase_       :    List[Any]								=			self.model_tester.prepare_config_and_inputs_for_common()

        lowerCAmelCase_       :    Dict								=			[
            {'title': 'multi_label_classification', 'num_labels': 2, 'dtype': torch.float},
            {'title': 'single_label_classification', 'num_labels': 1, 'dtype': torch.long},
            {'title': 'regression', 'num_labels': 1, 'dtype': torch.float},
        ]

        for model_class in self.all_model_classes:
          if (
              model_class
              not in [
                  *get_values(SCREAMING_SNAKE_CASE_	),
              ]
              or model_class.__name__ == "LevitForImageClassificationWithTeacher"
          ):
            continue

          for problem_type in problem_types:
            with self.subTest(msg=F"Testing {model_class} with {problem_type['title']}"	):
              lowerCAmelCase_       :    Tuple								=			problem_type['title']
              lowerCAmelCase_       :    Optional[int]								=			problem_type['num_labels']

              lowerCAmelCase_       :    str								=			model_class(SCREAMING_SNAKE_CASE_	)
              model.to(SCREAMING_SNAKE_CASE_	)
              model.train()

              lowerCAmelCase_       :    List[Any]								=			self._prepare_for_class(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_   ,      return_labels=SCREAMING_SNAKE_CASE_	)

              if problem_type["num_labels"] > 1:
                lowerCAmelCase_       :    List[Any]								=			inputs['labels'].unsqueeze(1	).repeat(1   ,      problem_type['num_labels']	)

              lowerCAmelCase_       :    List[Any]								=			inputs['labels'].to(problem_type['dtype']	)

              # This tests that we do not trigger the warning form PyTorch "Using a target size that is different
              # to the input size. This will likely lead to incorrect results due to broadcasting. Please ensure
              # they have the same size." which is a symptom something in wrong for the regression problem.
              # See https://github.com/huggingface/transformers/issues/11780
              with warnings.catch_warnings(record=SCREAMING_SNAKE_CASE_	) as warning_list:
                lowerCAmelCase_       :    Tuple								=			model(**SCREAMING_SNAKE_CASE_	).loss
              for w in warning_list:
                if "Using a target size that is different to the input size" in str(w.message	):
                  raise ValueError(
                      F"Something is going wrong in the regression problem: intercepted {w.message}"	)

              loss.backward()





      @slow
      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]	):
        for model_name in LEVIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          lowerCAmelCase_       :    Any								=			LevitModel.from_pretrained(SCREAMING_SNAKE_CASE_	)
          self.assertIsNotNone(SCREAMING_SNAKE_CASE_	)




def     UpperCamelCase_   (   )						->       List[Any]:

  """simple docstring"""




  lowerCAmelCase_       :    Dict								=			Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'						)
  return image







@require_torch
@require_vision
class 				UpperCamelCase__       (     unittest.TestCase  ):




      """simple docstring"""




      @cached_property
      def 					SCREAMING_SNAKE_CASE__			(				self    :				Any	):
        return LevitImageProcessor.from_pretrained(LEVIT_PRETRAINED_MODEL_ARCHIVE_LIST[0]	)





      @slow
      def 					SCREAMING_SNAKE_CASE__			(				self    :				str	):
        lowerCAmelCase_       :    Union[str, Any]								=			LevitForImageClassificationWithTeacher.from_pretrained(LEVIT_PRETRAINED_MODEL_ARCHIVE_LIST[0]	).to(
            SCREAMING_SNAKE_CASE_	)

        lowerCAmelCase_       :    str								=			self.default_image_processor
        lowerCAmelCase_       :    List[Any]								=			prepare_img()
        lowerCAmelCase_       :    List[Any]								=			image_processor(images=SCREAMING_SNAKE_CASE_   ,      return_tensors='pt'	).to(SCREAMING_SNAKE_CASE_	)

        # forward pass
        with torch.no_grad():
          lowerCAmelCase_       :    Optional[Any]								=			model(**SCREAMING_SNAKE_CASE_	)

        # verify the logits
        lowerCAmelCase_       :    Optional[int]								=			torch.Size((1, 1_0_0_0)	)
        self.assertEqual(outputs.logits.shape   ,      SCREAMING_SNAKE_CASE_	)

        lowerCAmelCase_       :    str								=			torch.tensor([1.04_48, -0.37_45, -1.83_17]	).to(SCREAMING_SNAKE_CASE_	)

        self.assertTrue(torch.allclose(outputs.logits[0, :3]   ,      SCREAMING_SNAKE_CASE_   ,      atol=1E-4	)	)










"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__		:    List[Any]				   =     logging.get_logger(__name__)

lowercase__		:    List[str]				   =     {
    """s-JoL/Open-Llama-V1""": """https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json""",
}







class 				UpperCamelCase__       (     lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE      =					"""open-llama"""



      def __init__(				self    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				Tuple=1_0_0_0_0_0   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]=4_0_9_6   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=1_1_0_0_8   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=3_2   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=3_2   ,      SCREAMING_SNAKE_CASE_    :				str="silu"   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=2_0_4_8   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=0.02   ,      SCREAMING_SNAKE_CASE_    :				int=1E-6   ,      SCREAMING_SNAKE_CASE_    :				List[str]=True   ,      SCREAMING_SNAKE_CASE_    :				Dict=0   ,      SCREAMING_SNAKE_CASE_    :				Dict=1   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=2   ,      SCREAMING_SNAKE_CASE_    :				str=False   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]=True   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=0.1   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=0.1   ,      SCREAMING_SNAKE_CASE_    :				Dict=True   ,      SCREAMING_SNAKE_CASE_    :				Any=True   ,      SCREAMING_SNAKE_CASE_    :				Tuple=None   ,      **SCREAMING_SNAKE_CASE_    :				Optional[int]   ,      ):
        lowerCAmelCase_       :    Union[str, Any]								=			vocab_size
        lowerCAmelCase_       :    int								=			max_position_embeddings
        lowerCAmelCase_       :    Union[str, Any]								=			hidden_size
        lowerCAmelCase_       :    Optional[Any]								=			intermediate_size
        lowerCAmelCase_       :    List[Any]								=			num_hidden_layers
        lowerCAmelCase_       :    Optional[int]								=			num_attention_heads
        lowerCAmelCase_       :    List[Any]								=			hidden_act
        lowerCAmelCase_       :    List[Any]								=			initializer_range
        lowerCAmelCase_       :    List[str]								=			rms_norm_eps
        lowerCAmelCase_       :    List[Any]								=			use_cache
        lowerCAmelCase_       :    Optional[int]								=			kwargs.pop(
            'use_memorry_efficient_attention'   ,      SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    List[Any]								=			hidden_dropout_prob
        lowerCAmelCase_       :    List[str]								=			attention_dropout_prob
        lowerCAmelCase_       :    Tuple								=			use_stable_embedding
        lowerCAmelCase_       :    Optional[Any]								=			shared_input_output_embedding
        lowerCAmelCase_       :    Optional[Any]								=			rope_scaling
        self._rope_scaling_validation()

        super().__init__(
            pad_token_id=SCREAMING_SNAKE_CASE_   ,      bos_token_id=SCREAMING_SNAKE_CASE_   ,      eos_token_id=SCREAMING_SNAKE_CASE_   ,      tie_word_embeddings=SCREAMING_SNAKE_CASE_   ,      **SCREAMING_SNAKE_CASE_   ,      )





      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[str]	):
        if self.rope_scaling is None:
          return

        if not isinstance(self.rope_scaling   ,      SCREAMING_SNAKE_CASE_	) or len(self.rope_scaling	) != 2:
          raise ValueError(
              '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
              F"got {self.rope_scaling}"	)
        lowerCAmelCase_       :    int								=			self.rope_scaling.get('type'   ,      SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    Union[str, Any]								=			self.rope_scaling.get('factor'   ,      SCREAMING_SNAKE_CASE_	)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
          raise ValueError(
              F"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"	)
        if rope_scaling_factor is None or not isinstance(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_	) or rope_scaling_factor <= 1.0:
          raise ValueError(F"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"	)





