


from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase    		=       logging.get_logger(__name__)

__UpperCamelCase    		=       {}







class        lowerCamelCase__	(							UpperCAmelCase      ):






			"""simple docstring"""






			_UpperCamelCase		:					Tuple    =		'llama'
			_UpperCamelCase		:					int    =		['past_key_values']




			def __init__(							self      ,						snake_case=32000      ,						snake_case=4096      ,						snake_case=11008      ,						snake_case=32      ,						snake_case=32      ,						snake_case=None      ,						snake_case="silu"      ,						snake_case=2048      ,						snake_case=0.02      ,						snake_case=1E-6      ,						snake_case=True      ,						snake_case=0      ,						snake_case=1      ,						snake_case=2      ,						snake_case=1      ,						snake_case=False      ,						snake_case=None      ,						**snake_case      ,						):

						'''simple docstring'''


						UpperCamelCase__					=       vocab_size
						UpperCamelCase__					=       max_position_embeddings
						UpperCamelCase__					=       hidden_size
						UpperCamelCase__					=       intermediate_size
						UpperCamelCase__					=       num_hidden_layers
						UpperCamelCase__					=       num_attention_heads

						# for backward compatibility
						if num_key_value_heads is None:
									UpperCamelCase__					=       num_attention_heads

						UpperCamelCase__					=       num_key_value_heads
						UpperCamelCase__					=       hidden_act
						UpperCamelCase__					=       initializer_range
						UpperCamelCase__					=       rms_norm_eps
						UpperCamelCase__					=       pretraining_tp
						UpperCamelCase__					=       use_cache
						UpperCamelCase__					=       rope_scaling
						self._rope_scaling_validation()

						super().__init__(
						    pad_token_id=snake_case      ,						bos_token_id=snake_case      ,						eos_token_id=snake_case      ,						tie_word_embeddings=snake_case      ,						**snake_case      ,						)







			def 				snake_case__				(							self     ):

						'''simple docstring'''


						if self.rope_scaling is None:
									return

						if not isinstance(self.rope_scaling      ,						snake_case     ) or len(self.rope_scaling     ) != 2:
									raise ValueError(
									    "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
									    F'''got {self.rope_scaling}'''     )
						UpperCamelCase__					=       self.rope_scaling.get("type"      ,						snake_case     )
						UpperCamelCase__					=       self.rope_scaling.get("factor"      ,						snake_case     )
						if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
									raise ValueError(
									    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''     )
						if rope_scaling_factor is None or not isinstance(snake_case      ,						snake_case     ) or rope_scaling_factor <= 1.0:
									raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''     )






from abc import ABC, abstractmethod
from argparse import ArgumentParser







class        lowerCamelCase__	(							UpperCAmelCase      ):






			"""simple docstring"""






			@staticmethod
			@abstractmethod
			def 				snake_case__				(							snake_case     ):

						'''simple docstring'''


						raise NotImplementedError()







			@abstractmethod
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						raise NotImplementedError()


