





from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_							=			logging.get_logger(__name__)

snake_case_							=			{
    '''s-JoL/Open-Llama-V1''': '''https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json''',
}
class        snake_case_		(			__lowerCamelCase):
		lowerCamelCase    :Any						=     "open-llama"







		def __init__(		self			,     __lowercase=1_0_0_0_0_0			,     __lowercase=4_0_9_6			,     __lowercase=1_1_0_0_8			,     __lowercase=3_2			,     __lowercase=3_2			,     __lowercase="silu"			,     __lowercase=2_0_4_8			,     __lowercase=0.0_2			,     __lowercase=1e-6			,     __lowercase=True			,     __lowercase=0			,     __lowercase=1			,     __lowercase=2			,     __lowercase=False			,     __lowercase=True			,     __lowercase=0.1			,     __lowercase=0.1			,     __lowercase=True			,     __lowercase=True			,     __lowercase=None			,     **__lowercase			,     )				->   Optional[int]:
				lowerCamelCase   : Optional[Any]								=vocab_size
				lowerCamelCase   : str								=max_position_embeddings
				lowerCamelCase   : Optional[int]								=hidden_size
				lowerCamelCase   : int								=intermediate_size
				lowerCamelCase   : Tuple								=num_hidden_layers
				lowerCamelCase   : List[Any]								=num_attention_heads
				lowerCamelCase   : Optional[int]								=hidden_act
				lowerCamelCase   : List[str]								=initializer_range
				lowerCamelCase   : Optional[int]								=rms_norm_eps
				lowerCamelCase   : List[str]								=use_cache
				lowerCamelCase   : List[str]								=kwargs.pop(
				    '''use_memorry_efficient_attention'''			,     SCREAMING_SNAKE_CASE_   )
				lowerCamelCase   : List[str]								=hidden_dropout_prob
				lowerCamelCase   : List[str]								=attention_dropout_prob
				lowerCamelCase   : str								=use_stable_embedding
				lowerCamelCase   : Optional[Any]								=shared_input_output_embedding
				lowerCamelCase   : List[Any]								=rope_scaling
				self._rope_scaling_validation()

				super().__init__(
				    pad_token_id=SCREAMING_SNAKE_CASE_			,     bos_token_id=SCREAMING_SNAKE_CASE_			,     eos_token_id=SCREAMING_SNAKE_CASE_			,     tie_word_embeddings=SCREAMING_SNAKE_CASE_			,     **SCREAMING_SNAKE_CASE_			,     )







		def 			__lowercase   (		self   )				->   int:
				if self.rope_scaling is None:
						return

				if not isinstance(self.rope_scaling			,     SCREAMING_SNAKE_CASE_   ) or len(self.rope_scaling   ) != 2:
						raise ValueError(
						    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
						    F"got {self.rope_scaling}"   )
				lowerCamelCase   : List[str]								=self.rope_scaling.get('''type'''			,     SCREAMING_SNAKE_CASE_   )
				lowerCamelCase   : str								=self.rope_scaling.get('''factor'''			,     SCREAMING_SNAKE_CASE_   )
				if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
						raise ValueError(
						    F"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"   )
				if rope_scaling_factor is None or not isinstance(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_   ) or rope_scaling_factor <= 1.0:
						raise ValueError(F"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"   )








from typing import Optional, Union

import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models.modeling_utils import ModelMixin
class        snake_case_		(			_A						,					_A):







		@register_to_config
		def __init__(		self			,     __lowercase = 7_6_8			,     )				->   List[Any]:
				super().__init__()

				lowerCamelCase   : str								=nn.Parameter(torch.zeros(1			,     __lowercase   )   )
				lowerCamelCase   : Union[str, Any]								=nn.Parameter(torch.ones(1			,     __lowercase   )   )







		def 			__lowercase   (		self			,     __lowercase = None			,     __lowercase = None			,     )				->   List[str]:
				lowerCamelCase   : Tuple								=nn.Parameter(self.mean.to(__lowercase   ).to(__lowercase   )   )
				lowerCamelCase   : Dict								=nn.Parameter(self.std.to(__lowercase   ).to(__lowercase   )   )
				return self







		def 			__lowercase   (		self			,     __lowercase   )				->   Optional[Any]:
				lowerCamelCase   : Any								=(embeds - self.mean) * 1.0 / self.std
				return embeds







		def 			__lowercase   (		self			,     __lowercase   )				->   Optional[int]:
				lowerCamelCase   : Any								=(embeds * self.std) + self.mean
				return embeds

