



'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase:   List[str]					=		logging.get_logger(__name__)

lowerCAmelCase:   Optional[Any]					=		{}

class 				a__(   lowerCamelCase__      ):
    lowercase__					=       """llama"""
    lowercase__					=       ["""past_key_values"""]







    def __init__( self					:    List[Any]		,     __snake_case					:    Dict=3_20_00		,     __snake_case					:    Union[str, Any]=40_96		,     __snake_case					:    Optional[Any]=1_10_08		,     __snake_case					:    Dict=32		,     __snake_case					:    Optional[int]=32		,     __snake_case					:    Union[str, Any]=None		,     __snake_case					:    Dict="silu"		,     __snake_case					:    List[Any]=20_48		,     __snake_case					:    Union[str, Any]=0.02		,     __snake_case					:    List[Any]=1e-6		,     __snake_case					:    str=True		,     __snake_case					:    List[Any]=0		,     __snake_case					:    Optional[int]=1		,     __snake_case					:    List[Any]=2		,     __snake_case					:    Union[str, Any]=1		,     __snake_case					:    Any=False		,     __snake_case					:    List[str]=None		,     **__snake_case					:    Tuple		,     ):
      a		:   List[Any]	  =	vocab_size
      a		:   List[str]	  =	max_position_embeddings
      a		:   List[Any]	  =	hidden_size
      a		:   Optional[int]	  =	intermediate_size
      a		:   Union[str, Any]	  =	num_hidden_layers
      a		:   Dict	  =	num_attention_heads

      # for backward compatibility
      if num_key_value_heads is None:
        a		:   Any	  =	num_attention_heads

      a		:   int	  =	num_key_value_heads
      a		:   Any	  =	hidden_act
      a		:   Union[str, Any]	  =	initializer_range
      a		:   Optional[int]	  =	rms_norm_eps
      a		:   List[str]	  =	pretraining_tp
      a		:   Optional[int]	  =	use_cache
      a		:   int	  =	rope_scaling
      self._rope_scaling_validation()

      super().__init__(
          pad_token_id=__snake_case		,     bos_token_id=__snake_case		,     eos_token_id=__snake_case		,     tie_word_embeddings=__snake_case		,     **__snake_case		,     )







    def      lowercase_			( self					:    Optional[Any]      ):
      if self.rope_scaling is None:
        return

      if not isinstance(self.rope_scaling		,     __snake_case      ) or len(self.rope_scaling      ) != 2:
        raise ValueError(
            '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
            F"""got {self.rope_scaling}"""      )
      a		:   List[Any]	  =	self.rope_scaling.get('type'		,     __snake_case      )
      a		:   str	  =	self.rope_scaling.get('factor'		,     __snake_case      )
      if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
        raise ValueError(
            F"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""      )
      if rope_scaling_factor is None or not isinstance(__snake_case		,     __snake_case      ) or rope_scaling_factor <= 1.0:
        raise ValueError(F"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""      )




'''simple docstring'''






import numpy as np
def  lowerCamelCase__			(		_A							):
  return 1 / (1 + np.exp(-vector							))
def  lowerCamelCase__			(		_A							):
  return vector * sigmoid(1.702 * vector							)


if __name__ == "__main__":
     import doctest

     doctest.testmod()