


'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__						:   Any				=    logging.get_logger(__name__)

A__						:   Union[str, Any]				=    {
    """s-JoL/Open-Llama-V1""": """https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json""",
}





class   UpperCAmelCase_    (A__    ):





    """simple docstring"""






    lowerCamelCase    :   Optional[Any]					=     'open-llama'







    def __init__(				self     ,	SCREAMING_SNAKE_CASE_=10_00_00     ,	SCREAMING_SNAKE_CASE_=40_96     ,	SCREAMING_SNAKE_CASE_=1_10_08     ,	SCREAMING_SNAKE_CASE_=32     ,	SCREAMING_SNAKE_CASE_=32     ,	SCREAMING_SNAKE_CASE_="silu"     ,	SCREAMING_SNAKE_CASE_=20_48     ,	SCREAMING_SNAKE_CASE_=0.0_2     ,	SCREAMING_SNAKE_CASE_=1E-6     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=0     ,	SCREAMING_SNAKE_CASE_=1     ,	SCREAMING_SNAKE_CASE_=2     ,	SCREAMING_SNAKE_CASE_=False     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=0.1     ,	SCREAMING_SNAKE_CASE_=0.1     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=None     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	str:
         __lowerCamelCase  :      str														=						vocab_size
         __lowerCamelCase  :      int														=						max_position_embeddings
         __lowerCamelCase  :      Optional[int]														=						hidden_size
         __lowerCamelCase  :      Optional[Any]														=						intermediate_size
         __lowerCamelCase  :      Optional[Any]														=						num_hidden_layers
         __lowerCamelCase  :      List[Any]														=						num_attention_heads
         __lowerCamelCase  :      List[str]														=						hidden_act
         __lowerCamelCase  :      int														=						initializer_range
         __lowerCamelCase  :      Optional[int]														=						rms_norm_eps
         __lowerCamelCase  :      Dict														=						use_cache
         __lowerCamelCase  :      Dict														=						kwargs.pop(
             'use_memorry_efficient_attention'     ,	_lowerCamelCase      )
         __lowerCamelCase  :      Optional[int]														=						hidden_dropout_prob
         __lowerCamelCase  :      Optional[Any]														=						attention_dropout_prob
         __lowerCamelCase  :      List[str]														=						use_stable_embedding
         __lowerCamelCase  :      Tuple														=						shared_input_output_embedding
         __lowerCamelCase  :      Optional[Any]														=						rope_scaling
         self._rope_scaling_validation()

         super().__init__(
             pad_token_id=_lowerCamelCase     ,	bos_token_id=_lowerCamelCase     ,	eos_token_id=_lowerCamelCase     ,	tie_word_embeddings=_lowerCamelCase     ,	**_lowerCamelCase     ,	)







    def  lowercase_							(				self      )							->	str:



         if self.rope_scaling is None:
              return

         if not isinstance(self.rope_scaling     ,	_lowerCamelCase      ) or len(self.rope_scaling      ) != 2:
              raise ValueError(
                  '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                  f'got {self.rope_scaling}'      )
         __lowerCamelCase  :      int														=						self.rope_scaling.get('type'     ,	_lowerCamelCase      )
         __lowerCamelCase  :      Any														=						self.rope_scaling.get('factor'     ,	_lowerCamelCase      )
         if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
              raise ValueError(
                  f'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'      )
         if rope_scaling_factor is None or not isinstance(_lowerCamelCase     ,	_lowerCamelCase      ) or rope_scaling_factor <= 1.0:
              raise ValueError(f'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'      )








"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


__SCREAMING_SNAKE_CASE							=     {
    'configuration_mega': ['MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MegaConfig', 'MegaOnnxConfig'],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     [
     'MEGA_PRETRAINED_MODEL_ARCHIVE_LIST',
     'MegaForCausalLM',
     'MegaForMaskedLM',
     'MegaForMultipleChoice',
     'MegaForQuestionAnswering',
     'MegaForSequenceClassification',
     'MegaForTokenClassification',
     'MegaModel',
     'MegaPreTrainedModel',
 ]

if TYPE_CHECKING:
 from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_mega import (
      MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
      MegaForCausalLM,
      MegaForMaskedLM,
      MegaForMultipleChoice,
      MegaForQuestionAnswering,
      MegaForSequenceClassification,
      MegaForTokenClassification,
      MegaModel,
      MegaPreTrainedModel,
  )

else:
 import sys

 __SCREAMING_SNAKE_CASE							=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
