
"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {
    's-JoL/Open-Llama-V1': 'https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json',
}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      '''open-llama'''



     def __init__(							self :Dict      ,  _lowerCamelCase :Tuple=100_000      ,  _lowerCamelCase :Tuple=4_096      ,  _lowerCamelCase :str=11_008      ,  _lowerCamelCase :Dict=32      ,  _lowerCamelCase :Any=32      ,  _lowerCamelCase :int="silu"      ,  _lowerCamelCase :int=2_048      ,  _lowerCamelCase :Optional[int]=0.02      ,  _lowerCamelCase :Any=1E-6      ,  _lowerCamelCase :Optional[int]=True      ,  _lowerCamelCase :Any=0      ,  _lowerCamelCase :Optional[Any]=1      ,  _lowerCamelCase :Any=2      ,  _lowerCamelCase :str=False      ,  _lowerCamelCase :Any=True      ,  _lowerCamelCase :Optional[int]=0.1      ,  _lowerCamelCase :Any=0.1      ,  _lowerCamelCase :Optional[int]=True      ,  _lowerCamelCase :str=True      ,  _lowerCamelCase :Union[str, Any]=None      ,  **_lowerCamelCase :Optional[int]      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=vocab_size
      UpperCamelCase_     :					int   	=max_position_embeddings
      UpperCamelCase_     :					Optional[int]   	=hidden_size
      UpperCamelCase_     :					Optional[Any]   	=intermediate_size
      UpperCamelCase_     :					Optional[Any]   	=num_hidden_layers
      UpperCamelCase_     :					List[Any]   	=num_attention_heads
      UpperCamelCase_     :					List[str]   	=hidden_act
      UpperCamelCase_     :					int   	=initializer_range
      UpperCamelCase_     :					Optional[int]   	=rms_norm_eps
      UpperCamelCase_     :					Dict   	=use_cache
      UpperCamelCase_     :					Dict   	=kwargs.pop(
          'use_memorry_efficient_attention'      ,  _lowerCamelCase   )
      UpperCamelCase_     :					Optional[int]   	=hidden_dropout_prob
      UpperCamelCase_     :					Optional[Any]   	=attention_dropout_prob
      UpperCamelCase_     :					List[str]   	=use_stable_embedding
      UpperCamelCase_     :					Tuple   	=shared_input_output_embedding
      UpperCamelCase_     :					Optional[Any]   	=rope_scaling
      self._rope_scaling_validation()

      super().__init__(
          pad_token_id=_lowerCamelCase      ,  bos_token_id=_lowerCamelCase      ,  eos_token_id=_lowerCamelCase      ,  tie_word_embeddings=_lowerCamelCase      ,  **_lowerCamelCase      ,  )




     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      if self.rope_scaling is None:
       return

      if not isinstance(self.rope_scaling      ,  _lowerCamelCase   ) or len(self.rope_scaling   ) != 2:
       raise ValueError(
           '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
           f'''got {self.rope_scaling}'''   )
      UpperCamelCase_     :					int   	=self.rope_scaling.get('type'      ,  _lowerCamelCase   )
      UpperCamelCase_     :					Any   	=self.rope_scaling.get('factor'      ,  _lowerCamelCase   )
      if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
       raise ValueError(
           f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''   )
      if rope_scaling_factor is None or not isinstance(_lowerCamelCase      ,  _lowerCamelCase   ) or rope_scaling_factor <= 1.0:
       raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''   )


"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


__SCREAMING_SNAKE_CASE							=     {
    'configuration_mega': ['MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MegaConfig', 'MegaOnnxConfig'],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     [
     'MEGA_PRETRAINED_MODEL_ARCHIVE_LIST',
     'MegaForCausalLM',
     'MegaForMaskedLM',
     'MegaForMultipleChoice',
     'MegaForQuestionAnswering',
     'MegaForSequenceClassification',
     'MegaForTokenClassification',
     'MegaModel',
     'MegaPreTrainedModel',
 ]

if TYPE_CHECKING:
 from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_mega import (
      MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
      MegaForCausalLM,
      MegaForMaskedLM,
      MegaForMultipleChoice,
      MegaForQuestionAnswering,
      MegaForSequenceClassification,
      MegaForTokenClassification,
      MegaModel,
      MegaPreTrainedModel,
  )

else:
 import sys

 __SCREAMING_SNAKE_CASE							=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
