


'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_      =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_      =			{
    's-JoL/Open-Llama-V1': 'https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json',
}


class  lowerCAmelCase_						(       __lowerCamelCase      ):





  """simple docstring"""



  a_       :Dict												="""open-llama"""

  def __init__(					self     :							Optional[int]				,							SCREAMING_SNAKE_CASE__     :							Any=1_0_0_0_0_0				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]=4_0_9_6				,							SCREAMING_SNAKE_CASE__     :							int=1_1_0_0_8				,							SCREAMING_SNAKE_CASE__     :							Dict=3_2				,							SCREAMING_SNAKE_CASE__     :							List[Any]=3_2				,							SCREAMING_SNAKE_CASE__     :							Optional[int]="silu"				,							SCREAMING_SNAKE_CASE__     :							Optional[int]=2_0_4_8				,							SCREAMING_SNAKE_CASE__     :							str=0.0_2				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]=1E-6				,							SCREAMING_SNAKE_CASE__     :							int=True				,							SCREAMING_SNAKE_CASE__     :							Any=0				,							SCREAMING_SNAKE_CASE__     :							Optional[int]=1				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]=2				,							SCREAMING_SNAKE_CASE__     :							Any=False				,							SCREAMING_SNAKE_CASE__     :							List[Any]=True				,							SCREAMING_SNAKE_CASE__     :							Tuple=0.1				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]=0.1				,							SCREAMING_SNAKE_CASE__     :							Tuple=True				,							SCREAMING_SNAKE_CASE__     :							str=True				,							SCREAMING_SNAKE_CASE__     :							List[Any]=None				,							**SCREAMING_SNAKE_CASE__     :							Dict				,							):





       '''simple docstring'''

       __a       =   vocab_size
       __a       =   max_position_embeddings
       __a       =   hidden_size
       __a       =   intermediate_size
       __a       =   num_hidden_layers
       __a       =   num_attention_heads
       __a       =   hidden_act
       __a       =   initializer_range
       __a       =   rms_norm_eps
       __a       =   use_cache
       __a       =   kwargs.pop(
           """use_memorry_efficient_attention"""				,							UpperCAmelCase_						)
       __a       =   hidden_dropout_prob
       __a       =   attention_dropout_prob
       __a       =   use_stable_embedding
       __a       =   shared_input_output_embedding
       __a       =   rope_scaling
       self._rope_scaling_validation()

       super().__init__(
           pad_token_id=UpperCAmelCase_				,							bos_token_id=UpperCAmelCase_				,							eos_token_id=UpperCAmelCase_				,							tie_word_embeddings=UpperCAmelCase_				,							**UpperCAmelCase_				,							)


  def __a						(					self     :							List[str]						):





       '''simple docstring'''

       if self.rope_scaling is None:
            return

       if not isinstance(self.rope_scaling				,							UpperCAmelCase_						) or len(self.rope_scaling						) != 2:
            raise ValueError(
                """`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, """
                f'''got {self.rope_scaling}'''						)
       __a       =   self.rope_scaling.get("""type"""				,							UpperCAmelCase_						)
       __a       =   self.rope_scaling.get("""factor"""				,							UpperCAmelCase_						)
       if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
            raise ValueError(
                f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''						)
       if rope_scaling_factor is None or not isinstance(UpperCAmelCase_				,							UpperCAmelCase_						) or rope_scaling_factor <= 1.0:
            raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''						)










'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


SCREAMING_SNAKE_CASE_      =			{
    'configuration_rag': ['RagConfig'],
    'retrieval_rag': ['RagRetriever'],
    'tokenization_rag': ['RagTokenizer'],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE_      =			[
     'RagModel',
     'RagPreTrainedModel',
     'RagSequenceForGeneration',
     'RagTokenForGeneration',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE_      =			[
     'TFRagModel',
     'TFRagPreTrainedModel',
     'TFRagSequenceForGeneration',
     'TFRagTokenForGeneration',
 ]


if TYPE_CHECKING:
 from .configuration_rag import RagConfig
 from .retrieval_rag import RagRetriever
 from .tokenization_rag import RagTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_rag import RagModel, RagPreTrainedModel, RagSequenceForGeneration, RagTokenForGeneration

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_rag import (
      TFRagModel,
      TFRagPreTrainedModel,
      TFRagSequenceForGeneration,
      TFRagTokenForGeneration,
  )

else:
 import sys

 SCREAMING_SNAKE_CASE_      =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






