




'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_		:     Optional[Any]          =		logging.get_logger(__name__)

UpperCamelCase_		:     Optional[int]          =		{}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       Union[str, Any]       							=		"""llama"""
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		["""past_key_values"""]






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=32_000   ,_SCREAMING_SNAKE_CASE=4_096   ,_SCREAMING_SNAKE_CASE=11_008   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE="silu"   ,_SCREAMING_SNAKE_CASE=2_048   ,_SCREAMING_SNAKE_CASE=0.0_2   ,_SCREAMING_SNAKE_CASE=1e-6   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=0   ,_SCREAMING_SNAKE_CASE=1   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=1   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=None   ,**_SCREAMING_SNAKE_CASE   ,)				->     Tuple:
          _snake_case				= vocab_size
          _snake_case				= max_position_embeddings
          _snake_case				= hidden_size
          _snake_case				= intermediate_size
          _snake_case				= num_hidden_layers
          _snake_case				= num_attention_heads

          # for backward compatibility
          if num_key_value_heads is None:
               _snake_case				= num_attention_heads

          _snake_case				= num_key_value_heads
          _snake_case				= hidden_act
          _snake_case				= initializer_range
          _snake_case				= rms_norm_eps
          _snake_case				= pretraining_tp
          _snake_case				= use_cache
          _snake_case				= rope_scaling
          self._rope_scaling_validation()

          super().__init__(
              pad_token_id=_SCREAMING_SNAKE_CASE   ,bos_token_id=_SCREAMING_SNAKE_CASE   ,eos_token_id=_SCREAMING_SNAKE_CASE   ,tie_word_embeddings=_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE   ,)






     def 							_lowercase      (					self       )				->     Optional[int]:
          if self.rope_scaling is None:
               return

          if not isinstance(self.rope_scaling   ,_SCREAMING_SNAKE_CASE       ) or len(self.rope_scaling       ) != 2:
               raise ValueError(
                   "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
                   f"""got {self.rope_scaling}"""       )
          _snake_case				= self.rope_scaling.get("type"   ,_SCREAMING_SNAKE_CASE       )
          _snake_case				= self.rope_scaling.get("factor"   ,_SCREAMING_SNAKE_CASE       )
          if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
               raise ValueError(
                   f"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""       )
          if rope_scaling_factor is None or not isinstance(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       ) or rope_scaling_factor <= 1.0:
               raise ValueError(f"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""       )








'''simple docstring'''





import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
UpperCamelCase_		:     Optional[int]          =		object()

# For specifying empty leaf dict `{}`
UpperCamelCase_		:     Optional[int]          =		object()





def       __a      (  _UpperCamelCase:       Union[str, Any]   ,		_UpperCamelCase:       Optional[Any]       )		->					Optional[Any]:

     """simple docstring"""




     _snake_case				= tuple((re.compile(x + "$"       ) for x in qs)       )
     for i in range(len(_UpperCamelCase       ) - len(_UpperCamelCase       ) + 1       ):
          _snake_case				= [x.match(_UpperCamelCase       ) for x, y in zip(_UpperCamelCase   ,		ks[i:]       )]
          if matches and all(_UpperCamelCase       ):
               return True
     return False





def       __a      (  _UpperCamelCase:       Optional[Any]       )		->					Union[str, Any]:

     """simple docstring"""




     def replace(_UpperCamelCase:       Tuple   ,		_UpperCamelCase:       List[str]       ):
          for rule, replacement in rules:
               if _match(_UpperCamelCase   ,		_UpperCamelCase       ):
                    return replacement
          return val

     return replace





def       __a      (  )		->					Any:

     """simple docstring"""




     return [
         # embeddings
         (("transformer", "wpe", "embedding"), P("mp"   ,		_UpperCamelCase       )),
         (("transformer", "wte", "embedding"), P("mp"   ,		_UpperCamelCase       )),
         # atention
         (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(_UpperCamelCase   ,		"mp"       )),
         (("attention", "out_proj", "kernel"), P("mp"   ,		_UpperCamelCase       )),
         (("attention", "out_proj", "bias"), None),
         # mlp
         (("mlp", "c_fc", "kernel"), P(_UpperCamelCase   ,		"mp"       )),
         (("mlp", "c_fc", "bias"), P("mp"       )),
         (("mlp", "c_proj", "kernel"), P("mp"   ,		_UpperCamelCase       )),
         (("mlp", "c_proj", "bias"), None),
         # layer norms
         ((r"ln_\d+", "bias"), None),
         ((r"\d+", r"ln_\d+", "scale"), None),
         (("ln_f", "bias"), None),
         (("ln_f", "scale"), None),
     ]





def       __a      (  _UpperCamelCase:       Union[str, Any]       )		->					Any:

     """simple docstring"""




     _snake_case				= _get_partition_rules()
     _snake_case				= _replacement_rules(_UpperCamelCase       )
     _snake_case				= {k: _unmatched for k in flatten_dict(_UpperCamelCase       )}
     _snake_case				= {k: replace(_UpperCamelCase   ,		_UpperCamelCase       ) for k, v in initd.items()}
     assert _unmatched not in result.values(), "Incomplete partition spec."
     return freeze(unflatten_dict(_UpperCamelCase       )       )


