


import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
_lowerCAmelCase:						Any   				=       object()

# For specifying empty leaf dict `{}`
_lowerCAmelCase:						Dict   				=       object()






def   _lowercase(					__a				:						Tuple    ,     __a				:						Optional[Any]     ):
							a__      =tuple((re.compile(x + '$'     ) for x in qs)     )
							for i in range(len(__a     ) - len(__a     ) + 1     ):
														a__      =[x.match(__a     ) for x, y in zip(__a    ,     ks[i:]     )]
														if matches and all(__a     ):
																					return True
							return False






def   _lowercase(					__a				:						Any     ):
							def replace(__a				:						List[str]    ,     __a				:						Tuple     ):
														for rule, replacement in rules:
																					if _match(__a    ,     __a     ):
																												return replacement
														return val

							return replace






def   _lowercase(					):
							return [
							    # embeddings
							    (("transformer", "wpe", "embedding"), P('mp'    ,     __a     )),
							    (("transformer", "wte", "embedding"), P('mp'    ,     __a     )),
							    # atention
							    (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(__a    ,     'mp'     )),
							    (("attention", "out_proj", "kernel"), P('mp'    ,     __a     )),
							    (("attention", "out_proj", "bias"), None),
							    # mlp
							    (("mlp", "c_fc", "kernel"), P(__a    ,     'mp'     )),
							    (("mlp", "c_fc", "bias"), P('mp'     )),
							    (("mlp", "c_proj", "kernel"), P('mp'    ,     __a     )),
							    (("mlp", "c_proj", "bias"), None),
							    # layer norms
							    ((r"ln_\d+", "bias"), None),
							    ((r"\d+", r"ln_\d+", "scale"), None),
							    (("ln_f", "bias"), None),
							    (("ln_f", "scale"), None),
							]







def   _lowercase(					__a				:						int     ):
							a__      =_get_partition_rules()
							a__      =_replacement_rules(__a     )
							a__      ={k: _unmatched for k in flatten_dict(__a     )}
							a__      ={k: replace(__a    ,     __a     ) for k, v in initd.items()}
							assert _unmatched not in result.values(), "Incomplete partition spec."
							return freeze(unflatten_dict(__a     )     )








import numpy as np







def   __lowerCamelCase   (				__a  :np.array       )						->							np.array:






		"""simple docstring"""

		return 1 / (1 + np.exp(-vector       ))







def   __lowerCamelCase   (				__a  :np.array       )						->							np.array:






		"""simple docstring"""

		return vector * sigmoid(1.702 * vector       )


if __name__ == "__main__":
					import doctest

					doctest.testmod()



