





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_							=				logging.get_logger(__name__)

UpperCAmelCase_							=				{
    """facebook/nllb-moe-54B""": """https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json""",
}
class     UpperCamelCase__					(     lowerCamelCase__				):






		'''simple docstring'''






		__a       :       Optional[Any]       =	"""nllb-moe"""
		__a       :       str       =	["""past_key_values"""]
		__a       :       List[Any]       =	{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}






		def __init__(							self,   snake_case__=12_81_12,   snake_case__=10_24,   snake_case__=12,   snake_case__=40_96,   snake_case__=16,   snake_case__=12,   snake_case__=40_96,   snake_case__=16,   snake_case__=0.05,   snake_case__=0.05,   snake_case__=True,   snake_case__=True,   snake_case__="relu",   snake_case__=10_24,   snake_case__=0.1,   snake_case__=0.1,   snake_case__=0.0,   snake_case__=0.02,   snake_case__=2,   snake_case__=True,   snake_case__=False,   snake_case__="float32",   snake_case__=False,   snake_case__=1_28,   snake_case__=64,   snake_case__=4,   snake_case__=4,   snake_case__=0.001,   snake_case__=0.001,   snake_case__="all",   snake_case__=False,   snake_case__=False,   snake_case__=1.0,   snake_case__=0.2,   snake_case__=1,   snake_case__=0,   snake_case__=2,   snake_case__=False,   **snake_case__,   )       -> str:






									"""simple docstring"""
									lowercase_	:   Optional[Any]  				=     vocab_size
									lowercase_	:   Tuple  				=     max_position_embeddings
									lowercase_	:   str  				=     d_model
									lowercase_	:   Optional[Any]  				=     encoder_ffn_dim
									lowercase_	:   str  				=     encoder_layers
									lowercase_	:   Dict  				=     encoder_attention_heads
									lowercase_	:   str  				=     decoder_ffn_dim
									lowercase_	:   Dict  				=     decoder_layers
									lowercase_	:   Optional[Any]  				=     decoder_attention_heads
									lowercase_	:   int  				=     dropout
									lowercase_	:   List[Any]  				=     attention_dropout
									lowercase_	:   List[Any]  				=     activation_dropout
									lowercase_	:   Optional[int]  				=     activation_function
									lowercase_	:   Any  				=     init_std
									lowercase_	:   Any  				=     encoder_layerdrop
									lowercase_	:   int  				=     decoder_layerdrop
									lowercase_	:   Optional[int]  				=     use_cache
									lowercase_	:   Optional[Any]  				=     encoder_layers
									lowercase_	:   Any  				=     scale_embedding  # scale factor will be sqrt(d_model) if True
									lowercase_	:   Tuple  				=     router_z_loss_coef
									lowercase_	:   Union[str, Any]  				=     router_aux_loss_coef
									lowercase_	:   List[str]  				=     decoder_sparse_step
									lowercase_	:   Optional[int]  				=     encoder_sparse_step
									lowercase_	:   List[Any]  				=     num_experts
									lowercase_	:   Dict  				=     expert_capacity
									lowercase_	:   str  				=     router_bias
									if router_dtype not in ["float32", "float16", "bfloat16"]:
																raise ValueError(f"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""  )
									lowercase_	:   Any  				=     router_dtype

									lowercase_	:   str  				=     router_ignore_padding_tokens
									lowercase_	:   int  				=     batch_prioritized_routing
									lowercase_	:   Optional[Any]  				=     second_expert_policy
									lowercase_	:   List[Any]  				=     normalize_router_prob_before_dropping
									lowercase_	:   Optional[Any]  				=     moe_eval_capacity_token_fraction
									lowercase_	:   Optional[Any]  				=     moe_token_dropout
									lowercase_	:   Any  				=     output_router_logits
									super().__init__(
									    pad_token_id=snake_case__,   bos_token_id=snake_case__,   eos_token_id=snake_case__,   is_encoder_decoder=snake_case__,   decoder_start_token_id=snake_case__,   **snake_case__,   )






from __future__ import annotations

import numpy as np





def  __magic_name__  (				lowercase				)				-> Tuple:

							"""simple docstring"""



							return np.maximum(0				,       lowercase				)


if __name__ == "__main__":
							print(np.array(relu([-1, 0, 5])))  # --> [0, 0, 5]