





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase							:       str					 =	logging.get_logger(__name__)

lowerCAmelCase							:       Tuple					 =	{
    """facebook/nllb-moe-54B""": """https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json""",
}





class 				a			(      __lowercase			):
				SCREAMING_SNAKE_CASE__     :   Tuple		      =     '''nllb-moe'''
				SCREAMING_SNAKE_CASE__     :   Dict		      =     ['''past_key_values''']
				SCREAMING_SNAKE_CASE__     :   List[Any]		      =     {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}






				def __init__(  self			, _lowerCAmelCase=128112			, _lowerCAmelCase=1024			, _lowerCAmelCase=12			, _lowerCAmelCase=4096			, _lowerCAmelCase=16			, _lowerCAmelCase=12			, _lowerCAmelCase=4096			, _lowerCAmelCase=16			, _lowerCAmelCase=0.05			, _lowerCAmelCase=0.05			, _lowerCAmelCase=True			, _lowerCAmelCase=True			, _lowerCAmelCase="relu"			, _lowerCAmelCase=1024			, _lowerCAmelCase=0.1			, _lowerCAmelCase=0.1			, _lowerCAmelCase=0.0			, _lowerCAmelCase=0.02			, _lowerCAmelCase=2			, _lowerCAmelCase=True			, _lowerCAmelCase=False			, _lowerCAmelCase="float32"			, _lowerCAmelCase=False			, _lowerCAmelCase=128			, _lowerCAmelCase=64			, _lowerCAmelCase=4			, _lowerCAmelCase=4			, _lowerCAmelCase=0.001			, _lowerCAmelCase=0.001			, _lowerCAmelCase="all"			, _lowerCAmelCase=False			, _lowerCAmelCase=False			, _lowerCAmelCase=1.0			, _lowerCAmelCase=0.2			, _lowerCAmelCase=1			, _lowerCAmelCase=0			, _lowerCAmelCase=2			, _lowerCAmelCase=False			, **_lowerCAmelCase			, ):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   vocab_size
										__SCREAMING_SNAKE_CASE:				str           =   max_position_embeddings
										__SCREAMING_SNAKE_CASE:				Any           =   d_model
										__SCREAMING_SNAKE_CASE:				Any           =   encoder_ffn_dim
										__SCREAMING_SNAKE_CASE:				str           =   encoder_layers
										__SCREAMING_SNAKE_CASE:				List[Any]           =   encoder_attention_heads
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   decoder_ffn_dim
										__SCREAMING_SNAKE_CASE:				Any           =   decoder_layers
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   decoder_attention_heads
										__SCREAMING_SNAKE_CASE:				List[str]           =   dropout
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   attention_dropout
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   activation_dropout
										__SCREAMING_SNAKE_CASE:				Any           =   activation_function
										__SCREAMING_SNAKE_CASE:				Any           =   init_std
										__SCREAMING_SNAKE_CASE:				int           =   encoder_layerdrop
										__SCREAMING_SNAKE_CASE:				List[Any]           =   decoder_layerdrop
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   use_cache
										__SCREAMING_SNAKE_CASE:				Any           =   encoder_layers
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   scale_embedding  # scale factor will be sqrt(d_model) if True
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   router_z_loss_coef
										__SCREAMING_SNAKE_CASE:				Tuple           =   router_aux_loss_coef
										__SCREAMING_SNAKE_CASE:				Dict           =   decoder_sparse_step
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   encoder_sparse_step
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   num_experts
										__SCREAMING_SNAKE_CASE:				List[str]           =   expert_capacity
										__SCREAMING_SNAKE_CASE:				Tuple           =   router_bias
										if router_dtype not in ["float32", "float16", "bfloat16"]:
																raise ValueError(f"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   router_dtype

										__SCREAMING_SNAKE_CASE:				Tuple           =   router_ignore_padding_tokens
										__SCREAMING_SNAKE_CASE:				List[str]           =   batch_prioritized_routing
										__SCREAMING_SNAKE_CASE:				List[str]           =   second_expert_policy
										__SCREAMING_SNAKE_CASE:				int           =   normalize_router_prob_before_dropping
										__SCREAMING_SNAKE_CASE:				Tuple           =   moe_eval_capacity_token_fraction
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   moe_token_dropout
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   output_router_logits
										super().__init__(
										    pad_token_id=_lowerCAmelCase			, bos_token_id=_lowerCAmelCase			, eos_token_id=_lowerCAmelCase			, is_encoder_decoder=_lowerCAmelCase			, decoder_start_token_id=_lowerCAmelCase			, **_lowerCAmelCase			, )










from ..utils import DummyObject, requires_backends





class 				a			(      metaclass=__lowercase			):
				SCREAMING_SNAKE_CASE__     :   List[Any]		      =     ['''torch''', '''scipy''']






				def __init__(  self			, *_lowerCAmelCase			, **_lowerCAmelCase					):

										"""simple docstring"""




										requires_backends(self			, ['''torch''', '''scipy''']					)






				@classmethod
				def 			snake_case_    (  cls			, *_lowerCAmelCase			, **_lowerCAmelCase					):

										"""simple docstring"""




										requires_backends(cls			, ['''torch''', '''scipy''']					)






				@classmethod
				def 			snake_case_    (  cls			, *_lowerCAmelCase			, **_lowerCAmelCase					):

										"""simple docstring"""




										requires_backends(cls			, ['''torch''', '''scipy''']					)



