


from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase       :       Dict						       =				logging.get_logger(__name__)

__UpperCamelCase       :       Optional[Any]						       =				{
    "facebook/nllb-moe-54B": "https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json",
}

class       __magic_name__	(     __lowerCAmelCase):
     A:		Dict			=      "nllb-moe"
     A:		Any			=      ["past_key_values"]
     A:		Optional[int]			=      {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}


     def __init__(      self							:     Optional[int]							,	lowerCamelCase__							:     Union[str, Any]=128112							,	lowerCamelCase__							:     Any=1024							,	lowerCamelCase__							:     Optional[Any]=12							,	lowerCamelCase__							:     str=4096							,	lowerCamelCase__							:     List[Any]=16							,	lowerCamelCase__							:     Dict=12							,	lowerCamelCase__							:     Optional[Any]=4096							,	lowerCamelCase__							:     int=16							,	lowerCamelCase__							:     Dict=0.05							,	lowerCamelCase__							:     Dict=0.05							,	lowerCamelCase__							:     Optional[int]=True							,	lowerCamelCase__							:     List[Any]=True							,	lowerCamelCase__							:     Tuple="relu"							,	lowerCamelCase__							:     Any=1024							,	lowerCamelCase__							:     Any=0.1							,	lowerCamelCase__							:     Tuple=0.1							,	lowerCamelCase__							:     Optional[Any]=0.0							,	lowerCamelCase__							:     Dict=0.02							,	lowerCamelCase__							:     int=2							,	lowerCamelCase__							:     Optional[int]=True							,	lowerCamelCase__							:     str=False							,	lowerCamelCase__							:     Dict="float32"							,	lowerCamelCase__							:     Dict=False							,	lowerCamelCase__							:     List[str]=128							,	lowerCamelCase__							:     Any=64							,	lowerCamelCase__							:     Union[str, Any]=4							,	lowerCamelCase__							:     List[Any]=4							,	lowerCamelCase__							:     str=0.001							,	lowerCamelCase__							:     List[Any]=0.001							,	lowerCamelCase__							:     Union[str, Any]="all"							,	lowerCamelCase__							:     List[Any]=False							,	lowerCamelCase__							:     Tuple=False							,	lowerCamelCase__							:     Tuple=1.0							,	lowerCamelCase__							:     List[str]=0.2							,	lowerCamelCase__							:     str=1							,	lowerCamelCase__							:     Any=0							,	lowerCamelCase__							:     int=2							,	lowerCamelCase__							:     Any=False							,	**lowerCamelCase__							:     Any							,	)  ->      Tuple:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[int]          =   vocab_size
      UpperCamelCase__					:    List[Any]          =   max_position_embeddings
      UpperCamelCase__					:    str          =   d_model
      UpperCamelCase__					:    Union[str, Any]          =   encoder_ffn_dim
      UpperCamelCase__					:    Optional[int]          =   encoder_layers
      UpperCamelCase__					:    Tuple          =   encoder_attention_heads
      UpperCamelCase__					:    Union[str, Any]          =   decoder_ffn_dim
      UpperCamelCase__					:    Optional[int]          =   decoder_layers
      UpperCamelCase__					:    List[str]          =   decoder_attention_heads
      UpperCamelCase__					:    Union[str, Any]          =   dropout
      UpperCamelCase__					:    Union[str, Any]          =   attention_dropout
      UpperCamelCase__					:    Tuple          =   activation_dropout
      UpperCamelCase__					:    List[str]          =   activation_function
      UpperCamelCase__					:    int          =   init_std
      UpperCamelCase__					:    str          =   encoder_layerdrop
      UpperCamelCase__					:    Any          =   decoder_layerdrop
      UpperCamelCase__					:    str          =   use_cache
      UpperCamelCase__					:    str          =   encoder_layers
      UpperCamelCase__					:    List[Any]          =   scale_embedding  # scale factor will be sqrt(d_model) if True
      UpperCamelCase__					:    Optional[Any]          =   router_z_loss_coef
      UpperCamelCase__					:    List[str]          =   router_aux_loss_coef
      UpperCamelCase__					:    Optional[int]          =   decoder_sparse_step
      UpperCamelCase__					:    List[str]          =   encoder_sparse_step
      UpperCamelCase__					:    Any          =   num_experts
      UpperCamelCase__					:    int          =   expert_capacity
      UpperCamelCase__					:    Optional[int]          =   router_bias
      if router_dtype not in ["float32", "float16", "bfloat16"]:
       raise ValueError(F"`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"			)
      UpperCamelCase__					:    Tuple          =   router_dtype

      UpperCamelCase__					:    Optional[Any]          =   router_ignore_padding_tokens
      UpperCamelCase__					:    Dict          =   batch_prioritized_routing
      UpperCamelCase__					:    Union[str, Any]          =   second_expert_policy
      UpperCamelCase__					:    Dict          =   normalize_router_prob_before_dropping
      UpperCamelCase__					:    List[str]          =   moe_eval_capacity_token_fraction
      UpperCamelCase__					:    Dict          =   moe_token_dropout
      UpperCamelCase__					:    Any          =   output_router_logits
      super().__init__(
          pad_token_id=lowerCamelCase__							,	bos_token_id=lowerCamelCase__							,	eos_token_id=lowerCamelCase__							,	is_encoder_decoder=lowerCamelCase__							,	decoder_start_token_id=lowerCamelCase__							,	**lowerCamelCase__							,	)






from typing import TYPE_CHECKING

from ..utils import _LazyModule


__UpperCamelCase       :       int						       =				{
    "config": [
        "EXTERNAL_DATA_FORMAT_SIZE_LIMIT",
        "OnnxConfig",
        "OnnxConfigWithPast",
        "OnnxSeq2SeqConfigWithPast",
        "PatchingSpec",
    ],
    "convert": ["export", "validate_model_outputs"],
    "features": ["FeaturesManager"],
    "utils": ["ParameterFormat", "compute_serialized_parameters_size"],
}


if TYPE_CHECKING:
       from .config import (
           EXTERNAL_DATA_FORMAT_SIZE_LIMIT,
           OnnxConfig,
           OnnxConfigWithPast,
           OnnxSeqaSeqConfigWithPast,
           PatchingSpec,
       )
       from .convert import export, validate_model_outputs
       from .features import FeaturesManager
       from .utils import ParameterFormat, compute_serialized_parameters_size

else:
       import sys

       __UpperCamelCase       :       List[str]						       =				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


