
"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_            =    logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            =    {
    '''facebook/nllb-moe-54B''': '''https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json''',
}
class   _UpperCAmelCase				(		SCREAMING_SNAKE_CASE_				):
      __SCREAMING_SNAKE_CASE :				str				   =						"nllb-moe"
      __SCREAMING_SNAKE_CASE :				Union[str, Any]				   =						["past_key_values"]
      __SCREAMING_SNAKE_CASE :				Optional[Any]				   =						{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}




      def __init__(				self ,							lowercase_=1_2_8_1_1_2 ,							lowercase_=1_0_2_4 ,							lowercase_=1_2 ,							lowercase_=4_0_9_6 ,							lowercase_=1_6 ,							lowercase_=1_2 ,							lowercase_=4_0_9_6 ,							lowercase_=1_6 ,							lowercase_=0.0_5 ,							lowercase_=0.0_5 ,							lowercase_=True ,							lowercase_=True ,							lowercase_="relu" ,							lowercase_=1_0_2_4 ,							lowercase_=0.1 ,							lowercase_=0.1 ,							lowercase_=0.0 ,							lowercase_=0.0_2 ,							lowercase_=2 ,							lowercase_=True ,							lowercase_=False ,							lowercase_="float32" ,							lowercase_=False ,							lowercase_=1_2_8 ,							lowercase_=6_4 ,							lowercase_=4 ,							lowercase_=4 ,							lowercase_=0.0_0_1 ,							lowercase_=0.0_0_1 ,							lowercase_="all" ,							lowercase_=False ,							lowercase_=False ,							lowercase_=1.0 ,							lowercase_=0.2 ,							lowercase_=1 ,							lowercase_=0 ,							lowercase_=2 ,							lowercase_=False ,							**lowercase_ ,							)    ->       Any:
          UpperCAmelCase          =       vocab_size
          UpperCAmelCase          =       max_position_embeddings
          UpperCAmelCase          =       d_model
          UpperCAmelCase          =       encoder_ffn_dim
          UpperCAmelCase          =       encoder_layers
          UpperCAmelCase          =       encoder_attention_heads
          UpperCAmelCase          =       decoder_ffn_dim
          UpperCAmelCase          =       decoder_layers
          UpperCAmelCase          =       decoder_attention_heads
          UpperCAmelCase          =       dropout
          UpperCAmelCase          =       attention_dropout
          UpperCAmelCase          =       activation_dropout
          UpperCAmelCase          =       activation_function
          UpperCAmelCase          =       init_std
          UpperCAmelCase          =       encoder_layerdrop
          UpperCAmelCase          =       decoder_layerdrop
          UpperCAmelCase          =       use_cache
          UpperCAmelCase          =       encoder_layers
          UpperCAmelCase          =       scale_embedding  # scale factor will be sqrt(d_model) if True
          UpperCAmelCase          =       router_z_loss_coef
          UpperCAmelCase          =       router_aux_loss_coef
          UpperCAmelCase          =       decoder_sparse_step
          UpperCAmelCase          =       encoder_sparse_step
          UpperCAmelCase          =       num_experts
          UpperCAmelCase          =       expert_capacity
          UpperCAmelCase          =       router_bias
          if router_dtype not in ["float32", "float16", "bfloat16"]:
              raise ValueError(F"`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"				)
          UpperCAmelCase          =       router_dtype

          UpperCAmelCase          =       router_ignore_padding_tokens
          UpperCAmelCase          =       batch_prioritized_routing
          UpperCAmelCase          =       second_expert_policy
          UpperCAmelCase          =       normalize_router_prob_before_dropping
          UpperCAmelCase          =       moe_eval_capacity_token_fraction
          UpperCAmelCase          =       moe_token_dropout
          UpperCAmelCase          =       output_router_logits
          super().__init__(
              pad_token_id=lowercase_ ,							bos_token_id=lowercase_ ,							eos_token_id=lowercase_ ,							is_encoder_decoder=lowercase_ ,							decoder_start_token_id=lowercase_ ,							**lowercase_ ,							)





"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


SCREAMING_SNAKE_CASE_            =    {
    '''configuration_roc_bert''': ['''ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''RoCBertConfig'''],
    '''tokenization_roc_bert''': ['''RoCBertTokenizer'''],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   pass

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE_            =    [
       '''ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''RoCBertForCausalLM''',
       '''RoCBertForMaskedLM''',
       '''RoCBertForMultipleChoice''',
       '''RoCBertForPreTraining''',
       '''RoCBertForQuestionAnswering''',
       '''RoCBertForSequenceClassification''',
       '''RoCBertForTokenClassification''',
       '''RoCBertLayer''',
       '''RoCBertModel''',
       '''RoCBertPreTrainedModel''',
       '''load_tf_weights_in_roc_bert''',
   ]

if TYPE_CHECKING:
   from .configuration_roc_bert import ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RoCBertConfig
   from .tokenization_roc_bert import RoCBertTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      raise OptionalDependencyNotAvailable()

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_roc_bert import (
          ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          RoCBertForCausalLM,
          RoCBertForMaskedLM,
          RoCBertForMultipleChoice,
          RoCBertForPreTraining,
          RoCBertForQuestionAnswering,
          RoCBertForSequenceClassification,
          RoCBertForTokenClassification,
          RoCBertLayer,
          RoCBertModel,
          RoCBertPreTrainedModel,
          load_tf_weights_in_roc_bert,
      )


else:
   import sys

   SCREAMING_SNAKE_CASE_            =    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



