class_docstr: MLP.
  
  MLP will take the input with h hidden state, project it to 4*h
  hidden dimension, perform nonlinear transformation, and project the
  state back into h hidden dimension.
class_docstr_tok: ["MLP",".","MLP","will","take","the","input","with","h","hidden","state",",","project","it","to","4","*","h","hidden","dimension",",","perform","nonlinear","transformation",",","and","project","the","state","back","into","h","hidden","dimension","."]
code: def __init__(
      self,
      config: ModelParallelConfig,
      init_method,
      output_layer_init_method,
      hidden_size,
      ffn_hidden_size,
      dtype=torch.float32,
      bias_activation_fusion=True,
      openai_gelu=False,
      onnx_safe=False,
      activation="gelu",
      bias=True,
      transformer_block_type="pre_ln",
      normalization="layernorm",
      layernorm_epsilon=1e-5,
      persist_layer_norm=False,
      dropout=0.0,
  ):
      super(ParallelMLP, self).__init__(config=config)
      self.activation = activation
      self.bias = bias
      self.transformer_block_type = transformer_block_type
      self.normalization = normalization
      self.layernorm_epsilon = layernorm_epsilon
      self.persist_layer_norm = persist_layer_norm
      self.activation = activation
      self.dropout = dropout
      self.dtype = dtype
      self.set_accepted_adapter_types(
          [MLPInfusedAdapterConfig._target_]
      )
  
      supported_activations = [
          "gelu",
          "geglu",
          "reglu",
          "swiglu",
          "squared-relu",
          "fast-geglu",
          "fast-swiglu",
          "fast-reglu",
          "approx-gelu",
      ]
  
      if activation not in supported_activations:
          raise ValueError(
              f"Activation {activation} not supported. Supported activations are {supported_activations}"
          )
  
      self.fast_glu_activation = activation in [
          "fast-geglu",
          "fast-swiglu",
          "fast-reglu",
      ]
  
      # Project to 4h.
      self.dense_h_to_4h = tensor_parallel.ColumnParallelLinear(
          hidden_size,
          ffn_hidden_size * 2
          if self.fast_glu_activation
          else ffn_hidden_size,  # NOTE: When using geglu, divide ffn dim by 2/3 to keep overall params the same.
          config=config,
          gather_output=False,
          init_method=init_method,
          skip_bias_add=True,
          bias=bias,
      )
  
      if activation in ["geglu", "reglu", "swiglu"]:
          # Separate linear layer for *GLU activations.
          # Source: https://github.com/huggingface/transformers/blob/bee361c6f1f7704f8c688895f2f86f6e5ff84727/src/transformers/models/t5/modeling_t5.py#L292
          self.dense_h_to_4h_2 = tensor_parallel.ColumnParallelLinear(
              hidden_size,
              ffn_hidden_size,  # NOTE: When using *glu, divide ffn dim by 2/3 to keep overall params the same.
              config=config,
              gather_output=False,
              init_method=init_method,
              skip_bias_add=True,
              bias=bias,
          )
  
      self.glu_activation_family = activation in [
          "geglu",
          "reglu",
          "swiglu",
          "fast-geglu",
          "fast-reglu",
          "fast-swiglu",
      ]
      bias_activation_fusion_unavailable = activation in [
          "reglu",
          "swiglu",
      ]
  
      if bias_activation_fusion_unavailable and bias_activation_fusion:
          raise ValueError(
              f"Cannot use bias_activation_fusion with {activation} activation. Please turn bias gelu fusion off."
          )
  
      if (
          self.glu_activation_family
          and onnx_safe
          and self.bias_activation_fusion
      ):
          raise ValueError(
              f"Cannot use onnx_safe with specificed activation function and bias_activation_fusion : {activation} Please turn onnx safe off."
          )
  
      if bias_activation_fusion and not bias:
          raise ValueError(
              f"Cannot use bias_activation_fusion without bias terms. Please set bias=True or bias_activation_fusion=False."
          )
  
      self.bias_activation_fusion = bias_activation_fusion
  
      # Give openai_gelu precedence over other activations if set, for HF compatibility. Normally this is off and shouldn't affect regular model training.
      if openai_gelu:
          self.activation_func = openai_gelu_func
      elif activation in ["gelu", "geglu", "fast-geglu"]:
          self.activation_func = F.gelu
      elif activation == "approx-gelu":
          self.activation_func = ApproxGELUActivation
      elif onnx_safe:
          self.activation_func = erf_gelu
      elif activation in ["reglu", "fast-reglu"]:
          self.activation_func = F.relu
      elif activation in ["swiglu", "fast-swiglu"]:
          # SiLU or sigmoid linear unit is the same as swish with beta = 1 (which is what https://arxiv.org/pdf/2002.05202.pdf uses.)
          self.activation_func = F.silu
      elif activation == "squared-relu":
          self.activation_func = squared_relu
  
      # Project back to h.
      self.dense_4h_to_h = tensor_parallel.RowParallelLinear(
          ffn_hidden_size,
          hidden_size,
          config=config,
          input_is_parallel=True,
          init_method=output_layer_init_method,
          skip_bias_add=True,
          bias=bias,
      )
  
      # Normformer normalization
      if transformer_block_type == "normformer":
          if normalization == "layernorm":
              self.normalization = get_layer_norm(
                  ffn_hidden_size
                  // get_tensor_model_parallel_world_size(),
                  layernorm_epsilon,
                  persist_layer_norm,
              )
          elif normalization == "layernorm1p":
              self.normalization = LayerNorm1P(
                  ffn_hidden_size
                  // get_tensor_model_parallel_world_size(),
                  layernorm_epsilon,
                  sequence_parallel_enabled=config.sequence_parallel,
              )
          else:
              self.normalization = MixedFusedRMSNorm(
                  ffn_hidden_size
                  // get_tensor_model_parallel_world_size(),
                  layernorm_epsilon,
              )
  
imports: import torch
  import torch.nn.functional
  from nemo.collections.nlp.modules.common.megatron.adapters.parallel_adapters import AdapterName, MLPInfusedAdapterConfig
  from nemo.collections.nlp.modules.common.megatron.fused_bias_geglu import fused_bias_geglu
  from nemo.collections.nlp.modules.common.megatron.fused_bias_gelu import fused_bias_gelu
  from nemo.collections.nlp.modules.common.megatron.fused_layer_norm import get_layer_norm
  from nemo.collections.nlp.modules.common.megatron.layer_norm_1p import LayerNorm1P
  from nemo.collections.nlp.modules.common.megatron.module import MegatronModule
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, ApproxGELUActivation, erf_gelu
  from nemo.collections.nlp.modules.common.megatron.utils import openai_gelu
  from nemo.collections.nlp.modules.common.megatron.utils import squared_relu
  from nemo.core import adapter_mixins
label_desc: Use image node_id 1 to create a new ParallelMLP object from inherited base classes: MegatronModule, adapter_mixins with example: obj = ParallelMLP(config, init_method, output_layer_init_method, hidden_size, ffn_hidden_size, dtype, bias_activation_fusion, openai_gelu, onnx_safe, activation, bias, transformer_block_type, normalization, layernorm_epsilon, persist_layer_norm, dropout)