class_docstr: A two-layer Feed-Forward-Network with residual layer norm.
  
  Args:
      d_model (int): the size of input for the first-layer of the FFN.
      d_ff (int): the hidden layer size of the second-layer
          of the FNN.
      dropout (float): dropout probability in :math:`[0, 1)`.
      activation_fn (ActivationFunction): activation function used.
      layer_norm (string): 'standard' or 'rms'
class_docstr_tok: ["A","two-layer","Feed-Forward-Network","with","residual","layer","norm",".","Args",":","d_model","(","int",")",":","the","size","of","input","for","the","first-layer","of","the","FFN",".","d_ff","(","int",")",":","the","hidden","layer","size","of","the","second-layer","of","the","FNN",".","dropout","(","float",")",":","dropout","probability","in",":","math",":","`","[","0",",","1",")","`",".","activation_fn","(","ActivationFunction",")",":","activation","function","used",".","layer_norm","(","string",")",":","'standard","'","or","'rms","'"]
code: def __init__(
      self,
      d_model,
      d_ff,
      dropout=0.1,
      activation_fn=ActivationFunction.relu,
      add_ffnbias=True,
      parallel_residual=False,
      layer_norm="standard",
      norm_eps=1e-6,
      use_ckpting=[],
      parallel_gpu=1,
  ):
      super(PositionwiseFeedForward, self).__init__()
      assert (
          d_ff % parallel_gpu == 0
      ), "Model intermediate ffn size must be divisible by the number of partitions"
      self.w_1 = skip_init(
          nn.Linear,
          in_features=d_model,
          out_features=d_ff // parallel_gpu,
          bias=add_ffnbias,
      )
      self.w_2 = skip_init(
          nn.Linear,
          in_features=d_ff // parallel_gpu,
          out_features=d_model,
          bias=add_ffnbias,
      )
      if layer_norm == "standard" and not parallel_residual:
          self.layer_norm = nn.LayerNorm(d_model, eps=norm_eps)
      elif layer_norm == "rms" and not parallel_residual:
          self.layer_norm = RMSNorm(d_model, eps=norm_eps)
      elif not parallel_residual:
          raise ValueError(
              f"{layer_norm} layer norm type is not supported"
          )
      self.parallel_residual = parallel_residual
      self.dropout_p = dropout
      self.dropout_1 = nn.Dropout(dropout)
      self.dropout_2 = nn.Dropout(dropout)
      self.activation = ACTIVATION_FUNCTIONS[activation_fn]
      if activation_fn == "silu" or activation_fn == "gated-gelu":
          self.w_3 = skip_init(
              nn.Linear,
              in_features=d_model,
              out_features=d_ff // parallel_gpu,
              bias=add_ffnbias,
          )
      else:
          self.w_3 = None
      self.maybe_ckpt = (
          checkpoint if "ffn" in use_ckpting else lambda f, x: f(x)
      )
      self.parallel_gpu = parallel_gpu
  
file_path: OpenNMT-py/onmt/modules/position_ffn.py
imports: import torch.nn
  import torch.nn.functional
  from torch.utils.checkpoint import checkpoint
  from onmt.modules.rmsnorm import RMSNorm
  from torch.nn.utils import skip_init
  from torch.distributed import all_reduce
label_desc: Use image node_id 1 to create a new PositionwiseFeedForward object from inherited base classes: nn with example: obj = PositionwiseFeedForward(d_model, d_ff, dropout, activation_fn, add_ffnbias, parallel_residual, layer_norm, norm_eps, use_ckpting, parallel_gpu)