class_docstr: The Position-wise FFN layer used in Transformer-like architectures
  
  If pre_norm is True:
      norm(data) -> fc1 -> act -> act_dropout -> fc2 -> dropout -> res(+data)
  Else:
      data -> fc1 -> act -> act_dropout -> fc2 -> dropout -> norm(res(+data))
  Also, if we use gated projection. We will use
      fc1_1 * act(fc1_2(data)) to map the data
docstr: Parameters
  ----------
  units
  hidden_size
  activation_dropout
  dropout
  activation
  normalization
      layer_norm or no_norm
  layer_norm_eps
  pre_norm
      Pre-layer normalization as proposed in the paper:
      "[ACL2018] The Best of Both Worlds: Combining Recent Advances in
       Neural Machine Translation"
      This will stabilize the training of Transformers.
      You may also refer to
      "[Arxiv2020] Understanding the Difficulty of Training Transformers"
docstr_tok: ["Parameters","--","--","--","--","--","units","hidden_size","activation_dropout","dropout","activation","normalization","layer_norm","or","no_norm","layer_norm_eps","pre_norm","Pre-layer","normalization","as","proposed","in","the","paper",":","``","[","ACL2018","]","The","Best","of","Both","Worlds",":","Combining","Recent","Advances","in","Neural","Machine","Translation","''","This","will","stabilize","the","training","of","Transformers",".","You","may","also","refer","to","``","[","Arxiv2020","]","Understanding","the","Difficulty","of","Training","Transformers","''"]
code: def __init__(
      self,
      units: int = 512,
      hidden_size: int = 2048,
      activation_dropout: float = 0.0,
      dropout: float = 0.1,
      gated_proj: bool = False,
      activation="relu",
      normalization: str = "layer_norm",
      layer_norm_eps: float = 1e-5,
      pre_norm: bool = False,
  ):
      """
      Parameters
      ----------
      units
      hidden_size
      activation_dropout
      dropout
      activation
      normalization
          layer_norm or no_norm
      layer_norm_eps
      pre_norm
          Pre-layer normalization as proposed in the paper:
          "[ACL2018] The Best of Both Worlds: Combining Recent Advances in
           Neural Machine Translation"
          This will stabilize the training of Transformers.
          You may also refer to
          "[Arxiv2020] Understanding the Difficulty of Training Transformers"
      """
      super().__init__()
      self._pre_norm = pre_norm
      self._gated_proj = gated_proj
      self._kwargs = OrderedDict(
          [
              ("units", units),
              ("hidden_size", hidden_size),
              ("activation_dropout", activation_dropout),
              ("activation", activation),
              ("dropout", dropout),
              ("normalization", normalization),
              ("layer_norm_eps", layer_norm_eps),
              ("gated_proj", gated_proj),
              ("pre_norm", pre_norm),
          ]
      )
      self.dropout_layer = nn.Dropout(dropout)
      self.activation_dropout_layer = nn.Dropout(activation_dropout)
      self.ffn_1 = nn.Linear(
          in_features=units, out_features=hidden_size, bias=True
      )
      if self._gated_proj:
          self.ffn_1_gate = nn.Linear(
              in_features=units, out_features=hidden_size, bias=True
          )
      self.activation = get_activation(activation)
      self.ffn_2 = nn.Linear(
          in_features=hidden_size, out_features=units, bias=True
      )
      self.layer_norm = get_norm_layer(
          normalization=normalization,
          in_channels=units,
          epsilon=layer_norm_eps,
      )
      self.init_weights()
  
imports: import math
  from collections import OrderedDict
  import torch
  import numpy
  from torch import nn
  from typing import Union
  from .utils import to_torch_dtype
label_desc: Use image node_id 1 to create a new PositionwiseFFN object from inherited base classes: nn with example: obj = PositionwiseFFN(units, hidden_size, activation_dropout, dropout, gated_proj, activation, normalization, layer_norm_eps, pre_norm)