class_docstr: title: Position-wise Feed-Forward Network (FFN)
  summary: Documented reusable implementation of the position wise feedforward network.
  
  # Position-wise Feed-Forward Network (FFN)
  This is a [PyTorch](https://pytorch.org)  implementation
  of position-wise feedforward network used in transformer.
  FFN consists of two fully connected layers.
  Number of dimensions in the hidden layer $d_{ff}$, is generally set to around
  four times that of the token embedding $d_{model}$.
  So it is sometime also called the expand-and-contract network.
  There is an activation at the hidden layer, which is
  usually set to ReLU (Rectified Linear Unit) activation, $$\\max(0, x)$$
  That is, the FFN function is,
  $$FFN(x, W_1, W_2, b_1, b_2) = \\max(0, x W_1 + b_1) W_2 + b_2$$
  where $W_1$, $W_2$, $b_1$ and $b_2$ are learnable parameters.
  Sometimes the
  GELU (Gaussian Error Linear Unit) activation is also used instead of ReLU.
  $$x \\Phi(x)$$ where $\\Phi(x) = P(X \\le x), X \\sim \\mathcal{N}(0,1)$
  ### Gated Linear Units
  This is a generic implementation that supports different variants including
  [Gated Linear Units](https://arxiv.org/abs/2002.05202) (GLU).
class_docstr_tok: ["title",":","Position-wise","Feed-Forward","Network","(","FFN",")","summary",":","Documented","reusable","implementation","of","the","position","wise","feedforward","network",".","#","Position-wise","Feed-Forward","Network","(","FFN",")","This","is","a","[","PyTorch","]","(","https",":","\/\/pytorch.org",")","implementation","of","position-wise","feedforward","network","used","in","transformer",".","FFN","consists","of","two","fully","connected","layers",".","Number","of","dimensions","in","the","hidden","layer","$","d_","{","ff","}","$",",","is","generally","set","to","around","four","times","that","of","the","token","embedding","$","d_","{","model","}","$",".","So","it","is","sometime","also","called","the","expand-and-contract","network",".","There","is","an","activation","at","the","hidden","layer",",","which","is","usually","set","to","ReLU","(","Rectified","Linear","Unit",")","activation",",","$","$","\\\\max","(","0",",","x",")","$","$","That","is",",","the","FFN","function","is",",","$","$","FFN","(","x",",","W_1",",","W_2",",","b_1",",","b_2",")","=","\\\\max","(","0",",","x","W_1","+","b_1",")","W_2","+","b_2","$","$","where","$","W_1","$",",","$","W_2","$",",","$","b_1","$","and","$","b_2","$","are","learnable","parameters",".","Sometimes","the","GELU","(","Gaussian","Error","Linear","Unit",")","activation","is","also","used","instead","of","ReLU",".","$","$","x","\\\\Phi","(","x",")","$","$","where","$","\\\\Phi","(","x",")","=","P","(","X","\\\\le","x",")",",","X","\\\\sim","\\\\mathcal","{","N","}","(","0,1",")","$","#","#","#","Gated","Linear","Units","This","is","a","generic","implementation","that","supports","different","variants","including","[","Gated","Linear","Units","]","(","https",":","\/\/arxiv.org\/abs\/2002.05202",")","(","GLU",")","."]
code: def forward(self, x: torch.Tensor):
      # $f(x W_1 + b_1)$
      g = self.activation(self.layer1(x))
      # If gated, $f(x W_1 + b_1) \otimes (x V + b) $
      if self.is_gated:
          x = g * self.linear_v(x)
      # Otherwise
      else:
          x = g
      # Apply dropout
      x = self.dropout(x)
      # $(f(x W_1 + b_1) \otimes (x V + b)) W_2 + b_2$ or $f(x W_1 + b_1) W_2 + b_2$
      # depending on whether it is gated
      return self.layer2(x)
  
imports: import random
  from typing import Callable
  import torch
  import torch.nn
  from pytorch_tabular.models.common.layers.activations import RSoftmax, entmax15
label_desc: Use image node_id 2 for calling the PositionWiseFeedForward obj's underlying member method code with example usage: obj.forward(x) and returns: self