class_docstr: FeedForward module
  
  Args:
      dim_in (int): input dimension.
      dim_ff (int): middle dimension.
      dim_out (int, optional): output dimension. Defaults to None, which means dim_in = dim_out.
      dtype (optional): Defaults to torch.half.
      init_mean (float, optional): mean of :math:`\mathbf{W}\sim\mathcal{N}(\text{mean}, \text{std}^2)` for fully-connected module used in feed-forward layer. Defaults to 0.
      init_std (float, optional): std of :math:`\mathbf{W}\sim\mathcal{N}(\text{mean}, \text{std}^2)` for fully-connected module used in feed-forward layer. Defaults to 0.02.
      bias (bool, optional): whether to use bias term in fully-connected layers used in feed-forward module. Defaults to False.
      activate_fn (str, optional): Defaults to `gated_gelu`.
      dropout_p (int, optional): Defaults to 0.
class_docstr_tok: ["FeedForward","module","Args",":","dim_in","(","int",")",":","input","dimension",".","dim_ff","(","int",")",":","middle","dimension",".","dim_out","(","int",",","optional",")",":","output","dimension",".","Defaults","to","None",",","which","means","dim_in","=","dim_out",".","dtype","(","optional",")",":","Defaults","to","torch.half",".","init_mean","(","float",",","optional",")",":","mean","of",":","math",":","`","\\mathbf","{","W","}","\\sim\\mathcal","{","N","}","(","\\text","{","mean","}",",","\\text","{","std","}","^2",")","`","for","fully-connected","module","used","in","feed-forward","layer",".","Defaults","to","0.","init_std","(","float",",","optional",")",":","std","of",":","math",":","`","\\mathbf","{","W","}","\\sim\\mathcal","{","N","}","(","\\text","{","mean","}",",","\\text","{","std","}","^2",")","`","for","fully-connected","module","used","in","feed-forward","layer",".","Defaults","to","0.02.","bias","(","bool",",","optional",")",":","whether","to","use","bias","term","in","fully-connected","layers","used","in","feed-forward","module",".","Defaults","to","False",".","activate_fn","(","str",",","optional",")",":","Defaults","to","`","gated_gelu","`",".","dropout_p","(","int",",","optional",")",":","Defaults","to","0","."]
code: def __init__(
      self,
      dim_model: int,
      dim_ff: int,
      dtype=torch.half,
      dropout_p: Optional[float] = None,
  ):
      super().__init__()
  
      self.w_in = DenseGatedACT(
          dim_in=dim_model,
          dim_ff=dim_ff,
          dtype=dtype,
      )
  
      if dropout_p is not None:
          self.dropout = torch.nn.Dropout(dropout_p)
      else:
          self.dropout = None
  
      self.w_out = Linear(
          dim_in=dim_ff,
          dim_out=dim_model,
          dtype=dtype,
          scale_before=False,
      )
  
file_path: DeepKE/example/llm/CPM-Bee/src/cpm_live/native_layers/feedforward.py
imports: from typing import Optional
  import torch
  from .linear import Linear
label_desc: Use image node_id 1 to create a new FeedForward object from inherited base classes: torch.nn with example: obj = FeedForward(dim_model, dim_ff, dtype, dropout_p)