class_docstr: Represents a GLU-based Transformer feed-forward network as described in
  :cite:t:`https://doi.org/10.48550/arxiv.2002.05202`
class_docstr_tok: ["Represents","a","GLU-based","Transformer","feed-forward","network","as","described","in",":","cite",":","t",":","`","https",":","\/\/doi.org\/10.48550\/arxiv.2002.05202","`"]
docstr: :param model_dim:
      The dimensionality of the model.
  :param inner_dim:
      The non-scaled dimensionality of the inner projection layer.
  :param bias:
      If ``True``, all projections learn an additive bias.
  :param gate_activation:
      The activation to apply to outputs of the gate projection. If
      ``None``, :func:`~torch.nn.SiLU` will be used.
  :param inner_dim_scale:
      The scale factor for the dimensionality of the inner projection
      layer.
  :param inner_dim_to_multiple:
      The dimensionality of the inner projection layer is rounded up to
      the nearest multiple of this value.
  :param inner_dropout_p:
      The dropout probability on outputs of the inner projection layer.
docstr_tok: [":","param","model_dim",":","The","dimensionality","of","the","model",".",":","param","inner_dim",":","The","non-scaled","dimensionality","of","the","inner","projection","layer",".",":","param","bias",":","If","``","True","``",",","all","projections","learn","an","additive","bias",".",":","param","gate_activation",":","The","activation","to","apply","to","outputs","of","the","gate","projection",".","If","``","None","``",",",":","func",":","`","~torch.nn.SiLU","`","will","be","used",".",":","param","inner_dim_scale",":","The","scale","factor","for","the","dimensionality","of","the","inner","projection","layer",".",":","param","inner_dim_to_multiple",":","The","dimensionality","of","the","inner","projection","layer","is","rounded","up","to","the","nearest","multiple","of","this","value",".",":","param","inner_dropout_p",":","The","dropout","probability","on","outputs","of","the","inner","projection","layer","."]
code: def __init__(
      self,
      model_dim: int,
      inner_dim: int,
      bias: bool,
      *,
      gate_activation: Optional[Module] = None,
      inner_dim_scale: float = 2 / 3,
      inner_dim_to_multiple: int = 1,
      inner_dropout_p: float = 0.0,
      device: Optional[Device] = None,
      dtype: Optional[DataType] = None,
  ) -> None:
      """
      :param model_dim:
          The dimensionality of the model.
      :param inner_dim:
          The non-scaled dimensionality of the inner projection layer.
      :param bias:
          If ``True``, all projections learn an additive bias.
      :param gate_activation:
          The activation to apply to outputs of the gate projection. If
          ``None``, :func:`~torch.nn.SiLU` will be used.
      :param inner_dim_scale:
          The scale factor for the dimensionality of the inner projection
          layer.
      :param inner_dim_to_multiple:
          The dimensionality of the inner projection layer is rounded up to
          the nearest multiple of this value.
      :param inner_dropout_p:
          The dropout probability on outputs of the inner projection layer.
      """
      super().__init__(model_dim)
  
      self.inner_dim_scale = inner_dim_scale
  
      if inner_dim_scale != 1.0:
          inner_dim = int(inner_dim * inner_dim_scale)
  
      self.inner_dim_to_multiple = inner_dim_to_multiple
  
      if inner_dim_to_multiple != 1:
          inner_dim = inner_dim_to_multiple * (
              (inner_dim + inner_dim_to_multiple - 1)
              // inner_dim_to_multiple
          )
  
      self.gate_proj = Linear(
          model_dim, inner_dim, bias, device=device, dtype=dtype
      )
  
      if gate_activation is None:
          self.gate_activation = SiLU()
      else:
          self.gate_activation = gate_activation
  
      self.inner_proj = Linear(
          model_dim, inner_dim, bias, device=device, dtype=dtype
      )
  
      if inner_dropout_p > 0.0:
          self.inner_dropout = Dropout(inner_dropout_p)
      else:
          self.register_module("inner_dropout", None)
  
      self.output_proj = Linear(
          inner_dim, model_dim, bias, device=device, dtype=dtype
      )
  
imports: from abc import ABC, abstractmethod
  from typing import Optional, final
  from torch import Tensor
  from torch.nn import Dropout, Module, ReLU, SiLU
  from fairseq2.nn.normalization import LayerNorm
  from fairseq2.nn.projection import Linear
  from fairseq2.nn.transformer.layer_norm import LayerNormFactory, create_standard_layer_norm
  from fairseq2.nn.transformer.norm_order import TransformerNormOrder
  from fairseq2.typing import DataType, Device, finaloverride
label_desc: Use image node_id 1 to create a new GLUFeedForwardNetwork object from inherited base classes: FeedForwardNetwork with example: obj = GLUFeedForwardNetwork(model_dim, inner_dim, bias)