class_docstr: Note: The FF layer is not auto-scaled when using a GLU type activation.
  Better do this manually and choose a sensible intermed_size that is nicely divisible.
  
  The neox suggestion for approx. equal parameter count is int(4 * 2 / 3 * hidden_size) * 2 [this is ~5.33]
class_docstr_tok: ["Note",":","The","FF","layer","is","not","auto-scaled","when","using","a","GLU","type","activation",".","Better","do","this","manually","and","choose","a","sensible","intermed_size","that","is","nicely","divisible",".","The","neox","suggestion","for","approx",".","equal","parameter","count","is","int","(","4","*","2","\/","3","*","hidden_size",")","*","2","[","this","is","~5.33","]"]
code: def __init__(
      self,
      hidden_size,
      intermed_size,
      nonlin_fn=torch.nn.GELU,
      use_bias=True,
  ):
      super().__init__()
      self.dense_in = torch.nn.Linear(
          hidden_size, intermed_size, bias=use_bias
      )
      self.nonlin = nonlin_fn()
      if (
          isinstance(self.nonlin, GLU)
          or getattr(self.nonlin, "original_name", "") == "GLU"
      ):
          intermed_output_size = intermed_size // 2
      else:
          intermed_output_size = intermed_size
      self.dense_out = torch.nn.Linear(
          intermed_output_size, hidden_size, bias=use_bias
      )
  
imports: import torch
  import math
  from typing import Optional, Tuple
  from functools import partial
  from .embeddings import SinusoidalPositional, LearnablePositional, ScaledSinosoidal
  from .attention import get_attention_mechanism
label_desc: Use image node_id 1 to create a new FFNComponent object from inherited base classes: torch.nn with example: obj = FFNComponent(hidden_size, intermed_size, nonlin_fn, use_bias)