class_docstr: Note: The FF layer is not auto-scaled when using a GLU type activation.
  It actually turned out better not to scale it, so here the block is effectively smaller than may be expected.
  
  The neox suggestion for approx. equal parameter count is int(4 * 2 / 3 * hidden_size) * 2 [this is ~5.33]
class_docstr_tok: ["Note",":","The","FF","layer","is","not","auto-scaled","when","using","a","GLU","type","activation",".","It","actually","turned","out","better","not","to","scale","it",",","so","here","the","block","is","effectively","smaller","than","may","be","expected",".","The","neox","suggestion","for","approx",".","equal","parameter","count","is","int","(","4","*","2","\/","3","*","hidden_size",")","*","2","[","this","is","~5.33","]"]
code: def __init__(
      self,
      hidden_size,
      intermed_size,
      nonlin_fn=torch.nn.GELU,
      use_bias=True,
  ):
      super().__init__()
      self.dense_in = torch.nn.Linear(
          hidden_size, intermed_size, bias=use_bias
      )
      self.nonlin = nonlin_fn()
      if isinstance(self.nonlin, GLU):
          intermed_output_size = intermed_size // 2
      else:
          intermed_output_size = intermed_size
      self.dense_out = torch.nn.Linear(
          intermed_output_size, hidden_size, bias=use_bias
      )
  
imports: import torch
  from transformers import PretrainedConfig, PreTrainedModel
  from transformers import AutoConfig, AutoModel, AutoModelForMaskedLM, AutoModelForSequenceClassification, AutoModelForTokenClassification
  from typing import Optional
  from omegaconf import OmegaConf
  from .fff import FFF
  from .components import _get_norm_fn, _get_nonlin_fn, EmbeddingComponent, PoolingComponent, PredictionHeadComponent, GLU, get_extended_attention_mask, _init_module
  from .attention import get_attention_mechanism
label_desc: Use image node_id 1 to create a new FFNComponent object from inherited base classes: torch.nn with example: obj = FFNComponent(hidden_size, intermed_size, nonlin_fn, use_bias)