class_docstr: A transformer-encoder structure based on the components from above.
class_docstr_tok: ["A","transformer-encoder","structure","based","on","the","components","from","above","."]
code: def __init__(self, idx, cfg_arch):
      super().__init__()
      self.dropout = torch.nn.Dropout(
          cfg_arch.hidden_dropout_prob, inplace=False
      )
      self.norm1 = _get_norm_fn(cfg_arch.norm)(
          cfg_arch.hidden_size, eps=cfg_arch.norm_eps
      )
      self.norm2 = _get_norm_fn(cfg_arch.norm)(
          cfg_arch.hidden_size, eps=cfg_arch.norm_eps
      )
      self.attn = AttentionComponent(
          idx,
          cfg_arch.hidden_size,
          cfg_arch.attention,
          cfg_arch.use_bias,
      )
      self.LAYOUT = self.attn.LAYOUT
  
      UNDEFINED_VALUE = 3430323896892821
      if (
          OmegaConf.select(
              cfg_arch, "intermed_type", default=UNDEFINED_VALUE
          )
          == UNDEFINED_VALUE
          or cfg_arch.intermed_type == "ff"
      ):
          self.ffn = FFNComponent(
              cfg_arch.hidden_size,
              cfg_arch.intermed_size,
              _get_nonlin_fn(cfg_arch.nonlin),
              cfg_arch.use_bias,
          )
      elif cfg_arch.intermed_type == "fff":
          self.ffn = FFF(
              cfg_arch.hidden_size,
              cfg_arch.hidden_size,
              cfg_arch.intermed_depth,
              cfg_arch.intermed_size,
              _get_nonlin_fn(cfg_arch.nonlin),
          )
      else:
          raise ValueError(
              f"Invalid intermed_type {cfg_arch.intermed_type}"
          )
  
imports: import torch
  from transformers import PretrainedConfig, PreTrainedModel
  from transformers import AutoConfig, AutoModel, AutoModelForMaskedLM, AutoModelForSequenceClassification, AutoModelForTokenClassification
  from typing import Optional
  from omegaconf import OmegaConf
  from .fff import FFF
  from .components import _get_norm_fn, _get_nonlin_fn, EmbeddingComponent, PoolingComponent, PredictionHeadComponent, GLU, get_extended_attention_mask, _init_module
  from .attention import get_attention_mechanism
label_desc: Use image node_id 1 to create a new TransformerLayer object from inherited base classes: torch.nn with example: obj = TransformerLayer(idx, cfg_arch)