class_docstr: Transformer Encoder Layer
docstr: Parameters
  ----------
  units
  hidden_size
  num_heads
  attention_dropout_prob
  hidden_dropout_prob
  activation_dropout_prob
  layer_norm_eps
  pre_norm
      Whether to attach the normalization layer before attention layer
      If pre_norm:
          data -> norm(data) -> attn -> res(+data) -> ffn
      Else:
          data -> attn -> norm(res(+data)) -> ffn
  use_qkv_bias
      Whether to use bias for self attention
  activation
      The activation
  layout
      The layout
docstr_tok: ["Parameters","--","--","--","--","--","units","hidden_size","num_heads","attention_dropout_prob","hidden_dropout_prob","activation_dropout_prob","layer_norm_eps","pre_norm","Whether","to","attach","the","normalization","layer","before","attention","layer","If","pre_norm",":","data","-",">","norm","(","data",")","-",">","attn","-",">","res","(","+data",")","-",">","ffn","Else",":","data","-",">","attn","-",">","norm","(","res","(","+data",")",")","-",">","ffn","use_qkv_bias","Whether","to","use","bias","for","self","attention","activation","The","activation","layout","The","layout"]
code: def __init__(
      self,
      units: int = 512,
      hidden_size: int = 2048,
      num_heads: int = 8,
      attention_dropout_prob: float = 0.1,
      hidden_dropout_prob: float = 0.1,
      activation_dropout_prob: float = 0.0,
      layer_norm_eps: float = 1e-12,
      pre_norm: bool = False,
      use_qkv_bias: bool = True,
      activation: str = "relu",
      layout="NT",
  ):
      """
      Parameters
      ----------
      units
      hidden_size
      num_heads
      attention_dropout_prob
      hidden_dropout_prob
      activation_dropout_prob
      layer_norm_eps
      pre_norm
          Whether to attach the normalization layer before attention layer
          If pre_norm:
              data -> norm(data) -> attn -> res(+data) -> ffn
          Else:
              data -> attn -> norm(res(+data)) -> ffn
      use_qkv_bias
          Whether to use bias for self attention
      activation
          The activation
      layout
          The layout
      """
      super().__init__()
      self._units = units
      self._hidden_size = hidden_size
      self._num_heads = num_heads
      self._attention_dropout_prob = attention_dropout_prob
      self._hidden_dropout_prob = hidden_dropout_prob
      self._activation_dropout_prob = activation_dropout_prob
      self._pre_norm = pre_norm
      self._layout = layout
      assert layout in ["TN", "NT"], (
          "Invalid layout received = {}. "
          'Only "TN" and "NT" are accepted!'.format(layout)
      )
      assert (
          self._units % self._num_heads == 0
      ), "units must be divisive by the number of heads"
      self.dropout_layer = nn.Dropout(hidden_dropout_prob)
      self.attn_qkv = nn.Linear(
          out_features=3 * units, in_features=units, bias=use_qkv_bias
      )
      self.attention_proj = nn.Linear(
          out_features=units, in_features=units, bias=True
      )
      attention_layout = "NTK" if self._layout == "NT" else "TNK"
      self.attention_cell = MultiHeadAttentionCell(
          query_units=self._units,
          num_heads=self._num_heads,
          attention_dropout=self._attention_dropout_prob,
          scaled=True,
          layout=attention_layout,
      )
      self.layer_norm = nn.LayerNorm(
          eps=layer_norm_eps, normalized_shape=units
      )
      self.ffn = PositionwiseFFN(
          units=units,
          hidden_size=hidden_size,
          dropout=hidden_dropout_prob,
          activation_dropout=activation_dropout_prob,
          layer_norm_eps=layer_norm_eps,
          activation=activation,
          pre_norm=pre_norm,
      )
  
imports: from typing import Union, Optional, List
  from torch import nn
  import torch
  from ..layers import PositionwiseFFN
  from ..attention_cell import MultiHeadAttentionCell, gen_mem_attn_mask, gen_self_attn_mask
  from ..utils import to_torch_dtype
label_desc: Use image node_id 1 to create a new TransformerEncoderLayer object from inherited base classes: nn with example: obj = TransformerEncoderLayer(units, hidden_size, num_heads, attention_dropout_prob, hidden_dropout_prob, activation_dropout_prob, layer_norm_eps, pre_norm, use_qkv_bias, activation, layout)