class_docstr: Encoder layer implemented by fairseq.
  This version only removes the "args" parameter, no other changes
  
  In the original paper each operation (multi-head attention or FFN) is
  postprocessed with: `dropout -> add residual -> layernorm`.
  In the tensor2tensor code they suggest that learning is more robust when
  preprocessing each layer with layernorm and postprocessing with:
  `dropout -> add residual`. We default to the approach in the paper, but the
  tensor2tensor approach can be enabled by setting
  normalize_before to True.
class_docstr_tok: ["Encoder","layer","implemented","by","fairseq",".","This","version","only","removes","the","``","args","''","parameter",",","no","other","changes","In","the","original","paper","each","operation","(","multi-head","attention","or","FFN",")","is","postprocessed","with",":","`","dropout","-",">","add","residual","-",">","layernorm","`",".","In","the","tensor2tensor","code","they","suggest","that","learning","is","more","robust","when","preprocessing","each","layer","with","layernorm","and","postprocessing","with",":","`","dropout","-",">","add","residual","`",".","We","default","to","the","approach","in","the","paper",",","but","the","tensor2tensor","approach","can","be","enabled","by","setting","normalize_before","to","True","."]
code: def __init__(self, config, initial_weights=None, initial_biases=None):
      super().__init__()
      self.embed_dim = config.hidden_size
  
      self.self_attn = self.build_self_attention(
          self.embed_dim, config.nhead, config.attn_prob_dropout_ratio
      )
      self.self_attn_layer_norm = LayerNorm(self.embed_dim)
      self.dropout_module = Dropout(config.hidden_dropout_ratio)
      self.activation_fn = util.get_activation_fn(
          activation=config.activation_fn
      )
      self.activation_dropout_module = Dropout(
          float(config.activation_dropout_ratio)
      )
      self.normalize_before = config.pre_layer_norm
      self.fc1 = QuantLinear(
          self.embed_dim,
          config.intermediate_size,
      )
      self.fc2 = QuantLinear(
          config.intermediate_size,
          self.embed_dim,
          pre_activation=config.activation_fn,
      )
  
      self.final_layer_norm = LayerNorm(self.embed_dim)
  
      if initial_weights is None or initial_biases is None:
          return
  
      # load initial weights
      self.self_attn.qkv_proj.weight.data.copy_(
          copy_para(torch.cat(initial_weights[:3], 0), config.fp16)
      )
      self.self_attn.qkv_proj.bias.data.copy_(
          copy_para(torch.cat(initial_biases[:3], 0), config.fp16)
      )
      self.self_attn.out_proj.weight.data.copy_(
          copy_para(initial_weights[3], config.fp16)
      )
      self.self_attn.out_proj.bias.data.copy_(
          copy_para(initial_biases[3], config.fp16)
      )
      self.self_attn_layer_norm.weight.data.copy_(
          copy_para(initial_weights[4], config.fp16)
      )
      self.self_attn_layer_norm.bias.data.copy_(
          copy_para(initial_biases[4], config.fp16)
      )
      self.fc1.weight.data.copy_(
          copy_para(initial_weights[5], config.fp16)
      )
      self.fc1.bias.data.copy_(
          copy_para(initial_biases[5], config.fp16)
      )
      self.fc2.weight.data.copy_(
          copy_para(initial_weights[6], config.fp16)
      )
      self.fc2.bias.data.copy_(
          copy_para(initial_biases[6], config.fp16)
      )
      self.final_layer_norm.weight.data.copy_(
          copy_para(initial_weights[7], config.fp16)
      )
      self.final_layer_norm.bias.data.copy_(
          copy_para(initial_biases[7], config.fp16)
      )
  
imports: import __init__
  import math
  import uuid
  from typing import Dict, Optional, List
  import torch
  from torch import Tensor, nn
  from torch.nn import Parameter, LayerNorm, Dropout
  from csrc.pytorch import util
  from csrc.pytorch.layer_base import TransformerEmbeddingLayerBase, TransformerEncoderLayerBase, TransformerDecoderLayerBase
  from csrc.pytorch.quantization import QuantLinear, TensorQuantizer, act_quant_config, weight_quant_config, out_quant_config, emb_quant_config
label_desc: Use image node_id 1 to create a new TransformerEncoderLayer object from inherited base classes: TransformerEncoderLayerBase with example: obj = TransformerEncoderLayer(config, initial_weights, initial_biases)