class_docstr: A single layer of the transformer encoder.
  
  Args:
      d_model (int): the dimension of keys/values/queries in
                 MultiHeadedAttention, also the input size of
                 the first-layer of the PositionwiseFeedForward.
      heads (int): the number of head for MultiHeadedAttention.
      d_ff (int): the second-layer of the PositionwiseFeedForward.
      dropout (float): dropout probability(0-1.0).
      pos_ffn_activation_fn (ActivationFunction):
          activation function choice for PositionwiseFeedForward layer
      add_qkvbias (bool): whether to add bias to the Key/Value nn.Linear
      num_kv (int): number of heads for KV when different vs Q (multiquery)
      add_ffnbias (bool): whether to add bias to the FF nn.Linear
      parallel_residual (bool): Use parallel residual connections in each layer block, as used
          by the GPT-J and GPT-NeoX models
      layer_norm (string): type of layer normalization standard/rms
      norm_eps (float): layer norm epsilon
      use_ckpting (List): layers for which we checkpoint for backward
      parallel_gpu (int): Number of gpu for tensor parallelism
      rotary_interleave (bool): Interleave the head dimensions when rotary
          embeddings are applied
      rotary_theta (int): rotary base theta
      rotary_dim (int): rotary dim when different to dim per head
class_docstr_tok: ["A","single","layer","of","the","transformer","encoder",".","Args",":","d_model","(","int",")",":","the","dimension","of","keys\/values\/queries","in","MultiHeadedAttention",",","also","the","input","size","of","the","first-layer","of","the","PositionwiseFeedForward",".","heads","(","int",")",":","the","number","of","head","for","MultiHeadedAttention",".","d_ff","(","int",")",":","the","second-layer","of","the","PositionwiseFeedForward",".","dropout","(","float",")",":","dropout","probability","(","0-1.0",")",".","pos_ffn_activation_fn","(","ActivationFunction",")",":","activation","function","choice","for","PositionwiseFeedForward","layer","add_qkvbias","(","bool",")",":","whether","to","add","bias","to","the","Key\/Value","nn.Linear","num_kv","(","int",")",":","number","of","heads","for","KV","when","different","vs","Q","(","multiquery",")","add_ffnbias","(","bool",")",":","whether","to","add","bias","to","the","FF","nn.Linear","parallel_residual","(","bool",")",":","Use","parallel","residual","connections","in","each","layer","block",",","as","used","by","the","GPT-J","and","GPT-NeoX","models","layer_norm","(","string",")",":","type","of","layer","normalization","standard\/rms","norm_eps","(","float",")",":","layer","norm","epsilon","use_ckpting","(","List",")",":","layers","for","which","we","checkpoint","for","backward","parallel_gpu","(","int",")",":","Number","of","gpu","for","tensor","parallelism","rotary_interleave","(","bool",")",":","Interleave","the","head","dimensions","when","rotary","embeddings","are","applied","rotary_theta","(","int",")",":","rotary","base","theta","rotary_dim","(","int",")",":","rotary","dim","when","different","to","dim","per","head"]
code: def __init__(
      self,
      d_model,
      heads,
      d_ff,
      dropout,
      attention_dropout,
      max_relative_positions=0,
      relative_positions_buckets=0,
      pos_ffn_activation_fn=ActivationFunction.relu,
      add_qkvbias=False,
      num_kv=0,
      add_ffnbias=True,
      parallel_residual=False,
      layer_norm="standard",
      norm_eps=1e-6,
      use_ckpting=[],
      parallel_gpu=1,
      rotary_interleave=True,
      rotary_theta=1e4,
      rotary_dim=0,
  ):
      super(TransformerEncoderLayer, self).__init__()
  
      self.self_attn = MultiHeadedAttention(
          heads,
          d_model,
          dropout=attention_dropout,
          is_decoder=False,
          max_relative_positions=max_relative_positions,
          relative_positions_buckets=relative_positions_buckets,
          rotary_interleave=rotary_interleave,
          rotary_theta=rotary_theta,
          rotary_dim=rotary_dim,
          attn_type="self",
          add_qkvbias=add_qkvbias,
          num_kv=num_kv,
          use_ckpting=use_ckpting,
          parallel_gpu=parallel_gpu,
      )
      self.feed_forward = PositionwiseFeedForward(
          d_model,
          d_ff,
          dropout,
          pos_ffn_activation_fn,
          add_ffnbias,
          parallel_residual,
          layer_norm,
          norm_eps,
          use_ckpting=use_ckpting,
          parallel_gpu=parallel_gpu,
      )
      self.parallel_residual = parallel_residual
      if layer_norm == "standard":
          self.layer_norm = nn.LayerNorm(d_model, eps=norm_eps)
      elif layer_norm == "rms":
          self.layer_norm = RMSNorm(d_model, eps=norm_eps)
      else:
          raise ValueError(
              f"{layer_norm} layer norm type is not supported"
          )
      self.dropout_p = dropout
      self.dropout = nn.Dropout(dropout)
  
file_path: OpenNMT-py/onmt/encoders/transformer.py
imports: import torch.nn
  from onmt.encoders.encoder import EncoderBase
  from onmt.modules import MultiHeadedAttention
  from onmt.modules.position_ffn import PositionwiseFeedForward
  from onmt.modules.position_ffn import ActivationFunction
  from onmt.utils.misc import sequence_mask
label_desc: Use image node_id 1 to create a new TransformerEncoderLayer object from inherited base classes: nn with example: obj = TransformerEncoderLayer(d_model, heads, d_ff, dropout, attention_dropout, max_relative_positions, relative_positions_buckets, pos_ffn_activation_fn, add_qkvbias, num_kv, add_ffnbias, parallel_residual, layer_norm, norm_eps, use_ckpting, parallel_gpu, rotary_interleave, rotary_theta, rotary_dim)