class_docstr: A single layer of the transformer encoder.
  
  Args:
      d_model (int): the dimension of keys/values/queries in
                 MultiHeadedAttention, also the input size of
                 the first-layer of the PositionwiseFeedForward.
      heads (int): the number of head for MultiHeadedAttention.
      d_ff (int): the second-layer of the PositionwiseFeedForward.
      dropout (float): dropout probability(0-1.0).
      pos_ffn_activation_fn (ActivationFunction):
          activation function choice for PositionwiseFeedForward layer
      add_qkvbias (bool): whether to add bias to the Key/Value nn.Linear
      num_kv (int): number of heads for KV when different vs Q (multiquery)
      add_ffnbias (bool): whether to add bias to the FF nn.Linear
      parallel_residual (bool): Use parallel residual connections in each layer block, as used
          by the GPT-J and GPT-NeoX models
      layer_norm (string): type of layer normalization standard/rms
      norm_eps (float): layer norm epsilon
      use_ckpting (List): layers for which we checkpoint for backward
      parallel_gpu (int): Number of gpu for tensor parallelism
      rotary_interleave (bool): Interleave the head dimensions when rotary
          embeddings are applied
      rotary_theta (int): rotary base theta
      rotary_dim (int): rotary dim when different to dim per head
class_docstr_tok: ["A","single","layer","of","the","transformer","encoder",".","Args",":","d_model","(","int",")",":","the","dimension","of","keys\/values\/queries","in","MultiHeadedAttention",",","also","the","input","size","of","the","first-layer","of","the","PositionwiseFeedForward",".","heads","(","int",")",":","the","number","of","head","for","MultiHeadedAttention",".","d_ff","(","int",")",":","the","second-layer","of","the","PositionwiseFeedForward",".","dropout","(","float",")",":","dropout","probability","(","0-1.0",")",".","pos_ffn_activation_fn","(","ActivationFunction",")",":","activation","function","choice","for","PositionwiseFeedForward","layer","add_qkvbias","(","bool",")",":","whether","to","add","bias","to","the","Key\/Value","nn.Linear","num_kv","(","int",")",":","number","of","heads","for","KV","when","different","vs","Q","(","multiquery",")","add_ffnbias","(","bool",")",":","whether","to","add","bias","to","the","FF","nn.Linear","parallel_residual","(","bool",")",":","Use","parallel","residual","connections","in","each","layer","block",",","as","used","by","the","GPT-J","and","GPT-NeoX","models","layer_norm","(","string",")",":","type","of","layer","normalization","standard\/rms","norm_eps","(","float",")",":","layer","norm","epsilon","use_ckpting","(","List",")",":","layers","for","which","we","checkpoint","for","backward","parallel_gpu","(","int",")",":","Number","of","gpu","for","tensor","parallelism","rotary_interleave","(","bool",")",":","Interleave","the","head","dimensions","when","rotary","embeddings","are","applied","rotary_theta","(","int",")",":","rotary","base","theta","rotary_dim","(","int",")",":","rotary","dim","when","different","to","dim","per","head"]
docstr: Args:
      layer_in (FloatTensor): ``(batch_size, src_len, model_dim)``
      mask (LongTensor): ``(batch_size, 1, src_len)``
  
  Returns:
      (FloatTensor):
      * layer_out ``(batch_size, src_len, model_dim)``
docstr_tok: ["Args",":","layer_in","(","FloatTensor",")",":","``","(","batch_size",",","src_len",",","model_dim",")","``","mask","(","LongTensor",")",":","``","(","batch_size",",","1",",","src_len",")","``","Returns",":","(","FloatTensor",")",":","*","layer_out","``","(","batch_size",",","src_len",",","model_dim",")","``"]
code: def forward(self, layer_in, mask):
      """
      Args:
          layer_in (FloatTensor): ``(batch_size, src_len, model_dim)``
          mask (LongTensor): ``(batch_size, 1, src_len)``
  
      Returns:
          (FloatTensor):
          * layer_out ``(batch_size, src_len, model_dim)``
      """
      norm_layer_in = self.layer_norm(layer_in)
      context, _ = self.self_attn(
          norm_layer_in, norm_layer_in, norm_layer_in, mask=mask
      )
      if self.dropout_p > 0:
          context = self.dropout(context)
      if self.parallel_residual:
          # feed_forward applies residual, so we remove and apply residual with un-normed
          layer_out = (
              self.feed_forward(norm_layer_in)
              - norm_layer_in
              + layer_in
              + context
          )
      else:
          layer_out = context + layer_in
          layer_out = self.feed_forward(layer_out)
  
      return layer_out
  
file_path: OpenNMT-py/onmt/encoders/transformer.py
imports: import torch.nn
  from onmt.encoders.encoder import EncoderBase
  from onmt.modules import MultiHeadedAttention
  from onmt.modules.position_ffn import PositionwiseFeedForward
  from onmt.modules.position_ffn import ActivationFunction
  from onmt.utils.misc import sequence_mask
label_desc: Use image node_id 2 for calling the TransformerEncoderLayer obj's underlying member method code with example usage: obj.forward(layer_in, mask) and returns: layer_out