class_docstr: Transformer Decoder layer block in Pre-Norm style.
  Pre-Norm style is an improvement w.r.t. Original paper's Post-Norm style,
  providing better converge speed and performance. This is also the actual
  implementation in tensor2tensor and also avalable in fairseq.
  See https://tunz.kr/post/4 and :cite:`DeeperTransformer`.
class_docstr_tok: ["Transformer","Decoder","layer","block","in","Pre-Norm","style",".","Pre-Norm","style","is","an","improvement","w.r.t",".","Original","paper","'s","Post-Norm","style",",","providing","better","converge","speed","and","performance",".","This","is","also","the","actual","implementation","in","tensor2tensor","and","also","avalable","in","fairseq",".","See","https",":","\/\/tunz.kr\/post\/4","and",":","cite",":","`","DeeperTransformer","`","."]
docstr: A naive forward pass for transformer decoder.
  
  # T: could be 1 in the case of stepwise decoding or tgt_len
  
  Args:
      layer_in (FloatTensor): ``(batch_size, T, model_dim)``
      enc_out (FloatTensor): ``(batch_size, src_len, model_dim)``
      src_pad_mask (bool): ``(batch_size, 1, src_len)``
      tgt_pad_mask (bool): ``(batch_size, 1, T)``
      step (int or None): stepwise decoding counter
      future (bool): If set True, do not apply future_mask.
      return_attn (bool) : if set True requires attns output
  
  Returns:
      (FloatTensor, FloatTensor):
  
      * layer_out ``(batch_size, T, model_dim)``
      * attns ``(batch_size, head, T, src_len)``
docstr_tok: ["A","naive","forward","pass","for","transformer","decoder",".","#","T",":","could","be","1","in","the","case","of","stepwise","decoding","or","tgt_len","Args",":","layer_in","(","FloatTensor",")",":","``","(","batch_size",",","T",",","model_dim",")","``","enc_out","(","FloatTensor",")",":","``","(","batch_size",",","src_len",",","model_dim",")","``","src_pad_mask","(","bool",")",":","``","(","batch_size",",","1",",","src_len",")","``","tgt_pad_mask","(","bool",")",":","``","(","batch_size",",","1",",","T",")","``","step","(","int","or","None",")",":","stepwise","decoding","counter","future","(","bool",")",":","If","set","True",",","do","not","apply","future_mask",".","return_attn","(","bool",")",":","if","set","True","requires","attns","output","Returns",":","(","FloatTensor",",","FloatTensor",")",":","*","layer_out","``","(","batch_size",",","T",",","model_dim",")","``","*","attns","``","(","batch_size",",","head",",","T",",","src_len",")","``"]
code: def _forward(
      self,
      layer_in,
      enc_out,
      src_pad_mask,
      tgt_pad_mask,
      step=None,
      future=False,
      return_attn=False,
  ):
      """A naive forward pass for transformer decoder.
  
      # T: could be 1 in the case of stepwise decoding or tgt_len
  
      Args:
          layer_in (FloatTensor): ``(batch_size, T, model_dim)``
          enc_out (FloatTensor): ``(batch_size, src_len, model_dim)``
          src_pad_mask (bool): ``(batch_size, 1, src_len)``
          tgt_pad_mask (bool): ``(batch_size, 1, T)``
          step (int or None): stepwise decoding counter
          future (bool): If set True, do not apply future_mask.
          return_attn (bool) : if set True requires attns output
  
      Returns:
          (FloatTensor, FloatTensor):
  
          * layer_out ``(batch_size, T, model_dim)``
          * attns ``(batch_size, head, T, src_len)``
  
      """
      dec_mask = None
      src_pad_mask = src_pad_mask.unsqueeze(1)  # [B,1,1,slen]
  
      if layer_in.size(1) > 1:
          # masking is necessary when sequence length is greater than one
          dec_mask = self._compute_dec_mask(tgt_pad_mask, future)
          dec_mask = dec_mask.unsqueeze(1)
          dec_mask = dec_mask.expand(-1, -1, dec_mask.size(3), -1)
          src_pad_mask = src_pad_mask.expand(
              -1, -1, dec_mask.size(3), -1
          )
          # mask now are (batch x 1 x tlen x s or t len)
          # 1 = heads to be expanded in MHA
  
      norm_layer_in = self.layer_norm_1(layer_in)
  
      self_attn, _ = self._forward_self_attn(
          norm_layer_in, dec_mask, step, return_attn=return_attn
      )
      if self.dropout_p > 0:
          self_attn = self.dropout(self_attn)
      if self.parallel_residual:
          ctx_attn, attns = self.context_attn(
              enc_out,
              enc_out,
              norm_layer_in,
              mask=src_pad_mask,
              return_attn=return_attn,
          )
          # feed_forward applies residual, so we remove and apply residual with un-normed
          layer_out = (
              self.feed_forward(norm_layer_in)
              - norm_layer_in
              + layer_in
              + self_attn
              + ctx_attn
          )
      else:
          query = self_attn + layer_in
          norm_query = self.layer_norm_2(query)
          ctx_attn, attns = self.context_attn(
              enc_out,
              enc_out,
              norm_query,
              mask=src_pad_mask,
              return_attn=return_attn,
          )
          if self.dropout_p > 0:
              ctx_attn = self.dropout(ctx_attn)
          layer_out = self.feed_forward(ctx_attn + query)
  
      return layer_out, attns
  
imports: import torch
  import torch.nn
  from onmt.decoders.decoder import DecoderBase
  from onmt.modules import MultiHeadedAttention, AverageAttention
  from onmt.modules.position_ffn import PositionwiseFeedForward
  from onmt.modules.position_ffn import ActivationFunction
  from onmt.modules.moe import MoE
  from onmt.utils.misc import sequence_mask
  from onmt.modules.rmsnorm import RMSNorm
label_desc: Use image node_id 3 for calling the TransformerDecoderLayer obj's underlying member method code with example usage: obj._forward(layer_in, enc_out, src_pad_mask, tgt_pad_mask, step, future, return_attn) and returns: layer_out,  attns