class_docstr: Decoder layer block.
  
  In the original paper each operation (multi-head attention, encoder
  attention or FFN) is postprocessed with: `dropout -> add residual ->
  layernorm`. In the tensor2tensor code they suggest that learning is more
  robust when preprocessing each layer with layernorm and postprocessing with:
  `dropout -> add residual`. We default to the approach in the paper, but the
  tensor2tensor approach can be enabled by setting
  *args.decoder_normalize_before* to ``True``.
  
  Args:
      args (argparse.Namespace): parsed command-line arguments
      no_encoder_attn (bool, optional): whether to attend to encoder outputs
          (default: False).
class_docstr_tok: ["Decoder","layer","block",".","In","the","original","paper","each","operation","(","multi-head","attention",",","encoder","attention","or","FFN",")","is","postprocessed","with",":","`","dropout","-",">","add","residual","-",">","layernorm","`",".","In","the","tensor2tensor","code","they","suggest","that","learning","is","more","robust","when","preprocessing","each","layer","with","layernorm","and","postprocessing","with",":","`","dropout","-",">","add","residual","`",".","We","default","to","the","approach","in","the","paper",",","but","the","tensor2tensor","approach","can","be","enabled","by","setting","*","args.decoder_normalize_before","*","to","``","True","``",".","Args",":","args","(","argparse.Namespace",")",":","parsed","command-line","arguments","no_encoder_attn","(","bool",",","optional",")",":","whether","to","attend","to","encoder","outputs","(","default",":","False",")","."]
docstr: Args:
      input (Tuple):
          input[0] (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)`
          input[1] (Tensor): encoder output of shape `(batch, src_len, embed_dim)`
          input[2] (ByteTensor/FloatTensor): encoder padding mask -
              binary ByteTensor of shape `(batch, src_len)` where padding elements
              are indicated by ``1``.
  Returns:
      output (Tuple):
          output[0] (Tensor): encoded output of shape `(batch, src_len, embed_dim)`
          output[1] (ByteTensor/FloatTensor): encoder padding mask
          output[2] (LongTensor): previous decoder outputs
docstr_tok: ["Args",":","input","(","Tuple",")",":","input","[","0","]","(","Tensor",")",":","input","to","the","layer","of","shape","`","(","seq_len",",","batch",",","embed_dim",")","`","input","[","1","]","(","Tensor",")",":","encoder","output","of","shape","`","(","batch",",","src_len",",","embed_dim",")","`","input","[","2","]","(","ByteTensor\/FloatTensor",")",":","encoder","padding","mask","-","binary","ByteTensor","of","shape","`","(","batch",",","src_len",")","`","where","padding","elements","are","indicated","by","``","1","``",".","Returns",":","output","(","Tuple",")",":","output","[","0","]","(","Tensor",")",":","encoded","output","of","shape","`","(","batch",",","src_len",",","embed_dim",")","`","output","[","1","]","(","ByteTensor\/FloatTensor",")",":","encoder","padding","mask","output","[","2","]","(","LongTensor",")",":","previous","decoder","outputs"]
code: def forward(self, input):
      """
      Args:
          input (Tuple):
              input[0] (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)`
              input[1] (Tensor): encoder output of shape `(batch, src_len, embed_dim)`
              input[2] (ByteTensor/FloatTensor): encoder padding mask -
                  binary ByteTensor of shape `(batch, src_len)` where padding elements
                  are indicated by ``1``.
      Returns:
          output (Tuple):
              output[0] (Tensor): encoded output of shape `(batch, src_len, embed_dim)`
              output[1] (ByteTensor/FloatTensor): encoder padding mask
              output[2] (LongTensor): previous decoder outputs
      """
      # Note: incremental state is not yet supported
      mt_task = False
      if isinstance(input, tuple):
          x = input[0]
          encoder_out = input[1]
          encoder_padding_mask = input[2]
          incremental_state = None
          mt_task = True
      else:
          x = input
          encoder_out = None
          encoder_padding_mask = None
          incremental_state = None
  
      if incremental_state is None:
          self_attn_mask = self.buffered_future_mask(x)
      else:
          self_attn_mask = None
  
      # TODO: add back prev_self_attn_state, prev_attn_state,
      # self_attn_padding_mask
      prev_self_attn_state = None
      prev_attn_state = None
      self_attn_padding_mask = None
  
      residual = x
      x = self.maybe_layer_norm(
          self.self_attn_layer_norm, x, before=True
      )
      if prev_self_attn_state is not None:
          if incremental_state is None:
              incremental_state = {}
          prev_key, prev_value = prev_self_attn_state
          saved_state = {"prev_key": prev_key, "prev_value": prev_value}
          self.self_attn._set_input_buffer(
              incremental_state, saved_state
          )
      x, attn = self.self_attn(
          query=x,
          key=x,
          value=x,
          key_padding_mask=self_attn_padding_mask,
          incremental_state=incremental_state,
          need_weights=False,
          attn_mask=self_attn_mask,
      )
      x = F.dropout(x, p=self.dropout, training=self.training)
      x = residual + x
      x = self.maybe_layer_norm(
          self.self_attn_layer_norm, x, after=True
      )
  
      if self.encoder_attn is not None:
          residual = x
          x = self.maybe_layer_norm(
              self.encoder_attn_layer_norm, x, before=True
          )
          if prev_attn_state is not None:
              if incremental_state is None:
                  incremental_state = {}
              prev_key, prev_value = prev_attn_state
              saved_state = {
                  "prev_key": prev_key,
                  "prev_value": prev_value,
              }
              self.encoder_attn._set_input_buffer(
                  incremental_state, saved_state
              )
          x, attn = self.encoder_attn(
              query=x,
              key=encoder_out,
              value=encoder_out,
              key_padding_mask=encoder_padding_mask,
              incremental_state=incremental_state,
              static_kv=True,
              need_weights=(not self.training and self.need_attn),
          )
          x = F.dropout(x, p=self.dropout, training=self.training)
          x = residual + x
          x = self.maybe_layer_norm(
              self.encoder_attn_layer_norm, x, after=True
          )
  
      residual = x
      x = self.maybe_layer_norm(self.final_layer_norm, x, before=True)
      x = self.activation_fn(self.fc1(x))
      x = F.dropout(
          x, p=self.activation_dropout, training=self.training
      )
      x = self.fc2(x)
      x = F.dropout(x, p=self.dropout, training=self.training)
      x = residual + x
      x = self.maybe_layer_norm(self.final_layer_norm, x, after=True)
  
      if mt_task:
          return (x, encoder_out, encoder_padding_mask)
      return x
  
imports: import math
  from collections import namedtuple
  import torch
  import torch.nn
  import torch.nn.functional
  from fairseq import options, utils
  from fairseq.modules import AdaptiveSoftmax, LayerNorm, MultiheadAttention, PositionalEmbedding
label_desc: Use image node_id 3 for calling the TransformerDecoderLayer obj's underlying member method code with example usage: obj.forward(input) and returns: x, x,  encoder_out,  encoder_padding_mask