class_docstr: Building block of Transformer decoder.
  
  Args:
      hidden_size: size of the embeddings in the model, also known as d_model
      inner_size: number of neurons in the intermediate part of feed-forward
          net, usually is (4-8 x hidden_size) in the papers
      num_attention_heads: number of heads in multi-head attention
      attn_score_dropout: probability of dropout applied to attention scores
      attn_layer_dropout: probability of dropout applied to the output of the
          attention layers, but before layer normalization
      ffn_dropout: probability of dropout applied to FFN output
      hidden_act: activation function used between two linear layers in FFN
class_docstr_tok: ["Building","block","of","Transformer","decoder",".","Args",":","hidden_size",":","size","of","the","embeddings","in","the","model",",","also","known","as","d_model","inner_size",":","number","of","neurons","in","the","intermediate","part","of","feed-forward","net",",","usually","is","(","4-8","x","hidden_size",")","in","the","papers","num_attention_heads",":","number","of","heads","in","multi-head","attention","attn_score_dropout",":","probability","of","dropout","applied","to","attention","scores","attn_layer_dropout",":","probability","of","dropout","applied","to","the","output","of","the","attention","layers",",","but","before","layer","normalization","ffn_dropout",":","probability","of","dropout","applied","to","FFN","output","hidden_act",":","activation","function","used","between","two","linear","layers","in","FFN"]
code: def __init__(
      self,
      hidden_size: int,
      inner_size: int,
      num_attention_heads: int = 1,
      attn_score_dropout: float = 0.0,
      attn_layer_dropout: float = 0.0,
      ffn_dropout: float = 0.0,
      hidden_act: str = "relu",
      pre_ln: bool = False,
  ):
      super().__init__()
      self.pre_ln = pre_ln
      self.layer_norm_1 = nn.LayerNorm(hidden_size, eps=1e-5)
      self.first_sub_layer = MultiHeadAttention(
          hidden_size,
          num_attention_heads,
          attn_score_dropout,
          attn_layer_dropout,
      )
      self.layer_norm_2 = nn.LayerNorm(hidden_size, eps=1e-5)
      self.second_sub_layer = MultiHeadAttention(
          hidden_size,
          num_attention_heads,
          attn_score_dropout,
          attn_layer_dropout,
      )
      self.layer_norm_3 = nn.LayerNorm(hidden_size, eps=1e-5)
      self.third_sub_layer = PositionWiseFF(
          hidden_size, inner_size, ffn_dropout, hidden_act
      )
  
file_path: NeMo/nemo/collections/nlp/modules/common/transformer/transformer_decoders.py
imports: import copy
  import torch
  import torch.nn
  from nemo.collections.common.parts import form_attention_mask
  from nemo.collections.nlp.modules.common.transformer.transformer_modules import MultiHeadAttention, PositionWiseFF
label_desc: Use image node_id 1 to create a new TransformerDecoderBlock object from inherited base classes: nn with example: obj = TransformerDecoderBlock(hidden_size, inner_size, num_attention_heads, attn_score_dropout, attn_layer_dropout, ffn_dropout, hidden_act, pre_ln)