class_docstr: ## Transformer Decoder Layer with DeepNorm
  
  This implements a transformer decoder layer with DeepNorm.
  Encoder layers will have a similar form.
class_docstr_tok: ["#","#","Transformer","Decoder","Layer","with","DeepNorm","This","implements","a","transformer","decoder","layer","with","DeepNorm",".","Encoder","layers","will","have","a","similar","form","."]
docstr: :param d_model: is the token embedding size
  :param self_attn: is the self attention module
  :param feed_forward: is the feed forward module
  :param deep_norm_alpha: is $lpha$ coefficient in DeepNorm
  :param deep_norm_beta: is $eta$ constant for scaling weights initialization
docstr_tok: [":","param","d_model",":","is","the","token","embedding","size",":","param","self_attn",":","is","the","self","attention","module",":","param","feed_forward",":","is","the","feed","forward","module",":","param","deep_norm_alpha",":","is","$","\u0007lpha","$","coefficient","in","DeepNorm",":","param","deep_norm_beta",":","is","$","\beta","$","constant","for","scaling","weights","initialization"]
code: def __init__(
      self,
      *,
      d_model: int,
      self_attn: MultiHeadAttention,
      feed_forward: FeedForward,
      deep_norm_alpha: float,
      deep_norm_beta: float,
  ):
      """
      :param d_model: is the token embedding size
      :param self_attn: is the self attention module
      :param feed_forward: is the feed forward module
      :param deep_norm_alpha: is $\alpha$ coefficient in DeepNorm
      :param deep_norm_beta: is $\beta$ constant for scaling weights initialization
      """
      super().__init__()
  
      self.self_attn = self_attn
      self.feed_forward = feed_forward
      # DeepNorms after attention and feed forward network
      self.self_attn_norm = DeepNorm(deep_norm_alpha, [d_model])
      self.feed_forward_norm = DeepNorm(deep_norm_alpha, [d_model])
  
      # Scale weights after initialization
      with torch.no_grad():
          # Feed forward network linear transformations
          feed_forward.layer1.weight *= deep_norm_beta
          feed_forward.layer2.weight *= deep_norm_beta
  
          # Attention value projection
          self_attn.value.linear.weight *= deep_norm_beta
          # Attention output project
          self_attn.output.weight *= deep_norm_beta
  
      # The mask will be initialized on the first call
      self.mask = None
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/normalization/deep_norm/__init__.py
imports: from typing import Union, List
  import torch
  from torch import nn, Size
  from labml_nn.normalization.layer_norm import LayerNorm
  from labml_nn.transformers import MultiHeadAttention
  from labml_nn.transformers.feed_forward import FeedForward
  from labml_nn.transformers.utils import subsequent_mask
label_desc: Use image node_id 1 to create a new DeepNormTransformerLayer object from inherited base classes: nn with example: obj = DeepNormTransformerLayer()