class_docstr: Transformer Decoder layer block in Pre-Norm style.
  Pre-Norm style is an improvement w.r.t. Original paper's Post-Norm style,
  providing better converge speed and performance. This is also the actual
  implementation in tensor2tensor and also avalable in fairseq.
  See https://tunz.kr/post/4 and :cite:`DeeperTransformer`.
class_docstr_tok: ["Transformer","Decoder","layer","block","in","Pre-Norm","style",".","Pre-Norm","style","is","an","improvement","w.r.t",".","Original","paper","'s","Post-Norm","style",",","providing","better","converge","speed","and","performance",".","This","is","also","the","actual","implementation","in","tensor2tensor","and","also","avalable","in","fairseq",".","See","https",":","\/\/tunz.kr\/post\/4","and",":","cite",":","`","DeeperTransformer","`","."]
docstr: Args:
      See TransformerDecoderLayerBase
docstr_tok: ["Args",":","See","TransformerDecoderLayerBase"]
code: def __init__(
      self,
      d_model,
      heads,
      d_ff,
      dropout,
      attention_dropout,
      self_attn_type="scaled-dot",
      max_relative_positions=0,
      relative_positions_buckets=0,
      aan_useffn=False,
      full_context_alignment=False,
      alignment_heads=0,
      pos_ffn_activation_fn=ActivationFunction.relu,
      add_qkvbias=False,
      num_kv=0,
      add_ffnbias=True,
      parallel_residual=False,
      shared_layer_norm=False,
      layer_norm="standard",
      norm_eps=1e-6,
      use_ckpting=[],
      parallel_gpu=1,
      sliding_window=0,
      rotary_interleave=True,
      rotary_theta=1e4,
      rotary_dim=0,
      num_experts=0,
      num_experts_per_tok=2,
  ):
      """
      Args:
          See TransformerDecoderLayerBase
      """
      super(TransformerDecoderLayer, self).__init__(
          d_model,
          heads,
          d_ff,
          dropout,
          attention_dropout,
          self_attn_type,
          max_relative_positions,
          relative_positions_buckets,
          aan_useffn,
          full_context_alignment,
          alignment_heads,
          pos_ffn_activation_fn=pos_ffn_activation_fn,
          add_qkvbias=add_qkvbias,
          num_kv=num_kv,
          add_ffnbias=add_ffnbias,
          parallel_residual=parallel_residual,
          shared_layer_norm=shared_layer_norm,
          layer_norm=layer_norm,
          norm_eps=norm_eps,
          use_ckpting=use_ckpting,
          parallel_gpu=parallel_gpu,
          sliding_window=sliding_window,
          rotary_interleave=rotary_interleave,
          rotary_theta=rotary_theta,
          rotary_dim=rotary_dim,
          num_experts=num_experts,
          num_experts_per_tok=num_experts_per_tok,
      )
      self.context_attn = MultiHeadedAttention(
          heads,
          d_model,
          dropout=attention_dropout,
          attn_type="context",
          self_attn_type=self.self_attn_type,
          add_qkvbias=add_qkvbias,
          num_kv=num_kv,
          use_ckpting=use_ckpting,
          parallel_gpu=parallel_gpu,
      )
      if layer_norm == "standard":
          self.layer_norm_2 = nn.LayerNorm(d_model, eps=norm_eps)
      elif layer_norm == "rms":
          self.layer_norm_2 = RMSNorm(d_model, eps=norm_eps)
      else:
          raise ValueError(
              f"{layer_norm} layer norm type is not supported"
          )
  
imports: import torch
  import torch.nn
  from onmt.decoders.decoder import DecoderBase
  from onmt.modules import MultiHeadedAttention, AverageAttention
  from onmt.modules.position_ffn import PositionwiseFeedForward
  from onmt.modules.position_ffn import ActivationFunction
  from onmt.modules.moe import MoE
  from onmt.utils.misc import sequence_mask
  from onmt.modules.rmsnorm import RMSNorm
label_desc: Use image node_id 1 to create a new TransformerDecoderLayer object from inherited base classes: TransformerDecoderLayerBase with example: obj = TransformerDecoderLayer(d_model, heads, d_ff, dropout, attention_dropout, _attn_type, max_relative_positions, relative_positions_buckets, aan_useffn, full_context_alignment, alignment_heads, pos_ffn_activation_fn, add_qkvbias, num_kv, add_ffnbias, parallel_residual, shared_layer_norm, layer_norm, norm_eps, use_ckpting, parallel_gpu, sliding_window, rotary_interleave, rotary_theta, rotary_dim, num_experts, num_experts_per_tok)