class_docstr: Transformer decoder model.
      
class_docstr_tok: ["Transformer","decoder","model","."]
code: def __init__(
      self,
      config: ModelParallelConfig,
      init_method,
      output_layer_init_method,
      hidden_size,
      ffn_hidden_size,
      num_layers,
      num_attention_heads,
      apply_query_key_layer_scaling=True,
      kv_channels=None,
      pre_process=True,
      post_process=True,
      decoder_attn_mask_type=AttnMaskType.causal,
      hidden_dropout=0.1,
      attention_dropout=0.1,
      ffn_dropout=0.0,
      precision=16,
      fp32_residual_connection=False,
      activations_checkpoint_method=None,
      activations_checkpoint_num_layers=1,
      activations_checkpoint_granularity=None,
      layernorm_epsilon=1e-5,
      bias_activation_fusion=True,
      bias_dropout_add_fusion=True,
      masked_softmax_fusion=True,
      persist_layer_norm=False,
      openai_gelu=False,
      onnx_safe=False,
      activation="gelu",
      bias=True,
      normalization="layernorm",
      transformer_block_type="pre_ln",
      headscale=False,
      parent_model_type=ModelType.encoder_or_decoder,
      megatron_legacy=False,
      normalize_attention_scores=True,
      num_moe_experts=1,
      moe_frequency=1,
      moe_dropout=0.0,
      position_embedding_type="learned_absolute",
      use_flash_attention=False,
  ):
      super(MegatronTransformerDecoderModule, self).__init__(
          config=config
      )
  
      self.pre_process = pre_process
      self.post_process = post_process
      self.hidden_size = hidden_size
      self.num_layers = num_layers
      self.init_method = init_method
      self.model_attn_mask_type = decoder_attn_mask_type
      self.hidden_dropout = hidden_dropout
      self.output_layer_init_method = output_layer_init_method
      self.parent_model_type = parent_model_type
      self.normalization = normalization
  
      if kv_channels is None:
          assert (
              hidden_size % num_attention_heads == 0
          ), "hidden_size must be divisible by num_attention_heads if kv_channels is None"
          kv_channels = hidden_size // num_attention_heads
  
      # Transformer.
      self.model = ParallelTransformer(
          config=config,
          layer_type=LayerType.decoder,
          init_method=self.init_method,
          output_layer_init_method=self.output_layer_init_method,
          num_layers=self.num_layers,
          hidden_size=self.hidden_size,
          num_attention_heads=num_attention_heads,
          apply_query_key_layer_scaling=apply_query_key_layer_scaling,
          kv_channels=kv_channels,
          ffn_hidden_size=ffn_hidden_size,
          self_attn_mask_type=self.model_attn_mask_type,
          pre_process=self.pre_process,
          post_process=self.post_process,
          precision=precision,
          fp32_residual_connection=fp32_residual_connection,
          activations_checkpoint_method=activations_checkpoint_method,
          activations_checkpoint_num_layers=activations_checkpoint_num_layers,
          activations_checkpoint_granularity=activations_checkpoint_granularity,
          layernorm_epsilon=layernorm_epsilon,
          hidden_dropout=hidden_dropout,
          attention_dropout=attention_dropout,
          ffn_dropout=ffn_dropout,
          bias_activation_fusion=bias_activation_fusion,
          bias_dropout_add_fusion=bias_dropout_add_fusion,
          masked_softmax_fusion=masked_softmax_fusion,
          persist_layer_norm=persist_layer_norm,
          openai_gelu=openai_gelu,
          onnx_safe=onnx_safe,
          activation=activation,
          bias=bias,
          normalization=normalization,
          model_type=parent_model_type,
          transformer_block_type=transformer_block_type,
          headscale=headscale,
          megatron_legacy=megatron_legacy,
          normalize_attention_scores=normalize_attention_scores,
          num_moe_experts=num_moe_experts,
          moe_frequency=moe_frequency,
          moe_dropout=moe_dropout,
          position_embedding_type=position_embedding_type,
          use_flash_attention=use_flash_attention,
      )
      self._model_key = "model"
  
file_path: NeMo/nemo/collections/nlp/modules/common/megatron/megatron_transformer_decoder.py
imports: from nemo.collections.nlp.modules.common.megatron.layer_type import LayerType
  from nemo.collections.nlp.modules.common.megatron.megatron_decoder_module import MegatronDecoderModule
  from nemo.collections.nlp.modules.common.megatron.module import MegatronModule
  from nemo.collections.nlp.modules.common.megatron.transformer import ParallelTransformer
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, attn_mask_postprocess, build_attention_mask_3d
  from nemo.core.classes.exportable import Exportable
label_desc: Use image node_id 1 to create a new MegatronTransformerDecoderModule object from inherited base classes: MegatronModule, Exportable, MegatronDecoderModule with example: obj = MegatronTransformerDecoderModule(config, init_method, output_layer_init_method, hidden_size, ffn_hidden_size, num_layers, num_attention_heads, apply_query_key_layer_scaling, kv_channels, pre_process, post_process, decoder_attn_mask_type, hidden_dropout, attention_dropout, ffn_dropout, precision, fp32_residual_connection, activations_checkpoint_method, activations_checkpoint_num_layers, activations_checkpoint_granularity, layernorm_epsilon, bias_activation_fusion, bias_dropout_add_fusion, masked_softmax_fusion, persist_layer_norm, openai_gelu, onnx_safe, activation, bias, normalization, transformer_block_type, headscale, parent_model_type, megatron_legacy, normalize_attention_scores, num_moe_experts, moe_frequency, moe_dropout, position_embedding_type, use_flash_attention)