class_docstr: Transformer decoder model.
      
code: def __init__(
      self,
      config: ModelParallelConfig,
      init_method,
      output_layer_init_method,
      hidden_size,
      ffn_hidden_size,
      num_layers,
      num_attention_heads,
      apply_query_key_layer_scaling=True,
      kv_channels=None,
      layer_type=[],
      pre_process=True,
      post_process=True,
      hidden_dropout=0.1,
      attention_dropout=0.1,
      precision=16,
      fp32_residual_connection=False,
      activations_checkpoint_method=None,
      activations_checkpoint_num_layers=1,
      activations_checkpoint_granularity=None,
      layernorm_epsilon=1e-5,
      bias_activation_fusion=True,
      bias_dropout_add_fusion=True,
      masked_softmax_fusion=True,
      persist_layer_norm=False,
      openai_gelu=False,
      onnx_safe=False,
      activation="gelu",
      bias=True,
      normalization="layernorm",
      transformer_block_type="pre_ln",
      parent_model_type=ModelType.encoder_or_decoder,
      chunk_size=64,
      layer_number_offset=0,  # this is use only for attention norm_factor scaling
      normalize_attention_scores=True,
      megatron_legacy=False,
      turn_off_rop=False,
      version=1,  # model version
  ):
      super(MegatronRetrievalTransformerDecoderModule, self).__init__(
          config=config
      )
  
      self.pre_process = pre_process
      self.post_process = post_process
      self.hidden_size = hidden_size
      self.num_layers = num_layers
      self.init_method = init_method
      self.hidden_dropout = hidden_dropout
      self.output_layer_init_method = output_layer_init_method
      self.parent_model_type = parent_model_type
      self.turn_off_rop = turn_off_rop
      self.version = version
  
      if kv_channels is None:
          assert (
              hidden_size % num_attention_heads == 0
          ), "hidden_size must be divisible by num_attention_heads if kv_channels is None"
          kv_channels = hidden_size // num_attention_heads
  
      # Transformer.
      self.model = ParallelTransformer(
          config=config,
          init_method=self.init_method,
          output_layer_init_method=self.output_layer_init_method,
          num_layers=self.num_layers,
          hidden_size=self.hidden_size,
          num_attention_heads=num_attention_heads,
          apply_query_key_layer_scaling=apply_query_key_layer_scaling,
          kv_channels=kv_channels,
          layer_type=layer_type,
          ffn_hidden_size=ffn_hidden_size,
          self_attn_mask_type=AttnMaskType.padding,  # we use attention mask reset, enforce to use padding AttnMaskType, otherwise it has numeric issues
          pre_process=self.pre_process,
          post_process=self.post_process,
          precision=precision,
          fp32_residual_connection=fp32_residual_connection,
          activations_checkpoint_method=activations_checkpoint_method,
          activations_checkpoint_num_layers=activations_checkpoint_num_layers,
          activations_checkpoint_granularity=activations_checkpoint_granularity,
          layernorm_epsilon=layernorm_epsilon,
          hidden_dropout=hidden_dropout,
          attention_dropout=attention_dropout,
          bias_activation_fusion=bias_activation_fusion,
          bias_dropout_add_fusion=bias_dropout_add_fusion,
          masked_softmax_fusion=masked_softmax_fusion,
          persist_layer_norm=persist_layer_norm,
          openai_gelu=openai_gelu,
          onnx_safe=onnx_safe,
          activation=activation,
          bias=bias,
          normalization=normalization,
          transformer_block_type=transformer_block_type,
          model_type=parent_model_type,
          chunk_size=chunk_size,
          layer_number_offset=layer_number_offset,
          normalize_attention_scores=normalize_attention_scores,
          megatron_legacy=megatron_legacy,
      )
      rot_dim = (
          hidden_size // num_attention_heads
          if kv_channels is None
          else kv_channels
      )
      # partial rotary embeddings, which is better than full rotary
      # Wang and Komatsuzaki et al https://github.com/kingoflolz/mesh-transformer-jax/
      if not turn_off_rop:
          self.rotary_pos_emb = RotaryEmbedding(
              min(rot_dim, MIN_DIM_HEAD)
          )
      self.chunk_size = chunk_size
      self._model_key = "model"
  
imports: import torch
  import torch.nn.functional
  from einops import rearrange, repeat
  from nemo.collections.nlp.modules.common.megatron.module import MegatronModule
  from nemo.collections.nlp.modules.common.megatron.position_embedding import RotaryEmbedding
  from nemo.collections.nlp.modules.common.megatron.transformer import ParallelTransformer
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, build_attention_mask_3d
label_desc: Use image node_id 1 to create a new MegatronRetrievalTransformerDecoderModule object from inherited base classes: MegatronModule with example: obj = MegatronRetrievalTransformerDecoderModule(config, init_method, output_layer_init_method, hidden_size, ffn_hidden_size, num_layers, num_attention_heads, apply_query_key_layer_scaling, kv_channels, layer_type, pre_process, post_process, hidden_dropout, attention_dropout, precision, fp32_residual_connection, activations_checkpoint_method, activations_checkpoint_num_layers, activations_checkpoint_granularity, layernorm_epsilon, bias_activation_fusion, bias_dropout_add_fusion, masked_softmax_fusion, persist_layer_norm, openai_gelu, onnx_safe, activation, bias, normalization, transformer_block_type, parent_model_type, chunk_size, layer_number_offset, normalize_attention_scores, megatron_legacy, turn_off_rop, version)