code: def __init__(
      self,
      config: ModelParallelConfig,
      init_method,
      output_layer_init_method,
      layer_number,
      hidden_size,
      ffn_hidden_size,
      num_attention_heads,
      layer_type=LayerType.encoder,
      self_attn_mask_type=AttnMaskType.padding,
      fp32_residual_connection=False,
      precision=16,
      apply_query_key_layer_scaling=False,
      kv_channels=None,
      layernorm_epsilon=1e-5,
      hidden_dropout=0.1,
      bias_dropout_add_fusion=True,
      persist_layer_norm=False,
      bias_activation_fusion=True,
      openai_gelu=False,
      onnx_safe=False,
      masked_softmax_fusion=True,
      attention_dropout=0.1,
      ffn_dropout=0.0,
      activation="gelu",
      megatron_legacy=False,
      bias=True,
      chunk_size=64,
      normalization="layernorm",
      transformer_block_type="pre_ln",
      position_embedding_type="learned_absolute",
      multi_query_attention=False,
      headscale=False,
      activations_checkpoint_granularity=None,
      normalize_attention_scores=True,
      num_moe_experts=1,
      moe_frequency=1,
      moe_dropout=0.0,
      use_flash_attention=False,
  ):
      super(ParallelTransformerLayer, self).__init__(
          config=config,
          init_method=init_method,
          output_layer_init_method=output_layer_init_method,
          layer_number=layer_number,
          hidden_size=hidden_size,
          ffn_hidden_size=ffn_hidden_size,
          num_attention_heads=num_attention_heads,
          layer_type=layer_type,
          self_attn_mask_type=self_attn_mask_type,
          fp32_residual_connection=fp32_residual_connection,
          precision=precision,
          apply_query_key_layer_scaling=apply_query_key_layer_scaling,
          kv_channels=kv_channels,
          layernorm_epsilon=layernorm_epsilon,
          hidden_dropout=hidden_dropout,
          bias_dropout_add_fusion=bias_dropout_add_fusion,
          persist_layer_norm=persist_layer_norm,
          bias_activation_fusion=bias_activation_fusion,
          openai_gelu=openai_gelu,
          onnx_safe=onnx_safe,
          masked_softmax_fusion=masked_softmax_fusion,
          attention_dropout=attention_dropout,
          ffn_dropout=ffn_dropout,
          activation=activation,
          megatron_legacy=megatron_legacy,
          bias=bias,
          chunk_size=chunk_size,
          normalization=normalization,
          transformer_block_type=transformer_block_type,
          position_embedding_type=position_embedding_type,
          headscale=headscale,
          multi_query_attention=multi_query_attention,
          activations_checkpoint_granularity=activations_checkpoint_granularity,
          normalize_attention_scores=normalize_attention_scores,
          num_moe_experts=num_moe_experts,
          moe_frequency=moe_frequency,
          moe_dropout=moe_dropout,
          use_flash_attention=use_flash_attention,
      )
  
      # Dtype for forward pass - ignore amp O2
      self.dtype = utils_funcs.torch_dtype_from_precision(
          precision, megatron_amp_O2=None
      )
  
imports: from contextlib import nullcontext
  from typing import Any, Callable, Optional
  import torch
  import torch.nn
  from einops import rearrange
  from nemo.collections.common.parts.adapter_modules import LinearAdapterConfig
  from nemo.collections.nlp.modules.common.megatron.adapters.parallel_adapters import AdapterName, ParallelLinearAdapterConfig, ParallelLinearAdapterWeightTyingConfig
  from nemo.collections.nlp.modules.common.megatron.attention import ParallelAttention, ParallelChunkedCrossAttention
  from nemo.collections.nlp.modules.common.megatron.fused_bias_dropout_add import bias_dropout_add, bias_dropout_add_fused_inference, bias_dropout_add_fused_train, dropout_add
  from nemo.collections.nlp.modules.common.megatron.fused_layer_norm import get_layer_norm
  from nemo.collections.nlp.modules.common.megatron.layer_norm_1p import LayerNorm1P, LPLayerNorm
  from nemo.collections.nlp.modules.common.megatron.layer_type import LayerType
  from nemo.collections.nlp.modules.common.megatron.mlp import ParallelMLP, SwitchMLP
  from nemo.collections.nlp.modules.common.megatron.module import MegatronModule
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults
  from nemo.collections.nlp.parts import utils_funcs
  from nemo.core import adapter_mixins
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new ParallelTransformerLayer object from inherited base classes: ParallelTransformerLayer_ with example: obj = ParallelTransformerLayer(config, init_method, output_layer_init_method, layer_number, hidden_size, ffn_hidden_size, num_attention_heads, layer_type, _attn_mask_type, fp32_residual_connection, precision, apply_query_key_layer_scaling, kv_channels, layernorm_epsilon, hidden_dropout, bias_dropout_add_fusion, persist_layer_norm, bias_activation_fusion, openai_gelu, onnx_safe, masked_softmax_fusion, attention_dropout, ffn_dropout, activation, megatron_legacy, bias, chunk_size, normalization, transformer_block_type, position_embedding_type, multi_query_attention, headscale, activations_checkpoint_granularity, normalize_attention_scores, num_moe_experts, moe_frequency, moe_dropout, use_flash_attention)