class_docstr: Transformer encoder model.
      
class_docstr_tok: ["Transformer","encoder","model","."]
code: def __init__(
      self,
      config: ModelParallelConfig,
      init_method,
      output_layer_init_method,
      hidden_size,
      ffn_hidden_size,
      num_layers,
      num_attention_heads,
      apply_query_key_layer_scaling=True,
      kv_channels=None,
      pre_process=True,
      post_process=True,
      encoder_attn_mask_type=AttnMaskType.padding,
      hidden_dropout=0.1,
      attention_dropout=0.1,
      ffn_dropout=0.0,
      precision=16,
      fp32_residual_connection=False,
      activations_checkpoint_method=None,
      activations_checkpoint_num_layers=1,
      activations_checkpoint_granularity=None,
      layernorm_epsilon=1e-5,
      bias_activation_fusion=True,
      bias_dropout_add_fusion=True,
      masked_softmax_fusion=True,
      persist_layer_norm=False,
      openai_gelu=False,
      onnx_safe=False,
      activation="gelu",
      bias=True,
      normalization="layernorm",
      transformer_block_type="pre_ln",
      headscale=False,
      parent_model_type=ModelType.encoder_or_decoder,
      hidden_steps=32,
      num_self_attention_per_cross_attention=1,
      normalize_attention_scores=True,
      megatron_legacy=False,
  ):
      super(MegatronPerceiverEncoderModule, self).__init__(
          config=config
      )
  
      self.config = config
      self.pre_process = pre_process
      self.post_process = post_process
      self.hidden_size = hidden_size
      self.num_layers = num_layers
      self.init_method = init_method
      self.model_attn_mask_type = encoder_attn_mask_type
      self.hidden_dropout = hidden_dropout
      self.output_layer_init_method = output_layer_init_method
      self.parent_model_type = parent_model_type
      self.normalization = normalization
      self.transformer_block_type = transformer_block_type
      self.hidden_steps = hidden_steps
      self.num_self_attention_per_cross_attention = (
          num_self_attention_per_cross_attention
      )
      self.num_attention_heads = num_attention_heads
      self.apply_query_key_layer_scaling = apply_query_key_layer_scaling
      self.kv_channels = kv_channels
      self.ffn_hidden_size = ffn_hidden_size
      self.precision = precision
      self.fp32_residual_connection = fp32_residual_connection
      self.activations_checkpoint_method = activations_checkpoint_method
      self.activations_checkpoint_num_layers = (
          activations_checkpoint_num_layers
      )
      self.activations_checkpoint_granularity = (
          activations_checkpoint_granularity
      )
      self.layernorm_epsilon = layernorm_epsilon
      self.bias_activation_fusion = bias_activation_fusion
      self.bias_dropout_add_fusion = bias_dropout_add_fusion
      self.masked_softmax_fusion = masked_softmax_fusion
      self.persist_layer_norm = persist_layer_norm
      self.openai_gelu = openai_gelu
      self.onnx_safe = onnx_safe
      self.activation = activation
      self.bias = bias
      self.headscale = headscale
      self.hidden_dropout = hidden_dropout
      self.attention_dropout = attention_dropout
      self.normalization = normalization
      self.parent_model_type = parent_model_type
      self.transformer_block_type = transformer_block_type
      self.ffn_dropout = ffn_dropout
      self.normalize_attention_scores = normalize_attention_scores
      self.megatron_legacy = megatron_legacy
  
      assert self.num_self_attention_per_cross_attention >= 1
      assert self.hidden_steps >= 1
  
      self.init_hidden = torch.nn.Parameter(
          torch.nn.init.xavier_normal_(
              torch.empty(hidden_steps, hidden_size)
          )
      )
  
      self.cross_attn_layers = torch.nn.ModuleList(
          [
              self._build_cross_attn_layer()
              for _ in range(self.num_layers)
          ]
      )
      self.self_attn_layers = torch.nn.ModuleList(
          [
              self._build_self_attn_layer()
              for _ in range(
                  self.num_layers
                  * self.num_self_attention_per_cross_attention
              )
          ]
      )
      if normalization == "layernorm":
          self.final_layernorm = get_layer_norm(
              hidden_size, layernorm_epsilon, persist_layer_norm
          )
      else:
          self.final_layernorm = MixedFusedRMSNorm(
              hidden_size, layernorm_epsilon
          )
  
file_path: NeMo/nemo/collections/nlp/modules/common/megatron/megatron_perceiver_encoders.py
imports: import torch
  from nemo.collections.nlp.modules.common.megatron.fused_layer_norm import get_layer_norm
  from nemo.collections.nlp.modules.common.megatron.layer_type import LayerType
  from nemo.collections.nlp.modules.common.megatron.module import MegatronModule
  from nemo.collections.nlp.modules.common.megatron.transformer import ParallelTransformer
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, attn_mask_postprocess, build_attention_mask_3d
label_desc: Use image node_id 1 to create a new MegatronPerceiverEncoderModule object from inherited base classes: MegatronModule with example: obj = MegatronPerceiverEncoderModule(config, init_method, output_layer_init_method, hidden_size, ffn_hidden_size, num_layers, num_attention_heads, apply_query_key_layer_scaling, kv_channels, pre_process, post_process, encoder_attn_mask_type, hidden_dropout, attention_dropout, ffn_dropout, precision, fp32_residual_connection, activations_checkpoint_method, activations_checkpoint_num_layers, activations_checkpoint_granularity, layernorm_epsilon, bias_activation_fusion, bias_dropout_add_fusion, masked_softmax_fusion, persist_layer_norm, openai_gelu, onnx_safe, activation, bias, normalization, transformer_block_type, headscale, parent_model_type, hidden_steps, num__attention_per_cross_attention, normalize_attention_scores, megatron_legacy)