class_docstr: A single transformer layer.
  
  Transformer layer takes input with size [s, b, h] and returns an
  output of the same size.
class_docstr_tok: ["A","single","transformer","layer",".","Transformer","layer","takes","input","with","size","[","s",",","b",",","h","]","and","returns","an","output","of","the","same","size","."]
code: def __init__(
      self,
      config,
      init_method,
      output_layer_init_method,
      layer_number,
      hidden_size,
      ffn_hidden_size,
      num_attention_heads,
      layer_type=LayerType.encoder,
      self_attn_mask_type=AttnMaskType.padding,
      fp32_residual_connection=False,
      precision=16,
      apply_query_key_layer_scaling=True,
      kv_channels=None,
      layernorm_epsilon=1e-5,
      hidden_dropout=0.1,
      bias_dropout_add_fusion=True,
      persist_layer_norm=False,
      bias_activation_fusion=True,
      openai_gelu=False,
      onnx_safe=False,
      masked_softmax_fusion=True,
      attention_dropout=0.1,
      ffn_dropout=0.0,
      drop_path_rate=0.0,
      activation="gelu",
      megatron_legacy=False,
      bias=True,
      chunk_size=64,
      normalization="layernorm",
      transformer_block_type="pre_ln",
      headscale=False,
      activations_checkpoint_granularity=None,
      normalize_attention_scores=True,
      use_flash_attention=False,
  ):
      kwargs = locals()
      for key in ["self", "__class__"]:
          kwargs.pop(key)
      drop_path_rate = kwargs.pop("drop_path_rate")
      super(ParallelVisionTransformerLayer_, self).__init__(**kwargs)
  
      self.drop_path = (
          DropPath(drop_path_rate) if drop_path_rate > 0.0 else None
      )
  
file_path: NeMo/nemo/collections/vision/modules/common/megatron/vision_transformer.py
imports: import torch
  from nemo.collections.nlp.modules.common.megatron.layer_type import LayerType
  from nemo.collections.nlp.modules.common.megatron.module import MegatronModule
  from nemo.collections.nlp.modules.common.megatron.transformer import ParallelTransformer, ParallelTransformerLayer_
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults
label_desc: Use image node_id 1 to create a new ParallelVisionTransformerLayer_ object from inherited base classes: ParallelTransformerLayer_ with example: obj = ParallelVisionTransformerLayer_(config, init_method, output_layer_init_method, layer_number, hidden_size, ffn_hidden_size, num_attention_heads, layer_type, _attn_mask_type, fp32_residual_connection, precision, apply_query_key_layer_scaling, kv_channels, layernorm_epsilon, hidden_dropout, bias_dropout_add_fusion, persist_layer_norm, bias_activation_fusion, openai_gelu, onnx_safe, masked_softmax_fusion, attention_dropout, ffn_dropout, drop_path_rate, activation, megatron_legacy, bias, chunk_size, normalization, transformer_block_type, headscale, activations_checkpoint_granularity, normalize_attention_scores, use_flash_attention)