class_docstr: Bert Language model.
  Model returns [seq, batch, hidden] shape
class_docstr_tok: ["Bert","Language","model",".","Model","returns","[","seq",",","batch",",","hidden","]","shape"]
code: def __init__(
      self,
      config: ModelParallelConfig,
      vocab_size,
      hidden_size,
      max_position_embeddings,
      num_layers,
      num_attention_heads,
      ffn_hidden_size,
      apply_query_key_layer_scaling=True,
      kv_channels=None,
      num_tokentypes=0,
      parallel_output=True,
      pre_process=True,
      post_process=True,
      init_method_std=0.02,
      fp16_lm_cross_entropy=False,
      hidden_dropout=0.1,
      precision=16,
      fp32_residual_connection=False,
      activations_checkpoint_granularity=None,
      activations_checkpoint_method=None,
      activations_checkpoint_num_layers=1,
      activations_checkpoint_layers_per_pipeline=None,
      layernorm_epsilon=1e-5,
      masked_softmax_fusion=False,
      bias_gelu_fusion=True,
      bias_dropout_add_fusion=True,
      openai_gelu=False,
      onnx_safe=False,
      add_binary_head=True,
      megatron_legacy=False,
      sequence_parallel=False,
      position_embedding_type="learned_absolute",
  ):
      super(BertModel, self).__init__(config=config)
      self.fp16_lm_cross_entropy = fp16_lm_cross_entropy
      self.add_binary_head = add_binary_head
      self.parallel_output = parallel_output
      self.pre_process = pre_process
      self.post_process = post_process
      self.sequence_parallel = sequence_parallel
  
      init_method = init_method_normal(init_method_std)
      scaled_init_method = scaled_init_method_normal(
          init_method_std, num_layers
      )
  
      (
          self.language_model,
          self._language_model_key,
      ) = get_language_model(
          config=config,
          vocab_size=vocab_size,
          hidden_size=hidden_size,
          hidden_dropout=hidden_dropout,
          num_tokentypes=num_tokentypes,
          max_position_embeddings=max_position_embeddings,
          num_layers=num_layers,
          num_attention_heads=num_attention_heads,
          apply_query_key_layer_scaling=apply_query_key_layer_scaling,
          kv_channels=kv_channels,
          ffn_hidden_size=ffn_hidden_size,
          add_pooler=self.add_binary_head,
          encoder_attn_mask_type=AttnMaskType.padding,
          init_method=init_method,
          scaled_init_method=scaled_init_method,
          pre_process=self.pre_process,
          post_process=self.post_process,
          init_method_std=init_method_std,
          precision=precision,
          fp32_residual_connection=fp32_residual_connection,
          activations_checkpoint_granularity=activations_checkpoint_granularity,
          activations_checkpoint_method=activations_checkpoint_method,
          activations_checkpoint_num_layers=activations_checkpoint_num_layers,
          activations_checkpoint_layers_per_pipeline=activations_checkpoint_layers_per_pipeline,
          layernorm_epsilon=layernorm_epsilon,
          masked_softmax_fusion=masked_softmax_fusion,
          bias_activation_fusion=bias_gelu_fusion,
          bias_dropout_add_fusion=bias_dropout_add_fusion,
          openai_gelu=openai_gelu,
          onnx_safe=onnx_safe,
          megatron_legacy=megatron_legacy,
          position_embedding_type=position_embedding_type,
      )
  
      self.initialize_word_embeddings(
          init_method=init_method_normal(init_method_std),
          vocab_size=vocab_size,
          hidden_size=hidden_size,
      )
  
      if self.post_process:
          self.lm_head = BertLMHead(
              config,
              self.word_embeddings_weight().size(0),
              hidden_size,
              init_method,
              layernorm_epsilon,
              parallel_output,
              openai_gelu,
              onnx_safe,
          )
          self._lm_head_key = "lm_head"
          self.binary_head = None
          if self.add_binary_head:
              self.binary_head = get_linear_layer(
                  hidden_size, 2, init_method
              )
              self._binary_head_key = "binary_head"
  
file_path: NeMo/nemo/collections/nlp/models/language_modeling/megatron/bert_model.py
imports: import torch
  from nemo.collections.nlp.modules.common.megatron.language_model import get_language_model
  from nemo.collections.nlp.modules.common.megatron.module import MegatronModule
  from nemo.collections.nlp.modules.common.megatron.transformer import get_layer_norm
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, build_position_ids, erf_gelu, get_linear_layer, init_method_normal, openai_gelu, parallel_lm_logits, scaled_init_method_normal
label_desc: Use image node_id 1 to create a new BertModel object from inherited base classes: MegatronModule with example: obj = BertModel(config, vocab_size, hidden_size, max_position_embeddings, num_layers, num_attention_heads, ffn_hidden_size, apply_query_key_layer_scaling, kv_channels, num_tokentypes, parallel_output, pre_process, post_process, init_method_std, fp16_lm_cross_entropy, hidden_dropout, precision, fp32_residual_connection, activations_checkpoint_granularity, activations_checkpoint_method, activations_checkpoint_num_layers, activations_checkpoint_layers_per_pipeline, layernorm_epsilon, masked_softmax_fusion, bias_gelu_fusion, bias_dropout_add_fusion, openai_gelu, onnx_safe, add_binary_head, megatron_legacy, sequence_parallel, position_embedding_type)