class_docstr: Megatron Bert pretraining.
  Model returns [batch, seq, hidden] shape
class_docstr_tok: ["Megatron","Bert","pretraining",".","Model","returns","[","batch",",","seq",",","hidden","]","shape"]
code: def model_provider_func(self, pre_process, post_process):
      cfg = self.cfg
      num_tokentypes = 2 if cfg.bert_binary_head else 0
  
      if self.mcore_bert:
          from megatron.core.models.bert.bert_layer_specs import (
              bert_layer_with_transformer_engine_spec,
          )
          from megatron.core.models.bert.bert_model import (
              BertModel as MCoreBertModel,
          )
  
          model = MCoreBertModel(
              config=self.transformer_config,
              transformer_layer_spec=bert_layer_with_transformer_engine_spec,
              vocab_size=self.padded_vocab_size,
              max_sequence_length=cfg.max_position_embeddings,
              num_tokentypes=num_tokentypes,
              add_binary_head=cfg.bert_binary_head,
              share_embeddings_and_output_weights=self.cfg.get(
                  "share_embeddings_and_output_weights", True
              ),
              parallel_output=True,
              pre_process=pre_process,
              post_process=post_process,
          )
      else:
          model = BertModel(
              config=self.model_parallel_config,
              vocab_size=self.padded_vocab_size,
              hidden_size=cfg.hidden_size,
              max_position_embeddings=cfg.max_position_embeddings,
              num_layers=cfg.num_layers,
              num_attention_heads=cfg.num_attention_heads,
              apply_query_key_layer_scaling=cfg.get(
                  "apply_query_key_layer_scaling", True
              ),
              kv_channels=cfg.get("kv_channels", None),
              ffn_hidden_size=cfg.ffn_hidden_size,
              num_tokentypes=num_tokentypes,
              parallel_output=True,
              pre_process=pre_process,
              post_process=post_process,
              init_method_std=cfg.get("init_method_std", 0.02),
              fp16_lm_cross_entropy=cfg.get(
                  "fp16_lm_cross_entropy", False
              ),
              hidden_dropout=cfg.get("hidden_dropout", 0.1),
              precision=cfg.get("precision", 16),
              fp32_residual_connection=cfg.get(
                  "fp32_residual_connection", False
              ),
              activations_checkpoint_granularity=self.cfg.get(
                  "activations_checkpoint_granularity", None
              ),
              activations_checkpoint_method=self.cfg.get(
                  "activations_checkpoint_method", None
              ),
              activations_checkpoint_num_layers=self.cfg.get(
                  "activations_checkpoint_num_layers", 1
              ),
              activations_checkpoint_layers_per_pipeline=self.cfg.get(
                  "activations_checkpoint_layers_per_pipeline", None
              ),
              layernorm_epsilon=cfg.get("layernorm_epsilon", 1e-5),
              masked_softmax_fusion=cfg.get(
                  "masked_softmax_fusion", True
              ),
              bias_gelu_fusion=cfg.get("bias_gelu_fusion", True),
              bias_dropout_add_fusion=cfg.get(
                  "bias_dropout_add_fusion", True
              ),
              onnx_safe=cfg.get("onnx_safe", False),
              add_binary_head=cfg.bert_binary_head,
              megatron_legacy=cfg.get("megatron_legacy", False),
              position_embedding_type=self.cfg.get(
                  "position_embedding_type", "learned_absolute"
              ),
          )
  
      return model
  
imports: import itertools
  from typing import Any, Dict, List, Optional
  import torch
  import torch.nn.functional
  from omegaconf.dictconfig import DictConfig
  from pytorch_lightning.trainer.trainer import Trainer
  from nemo.collections.nlp.data.language_modeling.megatron import dataset_utils
  from nemo.collections.nlp.data.language_modeling.megatron.data_samplers import MegatronPretrainingRandomSampler, MegatronPretrainingSampler
  from nemo.collections.nlp.models.language_modeling.megatron.bert_model import BertModel
  from nemo.collections.nlp.models.language_modeling.megatron_base_model import MegatronBaseModel
  from nemo.collections.nlp.modules.common.megatron.build_model import build_model
  from nemo.collections.nlp.modules.common.megatron.module import Float16Module
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, average_losses_across_data_parallel_group, get_params_for_weight_decay_optimization
  from nemo.collections.nlp.parts.nlp_overrides import GradScaler
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.core.classes.common import PretrainedModelInfo
  from nemo.core.neural_types import ChannelType, MaskType, NeuralType
  from nemo.utils import AppState, logging
label_desc: Use image node_id 2 for calling the MegatronBertModel obj's underlying member method code with example usage: obj.model_provider_func(pre_process, post_process) and returns: model