class_docstr: Megatron base class. All NeMo Megatron models inherit from this class.
  
  - Initialize the model parallel world for nemo.
  - Turn on all of the nvidia optimizations.
  - If `cfg.tokenizer` is available, it loads the tokenizer and pad the vocab to the
    correct size for tensor model parallelism.
  - If using distributed optimizer, configure to be compatible
    with O2 level optimizations and/or model parallelism.
  - Perform gradient clipping: `grad_clip_pl_default` triggers
    the PyTorch Lightning default implementation, `with_distributed_adam` triggers
    the distributed optimizer's implementation, `megatron_amp_O2` triggers gradient clipping on the main grads,
    and otherwise gradient clipping is performed on the model grads.
class_docstr_tok: ["Megatron","base","class",".","All","NeMo","Megatron","models","inherit","from","this","class",".","-","Initialize","the","model","parallel","world","for","nemo",".","-","Turn","on","all","of","the","nvidia","optimizations",".","-","If","`","cfg.tokenizer","`","is","available",",","it","loads","the","tokenizer","and","pad","the","vocab","to","the","correct","size","for","tensor","model","parallelism",".","-","If","using","distributed","optimizer",",","configure","to","be","compatible","with","O2","level","optimizations","and\/or","model","parallelism",".","-","Perform","gradient","clipping",":","`","grad_clip_pl_default","`","triggers","the","PyTorch","Lightning","default","implementation",",","`","with_distributed_adam","`","triggers","the","distributed","optimizer","'s","implementation",",","`","megatron_amp_O2","`","triggers","gradient","clipping","on","the","main","grads",",","and","otherwise","gradient","clipping","is","performed","on","the","model","grads","."]
docstr: Certain configurations might be incompatible or discouraged.
  We can check for them here and override if necessary.
code: def _validate_and_override_config(self):
      """Certain configurations might be incompatible or discouraged.
      We can check for them here and override if necessary.
      """
      app_state = AppState()
  
      if (
          self.cfg.get("sequence_parallel", False)
          and self.cfg.get("tensor_model_parallel_size", 1) == 1
      ):
          logging.info(
              "Sequence parallel should only be used with tensor parallel size > 1. Setting sequence parallel to False"
          )
          with open_dict(self.cfg):
              self.cfg.sequence_parallel = False
  
      # Gradient accumulation fusion does not work with our baseline implementaiton of
      # async grad allreduce. This should be fixed!
      # For now we must disable it whenever using the baseline implementaion.
      # The distributed adam from apex does work with gradient accumulation fusion.
      distributed_fused_adam = (
          self.cfg.optim.get("name", "fused_adam")
          == "distributed_fused_adam"
      )
      pipeline_model_parallel_size = self.cfg.get(
          "pipeline_model_parallel_size", 1
      )
      data_parallel_size = app_state.data_parallel_size
  
      if self.cfg.get("gradient_accumulation_fusion", False):
          if (
              data_parallel_size > 1
              and pipeline_model_parallel_size == 1
              and not distributed_fused_adam
          ):
              logging.info(
                  "When not using pipeline model parallel, gradient accumulation fusion can only be used with distributed_fused_adam."
              )
              with open_dict(self.cfg):
                  self.cfg.gradient_accumulation_fusion = False
          if self.cfg.get("fsdp", False):
              logging.info(
                  "When using FSDP, gradient accumulation cannot be fused to gradient computation."
              )
              with open_dict(self.cfg):
                  self.cfg.gradient_accumulation_fusion = False
          if not self.cfg.get("megatron_amp_O2", False):
              logging.info(
                  "Gradient accumulation fusion can only be used with megatron amp O2 mixed precision."
              )
              with open_dict(self.cfg):
                  self.cfg.gradient_accumulation_fusion = False
  
      if self.cfg.get("use_emha", False):
          raise ValueError(
              "use_emha is not yet supported please set to False"
          )
  
      vp_size = self.cfg.get(
          "virtual_pipeline_model_parallel_size", None
      )
  
      if vp_size is not None:
          if vp_size == 1:
              self.cfg["virtual_pipeline_model_parallel_size"] = None
          else:
              assert (
                  self.cfg.num_layers
                  // self.cfg.pipeline_model_parallel_size
              ) % vp_size == 0, "Make sure the number of model chunks is the same across all pipeline stages."
  
      if self.cfg.get("ub_tp_comm_overlap", False):
          if not self.cfg.get("sequence_parallel", False):
              logging.info(
                  "Pipelined tensor-parallel communication overlap is available with sequence-parallelism."
                  "Setting `ub_tp_comm_overlap` to False."
              )
              with open_dict(self.cfg):
                  self.cfg.ub_tp_comm_overlap = False
  
          if self.cfg.get("fsdp", False):
              logging.info(
                  "Userbuffer tensor-parallel communication overlap is not available with FSDP."
                  "Setting `ub_tp_comm_overlap` to False."
              )
              with open_dict(self.cfg):
                  self.cfg.ub_tp_comm_overlap = False
  
      if self.cfg.get("fsdp", False) and self.cfg.get("fp8", False):
          raise ValueError("Torch FSDP does not support FP8.")
  
imports: import gc
  import itertools
  import os
  import re
  from dataclasses import fields
  from typing import Any, Dict, Optional, Union
  import omegaconf
  import torch
  from omegaconf import OmegaConf, open_dict
  from omegaconf.dictconfig import DictConfig
  from pytorch_lightning.plugins.precision import MixedPrecisionPlugin
  from pytorch_lightning.trainer.connectors.logger_connector.fx_validator import _FxValidator
  from pytorch_lightning.trainer.trainer import Trainer
  from nemo.collections.nlp.models.nlp_model import NLPModel
  from nemo.collections.nlp.modules.common.megatron.attention import HAVE_FLASH_ATTENTION
  from nemo.collections.nlp.modules.common.megatron.clip_grads import clip_grad_norm_distributed_optimizer, clip_grad_norm_fp32
  from nemo.collections.nlp.modules.common.megatron.megatron_init import initialize_model_parallel_for_nemo
  from nemo.collections.nlp.modules.common.megatron.module import Float16Module
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_nmt_tokenizer
  from nemo.collections.nlp.parts import utils_funcs
  from nemo.collections.nlp.parts.nlp_overrides import NEMO_MEGATRON_MODEL_PARALLEL_APPSTATE_OVERRIDE, GradScaler
  from nemo.collections.nlp.parts.utils_funcs import activation_to_func
  from nemo.core.optim import MainParamsOptimizerWrapper, prepare_lr_scheduler
  from nemo.utils import AppState, logging, str_to_dtype
  from nemo.utils.get_rank import is_global_rank_zero
label_desc: Use image node_id 27 for calling the MegatronBaseModel obj's underlying member method code with example usage: obj._validate_and_override_config() without return types