class_docstr: Megatron Bert pretraining.
  Model returns [batch, seq, hidden] shape
class_docstr_tok: ["Megatron","Bert","pretraining",".","Model","returns","[","batch",",","seq",",","hidden","]","shape"]
docstr: All-reduce layernorm parameters across model parallel nodes when sequence parallelism is used.
  Modified from megatron-lm:
  https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/blob/3f91f09bb2ab32f9904b47f46f19d2fc3f518ed8/megatron/training.py#L425
code: def allreduce_sequence_parallel_gradients(self):
      """All-reduce layernorm parameters across model parallel nodes when sequence parallelism is used.
      Modified from megatron-lm:
      https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/blob/3f91f09bb2ab32f9904b47f46f19d2fc3f518ed8/megatron/training.py#L425
      """
  
      grads = []
      if isinstance(self.model, list):
          for module in self.model:
              self._append_sequence_parallel_module_grads(module, grads)
      else:
          self._append_sequence_parallel_module_grads(self.model, grads)
  
      coalesced = torch._utils._flatten_dense_tensors(grads)
      torch.distributed.all_reduce(
          coalesced,
          group=parallel_state.get_tensor_model_parallel_group(),
      )
      for buf, synced in zip(
          grads, torch._utils._unflatten_dense_tensors(coalesced, grads)
      ):
          buf.copy_(synced)
  
imports: import itertools
  from typing import Any, Dict, List, Optional
  import torch
  import torch.nn.functional
  from omegaconf.dictconfig import DictConfig
  from pytorch_lightning.trainer.trainer import Trainer
  from nemo.collections.nlp.data.language_modeling.megatron import dataset_utils
  from nemo.collections.nlp.data.language_modeling.megatron.data_samplers import MegatronPretrainingRandomSampler, MegatronPretrainingSampler
  from nemo.collections.nlp.models.language_modeling.megatron.bert_model import BertModel
  from nemo.collections.nlp.models.language_modeling.megatron_base_model import MegatronBaseModel
  from nemo.collections.nlp.modules.common.megatron.build_model import build_model
  from nemo.collections.nlp.modules.common.megatron.module import Float16Module
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, average_losses_across_data_parallel_group, get_params_for_weight_decay_optimization
  from nemo.collections.nlp.parts.nlp_overrides import GradScaler
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.core.classes.common import PretrainedModelInfo
  from nemo.core.neural_types import ChannelType, MaskType, NeuralType
  from nemo.utils import AppState, logging
label_desc: Use image node_id 21 for calling the MegatronBertModel obj's underlying member method code with example usage: obj.allreduce_sequence_parallel_gradients() without return types