class_docstr: Megatron Vision Transformer Model.
class_docstr_tok: ["Megatron","Vision","Transformer","Model","."]
docstr: All-reduce layernorm parameters across model parallel nodes when sequence parallelism is used.
  Modified from megatron-lm:
  https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/blob/3f91f09bb2ab32f9904b47f46f19d2fc3f518ed8/megatron/training.py#L425
docstr_tok: ["All-reduce","layernorm","parameters","across","model","parallel","nodes","when","sequence","parallelism","is","used",".","Modified","from","megatron-lm",":","https",":","\/\/gitlab-master.nvidia.com\/ADLR\/megatron-lm\/-\/blob\/3f91f09bb2ab32f9904b47f46f19d2fc3f518ed8\/megatron\/training.py","#","L425"]
code: def allreduce_sequence_parallel_gradients(self):
      """All-reduce layernorm parameters across model parallel nodes when sequence parallelism is used.
      Modified from megatron-lm:
      https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/blob/3f91f09bb2ab32f9904b47f46f19d2fc3f518ed8/megatron/training.py#L425
      """
  
      grads = []
      if isinstance(self.model, list):
          for module in self.model:
              self._append_sequence_parallel_module_grads(module, grads)
      else:
          self._append_sequence_parallel_module_grads(self.model, grads)
  
      coalesced = torch._utils._flatten_dense_tensors(grads)
      torch.distributed.all_reduce(
          coalesced,
          group=parallel_state.get_tensor_model_parallel_group(),
      )
      for buf, synced in zip(
          grads, torch._utils._unflatten_dense_tensors(coalesced, grads)
      ):
          buf.copy_(synced)
  
imports: import itertools
  from functools import partial
  from typing import Any, Optional
  import torch
  from omegaconf.dictconfig import DictConfig
  from pytorch_lightning.accelerators import CPUAccelerator
  from pytorch_lightning.trainer.trainer import Trainer
  from nemo.collections.nlp.data.language_modeling.megatron.data_samplers import MegatronPretrainingSampler
  from nemo.collections.nlp.models.language_modeling.megatron_base_model import MegatronBaseModel
  from nemo.collections.nlp.modules.common.megatron.build_model import build_model
  from nemo.collections.nlp.modules.common.megatron.module import Float16Module, MegatronModule
  from nemo.collections.nlp.modules.common.megatron.utils import average_losses_across_data_parallel_group, get_all_params_for_weight_decay_optimization, get_linear_layer, get_params_for_weight_decay_optimization, init_method_normal, scaled_init_method_normal
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank, torch_dtype_from_precision
  from nemo.collections.vision.data.megatron.data_samplers import MegatronVisionPretrainingRandomSampler
  from nemo.collections.vision.data.megatron.vit_dataset import build_train_valid_datasets
  from nemo.collections.vision.modules.vit.vit_backbone import VitBackbone, VitMlpHead
  from nemo.core.classes.common import PretrainedModelInfo
  from nemo.utils import logging
label_desc: Use image node_id 13 for calling the MegatronVitClassificationModel obj's underlying member method code with example usage: obj.allreduce_sequence_parallel_gradients() without return types