code: def _add_allreduce_graph(model):
      """Construct the graph that performs Allreduce on the gradients."""
      # Need to all-reduce the per-GPU gradients if training with more than 1 GPU
      all_params = model.TrainableParams()
      assert len(all_params) % cfg.NUM_GPUS == 0
      # The model parameters are replicated on each GPU, get the number
      # distinct parameter blobs (i.e., the number of parameter blobs on
      # each GPU)
      params_per_gpu = int(len(all_params) / cfg.NUM_GPUS)
      with c2_utils.CudaScope(0):
          # Iterate over distinct parameter blobs
          for i in range(params_per_gpu):
              # Gradients from all GPUs for this parameter blob
              gradients = [
                  model.param_to_grad[p]
                  for p in all_params[i::params_per_gpu]
              ]
              if len(gradients) > 0:
                  if cfg.USE_NCCL:
                      model.net.NCCLAllreduce(gradients, gradients)
                  else:
                      muji.Allreduce(
                          model.net, gradients, reduced_affix=""
                      )
  
file_path: dense-pose/detectron/modeling/optimizer.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from __future__ import unicode_literals
  import logging
  from caffe2.python import muji
  from detectron.core.config import cfg
  import detectron.utils.c2
label_desc: Use image node_id 3 for calling a global function with example usage: _add_allreduce_graph(model) without return types