code: def avg_grads(tower_grads):
      """Calculate the average gradient for each shared variable across all
      towers.
      Note that this function provides a synchronization point across all towers.
      Args:
        tower_grads: List of lists of (gradient, variable) tuples. The outer list
          is over individual gradients. The inner list is over the gradient
          calculation for each tower.
      Returns:
         List of pairs of (gradient, variable) where the gradient has been
         averaged across all towers.
  
      Modified from this tutorial: https://tinyurl.com/n3jr2vm
      """
      if len(tower_grads) == 1:
          return tower_grads[0]
      average_grads = []
      for grad_and_vars in zip(*tower_grads):
          # Note that each grad_and_vars looks like the following:
          #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
          grads = [g for g, _ in grad_and_vars]
  
          # Average over the 'tower' dimension.
          grad = tf.add_n(grads) / len(grads)
  
          # Keep in mind that the Variables are redundant because they are shared
          # across towers. So .. we will just return the first tower's pointer to
          # the Variable.
          v = grad_and_vars[0][1]
          assert all(
              v is grad_and_var[1] for grad_and_var in grad_and_vars
          )
          grad_and_var = (grad, v)
          average_grads.append(grad_and_var)
      return average_grads
  
file_path: cleverhans/cleverhans_v3.1.0/cleverhans/train.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from __future__ import unicode_literals
  import logging
  import os
  import time
  import warnings
  import math
  import numpy
  from six.moves import xrange
  import tensorflow
  from cleverhans import canary
  from cleverhans.utils import _ArgsWrapper, create_logger
  from cleverhans.utils import safe_zip
  from cleverhans.utils_tf import infer_devices
  from cleverhans.utils_tf import initialize_uninitialized_global_variables
label_desc: Use image node_id 2 for calling a global function with example usage: avg_grads(tower_grads) and returns: average_grads, tower_grads