code: def average_gradients(tower_grads):
      average_grads = []
      for grad_and_vars in zip(*tower_grads):
          # Note that each grad_and_vars looks like the following:
          #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
          grads = []
          for g, _ in grad_and_vars:
              # Add 0 dimension to the gradients to represent the tower.
              expanded_g = tf_v1.expand_dims(g, 0)
  
              # Append on a 'tower' dimension which we will average over below.
              grads.append(expanded_g)
  
          # Average over the 'tower' dimension.
          grad = tf_v1.concat(grads, 0)
          grad = tf_v1.reduce_mean(grad, 0)
  
          # Keep in mind that the Variables are redundant because they are shared
          # across towers. So .. we will just return the first tower's pointer to
          # the Variable.
          v = grad_and_vars[0][1]
          grad_and_var = (grad, v)
          average_grads.append(grad_and_var)
      return average_grads
  
file_path: DALI/dali/test/python/test_dali_tf_dataset_mnist_graph.py
imports: import tensorflow
  import tensorflow.compat.v1
  from nose import with_setup
  import test_dali_tf_dataset_mnist
  from nose_utils import raises
label_desc: Use image node_id 9 for calling a global function with example usage: average_gradients(tower_grads) and returns: average_grads