'Average grads from towers.
The device where the average happens is chosen with round-robin.
Args:
tower_grads: Ngpu x Nvar x 2
Returns:
Nvar x 2'

@staticmethod
def _average_grads(tower_grads, devices):

    nr_device = len(devices)
    if (nr_device == 1):
        return tower_grads[0]
    new_tower_grads = []
    with tf.name_scope('AvgGrad'):
        for (i, grad_and_vars) in enumerate(zip(*tower_grads)):
            with tf.device(devices[(i % nr_device)]):
                v = grad_and_vars[0][1]
                all_grads = [g for (g, _) in grad_and_vars]
                grad = tf.multiply(tf.add_n(all_grads), (1.0 / nr_device))
                new_tower_grads.append((grad, v))
    return new_tower_grads
