def sum_grad_and_var_all_reduce(grad_and_vars, num_workers, alg, gpu_indices, aux_devices=None, num_shards=1):
    'Apply all-reduce algorithm over specified gradient tensors.'
    with ops.name_scope('allreduce'):
        scaled_grads = [g for (g, _) in grad_and_vars]
        if (alg == 'nccl'):
            summed_grads = nccl_ops.all_sum(scaled_grads)
        elif (alg == 'xring'):
            summed_grads = all_reduce.build_ring_all_reduce(scaled_grads, num_workers, num_shards, gpu_indices, math_ops.add)
        elif (alg == 'nccl/xring'):
            summed_grads = all_reduce.build_nccl_then_ring(scaled_grads, num_shards, math_ops.add)
        elif (alg == 'nccl/rechd'):
            summed_grads = all_reduce.build_nccl_then_recursive_hd(scaled_grads, math_ops.add)
        elif (alg == 'nccl/pscpu'):
            summed_grads = all_reduce.build_nccl_then_shuffle(scaled_grads, aux_devices, math_ops.add, math_ops.add_n)
        elif (alg == 'pscpu/pscpu'):
            second_gather_devices = aux_devices[:num_shards]
            summed_grads = all_reduce.build_shuffle_then_shuffle(scaled_grads, aux_devices, second_gather_devices, math_ops.add_n)
        elif (alg in ['pscpu', 'psgpu']):
            summed_grads = all_reduce.build_shuffle_all_reduce(scaled_grads, aux_devices, math_ops.add_n)
        else:
            raise ValueError('unsupported all_reduce alg: ', alg)
    result = []
    for ((_, v), g) in zip(grad_and_vars, summed_grads):
        result.append([g, v])
    return result
Apply all-reduce algorithm over specified gradient tensors.
def sum_grad_and_var_all_reduce(grad_and_vars, num_workers, alg, gpu_indices, aux_devices=None, num_shards=1):
    
    with ops.name_scope('allreduce'):
        scaled_grads = [g for (g, _) in grad_and_vars]
        if (alg == 'nccl'):
            summed_grads = nccl_ops.all_sum(scaled_grads)
        elif (alg == 'xring'):
            summed_grads = all_reduce.build_ring_all_reduce(scaled_grads, num_workers, num_shards, gpu_indices, math_ops.add)
        elif (alg == 'nccl/xring'):
            summed_grads = all_reduce.build_nccl_then_ring(scaled_grads, num_shards, math_ops.add)
        elif (alg == 'nccl/rechd'):
            summed_grads = all_reduce.build_nccl_then_recursive_hd(scaled_grads, math_ops.add)
        elif (alg == 'nccl/pscpu'):
            summed_grads = all_reduce.build_nccl_then_shuffle(scaled_grads, aux_devices, math_ops.add, math_ops.add_n)
        elif (alg == 'pscpu/pscpu'):
            second_gather_devices = aux_devices[:num_shards]
            summed_grads = all_reduce.build_shuffle_then_shuffle(scaled_grads, aux_devices, second_gather_devices, math_ops.add_n)
        elif (alg in ['pscpu', 'psgpu']):
            summed_grads = all_reduce.build_shuffle_all_reduce(scaled_grads, aux_devices, math_ops.add_n)
        else:
            raise ValueError('unsupported all_reduce alg: ', alg)
    result = []
    for ((_, v), g) in zip(grad_and_vars, summed_grads):
        result.append([g, v])
    return result