def build_collective_reduce(input_tensors, num_workers, collective_keys, reduction_op='Add', unary_op='Id'):
    'Build a subgraph that does one full all-reduce, using the collective Op.\n\n  Args:\n    input_tensors: tensors within a single worker graph that are to be reduced\n      together; must be one per device.\n    num_workers: total number of workers with identical independent graphs that\n      will be doing this same reduction.  The reduction will actually include\n      the corresponding tensors at all these workers.\n    collective_keys: a CollectiveKeys object.\n    reduction_op: string naming the reduction op.\n    unary_op: string naming the unary final op.\n\n  Returns:\n    An array of final tensors, one per device, computed by the full reduction.\n\n  Raises:\n    ValueError: There must be at least two tensors over all the workers.\n  '
    group_size = (len(input_tensors) * num_workers)
    if (group_size < 2):
        raise ValueError('num_workers * len(input_tensors) must be 2 or greater')
    devices = [t.device for t in input_tensors]
    num_devices = len(devices)
    group_key = collective_keys.get_group_key(devices)
    instance_key = collective_keys.get_instance_key()
    out_tensors = []
    subdiv_offsets = [0]
    for d in range(num_devices):
        with ops.device(devices[d]):
            reduce_op = collective_ops.all_reduce(input_tensors[d], group_size, group_key, instance_key, reduction_op, unary_op, subdiv_offsets)
            out_tensors.append(reduce_op)
    return out_tensors
Build a subgraph that does one full all-reduce, using the collective Op.

Args:
  input_tensors: tensors within a single worker graph that are to be reduced
    together; must be one per device.
  num_workers: total number of workers with identical independent graphs that
    will be doing this same reduction.  The reduction will actually include
    the corresponding tensors at all these workers.
  collective_keys: a CollectiveKeys object.
  reduction_op: string naming the reduction op.
  unary_op: string naming the unary final op.

Returns:
  An array of final tensors, one per device, computed by the full reduction.

Raises:
  ValueError: There must be at least two tensors over all the workers.
def build_collective_reduce(input_tensors, num_workers, collective_keys, reduction_op='Add', unary_op='Id'):
    'Build a subgraph that does one full all-reduce, using the collective Op.\n\n  Args:\n    input_tensors: tensors within a single worker graph that are to be reduced\n      together; must be one per device.\n    num_workers: total number of workers with identical independent graphs that\n      will be doing this same reduction.  The reduction will actually include\n      the corresponding tensors at all these workers.\n    collective_keys: a CollectiveKeys object.\n    reduction_op: string naming the reduction op.\n    unary_op: string naming the unary final op.\n\n  Returns:\n    An array of final tensors, one per device, computed by the full reduction.\n\n  Raises:\n    ValueError: There must be at least two tensors over all the workers.\n  '
    group_size = (len(input_tensors) * num_workers)
    if (group_size < 2):
        raise ValueError('num_workers * len(input_tensors) must be 2 or greater')
    devices = [t.device for t in input_tensors]
    num_devices = len(devices)
    group_key = collective_keys.get_group_key(devices)
    instance_key = collective_keys.get_instance_key()
    out_tensors = []
    subdiv_offsets = [0]
    for d in range(num_devices):
        with ops.device(devices[d]):
            reduce_op = collective_ops.all_reduce(input_tensors[d], group_size, group_key, instance_key, reduction_op, unary_op, subdiv_offsets)
            out_tensors.append(reduce_op)
    return out_tensors