class_docstr: All-reduce cross device ops using collective ops.
  
  In the between-graph replicated training, it will still do all-reduces across
  all workers and then put results on the right destinations.
class_docstr_tok: ["All-reduce","cross","device","ops","using","collective","ops",".","In","the","between-graph","replicated","training",",","it","will","still","do","all-reduces","across","all","workers","and","then","put","results","on","the","right","destinations","."]
docstr: All-reduce across all workers in a batch.
code: def _do_batch_all_reduce_dense(self, reduce_op, per_replica_values):
      """All-reduce across all workers in a batch."""
  
      chunked_gv = self._make_gradient_chunks(
          per_replica_values, self._num_packs
      )
  
      batch_size = len(per_replica_values)
      # Pass self._communication to the runtime as a communication hint.
      communication_hint = self._communication.value
      # For now, we use NCCL only when batch_size > 1 and num_packs is 1.
      # TODO(b/132575814): switch to NCCL for all collectives when communication
      # is NCCL.
      if self._communication == CollectiveCommunication.NCCL and (
          batch_size == 1 or self._num_packs != 1
      ):
          communication_hint = CollectiveCommunication.AUTO.value
  
      logging.log_first_n(
          logging.INFO,
          "Collective batch_all_reduce: %d all-reduces, "
          "num_workers = %d, communication_hint = %s"
          % (batch_size, self._num_workers, communication_hint),
          10,
      )
  
      reduced_gv_list = []
      for chunk in chunked_gv:
          # By placing all collective ops in a chunk under single name scope, we
          # ensure they will be picked up by the `ScopedAllocator` grappler
          # optimizer and packed into a single all-reduce.
          with ops.name_scope("allreduce"):
              for grad_and_vars in chunk:
                  # Gradients for the same variable but from different devices.
                  scaled_grads = [g for g, _ in grad_and_vars]
                  # collective_reduced = cross_device_utils.build_collective_reduce(
                  #     scaled_grads, self._num_workers, self._collective_keys, "Add",
                  #     "Id", communication_hint)
                  collective_reduced = my_build_collective_reduce(
                      scaled_grads,
                      self._num_workers,
                      self._collective_keys,
                      "Add",
                      "Id",
                      communication_hint,
                  )
                  result = []
                  for (_, v), g in zip(
                      grad_and_vars, collective_reduced
                  ):
                      result.append([g, v])
                  reduced_gv_list.append(result)
  
      new_device_grads = [list(x) for x in zip(*reduced_gv_list)]
      return _ungroup_and_make_mirrored(
          new_device_grads,
          per_replica_values[0],
          reduce_op,
          num_between_graph_workers=self._num_workers,
      )
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import enum
  import six
  from tensorflow.python.client import device_lib
  from tensorflow.python.distribute import cross_device_utils
  from tensorflow.python.distribute import device_util
  from tensorflow.python.distribute import reduce_util
  from tensorflow.python.distribute import values
  from tensorflow.python.eager import context
  from tensorflow.python.framework import kernels
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import tensor_util
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import resource_variable_ops
  from tensorflow.python.platform import tf_logging
  from tensorflow.python.util.tf_export import tf_export
  from tensorflow.tools.docs import doc_controls
  import tensorflow
  from tensorflow.python.distribute.cross_device_ops import CollectiveAllReduce
  import tensorflow.python.distribute.cross_device_ops
  from byteps.tensorflow.ops import broadcast, _push_pull
  from byteps.tensorflow import local_rank, size
label_desc: Use image node_id 7 for calling the MyCollectiveAllReduce obj's underlying member method code with example usage: obj._do_batch_all_reduce_dense(reduce_op, per_replica_values) and returns: _ungroup_and_make_mirrored