class_docstr: All-reduce cross device ops using collective ops.
  
  In the between-graph replicated training, it will still do all-reduces across
  all workers and then put results on the right destinations.
class_docstr_tok: ["All-reduce","cross","device","ops","using","collective","ops",".","In","the","between-graph","replicated","training",",","it","will","still","do","all-reduces","across","all","workers","and","then","put","results","on","the","right","destinations","."]
docstr: Make `per_replica_values` into chunks.
code: def _make_gradient_chunks(self, per_replica_values, num_packs):
      """Make `per_replica_values` into chunks."""
      chunked_by_device = _group_value_by_device(per_replica_values)
      chunked_by_var = list(zip(*chunked_by_device))
      # chunked_by_var is chunked by variables and takes the following format:
      # [((grad0_gpu0, v0_gpu0), (grad0_gpu1, v0_gpu1), (grad0_gpu2, v0_gpu2) ..),
      #  ((grad1_gpu0, v1_gpu0), (grad1_gpu1, v1_gpu1), (grad1_gpu0, v1_gpu2) ..),
      #  ((grad2_gpu0, v2_gpu0), (grad2_gpu1, v2_gpu1), (grad2_gpu0, v2_gpu2) ..),
      #  ...
      # ]
  
      # No chunking if number of variables is fewer than number of packs.
      if len(chunked_by_var) < num_packs:
          return [chunked_by_var]
  
      # First n-1 chunks get `chunk_size` grads, last chunk gets leftover grads.
      # This strategy can cause the last chunk to have larger size compared to the
      # first n-1 chunks.  Alternatively, we can increment chunk_size by 1 to get
      # slightly larger first n-1 chunks and smaller last chunk.
      # TODO(ayushd): compare different packing strategies.
      chunk_size = len(chunked_by_var) // num_packs
      leftover_size = len(chunked_by_var) - chunk_size * (num_packs - 1)
      assert leftover_size > 0
      chunked_gv = [
          chunked_by_var[x : x + chunk_size]
          for x in range(
              0, len(chunked_by_var) - leftover_size, chunk_size
          )
      ]
      chunked_gv.append(chunked_by_var[-leftover_size:])
  
      return chunked_gv
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import enum
  import six
  from tensorflow.python.client import device_lib
  from tensorflow.python.distribute import cross_device_utils
  from tensorflow.python.distribute import device_util
  from tensorflow.python.distribute import reduce_util
  from tensorflow.python.distribute import values
  from tensorflow.python.eager import context
  from tensorflow.python.framework import kernels
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import tensor_util
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import resource_variable_ops
  from tensorflow.python.platform import tf_logging
  from tensorflow.python.util.tf_export import tf_export
  from tensorflow.tools.docs import doc_controls
  import tensorflow
  from tensorflow.python.distribute.cross_device_ops import CollectiveAllReduce
  import tensorflow.python.distribute.cross_device_ops
  from byteps.tensorflow.ops import broadcast, _push_pull
  from byteps.tensorflow import local_rank, size
label_desc: Use image node_id 5 for calling the MyCollectiveAllReduce obj's underlying member method code with example usage: obj._make_gradient_chunks(per_replica_values, num_packs) and returns: chunked_gv, list