class_docstr: Distributed replicated training.
  Each worker process builds the same model on one or more GPUs.
  Gradients across GPUs are averaged within the worker,
  and get synchronously applied to the global copy of variables located on PS.
  Then each worker copy the latest variables from PS back to local.
  
  It is an equivalent of ``--variable_update=distributed_replicated`` in
  `tensorflow/benchmarks <https://github.com/tensorflow/benchmarks>`_.
  Note that the performance of this trainer is still not satisfactory,
  and TensorFlow team is not actively maintaining distributed training features.
  Check :class:`HorovodTrainer` and
  `ResNet-Horovod <https://github.com/tensorpack/benchmarks/tree/master/ResNet-Horovod>`_
  for better distributed training support.
  
  Note:
      1. Gradients are not averaged across workers, but applied to PS variables
         directly (either with or without locking depending on the optimizer).
      2. Some details about collections: all variables created inside tower
         will become local variables,
         and a clone will be made in global variables for all trainable/model variables.
  
  Example:
  
      .. code-block:: python
  
          # Create the server object like this:
          hosts = ['host1.com', 'host2.com']
          cluster_spec = tf.train.ClusterSpec({
              'ps': [h + ':2222' for h in hosts],
              'worker': [h + ':2223' for h in hosts]
          })
          server = tf.train.Server(
              cluster_spec, job_name=args.job, task_index=args.task,
              config=get_default_sess_config())
          # initialize trainer with this server object
  
      .. code-block:: none
  
          # Start training like this:
          (host1)$ ./train.py --job worker --task 0
          (host1)$ CUDA_VISIBLE_DEVICES= ./train.py --job ps --task 0
          (host2)$ ./train.py --job worker --task 1
          (host2)$ CUDA_VISIBLE_DEVICES= ./train.py --job ps --task 1
class_docstr_tok: ["Distributed","replicated","training",".","Each","worker","process","builds","the","same","model","on","one","or","more","GPUs",".","Gradients","across","GPUs","are","averaged","within","the","worker",",","and","get","synchronously","applied","to","the","global","copy","of","variables","located","on","PS",".","Then","each","worker","copy","the","latest","variables","from","PS","back","to","local",".","It","is","an","equivalent","of","``","--","variable_update=distributed_replicated","``","in","`","tensorflow\/benchmarks","<","https",":","\/\/github.com\/tensorflow\/benchmarks",">","`","_",".","Note","that","the","performance","of","this","trainer","is","still","not","satisfactory",",","and","TensorFlow","team","is","not","actively","maintaining","distributed","training","features",".","Check",":","class",":","`","HorovodTrainer","`","and","`","ResNet-Horovod","<","https",":","\/\/github.com\/tensorpack\/benchmarks\/tree\/master\/ResNet-Horovod",">","`","_","for","better","distributed","training","support",".","Note",":","1",".","Gradients","are","not","averaged","across","workers",",","but","applied","to","PS","variables","directly","(","either","with","or","without","locking","depending","on","the","optimizer",")",".","2",".","Some","details","about","collections",":","all","variables","created","inside","tower","will","become","local","variables",",","and","a","clone","will","be","made","in","global","variables","for","all","trainable\/model","variables",".","Example",":","..","code-block",":",":","python","#","Create","the","server","object","like","this",":","hosts","=","[","'host1.com","'",",","'host2.com","'","]","cluster_spec","=","tf.train.ClusterSpec","(","{","'ps","'",":","[","h","+","':2222","'","for","h","in","hosts","]",",","'worker","'",":","[","h","+","':2223","'","for","h","in","hosts","]","}",")","server","=","tf.train.Server","(","cluster_spec",",","job_name=args.job",",","task_index=args.task",",","config=get_default_sess_config","(",")",")","#","initialize","trainer","with","this","server","object","..","code-block",":",":","none","#","Start","training","like","this",":","(","host1",")","$",".\/train.py","--","job","worker","--","task","0","(","host1",")","$","CUDA_VISIBLE_DEVICES=",".\/train.py","--","job","ps","--","task","0","(","host2",")","$",".\/train.py","--","job","worker","--","task","1","(","host2",")","$","CUDA_VISIBLE_DEVICES=",".\/train.py","--","job","ps","--","task","1"]
docstr: Apply averaged gradients to ps vars, and then copy the updated
  variables back to each tower.
  
  Args:
      raw_grad_list: Ngpu x Nvar x 2 gradient list from all towers
      ps_var_grads: Nvar x 2 (grad, ps_var)
  
  Returns:
      list of copy ops
docstr_tok: ["Apply","averaged","gradients","to","ps","vars",",","and","then","copy","the","updated","variables","back","to","each","tower",".","Args",":","raw_grad_list",":","Ngpu","x","Nvar","x","2","gradient","list","from","all","towers","ps_var_grads",":","Nvar","x","2","(","grad",",","ps_var",")","Returns",":","list","of","copy","ops"]
code: def _apply_gradients_and_copy(self, opt, raw_grad_list, ps_var_grads):
      """
      Apply averaged gradients to ps vars, and then copy the updated
      variables back to each tower.
  
      Args:
          raw_grad_list: Ngpu x Nvar x 2 gradient list from all towers
          ps_var_grads: Nvar x 2 (grad, ps_var)
  
      Returns:
          list of copy ops
      """
      # TODO do this for variables together?
      with tf.name_scope("apply_gradients"):
          var_update_ops = []
          for vid, (g, v) in enumerate(ps_var_grads):
              # TODO do we put momentum variables into local or global?
              apply_gradient_op = opt.apply_gradients([(g, v)])
              barrier = self._add_sync_queues_and_barrier(
                  "param_update_barrier_{}".format(vid),
                  [apply_gradient_op],
              )
              with tf.control_dependencies([barrier]), tf.device(
                  self.cpu_device
              ):
                  updated_value = v.read_value()
                  for towerid in range(self.nr_gpu):
                      var_update_ops.append(
                          raw_grad_list[towerid][vid][1].assign(
                              updated_value
                          )
                      )
          return var_update_ops
  
imports: import re
  import tensorflow
  from ..tfutils.common import get_global_step_var, get_op_tensor_name
  from ..utils import logger
  from ..utils.argtools import memoized
  from .training import DataParallelBuilder, GraphBuilder
  from .utils import OverrideCachingDevice, split_grad_list, allreduce_grads_naive, override_to_local_variable
label_desc: Use image node_id 5 for calling the DistributedReplicatedBuilder obj's underlying member method code with example usage: obj._apply_gradients_and_copy(opt, raw_grad_list, ps_var_grads) and returns: var_update_ops