class_docstr: Distributed replicated training.
  Each worker process builds the same model on one or more GPUs.
  Gradients across GPUs are averaged within the worker,
  and get synchronously applied to the global copy of variables located on PS.
  Then each worker copy the latest variables from PS back to local.
  
  It is an equivalent of ``--variable_update=distributed_replicated`` in
  `tensorflow/benchmarks <https://github.com/tensorflow/benchmarks>`_.
  Note that the performance of this trainer is still not satisfactory,
  and TensorFlow team is not actively maintaining distributed training features.
  Check :class:`HorovodTrainer` and
  `ResNet-Horovod <https://github.com/tensorpack/benchmarks/tree/master/ResNet-Horovod>`_
  for better distributed training support.
  
  Note:
      1. Gradients are not averaged across workers, but applied to PS variables
         directly (either with or without locking depending on the optimizer).
      2. Some details about collections: all variables created inside tower
         will become local variables,
         and a clone will be made in global variables for all trainable/model variables.
  
  Example:
  
      .. code-block:: python
  
          # Create the server object like this:
          hosts = ['host1.com', 'host2.com']
          cluster_spec = tf.train.ClusterSpec({
              'ps': [h + ':2222' for h in hosts],
              'worker': [h + ':2223' for h in hosts]
          })
          server = tf.train.Server(
              cluster_spec, job_name=args.job, task_index=args.task,
              config=get_default_sess_config())
          # initialize trainer with this server object
  
      .. code-block:: none
  
          # Start training like this:
          (host1)$ ./train.py --job worker --task 0
          (host1)$ CUDA_VISIBLE_DEVICES= ./train.py --job ps --task 0
          (host2)$ ./train.py --job worker --task 1
          (host2)$ CUDA_VISIBLE_DEVICES= ./train.py --job ps --task 1
class_docstr_tok: ["Distributed","replicated","training",".","Each","worker","process","builds","the","same","model","on","one","or","more","GPUs",".","Gradients","across","GPUs","are","averaged","within","the","worker",",","and","get","synchronously","applied","to","the","global","copy","of","variables","located","on","PS",".","Then","each","worker","copy","the","latest","variables","from","PS","back","to","local",".","It","is","an","equivalent","of","``","--","variable_update=distributed_replicated","``","in","`","tensorflow\/benchmarks","<","https",":","\/\/github.com\/tensorflow\/benchmarks",">","`","_",".","Note","that","the","performance","of","this","trainer","is","still","not","satisfactory",",","and","TensorFlow","team","is","not","actively","maintaining","distributed","training","features",".","Check",":","class",":","`","HorovodTrainer","`","and","`","ResNet-Horovod","<","https",":","\/\/github.com\/tensorpack\/benchmarks\/tree\/master\/ResNet-Horovod",">","`","_","for","better","distributed","training","support",".","Note",":","1",".","Gradients","are","not","averaged","across","workers",",","but","applied","to","PS","variables","directly","(","either","with","or","without","locking","depending","on","the","optimizer",")",".","2",".","Some","details","about","collections",":","all","variables","created","inside","tower","will","become","local","variables",",","and","a","clone","will","be","made","in","global","variables","for","all","trainable\/model","variables",".","Example",":","..","code-block",":",":","python","#","Create","the","server","object","like","this",":","hosts","=","[","'host1.com","'",",","'host2.com","'","]","cluster_spec","=","tf.train.ClusterSpec","(","{","'ps","'",":","[","h","+","':2222","'","for","h","in","hosts","]",",","'worker","'",":","[","h","+","':2223","'","for","h","in","hosts","]","}",")","server","=","tf.train.Server","(","cluster_spec",",","job_name=args.job",",","task_index=args.task",",","config=get_default_sess_config","(",")",")","#","initialize","trainer","with","this","server","object","..","code-block",":",":","none","#","Start","training","like","this",":","(","host1",")","$",".\/train.py","--","job","worker","--","task","0","(","host1",")","$","CUDA_VISIBLE_DEVICES=",".\/train.py","--","job","ps","--","task","0","(","host2",")","$",".\/train.py","--","job","worker","--","task","1","(","host2",")","$","CUDA_VISIBLE_DEVICES=",".\/train.py","--","job","ps","--","task","1"]
docstr: Get the op to sync local model_variables to PS.
docstr_tok: ["Get","the","op","to","sync","local","model_variables","to","PS","."]
code: def _get_sync_model_vars_op(self):
      """
      Get the op to sync local model_variables to PS.
      """
      ops = []
      for shadow_v, local_v in self._shadow_model_vars:
          ops.append(shadow_v.assign(local_v.read_value()))
      assert len(ops)
      return tf.group(
          *ops, name="sync_{}_model_variables_to_ps".format(len(ops))
      )
  
imports: import re
  import tensorflow
  from ..tfutils.common import get_global_step_var, get_op_tensor_name
  from ..utils import logger
  from ..utils.argtools import memoized
  from .training import DataParallelBuilder, GraphBuilder
  from .utils import OverrideCachingDevice, split_grad_list, allreduce_grads_naive, override_to_local_variable
label_desc: Use image node_id 7 for calling the DistributedReplicatedBuilder obj's underlying member method code with example usage: obj._get_sync_model_vars_op() and returns: tf