class_docstr: Distributed parameter server training.
  A single copy of parameters are scattered around PS.
  Gradients across GPUs are averaged within the worker, and applied to PS.
  Each worker also caches the variables for reading.
  
  It is an equivalent of ``--variable_update=parameter_server`` in
  `tensorflow/benchmarks <https://github.com/tensorflow/benchmarks>`_.
  However this implementation hasn't been well tested.
  It probably still has issues in model saving, etc.
  Also, TensorFlow team is not actively maintaining distributed training features.
  Check :class:`HorovodTrainer` and
  `ResNet-Horovod <https://github.com/tensorpack/benchmarks/tree/master/ResNet-Horovod>`_
  for better distributed training support.
  
  Note:
      1. Gradients are not averaged across workers, but applied to PS variables
         directly (either with or without locking depending on the optimizer).
class_docstr_tok: ["Distributed","parameter","server","training",".","A","single","copy","of","parameters","are","scattered","around","PS",".","Gradients","across","GPUs","are","averaged","within","the","worker",",","and","applied","to","PS",".","Each","worker","also","caches","the","variables","for","reading",".","It","is","an","equivalent","of","``","--","variable_update=parameter_server","``","in","`","tensorflow\/benchmarks","<","https",":","\/\/github.com\/tensorflow\/benchmarks",">","`","_",".","However","this","implementation","has","n't","been","well","tested",".","It","probably","still","has","issues","in","model","saving",",","etc",".","Also",",","TensorFlow","team","is","not","actively","maintaining","distributed","training","features",".","Check",":","class",":","`","HorovodTrainer","`","and","`","ResNet-Horovod","<","https",":","\/\/github.com\/tensorpack\/benchmarks\/tree\/master\/ResNet-Horovod",">","`","_","for","better","distributed","training","support",".","Note",":","1",".","Gradients","are","not","averaged","across","workers",",","but","applied","to","PS","variables","directly","(","either","with","or","without","locking","depending","on","the","optimizer",")","."]
docstr: Args:
      towers (list[int]): list of GPU ids.
      server (tf.train.Server): the server with ps and workers.
          job_name must be 'worker'.
      caching_device (str): either 'cpu' or 'gpu'
docstr_tok: ["Args",":","towers","(","list","[","int","]",")",":","list","of","GPU","ids",".","server","(","tf.train.Server",")",":","the","server","with","ps","and","workers",".","job_name","must","be","'worker","'",".","caching_device","(","str",")",":","either","'cpu","'","or","'gpu","'"]
code: def __init__(self, towers, server, caching_device):
      """
      Args:
          towers (list[int]): list of GPU ids.
          server (tf.train.Server): the server with ps and workers.
              job_name must be 'worker'.
          caching_device (str): either 'cpu' or 'gpu'
      """
      DataParallelBuilder.__init__(self, towers)
      DistributedBuilderBase.__init__(self, server)
  
      assert caching_device in ["cpu", "gpu"], caching_device
      self.caching_device = caching_device
  
      self.is_chief = self.task_index == 0
  
      worker_prefix = "/job:worker/task:%s" % self.task_index
      self.param_server_device = tf.train.replica_device_setter(
          worker_device=worker_prefix + "/cpu:0", cluster=self.cluster
      )
      self.cpu_device = "%s/cpu:0" % worker_prefix
      self.raw_devices = [
          "{}/gpu:{}".format(worker_prefix, k) for k in self.towers
      ]
  
      self.sync_queue_devices = [
          "/job:ps/task:%s/cpu:0" % i for i in range(self.num_ps)
      ]
  
imports: import re
  import tensorflow
  from ..tfutils.common import get_global_step_var, get_op_tensor_name
  from ..utils import logger
  from ..utils.argtools import memoized
  from .training import DataParallelBuilder, GraphBuilder
  from .utils import OverrideCachingDevice, split_grad_list, allreduce_grads_naive, override_to_local_variable
label_desc: Use image node_id 1 to create a new DistributedParameterServerBuilder object from inherited base classes: DataParallelBuilder, DistributedBuilderBase with example: obj = DistributedParameterServerBuilder(towers, server, caching_device)