class_docstr: Data-parallel training in "replicated" mode,
  where each GPU contains a replicate of the whole model.
  It will build one tower on each GPU under its own variable scope.
  Each gradient update is averaged or summed across or GPUs through NCCL.
  
  It is an equivalent of ``--variable_update=replicated`` in
  `tensorflow/benchmarks <https://github.com/tensorflow/benchmarks>`_.
class_docstr_tok: ["Data-parallel","training","in","``","replicated","''","mode",",","where","each","GPU","contains","a","replicate","of","the","whole","model",".","It","will","build","one","tower","on","each","GPU","under","its","own","variable","scope",".","Each","gradient","update","is","averaged","or","summed","across","or","GPUs","through","NCCL",".","It","is","an","equivalent","of","``","--","variable_update=replicated","``","in","`","tensorflow\/benchmarks","<","https",":","\/\/github.com\/tensorflow\/benchmarks",">","`","_","."]
docstr: Copy values of variables on GPU 0 to other GPUs.
docstr_tok: ["Copy","values","of","variables","on","GPU","0","to","other","GPUs","."]
code: def get_post_init_ops():
      """
      Copy values of variables on GPU 0 to other GPUs.
      """
      # literally all variables, because it's better to sync optimizer-internal variables as well
      all_vars = tfv1.global_variables() + tfv1.local_variables()
      var_by_name = {v.name: v for v in all_vars}
      trainable_names = {x.name for x in tfv1.trainable_variables()}
      post_init_ops = []
  
      def log_failure(name, reason):
          logger.warn(
              "[ReplicatedTrainer] Do not know how to sync variable '{}' across GPUs. "
              "Reason: {} ".format(name, reason)
          )
          assert (
              name not in trainable_names
          ), "The aforementioned variable is trainable, so this is probably a fatal error."
          logger.warn(
              "[ReplicatedTrainer] This variable is non-trainable. "
              "Ignore this warning if you know it's OK to leave it out-of-sync."
          )
  
      for v in all_vars:
          if not v.name.startswith("tower"):
              continue
          if v.name.startswith("tower0"):
              # in this trainer, the master name doesn't have the towerx/ prefix
              log_failure(
                  v.name,
                  "Name should not have prefix 'tower0' in this trainer!",
              )
              continue  # TODO some vars (EMA) may still startswith tower0
  
          split_name = v.name.split("/")
          prefix = split_name[0]
          realname = "/".join(split_name[1:])
          if prefix in realname:
              log_failure(
                  v.name,
                  "Prefix {} appears multiple times in its name!".format(
                      prefix
                  ),
              )
              continue
          copy_from = var_by_name.get(realname)
          if copy_from is not None:
              post_init_ops.append(v.assign(copy_from.read_value()))
          else:
              log_failure(
                  v.name,
                  "Cannot find {} in the graph!".format(realname),
              )
      logger.info(
          "'sync_variables_from_main_tower' includes {} operations.".format(
              len(post_init_ops)
          )
      )
      return tf.group(
          *post_init_ops, name="sync_variables_from_main_tower"
      )
  
imports: import copy
  import pprint
  import re
  from abc import ABCMeta, abstractmethod
  from contextlib import contextmanager
  import six
  import tensorflow
  from ..compat import tfv1
  from ..tfutils.common import get_tf_version_tuple
  from ..tfutils.gradproc import ScaleGradient
  from ..tfutils.tower import TrainTowerContext
  from ..utils import logger
  from ..utils.develop import HIDE_DOC
  from .utils import GradientPacker, LeastLoadedDeviceSetter, aggregate_grads_colocate, allreduce_grads_naive, allreduce_grads, allreduce_grads_hierarchical, merge_grad_list, override_to_local_variable, split_grad_list
label_desc: Use image node_id 4 for calling the SyncMultiGPUReplicatedBuilder obj's underlying member method code with example usage: obj.get_post_init_ops() and returns: tf