code: def get_sync_bn_mean_var(inputs, red_axis, sync_statistics):
      ctx = get_current_tower_context()
      batch_mean = tf.reduce_mean(inputs, axis=red_axis)
      batch_mean_square = tf.reduce_mean(
          tf.square(inputs), axis=red_axis
      )
  
      TF_version = get_tf_version_tuple()
  
      if sync_statistics == "nccl":
          num_dev = ctx.total
          if num_dev == 1:
              logger.warn(
                  "BatchNorm(sync_statistics='nccl') is used with only one tower!"
              )
          else:
              assert TF_version >= (1, 10), (
                  "Cross-GPU BatchNorm is only supported in TF>=1.10 ."
                  "Upgrade TF or apply this patch manually: https://github.com/tensorflow/tensorflow/pull/20360"
              )
              if TF_version >= (1, 15):
                  logger.warn(
                      "BatchNorm(sync_statistics='nccl') may produce incorrect results due "
                      "to bug in TF>=1.15: https://github.com/tensorflow/tensorflow/issues/41539"
                  )
  
              if TF_version <= (1, 12):
                  try:
                      from tensorflow.contrib.nccl.python.ops.nccl_ops import (
                          _validate_and_load_nccl_so,
                      )  # deprecated
                  except Exception:
                      pass
                  else:
                      _validate_and_load_nccl_so()
                  from tensorflow.contrib.nccl.ops import (
                      gen_nccl_ops,
                  )  # deprecated
              else:
                  from tensorflow.python.ops import gen_nccl_ops
              shared_name = re.sub(
                  "tower[0-9]+/", "", tf.get_variable_scope().name
              )
              batch_mean = gen_nccl_ops.nccl_all_reduce(
                  input=batch_mean,
                  reduction="sum",
                  num_devices=num_dev,
                  shared_name=shared_name + "_NCCL_mean",
              ) * (1.0 / num_dev)
              batch_mean_square = gen_nccl_ops.nccl_all_reduce(
                  input=batch_mean_square,
                  reduction="sum",
                  num_devices=num_dev,
                  shared_name=shared_name + "_NCCL_mean_square",
              ) * (1.0 / num_dev)
      elif sync_statistics == "horovod":
          # Require https://github.com/uber/horovod/pull/331
          import horovod.tensorflow as hvd
  
          if hvd.size() == 1:
              logger.warn(
                  "BatchNorm(sync_statistics='horovod') is used with only one process!"
              )
          else:
              import horovod
  
              hvd_version = tuple(
                  map(int, horovod.__version__.split(".")[:3])
              )
              assert hvd_version >= (
                  0,
                  13,
                  6,
              ), "sync_statistics=horovod needs horovod>=0.13.6 !"
  
              batch_mean = hvd.allreduce(batch_mean, average=True)
              batch_mean_square = hvd.allreduce(
                  batch_mean_square, average=True
              )
      batch_var = batch_mean_square - tf.square(batch_mean)
      return batch_mean, batch_var
  
file_path: tensorpack/tensorpack/models/batch_norm.py
imports: import re
  from ..compat import tfv1
  from tensorflow.python.training import moving_averages
  from ..tfutils.collection import backup_collection, restore_collection
  from ..tfutils.common import get_tf_version_tuple
  from ..tfutils.tower import get_current_tower_context
  from ..utils import logger
  from ..utils.argtools import get_data_format, log_once
  from .common import VariableHolder, layer_register
  from .tflayer import convert_to_tflayer_args, rename_get_variable
  from .utils import disable_autograph
label_desc: Use image node_id 3 for calling a global function with example usage: get_sync_bn_mean_var(inputs, red_axis, sync_statistics) and returns: batch_mean,  batch_var