message: training: print errors only in the main worker
diff: @@ -14,6 +14,8 @@ import torch.nn as nn
  import torch.multiprocessing as mp
  import torch.distributed as dist
  +WORKER_RANK = 0
  +
  def round_down(a, b):
  return a // b * b
  @@ -25,6 +27,7 @@ def round_nearest(a, b):
  # Prints an error message and exits
  def error(*args):
  + if WORKER_RANK == 0:
  print('Error:', *args)
  exit(1)
  @@ -102,6 +105,10 @@ def init_worker(rank, cfg):
  rank=rank, world_size=cfg.num_devices,
  init_method='env://')
  + # Set the global rank for this worker process
  + global WORKER_RANK
  + WORKER_RANK = rank
  +
  # Running in distributed mode
  return True
  else:
  