code: def main(unused_argv):
      # Horovod: initialize Horovod.
      hvd.init()
  
      # Keras automatically creates a cache directory in ~/.keras/datasets for
      # storing the downloaded MNIST data. This creates a race
      # condition among the workers that share the same filesystem. If the
      # directory already exists by the time this worker gets around to creating
      # it, ignore the resulting exception and continue.
      cache_dir = os.path.join(
          os.path.expanduser("~"), ".keras", "datasets"
      )
      if not os.path.exists(cache_dir):
          try:
              os.mkdir(cache_dir)
          except OSError as e:
              if e.errno == errno.EEXIST and os.path.isdir(cache_dir):
                  pass
              else:
                  raise
  
      # Download and load MNIST dataset.
      (train_data, train_labels), (
          eval_data,
          eval_labels,
      ) = keras.datasets.mnist.load_data("MNIST-data-%d" % hvd.rank())
  
      # The shape of downloaded data is (-1, 28, 28), hence we need to reshape it
      # into (-1, 784) to feed into our network. Also, need to normalize the
      # features between 0 and 1.
      train_data = np.reshape(train_data, (-1, 784)) / 255.0
      eval_data = np.reshape(eval_data, (-1, 784)) / 255.0
  
      # Horovod: pin GPU to be used to process local rank (one GPU per process)
      config = tf.ConfigProto()
      config.gpu_options.allow_growth = True
      config.gpu_options.visible_device_list = str(hvd.local_rank())
  
      # Horovod: save checkpoints only on worker 0 to prevent other workers from
      # corrupting them.
      model_dir = "./mnist_convnet_model" if hvd.rank() == 0 else None
  
      # Create the Estimator
      mnist_classifier = tf.estimator.Estimator(
          model_fn=cnn_model_fn,
          model_dir=model_dir,
          config=tf.estimator.RunConfig(session_config=config),
      )
  
      # Set up logging for predictions
      # Log the values in the "Softmax" tensor with label "probabilities"
      tensors_to_log = {"probabilities": "softmax_tensor"}
      logging_hook = tf.train.LoggingTensorHook(
          tensors=tensors_to_log, every_n_iter=500
      )
  
      # Horovod: BroadcastGlobalVariablesHook broadcasts initial variable states from
      # rank 0 to all other processes. This is necessary to ensure consistent
      # initialization of all workers when training is started with random weights or
      # restored from a checkpoint.
      bcast_hook = hvd.BroadcastGlobalVariablesHook(0)
  
      # Train the model
      train_input_fn = tf.estimator.inputs.numpy_input_fn(
          x={"x": train_data},
          y=train_labels,
          batch_size=100,
          num_epochs=None,
          shuffle=True,
      )
  
      # Horovod: adjust number of steps based on number of GPUs.
      mnist_classifier.train(
          input_fn=train_input_fn,
          steps=20000 // hvd.size(),
          hooks=[logging_hook, bcast_hook],
      )
  
      # Evaluate the model and print results
      eval_input_fn = tf.estimator.inputs.numpy_input_fn(
          x={"x": eval_data}, y=eval_labels, num_epochs=1, shuffle=False
      )
      eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
      print(eval_results)
  
file_path: horovod/examples/tensorflow/tensorflow_mnist_estimator.py
imports: import os
  import errno
  import numpy
  import tensorflow
  import horovod.tensorflow
  from tensorflow import keras
label_desc: Use image node_id 2 for calling a global function with example usage: main(unused_argv) without return types