import tensorflow as tf

        all_top_1_ops = tf.reduce_sum(all_top_1_ops)
        all_top_5_ops = tf.reduce_sum(all_top_5_ops)
        fetches = [all_top_1_ops, all_top_5_ops] + enqueue_ops
      return (enqueue_ops, fetches)
    extra_nccl_ops = []
    apply_gradient_devices, gradient_state = (
        self.variable_mgr.preprocess_device_grads(device_grads))

    training_ops = []
    for d, device in enumerate(apply_gradient_devices):
      with tf.device(device):
        total_loss = tf.reduce_mean(losses)
        avg_grads = self.variable_mgr.get_gradients_to_apply(d, gradient_state)

        gradient_clip = FLAGS.gradient_clip
        learning_rate = self.model_conf.get_learning_rate()
        if self.dataset and FLAGS.num_epochs_per_decay > 0:
          num_batches_per_epoch = (
              self.dataset.num_examples_per_epoch() / self.batch_size)
          decay_steps = int(num_batches_per_epoch * FLAGS.num_epochs_per_decay)

          # Decay the learning rate exponentially based on the number of steps.
