import tensorflow as tf

                for grad, var in itertools.chain(*tower_gradvars):
                    if grad is not None:
                        all_grads.setdefault(var, []).append(grad)
                for var, grads in all_grads.items():
                    if len(grads) == 1:
                        avg_grad = grads[0]
                    else:
                        avg_grad = tf.multiply(tf.add_n(grads), 1. / len(grads))
                    gradvars.append((avg_grad, var))
                self.loss = tf.reduce_mean(tower_losses)
                tf.summary.scalar('loss', self.loss)

            # Create optimizer ops
            self.global_step = tf.Variable(0, trainable=False, name='global_step')
            opt = tf.train.RMSPropOptimizer(self.config['learning_rate'])
            with tf.control_dependencies(update_ops):
                self.trainer = opt.apply_gradients(
                        gradvars, global_step=self.global_step)

    def _eval_graph(self, data):
        tower_metrics = self._gpu_tower(data, Mode.EVAL)
        with tf.device('/cpu:0'):
            self.metrics = {m: tf.reduce_mean(tf.stack([t[m] for t in tower_metrics]))
                            for m in tower_metrics[0]}

    def _pred_graph(self, data):
        with tf.name_scope('pred'):
            with tf.device('/gpu:0'):
