'Sets self._train_op, op to run for training.'

def _add_train_op(self):

    hps = self._hps
    self._lr_rate = tf.maximum(hps.min_lr, tf.train.exponential_decay(hps.lr, self.global_step, 30000, 0.98))
    tvars = tf.trainable_variables()
    with tf.device(self._get_gpu((self._num_gpus - 1))):
        (grads, global_norm) = tf.clip_by_global_norm(tf.gradients(self._loss, tvars), hps.max_grad_norm)
    tf.summary.scalar('global_norm', global_norm)
    optimizer = tf.train.GradientDescentOptimizer(self._lr_rate)
    tf.summary.scalar('learning    rate', self._lr_rate)
    self._train_op = optimizer.apply_gradients(zip(grads, tvars), global_step=self.global_step, name='train_step')
