import tensorflow as tf

        ratio = tf.clip_by_value(ratio, 0, 10)
        surr1 = batch['advantage'] * ratio
        surr2 = batch['advantage'] * tf.clip_by_value(ratio, 1 - epsilon_decay, 1 + epsilon_decay)
        loss_pg = - 2.0 * tf.reduce_mean(tf.minimum(surr1, surr2))
