import tensorflow as tf

        epsilon_decay = tf.train.polynomial_decay(self.EPSILON, self.global_step, self.EPS_LEN, 0.1, power=0)
        ratio = tf.maximum(pi.prob(batch['actions']), 1e-6) / tf.maximum(pi_old.prob(batch['actions']), 1e-6)
        ratio = tf.clip_by_value(ratio, 0, 10)
        surr1 = batch['advantage'] * ratio
