import tensorflow as tf

        ratio = action_op.prob(self.tfa) / \
            (old_action_op.prob(self.tfa) + 1e-5)
        # 替代損失
        surr = ratio * self.tfadv
        # 減少代理損失
        self.aloss = -tf.reduce_mean(tf.minimum(
            surr,
            tf.clip_by_value(ratio, 1. - EPSILON, 1. + EPSILON) * self.tfadv))
        self.atrain_op = tf.train.AdamOptimizer(A_LR).minimize(self.aloss)
        # log
        self.train_writer = tf.summary.FileWriter("logs/", self.sess.graph)
        self.sess.run(tf.global_variables_initializer())
        self.tableAction = self.createActionTable()

