import tensorflow as tf

        surr2 = batch['advantage'] * tf.clip_by_value(ratio, 1 - epsilon_decay, 1 + epsilon_decay)
        loss_pg = - 2.0 * tf.reduce_mean(tf.minimum(surr1, surr2))
        loss_vf = 0.5 * tf.reduce_mean(tf.square(batch['rewards'] - self.vf))
        loss_entropy = - 0.01 * tf.reduce_mean(pi.entropy())
        loss = loss_pg + loss_vf + loss_entropy
        opt = tf.train.AdamOptimizer(self.LR)
        self.train_op = opt.minimize(loss, global_step=self.global_step, var_list=pi_params + vf_params)

        self.pi_new_params = [oldp.assign(p) for p, oldp in zip(pi_params, pi_old_params)]
        self.vf_new_params = [oldp.assign(p) for p, oldp in zip(vf_params, vf_old_params)]
        self.sess.run(tf.global_variables_initializer())

        # Tensorboard
        if summary_dir is not None:
            self.writer = tf.summary.FileWriter(summary_dir)
        tf.summary.scalar('Loss/Policy', loss_pg)
        tf.summary.scalar('Loss/Value', loss_vf)
        tf.summary.scalar('Loss/Entropy', loss_entropy)
        tf.summary.scalar('Loss/Total', loss)
        tf.summary.scalar('Var/Epsilon', epsilon_decay)
        tf.summary.scalar('Var/Policy Mode', tf.reduce_mean(pi.mode()))
        tf.summary.scalar('Var/Policy Sigma', tf.reduce_mean(pi.stddev()))
        tf.summary.scalar('Var/Value', tf.reduce_mean(self.vf))
        self.summarise = tf.summary.merge(tf.get_collection(tf.GraphKeys.SUMMARIES))

    # AC net
    def build_anet(self, state_in, name, reuse=False, batch_size=64):
        reg = None
        with tf.variable_scope(name, reuse=reuse):
            layer_a1 = tf.layers.dense(state_in, 512, tf.nn.relu, kernel_regularizer=reg)
