import tensorflow as tf

        self.vf, vf_params, self.vf_state_init, self.vf_state_final = self.build_cnet(batch['state'], 'vf')
        self.vf_eval, _, self.vf_eval_state_init, self.vf_eval_state_final = self.build_cnet(self.state, 'vf', reuse=True, batch_size=1)

        self.sample_action = tf.squeeze(pi_eval.sample(1), axis=0)
        self.eval_action = pi_eval.mode()
        self.global_step = tf.train.get_or_create_global_step()
        self.saver = tf.train.Saver()
        # Loss functions and training
        epsilon_decay = tf.train.polynomial_decay(self.EPSILON, self.global_step, self.EPS_LEN, 0.1, power=1)
        ratio = tf.maximum(pi.prob(batch['actions']), 1e-6) / tf.maximum(pi_old.prob(batch['actions']), 1e-6)
        ratio = tf.clip_by_value(ratio, 0, 10)
        surr1 = batch['advantage'] * ratio
        surr2 = batch['advantage'] * tf.clip_by_value(ratio, 1 - epsilon_decay, 1 + epsilon_decay)
        loss_pg = - 2.0 * tf.reduce_mean(tf.minimum(surr1, surr2))
        loss_vf = 0.5 * tf.reduce_mean(tf.square(batch['rewards'] - self.vf))
        loss_entropy = - 0.01 * tf.reduce_mean(pi.entropy())
        loss = loss_pg + loss_vf + loss_entropy
        opt = tf.train.AdamOptimizer(self.LR)
        self.train_op = opt.minimize(loss, global_step=self.global_step, var_list=pi_params + vf_params)

        self.pi_new_params = [oldp.assign(p) for p, oldp in zip(pi_params, pi_old_params)]
        self.vf_new_params = [oldp.assign(p) for p, oldp in zip(vf_params, vf_old_params)]
        self.sess.run(tf.global_variables_initializer())

        # Tensorboard
        if summary_dir is not None:
