import tensorflow as tf

            
            # Loss and train op
            self.loss = -self.normal_dist.log_prob(self.a_his) * self.target
            # Add cross entropy cost to encourage exploration
            self.loss -= entropy_beta * self.normal_dist.entropy()
            self.optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
            self.grads_and_vars = self.optimizer.compute_gradients(self.loss)
            self.grads=[];
            self.vars=[];
            for i in range(len(self.grads_and_vars)):
              self.grads.append(self.grads_and_vars[i][0]);
              self.vars.append(self.grads_and_vars[i][1]);
            self.grads=self.grads[-1*NUM_VARS:];
            self.vars=self.vars[-1*NUM_VARS:];
            self.train_op = self.optimizer.apply_gradients(
                self.grads_and_vars, global_step=tf.contrib.framework.get_global_step())
             
    def predict(self, state, sess=None):
        sess = sess or tf.get_default_session()
        return sess.run(self.action, { self.state: [state] })[0]

    def update(self, state, target, a_his, sess=None):
        sess = sess or tf.get_default_session()
        feed_dict = { self.state: state, self.target: target, self.a_his: a_his  }
        _, loss = sess.run([self.train_op, self.loss], feed_dict)
        return loss

class ValueEstimator_Pendulum():
    def __init__(self, learning_rate=0.1, par_idx=0,scope="value_estimator"):
        w_init = tf.random_normal_initializer(0.,.1);
