import tensorflow as tf

                self.grads.append(self.grads_and_vars[i][0]);
                self.vars.append(self.grads_and_vars[i][1]);
            self.grads=self.grads[-1*NUM_VARS:];
            self.vars=self.vars[-1*NUM_VARS:];
            self.train_op = self.optimizer.apply_gradients(
                                    self.grads_and_vars, global_step=tf.contrib.framework.get_global_step())

    def predict(self, state, sess=None):
        sess = sess or tf.get_default_session()
        state=featurize_state(state);
        return sess.run(self.action, { self.state: [state] })[0]

    def update(self, state, target, action, sess=None):
        sess = sess or tf.get_default_session()
        for st_idx in range(len(state)):
            state[st_idx]=featurize_state(state[st_idx]);
        feed_dict = { self.state: state, self.target: target, self.a_his: action  }
        _, loss = sess.run([self.train_op, self.loss], feed_dict)
        return loss

class ValueEstimator_MountainCarContinuous():
    def __init__(self, learning_rate=0.1, par_idx=0,scope="value_estimator"):
        w_init = tf.random_normal_initializer(0.,.1);
        with tf.variable_scope(scope+"_"+str(par_idx)):
            # state and target
            self.state = tf.placeholder(tf.float32, [None,400], "state")
