import tensorflow as tf

                self.expert_N_buffer.add((obs,action_list[i],reward_list[i],obs_,done))
                if len(n_step_buffer)== self.n_step_length:    
                    #self.expert_buffer.add(obs,action_list[i],reward_list[i],obs_,done_list[i],1)
                    one_step = n_step_buffer[0]
                    self.expert_buffer.add(one_step[0],one_step[1],one_step[2],one_step[3],one_step[4],1)

    def setup_model(self):
        with SetVerbosity(self.verbose):
            self.graph = tf.Graph()
            with self.graph.as_default():
                self.set_random_seed(self.seed)
                self.sess = tf_util.make_session(num_cpu=self.n_cpu_tf_sess, graph=self.graph)
                if self.prioritized_replay:
                    self.replay_buffer = PrioritizedReplayBuffer(self.buffer_size, alpha=self.prioritized_replay_alpha)
                    if self.prioritized_replay_beta_iters is None:
                        prioritized_replay_beta_iters = 100000
