import tensorflow as tf

        # to get q_tot_
        q_ = self.sess.run(self.q_next, feed_dict={self.s_: s_})
        q_m_ = np.max(q_, axis=1)
        q_tot_ = self.sess.run(self.Q_tot_, feed_dict={self.S_: S_, self.q_m_: q_m_})

        q_target = np.array(R) + (1 - np.array(done)) * self.gamma * np.squeeze(q_tot_, axis=-1)

        # import pdb; pdb.set_trace()
        tvars = tf.trainable_variables()
        tvars_vals_b = self.sess.run(tvars)
        # f = open("before.txt", "a")
        # for var, val in zip(tvars, tvars_vals):
        #     f.write(var,)
        # f.close()
        # update
        _, cost = self.sess.run([self._train_op, self.loss],
