import tensorflow as tf

                # with tf.variable_scope('layer_mix_eval'):
                #     lin1 = tf.matmul(tf.reshape(self.q_concat, shape=[-1, 1, self.n_agents]), self.w1) + tf.reshape(self.b1, shape=[-1, 1, 32])
                #     a1 = tf.nn.elu(lin1, name='a1')
                #     self.Q_tot = tf.reshape(tf.matmul(a1, self.w2), shape=[-1, 1]) + self.b2

                # with tf.variable_scope('layer_mix_target'):
                #     lin1_ = tf.matmul(tf.reshape(self.q_concat_, shape=[-1, 1, self.n_agents]), self.w1_) + tf.reshape(self.b1_, shape=[-1, 1, 32])
                #     a1_ = tf.nn.elu(lin1_, name='a1_')
                #     self.Q_tot_ = tf.reshape(tf.matmul(a1_, self.w2_), shape=[-1, 1]) + self.b2_

            # todo: add q_target, loss, train_op
            # with tf.variable_scope('q_target'):
                
            with tf.variable_scope('loss'):
                self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, tf.squeeze(self.Q_tot), name='TD_error'))
                # self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, self.Q_tot, name='TD_error'))

            with tf.variable_scope('train'):
                self._train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss)

    def act(self, state):
        if np.random.uniform() > self.epsilon :# pick the argmax action
            s = np.array(state)
            if len(s.shape) < 2:
                s = np.array(state)[np.newaxis, :]
            q_eval = self.sess.run(self.q_eval, feed_dict={self.s: s})
            action = np.argmax(q_eval, axis=-1).tolist()
        else:  # pick random action
