import tensorflow as tf


                    # Compute the policy loss
                    # Alternative: policy_kl_loss = tf.reduce_mean(logp_pi - min_qf_pi)
                    policy_kl_loss = tf.reduce_mean((self.ent_coef * logp_pi - min_qf_pi)*self.weight_ph)
                    actor_for_priority = tf.reduce_mean(self.ent_coef * logp_pi - min_qf_pi,1)
                    # NOTE: in the original implementation, they have an additional
                    # regularization loss for the Gaussian parameters
                    # this is not used for now
                    # policy_loss = (policy_kl_loss + policy_regularization_loss)
                    min_q = tf.minimum(dtm_qf1,dtm_qf2)

                    Q_filter = tf.cast((qf1 > min_q)|(qf2 > min_q),tf.float32)
                    #Q_filter_1 = tf.cast(qf1 > min_q,tf.float32)
                    #Q_filter_2 = tf.cast(qf2 > min_q,tf.float32)
                    im_loss1 = tf.square(self.actions_ph - self.deterministic_actions_ph)*Q_filter*self.is_demo_ph
                    #im_loss2 = tf.square(self.actions_ph - self.deterministic_actions_ph)*Q_filter_2*self.is_demo_ph
                    #actor_loss_di1 = tf.reduce_mean(im_loss1)
                    #actor_loss_di2 = tf.reduce_mean(im_loss2)
                    self.actor_loss_di = tf.reduce_mean(im_loss1)
                    imitation_for_priority = tf.reduce_mean(im_loss1,axis=1)
                    regularizerpi = tf.contrib.layers.l1_l2_regularizer(scale_l1=0.0, scale_l2=1e-5, scope="model/pi")
                    all_trainable_weights_pi = tf.trainable_variables('model/pi')
