code: def _critic_learn(self, obs, action, reward, next_obs, terminal):
      with paddle.no_grad():
          next_action, next_log_pro = self.sample(next_obs)
          q1_next, q2_next = self.target_model.value(
              next_obs, next_action
          )
          target_Q = paddle.minimum(
              q1_next, q2_next
          )  # a little different from SAC
          target_Q = reward + self.gamma * (1.0 - terminal) * target_Q
      cur_q1, cur_q2 = self.model.value(obs, action)
  
      qf1_loss = F.mse_loss(cur_q1, target_Q)
      qf2_loss = F.mse_loss(cur_q2, target_Q)
      mse = qf2_loss + qf1_loss
      ## add CQL
      random_actions_tensor = paddle.uniform(
          shape=[cur_q2.shape[0] * self.num_random, action.shape[-1]]
      )
      curr_actions_tensor, curr_log_pis = self._get_policy_actions(obs)
      new_curr_actions_tensor, new_log_pis = self._get_policy_actions(
          next_obs
      )
  
      q1_rand, q2_rand = self._get_tensor_values(
          obs, random_actions_tensor
      )
      q1_curr_actions, q2_curr_actions = self._get_tensor_values(
          obs, curr_actions_tensor
      )
      q1_next_actions, q2_next_actions = self._get_tensor_values(
          obs, new_curr_actions_tensor
      )
  
      cat_q1 = paddle.concat(
          [q1_rand, cur_q1.unsqueeze(1), q1_curr_actions], 1
      )
      cat_q2 = paddle.concat(
          [q2_rand, cur_q2.unsqueeze(1), q2_curr_actions], 1
      )
  
      if self.min_q_version == 3:
          # importance sampled version
          random_density = np.log(0.5 ** curr_actions_tensor.shape[-1])
          cat_q1 = paddle.concat(
              [
                  q1_rand - random_density,
                  q1_next_actions - new_log_pis.detach(),
                  q1_curr_actions - curr_log_pis.detach(),
              ],
              1,
          )
          cat_q2 = paddle.concat(
              [
                  q2_rand - random_density,
                  q2_next_actions - new_log_pis.detach(),
                  q2_curr_actions - curr_log_pis.detach(),
              ],
              1,
          )
  
      min_qf1_loss = (
          paddle.logsumexp(
              cat_q1 / self.temp,
              axis=1,
          ).mean()
          * self.min_q_weight
          * self.temp
      )
      min_qf2_loss = (
          paddle.logsumexp(
              cat_q2 / self.temp,
              axis=1,
          ).mean()
          * self.min_q_weight
          * self.temp
      )
  
      # Subtract the log likelihood of data
      min_qf1_loss -= cur_q1.mean() * self.min_q_weight
      min_qf2_loss -= cur_q2.mean() * self.min_q_weight
  
      if self.with_lagrange:
          self.alpha_prime = paddle.clip(
              self.log_alpha_prime.exp(), min=0.0, max=1000000.0
          )
          min_qf1_loss = self.alpha_prime * (
              min_qf1_loss - self.target_action_gap
          )
          min_qf2_loss = self.alpha_prime * (
              min_qf2_loss - self.target_action_gap
          )
  
          self.alpha_prime_optimizer.clear_grad()
          alpha_prime_loss = (-min_qf1_loss - min_qf2_loss) * 0.5
          alpha_prime_loss.backward(retain_graph=True)
          self.alpha_prime_optimizer.step()
  
      qf1_loss = qf1_loss + min_qf1_loss
      qf2_loss = qf2_loss + min_qf2_loss
      ## CQL done
  
      critic_loss = qf1_loss + qf2_loss
      self.critic_optimizer.clear_grad()
      critic_loss.backward(retain_graph=True)
      self.critic_optimizer.step()
      return critic_loss, mse
  
file_path: PARL/parl/algorithms/paddle/cql.py
imports: import parl
  import paddle
  from paddle.distribution import Normal
  import paddle.nn.functional
  from parl.utils.utils import check_model_method
  from copy import deepcopy
  import numpy
label_desc: Use image node_id 5 for calling the CQL obj's underlying member method code with example usage: obj._critic_learn(obs, action, reward, next_obs, terminal) and returns: critic_loss,  mse