docstr: CQL algorithm
  Args:
      model (parl.Model): forward network of actor and critic.
      gamma (float): discounted factor for reward computation
      tau (float): decay coefficient when updating the weights of self.target_model with self.model
      actor_lr (float): learning rate of the actor model
      critic_lr (float): learning rate of the critic model
      policy_eval_start (int): try doing behaivoral cloning at the beginning, 40000 or 10000 work similarly
      with_automatic_entropy_tuning (bool): train with automatic entropy tuning in Actor.
      with_lagrange (bool): train with lagrange
      lagrange_thresh (float): the value of tau, corresponds to the CQL(lagrange) version, suggest 10.0 in mujoco and 5.0 in Franka kitchen or Adroit domains
      min_q_version (int): min_q_version = 3 (CQL(H)), = 2 (CQL(rho)), will be set to <0 in cql if not using lagrange
      min_q_weight (float): the value of alpha in Critic loss, suggest 5.0 or 10.0 if not using lagrange
      alpha (float): the value of alpha(temperature parameter) in Actor loss, determines the relative importance of entropy term against the reward
docstr_tok: ["CQL","algorithm","Args",":","model","(","parl.Model",")",":","forward","network","of","actor","and","critic",".","gamma","(","float",")",":","discounted","factor","for","reward","computation","tau","(","float",")",":","decay","coefficient","when","updating","the","weights","of","self.target_model","with","self.model","actor_lr","(","float",")",":","learning","rate","of","the","actor","model","critic_lr","(","float",")",":","learning","rate","of","the","critic","model","policy_eval_start","(","int",")",":","try","doing","behaivoral","cloning","at","the","beginning",",","40000","or","10000","work","similarly","with_automatic_entropy_tuning","(","bool",")",":","train","with","automatic","entropy","tuning","in","Actor",".","with_lagrange","(","bool",")",":","train","with","lagrange","lagrange_thresh","(","float",")",":","the","value","of","tau",",","corresponds","to","the","CQL","(","lagrange",")","version",",","suggest","10.0","in","mujoco","and","5.0","in","Franka","kitchen","or","Adroit","domains","min_q_version","(","int",")",":","min_q_version","=","3","(","CQL","(","H",")",")",",","=","2","(","CQL","(","rho",")",")",",","will","be","set","to","<","0","in","cql","if","not","using","lagrange","min_q_weight","(","float",")",":","the","value","of","alpha","in","Critic","loss",",","suggest","5.0","or","10.0","if","not","using","lagrange","alpha","(","float",")",":","the","value","of","alpha","(","temperature","parameter",")","in","Actor","loss",",","determines","the","relative","importance","of","entropy","term","against","the","reward"]
code: def __init__(
      self,
      model,
      gamma=None,
      tau=None,
      actor_lr=None,
      critic_lr=None,
      policy_eval_start=40000,
      with_automatic_entropy_tuning=True,
      with_lagrange=False,
      lagrange_thresh=10.0,
      min_q_version=3,
      min_q_weight=5.0,
      alpha=1.0,
  ):
      """CQL algorithm
      Args:
          model (parl.Model): forward network of actor and critic.
          gamma (float): discounted factor for reward computation
          tau (float): decay coefficient when updating the weights of self.target_model with self.model
          actor_lr (float): learning rate of the actor model
          critic_lr (float): learning rate of the critic model
          policy_eval_start (int): try doing behaivoral cloning at the beginning, 40000 or 10000 work similarly
          with_automatic_entropy_tuning (bool): train with automatic entropy tuning in Actor.
          with_lagrange (bool): train with lagrange
          lagrange_thresh (float): the value of tau, corresponds to the CQL(lagrange) version, suggest 10.0 in mujoco and 5.0 in Franka kitchen or Adroit domains
          min_q_version (int): min_q_version = 3 (CQL(H)), = 2 (CQL(rho)), will be set to <0 in cql if not using lagrange
          min_q_weight (float): the value of alpha in Critic loss, suggest 5.0 or 10.0 if not using lagrange
          alpha (float): the value of alpha(temperature parameter) in Actor loss, determines the relative importance of entropy term against the reward
      """
      # check model method
      check_model_method(model, "value", self.__class__.__name__)
      check_model_method(model, "policy", self.__class__.__name__)
      check_model_method(
          model, "get_actor_params", self.__class__.__name__
      )
      check_model_method(
          model, "get_critic_params", self.__class__.__name__
      )
      assert isinstance(gamma, float)
      assert isinstance(tau, float)
      assert isinstance(actor_lr, float)
      assert isinstance(critic_lr, float)
      assert isinstance(with_automatic_entropy_tuning, bool)
      assert isinstance(with_lagrange, bool)
      assert isinstance(lagrange_thresh, float)
      assert isinstance(min_q_version, int)
      assert isinstance(min_q_weight, float)
      assert isinstance(alpha, float)
  
      self.gamma = gamma
      self.tau = tau
      self.actor_lr = actor_lr
      self.critic_lr = critic_lr
      self.alpha_prime = 0.0
      self.policy_eval_start = policy_eval_start
      self.with_automatic_entropy_tuning = with_automatic_entropy_tuning
      self.with_lagrange = with_lagrange
      self.lagrange_thresh = lagrange_thresh
      self.min_q_version = -1 if self.with_lagrange else min_q_version
      self.min_q_weight = min_q_weight
      self.alpha = alpha
      self.temp = 1.0
      self.num_random = 10
      self._current_steps = 0
      self.model = model
      self.target_model = deepcopy(self.model)
      self.actor_optimizer = paddle.optimizer.Adam(
          learning_rate=actor_lr,
          parameters=self.model.get_actor_params(),
      )
      self.critic_optimizer = paddle.optimizer.Adam(
          learning_rate=critic_lr,
          parameters=self.model.get_critic_params(),
      )
  
      if self.with_automatic_entropy_tuning:
          self.target_entropy = None
          self.log_alpha = paddle.zeros([1])
          self.log_alpha.stop_gradient = False
          self.alpha_optimizer = paddle.optimizer.Adam(
              learning_rate=actor_lr, parameters=[self.log_alpha]
          )
      if self.with_lagrange:
          self.target_action_gap = self.lagrange_thresh
          self.log_alpha_prime = paddle.zeros([1])
          self.log_alpha_prime.stop_gradient = False
          self.alpha_prime_optimizer = paddle.optimizer.Adam(
              learning_rate=critic_lr, parameters=[self.log_alpha_prime]
          )
  
file_path: PARL/parl/algorithms/paddle/cql.py
imports: import parl
  import paddle
  from paddle.distribution import Normal
  import paddle.nn.functional
  from parl.utils.utils import check_model_method
  from copy import deepcopy
  import numpy
label_desc: Use image node_id 1 to create a new CQL object from inherited base classes: parl with example: obj = CQL(model, gamma, tau, actor_lr, critic_lr, policy_eval_start, with_automatic_entropy_tuning, with_lagrange, lagrange_thresh, min_q_version, min_q_weight, alpha)