class_docstr: Critic Regularized Regression (CRR) algorithm trainer
  as described in https://arxiv.org/abs/2006.15134
docstr: Args:
      actor_network: states -> actions, trained to maximize value
      actor_network_target: copy of actor network for training stability
      q1_network: states -> q-value for all actions
      q1_network_target: copy of q-network for training stability
      q2_network (optional): double q-learning to stabilize training
          from overestimation bias. The presence of q2_network is specified
          in discrete_crr.py using the config parameter double_q_learning
      q2_network_target (optional): copy of q-network for training stability
      rl (optional): an instance of the RLParameter class, which
          defines relevant hyperparameters
      q_network_optimizer (optional): the optimizer class and
          optimizer hyperparameters for the q network(s) optimizer
      actor_network_optimizer (optional): see q_network_optimizer
      use_target_actor (optional): specifies whether target actor is used
      delayed_policy_update (optional): the ratio of q network updates
          to target and policy network updates
      beta: coefficient for KL-divergence policy constaint regularization of CRR
          see eq(5) in https://arxiv.org/pdf/2006.15134.pdf. With large beta, the output
          policy of CRR can not leaves too far away from the logged policy
  
      entropy_coeff: coefficient for entropy regularization
      clip_limit: threshold for importance sampling when compute entropy
          regularization using offline samples
      max_weight: the maximum possible action weight in the actor loss
  
      Explaination of entropy regularization:
      Entropy regularization punishes deterministic policy and encourages
      "unifom" policy. Entropy regularized MDP can be viewed as add the term
      (-entropy_coeff * pi_ratio * log_pi_b) to each reward. For detailed
      formulation of entropy regularized please see eq.(9) & eq.(10) in
      https://arxiv.org/pdf/2007.06558.pdf
docstr_tok: ["Args",":","actor_network",":","states","-",">","actions",",","trained","to","maximize","value","actor_network_target",":","copy","of","actor","network","for","training","stability","q1_network",":","states","-",">","q-value","for","all","actions","q1_network_target",":","copy","of","q-network","for","training","stability","q2_network","(","optional",")",":","double","q-learning","to","stabilize","training","from","overestimation","bias",".","The","presence","of","q2_network","is","specified","in","discrete_crr.py","using","the","config","parameter","double_q_learning","q2_network_target","(","optional",")",":","copy","of","q-network","for","training","stability","rl","(","optional",")",":","an","instance","of","the","RLParameter","class",",","which","defines","relevant","hyperparameters","q_network_optimizer","(","optional",")",":","the","optimizer","class","and","optimizer","hyperparameters","for","the","q","network","(","s",")","optimizer","actor_network_optimizer","(","optional",")",":","see","q_network_optimizer","use_target_actor","(","optional",")",":","specifies","whether","target","actor","is","used","delayed_policy_update","(","optional",")",":","the","ratio","of","q","network","updates","to","target","and","policy","network","updates","beta",":","coefficient","for","KL-divergence","policy","constaint","regularization","of","CRR","see","eq","(","5",")","in","https",":","\/\/arxiv.org\/pdf\/2006.15134.pdf",".","With","large","beta",",","the","output","policy","of","CRR","can","not","leaves","too","far","away","from","the","logged","policy","entropy_coeff",":","coefficient","for","entropy","regularization","clip_limit",":","threshold","for","importance","sampling","when","compute","entropy","regularization","using","offline","samples","max_weight",":","the","maximum","possible","action","weight","in","the","actor","loss","Explaination","of","entropy","regularization",":","Entropy","regularization","punishes","deterministic","policy","and","encourages","``","unifom","''","policy",".","Entropy","regularized","MDP","can","be","viewed","as","add","the","term","(","-entropy_coeff","*","pi_ratio","*","log_pi_b",")","to","each","reward",".","For","detailed","formulation","of","entropy","regularized","please","see","eq",".","(","9",")","&","eq",".","(","10",")","in","https",":","\/\/arxiv.org\/pdf\/2007.06558.pdf"]
code: def __init__(
      self,
      actor_network,
      actor_network_target,
      q1_network,
      q1_network_target,
      reward_network,
      q2_network=None,
      q2_network_target=None,
      q_network_cpe=None,
      q_network_cpe_target=None,
      metrics_to_score=None,
      evaluation: EvaluationParameters = field(  # noqa: B008
          default_factory=EvaluationParameters
      ),
      # Start CRRTrainerParameters. All parameters above should be
      # in the blocklist for CRRTrainerParameters in parameters.py
      rl: RLParameters = field(
          default_factory=RLParameters
      ),  # noqa: B008
      double_q_learning: bool = True,
      q_network_optimizer: Optimizer__Union = field(  # noqa: B008
          default_factory=Optimizer__Union.default
      ),
      actor_network_optimizer: Optimizer__Union = field(  # noqa: B008
          default_factory=Optimizer__Union.default
      ),
      use_target_actor: bool = False,
      actions: List[str] = field(default_factory=list),  # noqa: B008
      delayed_policy_update: int = 1,
      beta: float = 1.0,
      entropy_coeff: float = 0.0,
      clip_limit: float = 10.0,
      max_weight: float = 20.0,
  ) -> None:
      """
      Args:
          actor_network: states -> actions, trained to maximize value
          actor_network_target: copy of actor network for training stability
          q1_network: states -> q-value for all actions
          q1_network_target: copy of q-network for training stability
          q2_network (optional): double q-learning to stabilize training
              from overestimation bias. The presence of q2_network is specified
              in discrete_crr.py using the config parameter double_q_learning
          q2_network_target (optional): copy of q-network for training stability
          rl (optional): an instance of the RLParameter class, which
              defines relevant hyperparameters
          q_network_optimizer (optional): the optimizer class and
              optimizer hyperparameters for the q network(s) optimizer
          actor_network_optimizer (optional): see q_network_optimizer
          use_target_actor (optional): specifies whether target actor is used
          delayed_policy_update (optional): the ratio of q network updates
              to target and policy network updates
          beta: coefficient for KL-divergence policy constaint regularization of CRR
              see eq(5) in https://arxiv.org/pdf/2006.15134.pdf. With large beta, the output
              policy of CRR can not leaves too far away from the logged policy
  
          entropy_coeff: coefficient for entropy regularization
          clip_limit: threshold for importance sampling when compute entropy
              regularization using offline samples
          max_weight: the maximum possible action weight in the actor loss
  
          Explaination of entropy regularization:
          Entropy regularization punishes deterministic policy and encourages
          "unifom" policy. Entropy regularized MDP can be viewed as add the term
          (-entropy_coeff * pi_ratio * log_pi_b) to each reward. For detailed
          formulation of entropy regularized please see eq.(9) & eq.(10) in
          https://arxiv.org/pdf/2007.06558.pdf
      """
      super().__init__(
          rl,
          metrics_to_score=metrics_to_score,
          actions=actions,
          evaluation_parameters=evaluation,
      )
      self._actions = actions
      assert (
          self._actions is not None
      ), "Discrete-action CRR needs action names"
  
      self.rl_parameters = rl
      self.double_q_learning = double_q_learning
  
      self.use_target_actor = use_target_actor
  
      self.q1_network = q1_network
      self.q1_network_target = q1_network_target
      self.q_network_optimizer = q_network_optimizer
  
      self.q2_network = q2_network
      if self.q2_network is not None:
          assert (
              q2_network_target is not None
          ), "q2_network provided without a target network"
          self.q2_network_target = q2_network_target
  
      self.actor_network = actor_network
      self.actor_network_target = actor_network_target
      self.actor_network_optimizer = actor_network_optimizer
  
      self.delayed_policy_update = delayed_policy_update
  
      self._initialize_cpe(
          reward_network,
          q_network_cpe,
          q_network_cpe_target,
          optimizer=q_network_optimizer,
      )
      self.beta = beta
      self.entropy_coeff = entropy_coeff
      self.clip_limit = clip_limit
      self.max_weight = max_weight
  
imports: import logging
  from typing import List, Tuple
  import reagent.core.types
  import torch
  import torch.nn.functional
  from reagent.core.configuration import resolve_defaults
  from reagent.core.dataclasses import field
  from reagent.core.parameters import EvaluationParameters, RLParameters
  from reagent.optimizer import Optimizer__Union, SoftUpdate
  from reagent.training.dqn_trainer_base import DQNTrainerBaseLightning
  from torch import distributions
label_desc: Use image node_id 1 to create a new DiscreteCRRTrainer object from inherited base classes: DQNTrainerBaseLightning with example: obj = DiscreteCRRTrainer(actor_network, actor_network_target, q1_network, q1_network_target, reward_network, q2_network, q2_network_target, q_network_cpe, q_network_cpe_target, metrics_to_score, evaluation, rl, double_q_learning, q_network_optimizer, actor_network_optimizer, use_target_actor, actions, delayed_policy_update, beta, entropy_coeff, clip_limit, max_weight)