docstr: CRR training configuration
  
  Args:
      weight_type: weight type to use `bin` | `exp`.
      temperature: the exponent temperature used in exp weight type.
      max_weight: the max weight limit for exp weight type.
      advantage_type: The way we reduce q values to v_t values
          `max` | `mean` | `expectation`. `max` and `mean` work for both
          discrete and continuous action spaces while `expectation` only
          works for discrete action spaces.
          `max`: Uses max over sampled actions to estimate the value.
  
          .. math::
  
              A(s_t, a_t) = Q(s_t, a_t) - \max_{a^j} Q(s_t, a^j)
  
          where :math:`a^j` is `n_action_sample` times sampled from the
          policy :math:`\pi(a | s_t)`
          `mean`: Uses mean over sampled actions to estimate the value.
  
          .. math::
  
              A(s_t, a_t) = Q(s_t, a_t) - \frac{1}{m}\sum_{j=1}^{m}
              [Q(s_t, a^j)]
  
          where :math:`a^j` is `n_action_sample` times sampled from the
          policy :math:`\pi(a | s_t)`
          `expectation`: This uses categorical distribution to evaluate
          the expectation of the q values directly to estimate the value.
  
          .. math::
  
              A(s_t, a_t) = Q(s_t, a_t) - E_{a^j\sim \pi(a|s_t)}[Q(s_t,a^j)]
  
      n_action_sample: the number of actions to sample for v_t estimation.
      twin_q: if True, uses pessimistic q estimation.
      target_network_update_freq: The frequency at which we update the
          target copy of the model in terms of the number of gradient updates
          applied to the main model.
      actor_hiddens: The number of hidden units in the actor's fc network.
      actor_hidden_activation: The activation used in the actor's fc network.
      critic_hiddens: The number of hidden units in the critic's fc network.
      critic_hidden_activation: The activation used in the critic's fc network.
      tau: Polyak averaging coefficient
          (making it 1 is reduces it to a hard update).
      td_error_loss_fn: "huber" or "mse".
          Loss function for calculating critic error.
      categorical_distribution_temperature: Set the temperature parameter used
          by Categorical action distribution. A valid temperature is in the range
          of [0, 1]. Note that this mostly affects evaluation since critic error
          uses selected action for return calculation.
      actor_lr: Learning rate for the actor.
      critic_lr: Learning rate for the critic.
      **kwargs: forward compatibility kwargs
  
  Returns:
      This updated CRRConfig object.
docstr_tok: ["CRR","training","configuration","Args",":","weight_type",":","weight","type","to","use","`","bin","`","|","`","exp","`",".","temperature",":","the","exponent","temperature","used","in","exp","weight","type",".","max_weight",":","the","max","weight","limit","for","exp","weight","type",".","advantage_type",":","The","way","we","reduce","q","values","to","v_t","values","`","max","`","|","`","mean","`","|","`","expectation","`",".","`","max","`","and","`","mean","`","work","for","both","discrete","and","continuous","action","spaces","while","`","expectation","`","only","works","for","discrete","action","spaces",".","`","max","`",":","Uses","max","over","sampled","actions","to","estimate","the","value",".","..","math",":",":","A","(","s_t",",","a_t",")","=","Q","(","s_t",",","a_t",")","-","\\max_","{","a^j","}","Q","(","s_t",",","a^j",")","where",":","math",":","`","a^j","`","is","`","n_action_sample","`","times","sampled","from","the","policy",":","math",":","`","\\pi","(","a","|","s_t",")","`","`","mean","`",":","Uses","mean","over","sampled","actions","to","estimate","the","value",".","..","math",":",":","A","(","s_t",",","a_t",")","=","Q","(","s_t",",","a_t",")","-","\\frac","{","1","}","{","m","}","\\sum_","{","j=1","}","^","{","m","}","[","Q","(","s_t",",","a^j",")","]","where",":","math",":","`","a^j","`","is","`","n_action_sample","`","times","sampled","from","the","policy",":","math",":","`","\\pi","(","a","|","s_t",")","`","`","expectation","`",":","This","uses","categorical","distribution","to","evaluate","the","expectation","of","the","q","values","directly","to","estimate","the","value",".","..","math",":",":","A","(","s_t",",","a_t",")","=","Q","(","s_t",",","a_t",")","-","E_","{","a^j\\sim","\\pi","(","a|s_t",")","}","[","Q","(","s_t",",","a^j",")","]","n_action_sample",":","the","number","of","actions","to","sample","for","v_t","estimation",".","twin_q",":","if","True",",","uses","pessimistic","q","estimation",".","target_network_update_freq",":","The","frequency","at","which","we","update","the","target","copy","of","the","model","in","terms","of","the","number","of","gradient","updates","applied","to","the","main","model",".","actor_hiddens",":","The","number","of","hidden","units","in","the","actor","'s","fc","network",".","actor_hidden_activation",":","The","activation","used","in","the","actor","'s","fc","network",".","critic_hiddens",":","The","number","of","hidden","units","in","the","critic","'s","fc","network",".","critic_hidden_activation",":","The","activation","used","in","the","critic","'s","fc","network",".","tau",":","Polyak","averaging","coefficient","(","making","it","1","is","reduces","it","to","a","hard","update",")",".","td_error_loss_fn",":","``","huber","''","or","``","mse","''",".","Loss","function","for","calculating","critic","error",".","categorical_distribution_temperature",":","Set","the","temperature","parameter","used","by","Categorical","action","distribution",".","A","valid","temperature","is","in","the","range","of","[","0",",","1","]",".","Note","that","this","mostly","affects","evaluation","since","critic","error","uses","selected","action","for","return","calculation",".","actor_lr",":","Learning","rate","for","the","actor",".","critic_lr",":","Learning","rate","for","the","critic",".","*","*","kwargs",":","forward","compatibility","kwargs","Returns",":","This","updated","CRRConfig","object","."]
code: def training(
      self,
      *,
      weight_type: Optional[str] = NotProvided,
      temperature: Optional[float] = NotProvided,
      max_weight: Optional[float] = NotProvided,
      advantage_type: Optional[str] = NotProvided,
      n_action_sample: Optional[int] = NotProvided,
      twin_q: Optional[bool] = NotProvided,
      target_network_update_freq: Optional[int] = NotProvided,
      actor_hiddens: Optional[List[int]] = NotProvided,
      actor_hidden_activation: Optional[str] = NotProvided,
      critic_hiddens: Optional[List[int]] = NotProvided,
      critic_hidden_activation: Optional[str] = NotProvided,
      tau: Optional[float] = NotProvided,
      td_error_loss_fn: Optional[str] = NotProvided,
      categorical_distribution_temperature: Optional[
          float
      ] = NotProvided,
      actor_lr: Optional[float] = NotProvided,
      critic_lr: Optional[float] = NotProvided,
      **kwargs,
  ) -> "CRRConfig":
      r"""
      CRR training configuration
  
      Args:
          weight_type: weight type to use `bin` | `exp`.
          temperature: the exponent temperature used in exp weight type.
          max_weight: the max weight limit for exp weight type.
          advantage_type: The way we reduce q values to v_t values
              `max` | `mean` | `expectation`. `max` and `mean` work for both
              discrete and continuous action spaces while `expectation` only
              works for discrete action spaces.
              `max`: Uses max over sampled actions to estimate the value.
  
              .. math::
  
                  A(s_t, a_t) = Q(s_t, a_t) - \max_{a^j} Q(s_t, a^j)
  
              where :math:`a^j` is `n_action_sample` times sampled from the
              policy :math:`\pi(a | s_t)`
              `mean`: Uses mean over sampled actions to estimate the value.
  
              .. math::
  
                  A(s_t, a_t) = Q(s_t, a_t) - \frac{1}{m}\sum_{j=1}^{m}
                  [Q(s_t, a^j)]
  
              where :math:`a^j` is `n_action_sample` times sampled from the
              policy :math:`\pi(a | s_t)`
              `expectation`: This uses categorical distribution to evaluate
              the expectation of the q values directly to estimate the value.
  
              .. math::
  
                  A(s_t, a_t) = Q(s_t, a_t) - E_{a^j\sim \pi(a|s_t)}[Q(s_t,a^j)]
  
          n_action_sample: the number of actions to sample for v_t estimation.
          twin_q: if True, uses pessimistic q estimation.
          target_network_update_freq: The frequency at which we update the
              target copy of the model in terms of the number of gradient updates
              applied to the main model.
          actor_hiddens: The number of hidden units in the actor's fc network.
          actor_hidden_activation: The activation used in the actor's fc network.
          critic_hiddens: The number of hidden units in the critic's fc network.
          critic_hidden_activation: The activation used in the critic's fc network.
          tau: Polyak averaging coefficient
              (making it 1 is reduces it to a hard update).
          td_error_loss_fn: "huber" or "mse".
              Loss function for calculating critic error.
          categorical_distribution_temperature: Set the temperature parameter used
              by Categorical action distribution. A valid temperature is in the range
              of [0, 1]. Note that this mostly affects evaluation since critic error
              uses selected action for return calculation.
          actor_lr: Learning rate for the actor.
          critic_lr: Learning rate for the critic.
          **kwargs: forward compatibility kwargs
  
      Returns:
          This updated CRRConfig object.
      """
      super().training(**kwargs)
  
      if weight_type is not NotProvided:
          self.weight_type = weight_type
      if temperature is not NotProvided:
          self.temperature = temperature
      if max_weight is not NotProvided:
          self.max_weight = max_weight
      if advantage_type is not NotProvided:
          self.advantage_type = advantage_type
      if n_action_sample is not NotProvided:
          self.n_action_sample = n_action_sample
      if twin_q is not NotProvided:
          self.twin_q = twin_q
      if target_network_update_freq is not NotProvided:
          self.target_network_update_freq = target_network_update_freq
      if actor_hiddens is not NotProvided:
          self.actor_hiddens = actor_hiddens
      if actor_hidden_activation is not NotProvided:
          self.actor_hidden_activation = actor_hidden_activation
      if critic_hiddens is not NotProvided:
          self.critic_hiddens = critic_hiddens
      if critic_hidden_activation is not NotProvided:
          self.critic_hidden_activation = critic_hidden_activation
      if tau is not NotProvided:
          self.tau = tau
      if td_error_loss_fn is not NotProvided:
          self.td_error_loss_fn = td_error_loss_fn
      if categorical_distribution_temperature is not NotProvided:
          self.categorical_distribution_temperature = (
              categorical_distribution_temperature
          )
      if actor_lr is not NotProvided:
          self.actor_lr = actor_lr
      if critic_lr is not NotProvided:
          self.critic_lr = critic_lr
  
      return self
  
imports: import logging
  from typing import List, Optional, Type
  from ray.rllib.algorithms.algorithm import Algorithm
  from ray.rllib.algorithms.algorithm_config import AlgorithmConfig, NotProvided
  from ray.rllib.execution import synchronous_parallel_sample
  from ray.rllib.execution.train_ops import multi_gpu_train_one_step, train_one_step
  from ray.rllib.policy import Policy
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.metrics import LAST_TARGET_UPDATE_TS, NUM_AGENT_STEPS_SAMPLED, NUM_AGENT_STEPS_TRAINED, NUM_ENV_STEPS_SAMPLED, NUM_ENV_STEPS_TRAINED, NUM_TARGET_UPDATES, SAMPLE_TIMER, TARGET_NET_UPDATE_TIMER
  from ray.rllib.utils.typing import ResultDict
label_desc: Use image node_id 2 for calling the CRRConfig obj's underlying member method code with example usage: obj.training() and returns: self