class_docstr: Defines a configuration class from which a DDPG Trainer can be built.
  
  Example:
      >>> from ray.rllib.algorithms.ddpg.ddpg import DDPGConfig
      >>> config = DDPGConfig().training(lr=0.01).resources(num_gpus=1)
      >>> print(config.to_dict())  # doctest: +SKIP
      >>> # Build a Trainer object from the config and run one training iteration.
      >>> algo = config.build(env="Pendulum-v1") # doctest: +SKIP
      >>> algo.train()  # doctest: +SKIP
  
  Example:
      >>> from ray.rllib.algorithms.ddpg.ddpg import DDPGConfig
      >>> from ray import air
      >>> from ray import tune
      >>> config = DDPGConfig()
      >>> # Print out some default values.
      >>> print(config.lr) # doctest: +SKIP
      0.0004
      >>> # Update the config object.
      >>> config = config.training(lr=tune.grid_search([0.001, 0.0001]))
      >>> # Set the config object's env.
      >>> config = config.environment(env="Pendulum-v1")
      >>> # Use to_dict() to get the old-style python config dict
      >>> # when running with tune.
      >>> tune.Tuner(  # doctest: +SKIP
      ...     "DDPG",
      ...     run_config=air.RunConfig(stop={"episode_reward_mean": 200}),
      ...     param_space=config.to_dict(),
      ... ).fit()
class_docstr_tok: ["Defines","a","configuration","class","from","which","a","DDPG","Trainer","can","be","built",".","Example",":",">",">",">","from","ray.rllib.algorithms.ddpg.ddpg","import","DDPGConfig",">",">",">","config","=","DDPGConfig","(",")",".training","(","lr=0.01",")",".resources","(","num_gpus=1",")",">",">",">","print","(","config.to_dict","(",")",")","#","doctest",":","+SKIP",">",">",">","#","Build","a","Trainer","object","from","the","config","and","run","one","training","iteration",".",">",">",">","algo","=","config.build","(","env=","''","Pendulum-v1","''",")","#","doctest",":","+SKIP",">",">",">","algo.train","(",")","#","doctest",":","+SKIP","Example",":",">",">",">","from","ray.rllib.algorithms.ddpg.ddpg","import","DDPGConfig",">",">",">","from","ray","import","air",">",">",">","from","ray","import","tune",">",">",">","config","=","DDPGConfig","(",")",">",">",">","#","Print","out","some","default","values",".",">",">",">","print","(","config.lr",")","#","doctest",":","+SKIP","0.0004",">",">",">","#","Update","the","config","object",".",">",">",">","config","=","config.training","(","lr=tune.grid_search","(","[","0.001",",","0.0001","]",")",")",">",">",">","#","Set","the","config","object","'s","env",".",">",">",">","config","=","config.environment","(","env=","''","Pendulum-v1","''",")",">",">",">","#","Use","to_dict","(",")","to","get","the","old-style","python","config","dict",">",">",">","#","when","running","with","tune",".",">",">",">","tune.Tuner","(","#","doctest",":","+SKIP","...","``","DDPG","''",",","...","run_config=air.RunConfig","(","stop=","{","``","episode_reward_mean","''",":","200","}",")",",","...","param_space=config.to_dict","(",")",",","...",")",".fit","(",")"]
docstr: Sets the training related configuration.
  
  === Twin Delayed DDPG (TD3) and Soft Actor-Critic (SAC) tricks ===
  TD3: https://spinningup.openai.com/en/latest/algorithms/td3.html
  In addition to settings below, you can use "exploration_noise_type" and
  "exploration_gauss_act_noise" to get IID Gaussian exploration noise
  instead of OrnsteinUhlenbeck exploration noise.
  
  Args:
      twin_q: Use twin Q-net.
      policy_delay: Delayed policy update.
      smooth_target_policy: Target policy smoothing (this also replaces
          OrnsteinUhlenbeck exploration noise with IID Gaussian exploration
          noise, for now).
      target_noise: Gaussian stddev of target action noise for smoothing.
      target_noise_clip: Target noise limit (bound).
      use_state_preprocessor: Apply a state preprocessor with spec given by the
          "model" config option
          (like other RL algorithms). This is mostly useful if you have a weird
          observation shape, like an image. Disabled by default.
      actor_hiddens: Postprocess the policy network model output with these
          hidden layers. If use_state_preprocessor is False, then these will
          be the *only* hidden layers in the network.
      actor_hidden_activation: Hidden layers activation of the postprocessing
          stage of the policy network
      critic_hiddens: Postprocess the critic network model output with these
          hidden layers; again, if use_state_preprocessor is True, then the
          state will be preprocessed by the model specified with the "model"
          config option first.
      critic_hidden_activation: Hidden layers activation of the postprocessing
          state of the critic.
      n_step: N-step Q learning
      critic_lr: Learning rate for the critic (Q-function) optimizer.
      actor_lr: Learning rate for the actor (policy) optimizer.
      tau: Update the target by   au * policy + (1-       au) * target_policy
      use_huber: Conventionally, no need to clip gradients if using a huber loss
      huber_threshold: Threshold of a huber loss
      l2_reg: Weights for L2 regularization
      training_intensity: The intensity with which to update the model
          (vs collecting samples from
          the env). If None, uses the "natural" value of:
          `train_batch_size` / (`rollout_fragment_length` x `num_workers` x
          `num_envs_per_worker`).
          If provided, will make sure that the ratio between ts inserted into and
          sampled from the buffer matches the given value.
          Example:
              training_intensity=1000.0
              train_batch_size=250 rollout_fragment_length=1
              num_workers=1 (or 0) num_envs_per_worker=1
              -> natural value = 250 / 1 = 250.0
              -> will make sure that replay+train op will be executed 4x as
              often as rollout+insert op (4 * 250 = 1000).
          See: rllib/algorithms/dqn/dqn.py::calculate_rr_weights for further
          details.
  
  Returns:
      This updated DDPGConfig object.
docstr_tok: ["Sets","the","training","related","configuration",".","===","Twin","Delayed","DDPG","(","TD3",")","and","Soft","Actor-Critic","(","SAC",")","tricks","===","TD3",":","https",":","\/\/spinningup.openai.com\/en\/latest\/algorithms\/td3.html","In","addition","to","settings","below",",","you","can","use","``","exploration_noise_type","''","and","''","exploration_gauss_act_noise","''","to","get","IID","Gaussian","exploration","noise","instead","of","OrnsteinUhlenbeck","exploration","noise",".","Args",":","twin_q",":","Use","twin","Q-net",".","policy_delay",":","Delayed","policy","update",".","smooth_target_policy",":","Target","policy","smoothing","(","this","also","replaces","OrnsteinUhlenbeck","exploration","noise","with","IID","Gaussian","exploration","noise",",","for","now",")",".","target_noise",":","Gaussian","stddev","of","target","action","noise","for","smoothing",".","target_noise_clip",":","Target","noise","limit","(","bound",")",".","use_state_preprocessor",":","Apply","a","state","preprocessor","with","spec","given","by","the","``","model","''","config","option","(","like","other","RL","algorithms",")",".","This","is","mostly","useful","if","you","have","a","weird","observation","shape",",","like","an","image",".","Disabled","by","default",".","actor_hiddens",":","Postprocess","the","policy","network","model","output","with","these","hidden","layers",".","If","use_state_preprocessor","is","False",",","then","these","will","be","the","*","only","*","hidden","layers","in","the","network",".","actor_hidden_activation",":","Hidden","layers","activation","of","the","postprocessing","stage","of","the","policy","network","critic_hiddens",":","Postprocess","the","critic","network","model","output","with","these","hidden","layers",";","again",",","if","use_state_preprocessor","is","True",",","then","the","state","will","be","preprocessed","by","the","model","specified","with","the","``","model","''","config","option","first",".","critic_hidden_activation",":","Hidden","layers","activation","of","the","postprocessing","state","of","the","critic",".","n_step",":","N-step","Q","learning","critic_lr",":","Learning","rate","for","the","critic","(","Q-function",")","optimizer",".","actor_lr",":","Learning","rate","for","the","actor","(","policy",")","optimizer",".","tau",":","Update","the","target","by","au","*","policy","+","(","1-","au",")","*","target_policy","use_huber",":","Conventionally",",","no","need","to","clip","gradients","if","using","a","huber","loss","huber_threshold",":","Threshold","of","a","huber","loss","l2_reg",":","Weights","for","L2","regularization","training_intensity",":","The","intensity","with","which","to","update","the","model","(","vs","collecting","samples","from","the","env",")",".","If","None",",","uses","the","``","natural","''","value","of",":","`","train_batch_size","`","\/","(","`","rollout_fragment_length","`","x","`","num_workers","`","x","`","num_envs_per_worker","`",")",".","If","provided",",","will","make","sure","that","the","ratio","between","ts","inserted","into","and","sampled","from","the","buffer","matches","the","given","value",".","Example",":","training_intensity=1000.0","train_batch_size=250","rollout_fragment_length=1","num_workers=1","(","or","0",")","num_envs_per_worker=1","-",">","natural","value","=","250","\/","1","=","250.0","-",">","will","make","sure","that","replay+train","op","will","be","executed","4x","as","often","as","rollout+insert","op","(","4","*","250","=","1000",")",".","See",":","rllib\/algorithms\/dqn\/dqn.py",":",":calculate_rr_weights","for","further","details",".","Returns",":","This","updated","DDPGConfig","object","."]
code: def training(
      self,
      *,
      twin_q: Optional[bool] = NotProvided,
      policy_delay: Optional[int] = NotProvided,
      smooth_target_policy: Optional[bool] = NotProvided,
      target_noise: Optional[bool] = NotProvided,
      target_noise_clip: Optional[float] = NotProvided,
      use_state_preprocessor: Optional[bool] = NotProvided,
      actor_hiddens: Optional[List[int]] = NotProvided,
      actor_hidden_activation: Optional[str] = NotProvided,
      critic_hiddens: Optional[List[int]] = NotProvided,
      critic_hidden_activation: Optional[str] = NotProvided,
      n_step: Optional[int] = NotProvided,
      critic_lr: Optional[float] = NotProvided,
      actor_lr: Optional[float] = NotProvided,
      tau: Optional[float] = NotProvided,
      use_huber: Optional[bool] = NotProvided,
      huber_threshold: Optional[float] = NotProvided,
      l2_reg: Optional[float] = NotProvided,
      training_intensity: Optional[float] = NotProvided,
      **kwargs,
  ) -> "DDPGConfig":
      """Sets the training related configuration.
  
      === Twin Delayed DDPG (TD3) and Soft Actor-Critic (SAC) tricks ===
      TD3: https://spinningup.openai.com/en/latest/algorithms/td3.html
      In addition to settings below, you can use "exploration_noise_type" and
      "exploration_gauss_act_noise" to get IID Gaussian exploration noise
      instead of OrnsteinUhlenbeck exploration noise.
  
      Args:
          twin_q: Use twin Q-net.
          policy_delay: Delayed policy update.
          smooth_target_policy: Target policy smoothing (this also replaces
              OrnsteinUhlenbeck exploration noise with IID Gaussian exploration
              noise, for now).
          target_noise: Gaussian stddev of target action noise for smoothing.
          target_noise_clip: Target noise limit (bound).
          use_state_preprocessor: Apply a state preprocessor with spec given by the
              "model" config option
              (like other RL algorithms). This is mostly useful if you have a weird
              observation shape, like an image. Disabled by default.
          actor_hiddens: Postprocess the policy network model output with these
              hidden layers. If use_state_preprocessor is False, then these will
              be the *only* hidden layers in the network.
          actor_hidden_activation: Hidden layers activation of the postprocessing
              stage of the policy network
          critic_hiddens: Postprocess the critic network model output with these
              hidden layers; again, if use_state_preprocessor is True, then the
              state will be preprocessed by the model specified with the "model"
              config option first.
          critic_hidden_activation: Hidden layers activation of the postprocessing
              state of the critic.
          n_step: N-step Q learning
          critic_lr: Learning rate for the critic (Q-function) optimizer.
          actor_lr: Learning rate for the actor (policy) optimizer.
          tau: Update the target by \tau * policy + (1-\tau) * target_policy
          use_huber: Conventionally, no need to clip gradients if using a huber loss
          huber_threshold: Threshold of a huber loss
          l2_reg: Weights for L2 regularization
          training_intensity: The intensity with which to update the model
              (vs collecting samples from
              the env). If None, uses the "natural" value of:
              `train_batch_size` / (`rollout_fragment_length` x `num_workers` x
              `num_envs_per_worker`).
              If provided, will make sure that the ratio between ts inserted into and
              sampled from the buffer matches the given value.
              Example:
                  training_intensity=1000.0
                  train_batch_size=250 rollout_fragment_length=1
                  num_workers=1 (or 0) num_envs_per_worker=1
                  -> natural value = 250 / 1 = 250.0
                  -> will make sure that replay+train op will be executed 4x as
                  often as rollout+insert op (4 * 250 = 1000).
              See: rllib/algorithms/dqn/dqn.py::calculate_rr_weights for further
              details.
  
      Returns:
          This updated DDPGConfig object.
      """
      super().training(**kwargs)
  
      if twin_q is not NotProvided:
          self.twin_q = twin_q
      if policy_delay is not NotProvided:
          self.policy_delay = policy_delay
      if smooth_target_policy is not NotProvided:
          self.smooth_target_policy = smooth_target_policy
      if target_noise is not NotProvided:
          self.target_noise = target_noise
      if target_noise_clip is not NotProvided:
          self.target_noise_clip = target_noise_clip
      if use_state_preprocessor is not NotProvided:
          self.use_state_preprocessor = use_state_preprocessor
      if actor_hiddens is not NotProvided:
          self.actor_hiddens = actor_hiddens
      if actor_hidden_activation is not NotProvided:
          self.actor_hidden_activation = actor_hidden_activation
      if critic_hiddens is not NotProvided:
          self.critic_hiddens = critic_hiddens
      if critic_hidden_activation is not NotProvided:
          self.critic_hidden_activation = critic_hidden_activation
      if n_step is not NotProvided:
          self.n_step = n_step
      if critic_lr is not NotProvided:
          self.critic_lr = critic_lr
      if actor_lr is not NotProvided:
          self.actor_lr = actor_lr
      if tau is not NotProvided:
          self.tau = tau
      if use_huber is not NotProvided:
          self.use_huber = use_huber
      if huber_threshold is not NotProvided:
          self.huber_threshold = huber_threshold
      if l2_reg is not NotProvided:
          self.l2_reg = l2_reg
      if training_intensity is not NotProvided:
          self.training_intensity = training_intensity
  
      return self
  
file_path: ray/rllib_contrib/ddpg/src/rllib_ddpg/ddpg/ddpg.py
imports: import logging
  from typing import List, Optional, Type
  from ray.rllib.algorithms.algorithm_config import AlgorithmConfig, NotProvided
  from ray.rllib.algorithms.simple_q.simple_q import SimpleQ, SimpleQConfig
  from ray.rllib.policy.policy import Policy
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.deprecation import DEPRECATED_VALUE
label_desc: Use image node_id 2 for calling the DDPGConfig obj's underlying member method code with example usage: obj.training() and returns: self