class_docstr: A DDPG model based on https://arxiv.org/pdf/1509.02971.pdf.
  
  DDPG, also known as Deep Deterministic Policy Gradient, uses actor-critic
  method to optimize the policy and reward prediction. It uses a supervised
  method to update the critic network and policy gradient to update the actor
  network. And there are exploration strategy, replay buffer and target
  networks involved to stabilize the training process.
  
  Example:
      $ python garage/examples/tf/ddpg_pendulum.py
  
  Args:
      env_spec (EnvSpec): Environment specification.
      policy (Policy): Policy.
      qf (object): The q value network.
      replay_buffer (ReplayBuffer): Replay buffer.
      sampler (garage.sampler.Sampler): Sampler.
      steps_per_epoch (int): Number of train_once calls per epoch.
      n_train_steps (int): Training steps.
      buffer_batch_size (int): Batch size of replay buffer.
      max_episode_length_eval (int or None): Maximum length of episodes used
          for off-policy evaluation. If None, defaults to
          `env_spec.max_episode_length`.
      min_buffer_size (int): The minimum buffer size for replay buffer.
      exploration_policy (ExplorationPolicy): Exploration strategy.
      target_update_tau (float): Interpolation parameter for doing the
          soft target update.
      policy_lr (float): Learning rate for training policy network.
      qf_lr (float): Learning rate for training q value network.
      discount(float): Discount factor for the cumulative return.
      policy_weight_decay (float): L2 regularization factor for parameters
          of the policy network. Value of 0 means no regularization.
      qf_weight_decay (float): L2 regularization factor for parameters
          of the q value network. Value of 0 means no regularization.
      policy_optimizer (tf.compat.v1.train.Optimizer): Optimizer for training
          policy network.
      qf_optimizer (tf.compat.v1.train.Optimizer): Optimizer for training
          Q-function network.
      clip_pos_returns (bool): Whether or not clip positive returns.
      clip_return (float): Clip return to be in [-clip_return,
          clip_return].
      max_action (float): Maximum action magnitude.
      reward_scale (float): Reward scale.
      name (str): Name of the algorithm shown in computation graph.
class_docstr_tok: ["A","DDPG","model","based","on","https",":","\/\/arxiv.org\/pdf\/1509.02971.pdf",".","DDPG",",","also","known","as","Deep","Deterministic","Policy","Gradient",",","uses","actor-critic","method","to","optimize","the","policy","and","reward","prediction",".","It","uses","a","supervised","method","to","update","the","critic","network","and","policy","gradient","to","update","the","actor","network",".","And","there","are","exploration","strategy",",","replay","buffer","and","target","networks","involved","to","stabilize","the","training","process",".","Example",":","$","python","garage\/examples\/tf\/ddpg_pendulum.py","Args",":","env_spec","(","EnvSpec",")",":","Environment","specification",".","policy","(","Policy",")",":","Policy",".","qf","(","object",")",":","The","q","value","network",".","replay_buffer","(","ReplayBuffer",")",":","Replay","buffer",".","sampler","(","garage.sampler.Sampler",")",":","Sampler",".","steps_per_epoch","(","int",")",":","Number","of","train_once","calls","per","epoch",".","n_train_steps","(","int",")",":","Training","steps",".","buffer_batch_size","(","int",")",":","Batch","size","of","replay","buffer",".","max_episode_length_eval","(","int","or","None",")",":","Maximum","length","of","episodes","used","for","off-policy","evaluation",".","If","None",",","defaults","to","`","env_spec.max_episode_length","`",".","min_buffer_size","(","int",")",":","The","minimum","buffer","size","for","replay","buffer",".","exploration_policy","(","ExplorationPolicy",")",":","Exploration","strategy",".","target_update_tau","(","float",")",":","Interpolation","parameter","for","doing","the","soft","target","update",".","policy_lr","(","float",")",":","Learning","rate","for","training","policy","network",".","qf_lr","(","float",")",":","Learning","rate","for","training","q","value","network",".","discount","(","float",")",":","Discount","factor","for","the","cumulative","return",".","policy_weight_decay","(","float",")",":","L2","regularization","factor","for","parameters","of","the","policy","network",".","Value","of","0","means","no","regularization",".","qf_weight_decay","(","float",")",":","L2","regularization","factor","for","parameters","of","the","q","value","network",".","Value","of","0","means","no","regularization",".","policy_optimizer","(","tf.compat.v1.train.Optimizer",")",":","Optimizer","for","training","policy","network",".","qf_optimizer","(","tf.compat.v1.train.Optimizer",")",":","Optimizer","for","training","Q-function","network",".","clip_pos_returns","(","bool",")",":","Whether","or","not","clip","positive","returns",".","clip_return","(","float",")",":","Clip","return","to","be","in","[","-clip_return",",","clip_return","]",".","max_action","(","float",")",":","Maximum","action","magnitude",".","reward_scale","(","float",")",":","Reward","scale",".","name","(","str",")",":","Name","of","the","algorithm","shown","in","computation","graph","."]
code: def __init__(
      self,
      env_spec,
      policy,
      qf,
      replay_buffer,
      sampler,
      *,  # Everything after this is numbers.
      steps_per_epoch=20,
      n_train_steps=50,
      buffer_batch_size=64,
      min_buffer_size=int(1e4),
      max_episode_length_eval=None,
      exploration_policy=None,
      target_update_tau=0.01,
      discount=0.99,
      policy_weight_decay=0,
      qf_weight_decay=0,
      policy_optimizer=tf.compat.v1.train.AdamOptimizer,
      qf_optimizer=tf.compat.v1.train.AdamOptimizer,
      policy_lr=_Default(1e-4),
      qf_lr=_Default(1e-3),
      clip_pos_returns=False,
      clip_return=np.inf,
      max_action=None,
      reward_scale=1.0,
      name="DDPG",
  ):
      action_bound = env_spec.action_space.high
      self._max_action = (
          action_bound if max_action is None else max_action
      )
      self._tau = target_update_tau
      self._policy_weight_decay = policy_weight_decay
      self._qf_weight_decay = qf_weight_decay
      self._name = name
      self._clip_pos_returns = clip_pos_returns
      self._clip_return = clip_return
  
      self._episode_policy_losses = []
      self._episode_qf_losses = []
      self._epoch_ys = []
      self._epoch_qs = []
  
      self._target_policy = policy.clone("target_policy")
      self._target_qf = qf.clone("target_qf")
      self._policy_optimizer = policy_optimizer
      self._qf_optimizer = qf_optimizer
      self._policy_lr = policy_lr
      self._qf_lr = qf_lr
  
      self._min_buffer_size = min_buffer_size
      self._qf = qf
      self._steps_per_epoch = steps_per_epoch
      self._n_train_steps = n_train_steps
      self._buffer_batch_size = buffer_batch_size
      self._discount = discount
      self._reward_scale = reward_scale
      self.max_episode_length = env_spec.max_episode_length
      self._max_episode_length_eval = max_episode_length_eval
  
      if max_episode_length_eval is None:
          self._max_episode_length_eval = env_spec.max_episode_length
  
      self._eval_env = None
  
      self._env_spec = env_spec
      self._replay_buffer = replay_buffer
      self.policy = policy
      self.exploration_policy = exploration_policy
  
      self._sampler = sampler
  
      self._init_opt()
  
file_path: garage/src/garage/tf/algos/ddpg.py
imports: from dowel import logger, tabular
  import numpy
  import tensorflow
  from garage import _Default, log_performance, make_optimizer, obtain_evaluation_episodes
  from garage.np.algos import RLAlgorithm
  from garage.tf import compile_function, get_target_ops
label_desc: Use image node_id 1 to create a new DDPG object from inherited base classes: RLAlgorithm with example: obj = DDPG(env_spec, policy, qf, replay_buffer, sampler)