class_docstr: Implementation of TD3.
  
  Based on https://arxiv.org/pdf/1802.09477.pdf.
  
  Example:
      $ python garage/examples/tf/td3_pendulum.py
  
  Args:
      env_spec (EnvSpec): Environment.
      policy (Policy): Policy.
      qf (garage.tf.q_functions.QFunction): Q-function.
      qf2 (garage.tf.q_functions.QFunction): Q function to use
      replay_buffer (ReplayBuffer): Replay buffer.
      sampler (garage.sampler.Sampler): Sampler.
      target_update_tau (float): Interpolation parameter for doing the
          soft target update.
      policy_lr (float): Learning rate for training policy network.
      qf_lr (float): Learning rate for training q value network.
      policy_weight_decay (float): L2 weight decay factor for parameters
          of the policy network.
      qf_weight_decay (float): L2 weight decay factor for parameters
          of the q value network.
      policy_optimizer (tf.compat.v1.train.Optimizer): Optimizer for training
          policy network.
      qf_optimizer (tf.compat.v1.train.Optimizer): Optimizer for training
          Q-function network.
      clip_pos_returns (boolean): Whether or not clip positive returns.
      clip_return (float): Clip return to be in [-clip_return,
          clip_return].
      discount (float): Discount factor for the cumulative return.
      max_episode_length_eval (int or None): Maximum length of episodes used
          for off-policy evaluation. If `None`, defaults to
          `env_spec.max_episode_length`.
      max_action (float): Maximum action magnitude.
      name (str): Name of the algorithm shown in computation graph.
      steps_per_epoch (int): Number of batches of samples in each epoch.
      n_train_steps (int): Number of optimizations in each epoch cycle.
      buffer_batch_size (int): Size of replay buffer.
      min_buffer_size (int):
          Number of samples in replay buffer before first optimization.
      reward_scale (float): Scale to reward.
      exploration_policy_sigma (float): Action noise sigma.
      exploration_policy_clip (float): Action noise clip.
      actor_update_period (int): Action update period.
      exploration_policy (ExplorationPolicy): Exploration strategy.
class_docstr_tok: ["Implementation","of","TD3",".","Based","on","https",":","\/\/arxiv.org\/pdf\/1802.09477.pdf",".","Example",":","$","python","garage\/examples\/tf\/td3_pendulum.py","Args",":","env_spec","(","EnvSpec",")",":","Environment",".","policy","(","Policy",")",":","Policy",".","qf","(","garage.tf.q_functions.QFunction",")",":","Q-function",".","qf2","(","garage.tf.q_functions.QFunction",")",":","Q","function","to","use","replay_buffer","(","ReplayBuffer",")",":","Replay","buffer",".","sampler","(","garage.sampler.Sampler",")",":","Sampler",".","target_update_tau","(","float",")",":","Interpolation","parameter","for","doing","the","soft","target","update",".","policy_lr","(","float",")",":","Learning","rate","for","training","policy","network",".","qf_lr","(","float",")",":","Learning","rate","for","training","q","value","network",".","policy_weight_decay","(","float",")",":","L2","weight","decay","factor","for","parameters","of","the","policy","network",".","qf_weight_decay","(","float",")",":","L2","weight","decay","factor","for","parameters","of","the","q","value","network",".","policy_optimizer","(","tf.compat.v1.train.Optimizer",")",":","Optimizer","for","training","policy","network",".","qf_optimizer","(","tf.compat.v1.train.Optimizer",")",":","Optimizer","for","training","Q-function","network",".","clip_pos_returns","(","boolean",")",":","Whether","or","not","clip","positive","returns",".","clip_return","(","float",")",":","Clip","return","to","be","in","[","-clip_return",",","clip_return","]",".","discount","(","float",")",":","Discount","factor","for","the","cumulative","return",".","max_episode_length_eval","(","int","or","None",")",":","Maximum","length","of","episodes","used","for","off-policy","evaluation",".","If","`","None","`",",","defaults","to","`","env_spec.max_episode_length","`",".","max_action","(","float",")",":","Maximum","action","magnitude",".","name","(","str",")",":","Name","of","the","algorithm","shown","in","computation","graph",".","steps_per_epoch","(","int",")",":","Number","of","batches","of","samples","in","each","epoch",".","n_train_steps","(","int",")",":","Number","of","optimizations","in","each","epoch","cycle",".","buffer_batch_size","(","int",")",":","Size","of","replay","buffer",".","min_buffer_size","(","int",")",":","Number","of","samples","in","replay","buffer","before","first","optimization",".","reward_scale","(","float",")",":","Scale","to","reward",".","exploration_policy_sigma","(","float",")",":","Action","noise","sigma",".","exploration_policy_clip","(","float",")",":","Action","noise","clip",".","actor_update_period","(","int",")",":","Action","update","period",".","exploration_policy","(","ExplorationPolicy",")",":","Exploration","strategy","."]
code: def __init__(
      self,
      env_spec,
      policy,
      qf,
      qf2,
      replay_buffer,
      sampler,
      *,  # Everything after this is numbers.
      target_update_tau=0.01,
      policy_weight_decay=0,
      qf_weight_decay=0,
      policy_optimizer=tf.compat.v1.train.AdamOptimizer,
      qf_optimizer=tf.compat.v1.train.AdamOptimizer,
      policy_lr=_Default(1e-4),
      qf_lr=_Default(1e-3),
      clip_pos_returns=False,
      clip_return=np.inf,
      discount=0.99,
      max_episode_length_eval=None,
      max_action=None,
      name="TD3",
      steps_per_epoch=20,
      n_train_steps=50,
      buffer_batch_size=64,
      min_buffer_size=1e4,
      reward_scale=1.0,
      exploration_policy_sigma=0.2,
      actor_update_period=2,
      exploration_policy_clip=0.5,
      exploration_policy=None,
  ):
      action_bound = env_spec.action_space.high
      self._max_action = (
          action_bound if max_action is None else max_action
      )
      self._tau = target_update_tau
      self._policy_weight_decay = policy_weight_decay
      self._qf_weight_decay = qf_weight_decay
      self._name = name
      self._clip_pos_returns = clip_pos_returns
      self._clip_return = clip_return
  
      self._episode_policy_losses = []
      self._episode_qf_losses = []
      self._epoch_ys = []
      self._epoch_qs = []
  
      self._target_policy = policy.clone("target_policy")
      self._target_qf = qf.clone("target_qf")
  
      self.qf2 = qf2
      self.qf = qf
  
      self._exploration_policy_sigma = exploration_policy_sigma
      self._exploration_policy_clip = exploration_policy_clip
      self._actor_update_period = actor_update_period
      self._action_loss = None
  
      self._target_qf2 = qf2.clone("target_qf2")
      self._policy_optimizer = policy_optimizer
      self._qf_optimizer = qf_optimizer
      self._policy_lr = policy_lr
      self._qf_lr = qf_lr
  
      self._policy = policy
      self._n_train_steps = n_train_steps
  
      self._min_buffer_size = min_buffer_size
      self._qf = qf
      self._steps_per_epoch = steps_per_epoch
      self._n_train_steps = n_train_steps
      self._buffer_batch_size = buffer_batch_size
      self._discount = discount
      self._reward_scale = reward_scale
      self.max_episode_length = env_spec.max_episode_length
      if max_episode_length_eval is not None:
          self._max_episode_length_eval = max_episode_length_eval
      self._eval_env = None
  
      self._env_spec = env_spec
      self._replay_buffer = replay_buffer
      self.policy = policy
      self.exploration_policy = exploration_policy
  
      self._sampler = sampler
  
      self._init_opt()
  
file_path: garage/src/garage/tf/algos/td3.py
imports: from dowel import logger, tabular
  import numpy
  import tensorflow
  from garage import _Default, log_performance, make_optimizer, obtain_evaluation_episodes
  from garage.np.algos import RLAlgorithm
  from garage.tf import compile_function, get_target_ops
label_desc: Use image node_id 1 to create a new TD3 object from inherited base classes: RLAlgorithm with example: obj = TD3(env_spec, policy, qf, qf2, replay_buffer, sampler)