class_docstr: Implementation of TD3.
  
  Based on https://arxiv.org/pdf/1802.09477.pdf.
  
  Example:
      $ python garage/examples/tf/td3_pendulum.py
  
  Args:
      env_spec (EnvSpec): Environment.
      policy (Policy): Policy.
      qf (garage.tf.q_functions.QFunction): Q-function.
      qf2 (garage.tf.q_functions.QFunction): Q function to use
      replay_buffer (ReplayBuffer): Replay buffer.
      sampler (garage.sampler.Sampler): Sampler.
      target_update_tau (float): Interpolation parameter for doing the
          soft target update.
      policy_lr (float): Learning rate for training policy network.
      qf_lr (float): Learning rate for training q value network.
      policy_weight_decay (float): L2 weight decay factor for parameters
          of the policy network.
      qf_weight_decay (float): L2 weight decay factor for parameters
          of the q value network.
      policy_optimizer (tf.compat.v1.train.Optimizer): Optimizer for training
          policy network.
      qf_optimizer (tf.compat.v1.train.Optimizer): Optimizer for training
          Q-function network.
      clip_pos_returns (boolean): Whether or not clip positive returns.
      clip_return (float): Clip return to be in [-clip_return,
          clip_return].
      discount (float): Discount factor for the cumulative return.
      max_episode_length_eval (int or None): Maximum length of episodes used
          for off-policy evaluation. If `None`, defaults to
          `env_spec.max_episode_length`.
      max_action (float): Maximum action magnitude.
      name (str): Name of the algorithm shown in computation graph.
      steps_per_epoch (int): Number of batches of samples in each epoch.
      n_train_steps (int): Number of optimizations in each epoch cycle.
      buffer_batch_size (int): Size of replay buffer.
      min_buffer_size (int):
          Number of samples in replay buffer before first optimization.
      reward_scale (float): Scale to reward.
      exploration_policy_sigma (float): Action noise sigma.
      exploration_policy_clip (float): Action noise clip.
      actor_update_period (int): Action update period.
      exploration_policy (ExplorationPolicy): Exploration strategy.
class_docstr_tok: ["Implementation","of","TD3",".","Based","on","https",":","\/\/arxiv.org\/pdf\/1802.09477.pdf",".","Example",":","$","python","garage\/examples\/tf\/td3_pendulum.py","Args",":","env_spec","(","EnvSpec",")",":","Environment",".","policy","(","Policy",")",":","Policy",".","qf","(","garage.tf.q_functions.QFunction",")",":","Q-function",".","qf2","(","garage.tf.q_functions.QFunction",")",":","Q","function","to","use","replay_buffer","(","ReplayBuffer",")",":","Replay","buffer",".","sampler","(","garage.sampler.Sampler",")",":","Sampler",".","target_update_tau","(","float",")",":","Interpolation","parameter","for","doing","the","soft","target","update",".","policy_lr","(","float",")",":","Learning","rate","for","training","policy","network",".","qf_lr","(","float",")",":","Learning","rate","for","training","q","value","network",".","policy_weight_decay","(","float",")",":","L2","weight","decay","factor","for","parameters","of","the","policy","network",".","qf_weight_decay","(","float",")",":","L2","weight","decay","factor","for","parameters","of","the","q","value","network",".","policy_optimizer","(","tf.compat.v1.train.Optimizer",")",":","Optimizer","for","training","policy","network",".","qf_optimizer","(","tf.compat.v1.train.Optimizer",")",":","Optimizer","for","training","Q-function","network",".","clip_pos_returns","(","boolean",")",":","Whether","or","not","clip","positive","returns",".","clip_return","(","float",")",":","Clip","return","to","be","in","[","-clip_return",",","clip_return","]",".","discount","(","float",")",":","Discount","factor","for","the","cumulative","return",".","max_episode_length_eval","(","int","or","None",")",":","Maximum","length","of","episodes","used","for","off-policy","evaluation",".","If","`","None","`",",","defaults","to","`","env_spec.max_episode_length","`",".","max_action","(","float",")",":","Maximum","action","magnitude",".","name","(","str",")",":","Name","of","the","algorithm","shown","in","computation","graph",".","steps_per_epoch","(","int",")",":","Number","of","batches","of","samples","in","each","epoch",".","n_train_steps","(","int",")",":","Number","of","optimizations","in","each","epoch","cycle",".","buffer_batch_size","(","int",")",":","Size","of","replay","buffer",".","min_buffer_size","(","int",")",":","Number","of","samples","in","replay","buffer","before","first","optimization",".","reward_scale","(","float",")",":","Scale","to","reward",".","exploration_policy_sigma","(","float",")",":","Action","noise","sigma",".","exploration_policy_clip","(","float",")",":","Action","noise","clip",".","actor_update_period","(","int",")",":","Action","update","period",".","exploration_policy","(","ExplorationPolicy",")",":","Exploration","strategy","."]
docstr: Perform one step of policy optimization given one batch of samples.
  
  Args:
      itr (int): Iteration number.
      episodes (EpisodeBatch): Batch of episodes.
docstr_tok: ["Perform","one","step","of","policy","optimization","given","one","batch","of","samples",".","Args",":","itr","(","int",")",":","Iteration","number",".","episodes","(","EpisodeBatch",")",":","Batch","of","episodes","."]
code: def _train_once(self, itr, episodes):
      """Perform one step of policy optimization given one batch of samples.
  
      Args:
          itr (int): Iteration number.
          episodes (EpisodeBatch): Batch of episodes.
  
      """
      self._replay_buffer.add_episode_batch(episodes)
  
      epoch = itr / self._steps_per_epoch
  
      for _ in range(self._n_train_steps):
          if (
              self._replay_buffer.n_transitions_stored
              >= self._min_buffer_size
          ):
              qf_loss, y_s, qval, policy_loss = self._optimize_policy(
                  itr
              )
  
              self._episode_policy_losses.append(policy_loss)
              self._episode_qf_losses.append(qf_loss)
              self._epoch_ys.append(y_s)
              self._epoch_qs.append(qval)
  
      if itr % self._steps_per_epoch == 0:
          logger.log("Training finished")
  
          if (
              self._replay_buffer.n_transitions_stored
              >= self._min_buffer_size
          ):
              tabular.record("Epoch", epoch)
              tabular.record(
                  "Policy/AveragePolicyLoss",
                  np.mean(self._episode_policy_losses),
              )
              tabular.record(
                  "QFunction/AverageQFunctionLoss",
                  np.mean(self._episode_qf_losses),
              )
              tabular.record(
                  "QFunction/AverageQ", np.mean(self._epoch_qs)
              )
              tabular.record("QFunction/MaxQ", np.max(self._epoch_qs))
              tabular.record(
                  "QFunction/AverageAbsQ",
                  np.mean(np.abs(self._epoch_qs)),
              )
              tabular.record(
                  "QFunction/AverageY", np.mean(self._epoch_ys)
              )
              tabular.record("QFunction/MaxY", np.max(self._epoch_ys))
              tabular.record(
                  "QFunction/AverageAbsY",
                  np.mean(np.abs(self._epoch_ys)),
              )
  
file_path: garage/src/garage/tf/algos/td3.py
imports: from dowel import logger, tabular
  import numpy
  import tensorflow
  from garage import _Default, log_performance, make_optimizer, obtain_evaluation_episodes
  from garage.np.algos import RLAlgorithm
  from garage.tf import compile_function, get_target_ops
label_desc: Use image node_id 6 for calling the TD3 obj's underlying member method code with example usage: obj._train_once(itr, episodes) without return types