class_docstr: Implementation of TD3.
  
  Based on https://arxiv.org/pdf/1802.09477.pdf.
  
  Args:
      env_spec (EnvSpec): Environment specification.
      policy (garage.torch.policies.Policy): Policy (actor network).
      qf1 (garage.torch.q_functions.QFunction): Q function (critic network).
      qf2 (garage.torch.q_functions.QFunction): Q function (critic network).
      replay_buffer (ReplayBuffer): Replay buffer.
      sampler (garage.sampler.Sampler): Sampler.
      replay_buffer_size (int): Size of the replay buffer
      exploration_policy (garage.np.exploration_policies.ExplorationPolicy):
              Exploration strategy.
      uniform_random_policy
              (garage.np.exploration_policies.ExplorationPolicy):
              Uniform random exploration strategy.
      target_update_tau (float): Interpolation parameter for doing the
          soft target update.
      discount (float): Discount factor (gamma) for the cumulative return.
      reward_scaling (float): Reward scaling.
      update_actor_interval (int): Policy (Actor network) update interval.
      max_action (float): Maximum action magnitude.
      buffer_batch_size (int): Size of replay buffer.
      min_buffer_size (int): The minimum buffer size for replay buffer.
      policy_noise (float): Policy (actor) noise.
      policy_noise_clip (float): Noise clip.
      exploration_noise (float): Exploration noise.
      clip_return (float): Clip return to be in [-clip_return,
          clip_return].
      policy_lr (float): Learning rate for training policy network.
      qf_lr (float): Learning rate for training Q network.
      policy_optimizer (Union[type, tuple[type, dict]]): Type of optimizer
          for training policy network. This can be an optimizer type such as
          `torch.optim.Adam` or a tuple of type and dictionary, where
          dictionary contains arguments to initialize the optimizer
          e.g. `(torch.optim.Adam, {'lr' : 1e-3})`.
      qf_optimizer (Union[type, tuple[type, dict]]): Type of optimizer
          for training Q-value network. This can be an optimizer type such
          as `torch.optim.Adam` or a tuple of type and dictionary, where
          dictionary contains arguments to initialize the optimizer
          e.g. `(torch.optim.Adam, {'lr' : 1e-3})`.
      steps_per_epoch (int): Number of train_once calls per epoch.
      grad_steps_per_env_step (int): Number of gradient steps taken per
          environment step sampled.
      max_episode_length_eval (int or None): Maximum length of episodes used
          for off-policy evaluation. If None, defaults to
          `env_spec.max_episode_length`.
      num_evaluation_episodes (int): The number of evaluation
          trajectories used for computing eval stats at the end of every
          epoch.
      start_steps (int): The number of steps for warming up before
           selecting actions according to policy.
      update_after (int): The number of steps to perform before policy
          is updated.
      use_deterministic_evaluation (bool): True if the trained policy
          should be evaluated deterministically.
class_docstr_tok: ["Implementation","of","TD3",".","Based","on","https",":","\/\/arxiv.org\/pdf\/1802.09477.pdf",".","Args",":","env_spec","(","EnvSpec",")",":","Environment","specification",".","policy","(","garage.torch.policies.Policy",")",":","Policy","(","actor","network",")",".","qf1","(","garage.torch.q_functions.QFunction",")",":","Q","function","(","critic","network",")",".","qf2","(","garage.torch.q_functions.QFunction",")",":","Q","function","(","critic","network",")",".","replay_buffer","(","ReplayBuffer",")",":","Replay","buffer",".","sampler","(","garage.sampler.Sampler",")",":","Sampler",".","replay_buffer_size","(","int",")",":","Size","of","the","replay","buffer","exploration_policy","(","garage.np.exploration_policies.ExplorationPolicy",")",":","Exploration","strategy",".","uniform_random_policy","(","garage.np.exploration_policies.ExplorationPolicy",")",":","Uniform","random","exploration","strategy",".","target_update_tau","(","float",")",":","Interpolation","parameter","for","doing","the","soft","target","update",".","discount","(","float",")",":","Discount","factor","(","gamma",")","for","the","cumulative","return",".","reward_scaling","(","float",")",":","Reward","scaling",".","update_actor_interval","(","int",")",":","Policy","(","Actor","network",")","update","interval",".","max_action","(","float",")",":","Maximum","action","magnitude",".","buffer_batch_size","(","int",")",":","Size","of","replay","buffer",".","min_buffer_size","(","int",")",":","The","minimum","buffer","size","for","replay","buffer",".","policy_noise","(","float",")",":","Policy","(","actor",")","noise",".","policy_noise_clip","(","float",")",":","Noise","clip",".","exploration_noise","(","float",")",":","Exploration","noise",".","clip_return","(","float",")",":","Clip","return","to","be","in","[","-clip_return",",","clip_return","]",".","policy_lr","(","float",")",":","Learning","rate","for","training","policy","network",".","qf_lr","(","float",")",":","Learning","rate","for","training","Q","network",".","policy_optimizer","(","Union","[","type",",","tuple","[","type",",","dict","]","]",")",":","Type","of","optimizer","for","training","policy","network",".","This","can","be","an","optimizer","type","such","as","`","torch.optim.Adam","`","or","a","tuple","of","type","and","dictionary",",","where","dictionary","contains","arguments","to","initialize","the","optimizer","e.g",".","`","(","torch.optim.Adam",",","{","'lr","'",":","1e-3","}",")","`",".","qf_optimizer","(","Union","[","type",",","tuple","[","type",",","dict","]","]",")",":","Type","of","optimizer","for","training","Q-value","network",".","This","can","be","an","optimizer","type","such","as","`","torch.optim.Adam","`","or","a","tuple","of","type","and","dictionary",",","where","dictionary","contains","arguments","to","initialize","the","optimizer","e.g",".","`","(","torch.optim.Adam",",","{","'lr","'",":","1e-3","}",")","`",".","steps_per_epoch","(","int",")",":","Number","of","train_once","calls","per","epoch",".","grad_steps_per_env_step","(","int",")",":","Number","of","gradient","steps","taken","per","environment","step","sampled",".","max_episode_length_eval","(","int","or","None",")",":","Maximum","length","of","episodes","used","for","off-policy","evaluation",".","If","None",",","defaults","to","`","env_spec.max_episode_length","`",".","num_evaluation_episodes","(","int",")",":","The","number","of","evaluation","trajectories","used","for","computing","eval","stats","at","the","end","of","every","epoch",".","start_steps","(","int",")",":","The","number","of","steps","for","warming","up","before","selecting","actions","according","to","policy",".","update_after","(","int",")",":","The","number","of","steps","to","perform","before","policy","is","updated",".","use_deterministic_evaluation","(","bool",")",":","True","if","the","trained","policy","should","be","evaluated","deterministically","."]
docstr: Perform one iteration of training.
  
  Args:
      itr (int): Iteration number.
docstr_tok: ["Perform","one","iteration","of","training",".","Args",":","itr","(","int",")",":","Iteration","number","."]
code: def _train_once(self, itr):
      """Perform one iteration of training.
  
      Args:
          itr (int): Iteration number.
  
      """
      for grad_step_timer in range(self._grad_steps_per_env_step):
          if (
              self._replay_buffer.n_transitions_stored
              >= self._min_buffer_size
          ):
              # Sample from buffer
              samples = self._replay_buffer.sample_transitions(
                  self._buffer_batch_size
              )
              samples = as_torch_dict(samples)
  
              # Optimize
              qf_loss, y, q, policy_loss = torch_to_np(
                  self._optimize_policy(samples, grad_step_timer)
              )
  
              self._episode_policy_losses.append(policy_loss)
              self._episode_qf_losses.append(qf_loss)
              self._epoch_ys.append(y)
              self._epoch_qs.append(q)
  
      if itr % self._steps_per_epoch == 0:
          logger.log("Training finished")
          epoch = itr // self._steps_per_epoch
  
          if (
              self._replay_buffer.n_transitions_stored
              >= self._min_buffer_size
          ):
              tabular.record("Epoch", epoch)
              self._log_statistics()
  
imports: import copy
  from dowel import logger, tabular
  import numpy
  import torch
  import torch.nn.functional
  from garage import _Default, log_performance, make_optimizer, obtain_evaluation_episodes
  from garage.np.algos import RLAlgorithm
  from garage.torch import as_torch_dict, global_device, soft_update_model, torch_to_np
  from garage.torch._functions import zero_optim_grads
label_desc: Use image node_id 4 for calling the TD3 obj's underlying member method code with example usage: obj._train_once(itr) without return types