class_docstr: Implementation of TD3.
  
  Based on https://arxiv.org/pdf/1802.09477.pdf.
  
  Example:
      $ python garage/examples/tf/td3_pendulum.py
  
  Args:
      env_spec (EnvSpec): Environment.
      policy (Policy): Policy.
      qf (garage.tf.q_functions.QFunction): Q-function.
      qf2 (garage.tf.q_functions.QFunction): Q function to use
      replay_buffer (ReplayBuffer): Replay buffer.
      sampler (garage.sampler.Sampler): Sampler.
      target_update_tau (float): Interpolation parameter for doing the
          soft target update.
      policy_lr (float): Learning rate for training policy network.
      qf_lr (float): Learning rate for training q value network.
      policy_weight_decay (float): L2 weight decay factor for parameters
          of the policy network.
      qf_weight_decay (float): L2 weight decay factor for parameters
          of the q value network.
      policy_optimizer (tf.compat.v1.train.Optimizer): Optimizer for training
          policy network.
      qf_optimizer (tf.compat.v1.train.Optimizer): Optimizer for training
          Q-function network.
      clip_pos_returns (boolean): Whether or not clip positive returns.
      clip_return (float): Clip return to be in [-clip_return,
          clip_return].
      discount (float): Discount factor for the cumulative return.
      max_episode_length_eval (int or None): Maximum length of episodes used
          for off-policy evaluation. If `None`, defaults to
          `env_spec.max_episode_length`.
      max_action (float): Maximum action magnitude.
      name (str): Name of the algorithm shown in computation graph.
      steps_per_epoch (int): Number of batches of samples in each epoch.
      n_train_steps (int): Number of optimizations in each epoch cycle.
      buffer_batch_size (int): Size of replay buffer.
      min_buffer_size (int):
          Number of samples in replay buffer before first optimization.
      reward_scale (float): Scale to reward.
      exploration_policy_sigma (float): Action noise sigma.
      exploration_policy_clip (float): Action noise clip.
      actor_update_period (int): Action update period.
      exploration_policy (ExplorationPolicy): Exploration strategy.
class_docstr_tok: ["Implementation","of","TD3",".","Based","on","https",":","\/\/arxiv.org\/pdf\/1802.09477.pdf",".","Example",":","$","python","garage\/examples\/tf\/td3_pendulum.py","Args",":","env_spec","(","EnvSpec",")",":","Environment",".","policy","(","Policy",")",":","Policy",".","qf","(","garage.tf.q_functions.QFunction",")",":","Q-function",".","qf2","(","garage.tf.q_functions.QFunction",")",":","Q","function","to","use","replay_buffer","(","ReplayBuffer",")",":","Replay","buffer",".","sampler","(","garage.sampler.Sampler",")",":","Sampler",".","target_update_tau","(","float",")",":","Interpolation","parameter","for","doing","the","soft","target","update",".","policy_lr","(","float",")",":","Learning","rate","for","training","policy","network",".","qf_lr","(","float",")",":","Learning","rate","for","training","q","value","network",".","policy_weight_decay","(","float",")",":","L2","weight","decay","factor","for","parameters","of","the","policy","network",".","qf_weight_decay","(","float",")",":","L2","weight","decay","factor","for","parameters","of","the","q","value","network",".","policy_optimizer","(","tf.compat.v1.train.Optimizer",")",":","Optimizer","for","training","policy","network",".","qf_optimizer","(","tf.compat.v1.train.Optimizer",")",":","Optimizer","for","training","Q-function","network",".","clip_pos_returns","(","boolean",")",":","Whether","or","not","clip","positive","returns",".","clip_return","(","float",")",":","Clip","return","to","be","in","[","-clip_return",",","clip_return","]",".","discount","(","float",")",":","Discount","factor","for","the","cumulative","return",".","max_episode_length_eval","(","int","or","None",")",":","Maximum","length","of","episodes","used","for","off-policy","evaluation",".","If","`","None","`",",","defaults","to","`","env_spec.max_episode_length","`",".","max_action","(","float",")",":","Maximum","action","magnitude",".","name","(","str",")",":","Name","of","the","algorithm","shown","in","computation","graph",".","steps_per_epoch","(","int",")",":","Number","of","batches","of","samples","in","each","epoch",".","n_train_steps","(","int",")",":","Number","of","optimizations","in","each","epoch","cycle",".","buffer_batch_size","(","int",")",":","Size","of","replay","buffer",".","min_buffer_size","(","int",")",":","Number","of","samples","in","replay","buffer","before","first","optimization",".","reward_scale","(","float",")",":","Scale","to","reward",".","exploration_policy_sigma","(","float",")",":","Action","noise","sigma",".","exploration_policy_clip","(","float",")",":","Action","noise","clip",".","actor_update_period","(","int",")",":","Action","update","period",".","exploration_policy","(","ExplorationPolicy",")",":","Exploration","strategy","."]
docstr: Build the loss function and init the optimizer.
docstr_tok: ["Build","the","loss","function","and","init","the","optimizer","."]
code: def _init_opt(self):
      """Build the loss function and init the optimizer."""
      with tf.name_scope(self._name):
          # Create target policy (actor) and qf (critic) networks
          with tf.name_scope("inputs"):
              obs_dim = self._env_spec.observation_space.flat_dim
              y = tf.compat.v1.placeholder(
                  tf.float32, shape=(None, 1), name="input_y"
              )
              obs = tf.compat.v1.placeholder(
                  tf.float32,
                  shape=(None, obs_dim),
                  name="input_observation",
              )
              actions = tf.compat.v1.placeholder(
                  tf.float32,
                  shape=(None, self._env_spec.action_space.flat_dim),
                  name="input_action",
              )
  
          policy_network_outputs = self._target_policy.build(
              obs, name="policy"
          )
          target_qf_outputs = self._target_qf.build(
              obs, actions, name="qf"
          )
          target_qf2_outputs = self._target_qf2.build(
              obs, actions, name="qf"
          )
  
          self._target_policy_f_prob_online = compile_function(
              inputs=[obs], outputs=policy_network_outputs
          )
  
          self._target_qf_f_prob_online = compile_function(
              inputs=[obs, actions], outputs=target_qf_outputs
          )
  
          self._target_qf2_f_prob_online = compile_function(
              inputs=[obs, actions], outputs=target_qf2_outputs
          )
  
          # Set up target init and update functions
          with tf.name_scope("setup_target"):
              policy_init_op, policy_update_op = get_target_ops(
                  self.policy.get_global_vars(),
                  self._target_policy.get_global_vars(),
                  self._tau,
              )
              qf_init_ops, qf_update_ops = get_target_ops(
                  self.qf.get_global_vars(),
                  self._target_qf.get_global_vars(),
                  self._tau,
              )
              qf2_init_ops, qf2_update_ops = get_target_ops(
                  self.qf2.get_global_vars(),
                  self._target_qf2.get_global_vars(),
                  self._tau,
              )
              target_init_op = (
                  policy_init_op + qf_init_ops + qf2_init_ops
              )
              target_update_op = (
                  policy_update_op + qf_update_ops + qf2_update_ops
              )
  
          f_init_target = compile_function(
              inputs=[], outputs=target_init_op
          )
          f_update_target = compile_function(
              inputs=[], outputs=target_update_op
          )
  
          # Set up policy training function
          next_action = self.policy.build(obs, name="policy_action")
          next_qval = self.qf.build(
              obs, next_action, name="policy_action_qval"
          )
          with tf.name_scope("action_loss"):
              action_loss = -tf.reduce_mean(next_qval)
  
          with tf.name_scope("minimize_action_loss"):
              policy_optimizer = make_optimizer(
                  self._policy_optimizer,
                  learning_rate=self._policy_lr,
                  name="PolicyOptimizer",
              )
              policy_train_op = policy_optimizer.minimize(
                  action_loss, var_list=self.policy.get_trainable_vars()
              )
  
          f_train_policy = compile_function(
              inputs=[obs], outputs=[policy_train_op, action_loss]
          )
  
          # Set up qf training function
          qval = self.qf.build(obs, actions, name="q_value")
          q2val = self.qf2.build(obs, actions, name="q2_value")
          with tf.name_scope("qval1_loss"):
              qval1_loss = tf.reduce_mean(
                  tf.math.squared_difference(y, qval)
              )
          with tf.name_scope("qval2_loss"):
              qval2_loss = tf.reduce_mean(
                  tf.math.squared_difference(y, q2val)
              )
  
          with tf.name_scope("minimize_qf_loss"):
              qf_optimizer = make_optimizer(
                  self._qf_optimizer,
                  learning_rate=self._qf_lr,
                  name="QFunctionOptimizer",
              )
              qf_train_op = qf_optimizer.minimize(
                  qval1_loss, var_list=self.qf.get_trainable_vars()
              )
              qf2_train_op = qf_optimizer.minimize(
                  qval2_loss, var_list=self.qf2.get_trainable_vars()
              )
  
          f_train_qf = compile_function(
              inputs=[y, obs, actions],
              outputs=[qf_train_op, qval1_loss, qval],
          )
          f_train_qf2 = compile_function(
              inputs=[y, obs, actions],
              outputs=[qf2_train_op, qval2_loss, q2val],
          )
  
          self._f_train_policy = f_train_policy
          self._f_train_qf = f_train_qf
          self._f_init_target = f_init_target
          self._f_update_target = f_update_target
          self._f_train_qf2 = f_train_qf2
  
file_path: garage/src/garage/tf/algos/td3.py
imports: from dowel import logger, tabular
  import numpy
  import tensorflow
  from garage import _Default, log_performance, make_optimizer, obtain_evaluation_episodes
  from garage.np.algos import RLAlgorithm
  from garage.tf import compile_function, get_target_ops
label_desc: Use image node_id 2 for calling the TD3 obj's underlying member method code with example usage: obj._init_opt() without return types