text: import numpy as np
  import tensorflow as tf
  import gym
  import time
  from spinup.algos.td3 import core
  from spinup.algos.td3.td3_randtarg import td3 as true_td3
  from spinup.algos.td3.core import get_vars
  from spinup.utils.logx import EpochLogger
  
  """
  
  Exercise 1.3: TD3 Computation Graph
  
  Implement the core computation graph for the TD3 algorithm.
  
  As starter code, you are given the entirety of the TD3 algorithm except
  for the computation graph. Find "YOUR CODE HERE" to begin.
  
  To clarify: you will not write an "actor_critic" function for this 
  exercise. But you will use one to build the graph for computing the
  TD3 updates.
  
  """
  
  class ReplayBuffer:
      """
      A simple FIFO experience replay buffer for TD3 agents.
      """
  
      def __init__(self, obs_dim, act_dim, size):
          self.obs1_buf = np.zeros([size, obs_dim], dtype=np.float32)
          self.obs2_buf = np.zeros([size, obs_dim], dtype=np.float32)
          self.acts_buf = np.zeros([size, act_dim], dtype=np.float32)
          self.rews_buf = np.zeros(size, dtype=np.float32)
          self.done_buf = np.zeros(size, dtype=np.float32)
          self.ptr, self.size, self.max_size = 0, 0, size
  
      def store(self, obs, act, rew, next_obs, done):
          self.obs1_buf[self.ptr] = obs
          self.obs2_buf[self.ptr] = next_obs
          self.acts_buf[self.ptr] = act
          self.rews_buf[self.ptr] = rew
          self.done_buf[self.ptr] = done
          self.ptr = (self.ptr+1) % self.max_size
          self.size = min(self.size+1, self.max_size)
  
      def sample_batch(self, batch_size=32):
          idxs = np.random.randint(0, self.size, size=batch_size)
          return dict(obs1=self.obs1_buf[idxs],
                      obs2=self.obs2_buf[idxs],
                      acts=self.acts_buf[idxs],
                      rews=self.rews_buf[idxs],
                      done=self.done_buf[idxs])
  
  """
  
  TD3 (Twin Delayed DDPG)
  
  """
  def td3(env_fn, actor_critic=core.mlp_actor_critic, ac_kwargs=dict(), seed=0, 
          steps_per_epoch=5000, epochs=100, replay_size=int(1e6), gamma=0.99, 
          polyak=0.995, pi_lr=1e-3, q_lr=1e-3, batch_size=100, start_steps=10000, 
          act_noise=0.1, target_noise=0.2, noise_clip=0.5, policy_delay=2, 
          max_ep_len=1000, logger_kwargs=dict(), save_freq=1):
      """
  
      Args:
          env_fn : A function which creates a copy of the environment.
              The environment must satisfy the OpenAI Gym API.
  
          actor_critic: A function which takes in placeholder symbols 
              for state, ``x_ph``, and action, ``a_ph``, and returns the main 
              outputs from the agent's Tensorflow computation graph:
  
              ===========  ================  ======================================
              Symbol       Shape             Description
              ===========  ================  ======================================
              ``pi``       (batch, act_dim)  | Deterministically computes actions
                                             | from policy given states.
              ``q1``       (batch,)          | Gives one estimate of Q* for 
                                             | states in ``x_ph`` and actions in
                                             | ``a_ph``.
              ``q2``       (batch,)          | Gives another estimate of Q* for 
                                             | states in ``x_ph`` and actions in
                                             | ``a_ph``.
              ``q1_pi``    (batch,)          | Gives the composition of ``q1`` and 
                                             | ``pi`` for states in ``x_ph``: 
                                             | q1(x, pi(x)).
              ===========  ================  ======================================
  
          ac_kwargs (dict): Any kwargs appropriate for the actor_critic 
              function you provided to TD3.
  
          seed (int): Seed for random number generators.
  
          steps_per_epoch (int): Number of steps of interaction (state-action pairs) 
              for the agent and the environment in each epoch.
  
          epochs (int): Number of epochs to run and train agent.
  
          replay_size (int): Maximum length of replay buffer.
  
          gamma (float): Discount factor. (Always between 0 and 1.)
  
          polyak (float): Interpolation factor in polyak averaging for target 
              networks. Target networks are updated towards main networks 
              according to:
  
              .. math:: \\theta_{\\text{targ}} \\leftarrow 
                  \\rho \\theta_{\\text{targ}} + (1-\\rho) \\theta
  
              where :math:`\\rho` is polyak. (Always between 0 and 1, usually 
              close to 1.)
  
          pi_lr (float): Learning rate for policy.
  
          q_lr (float): Learning rate for Q-networks.
  
          batch_size (int): Minibatch size for SGD.
  
          start_steps (int): Number of steps for uniform-random action selection,
              before running real policy. Helps exploration.
  
          act_noise (float): Stddev for Gaussian exploration noise added to 
              policy at training time. (At test time, no noise is added.)
  
          target_noise (float): Stddev for smoothing noise added to target 
              policy.
  
          noise_clip (float): Limit for absolute value of target policy 
              smoothing noise.
  
          policy_delay (int): Policy will only be updated once every 
              policy_delay times for each update of the Q-networks.
  
          max_ep_len (int): Maximum length of trajectory / episode / rollout.
  
          logger_kwargs (dict): Keyword args for EpochLogger.
  
          save_freq (int): How often (in terms of gap between epochs) to save
              the current policy and value function.
  
      """
  
      logger = EpochLogger(**logger_kwargs)
      logger.save_config(locals())
  
      tf.set_random_seed(seed)
      np.random.seed(seed)
  
      env, test_env = env_fn(), env_fn()
      obs_dim = env.observation_space.shape[0]
      act_dim = env.action_space.shape[0]
  
      # Action limit for clamping: critically, assumes all dimensions share the same bound!
      act_limit = env.action_space.high[0]
  
      # Share information about action space with policy architecture
      ac_kwargs['action_space'] = env.action_space
  
      # Inputs to computation graph
      x_ph, a_ph, x2_ph, r_ph, d_ph = core.placeholders(obs_dim, act_dim, obs_dim, None, None)
  
      #=========================================================================#
      #                                                                         #
      #           All of your code goes in the space below.                     #
      #                                                                         #
      #=========================================================================#
  
      # Main outputs from computation graph
      with tf.variable_scope('main'):
          #######################
          #                     #
          #   YOUR CODE HERE    #
          #                     #
          #######################
          # pi, q1, q2, q1_pi = 
          pass
      
      # Target policy network
      with tf.variable_scope('target'):
          #######################
          #                     #
          #   YOUR CODE HERE    #
          #                     #
          #######################
          # pi_targ =
          pass
      
      # Target Q networks
      with tf.variable_scope('target', reuse=True):
  
          # Target policy smoothing, by adding clipped noise to target actions
          #######################
          #                     #
          #   YOUR CODE HERE    #
          #                     #
          #######################
  
          # Target Q-values, using action from smoothed target policy
          #######################
          #                     #
          #   YOUR CODE HERE    #
          #                     #
          #######################
          pass
  
      # Experience buffer
      replay_buffer = ReplayBuffer(obs_dim=obs_dim, act_dim=act_dim, size=replay_size)
  
      # Count variables
      var_counts = tuple(core.count_vars(scope) for scope in ['main/pi', 'main/q1', 'main/q2', 'main'])
      print('\nNumber of parameters: \t pi: %d, \t q1: %d, \t q2: %d, \t total: %d\n'%var_counts)
  
      # Bellman backup for Q functions, using Clipped Double-Q targets
      #######################
      #                     #
      #   YOUR CODE HERE    #
      #                     #
      #######################
  
      # TD3 losses
      #######################
      #                     #
      #   YOUR CODE HERE    #
      #                     #
      #######################
      # pi_loss = 
      # q1_loss = 
      # q2_loss = 
      # q_loss = 
  
      #=========================================================================#
      #                                                                         #
      #           All of your code goes in the space above.                     #
      #                                                                         #
      #=========================================================================#
  
      # Separate train ops for pi, q
      pi_optimizer = tf.train.AdamOptimizer(learning_rate=pi_lr)
      q_optimizer = tf.train.AdamOptimizer(learning_rate=q_lr)
      train_pi_op = pi_optimizer.minimize(pi_loss, var_list=get_vars('main/pi'))
      train_q_op = q_optimizer.minimize(q_loss, var_list=get_vars('main/q'))
  
      # Polyak averaging for target variables
      target_update = tf.group([tf.assign(v_targ, polyak*v_targ + (1-polyak)*v_main)
                                for v_main, v_targ in zip(get_vars('main'), get_vars('target'))])
  
      # Initializing targets to match main variables
      target_init = tf.group([tf.assign(v_targ, v_main)
                                for v_main, v_targ in zip(get_vars('main'), get_vars('target'))])
  
      sess = tf.Session()
      sess.run(tf.global_variables_initializer())
      sess.run(target_init)
  
      # Setup model saving
      logger.setup_tf_saver(sess, inputs={'x': x_ph, 'a': a_ph}, outputs={'pi': pi, 'q1': q1, 'q2': q2})
  
      def get_action(o, noise_scale):
          a = sess.run(pi, feed_dict={x_ph: o.reshape(1,-1)})[0]
          a += noise_scale * np.random.randn(act_dim)
          return np.clip(a, -act_limit, act_limit)
  
      def test_agent(n=10):
          for j in range(n):
              o, r, d, ep_ret, ep_len = test_env.reset(), 0, False, 0, 0
              while not(d or (ep_len == max_ep_len)):
                  # Take deterministic actions at test time (noise_scale=0)
                  o, r, d, _ = test_env.step(get_action(o, 0))
                  ep_ret += r
                  ep_len += 1
              logger.store(TestEpRet=ep_ret, TestEpLen=ep_len)
  
      start_time = time.time()
      o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
      total_steps = steps_per_epoch * epochs
  
      # Main loop: collect experience in env and update/log each epoch
      for t in range(total_steps):
  
          """
          Until start_steps have elapsed, randomly sample actions
          from a uniform distribution for better exploration. Afterwards, 
          use the learned policy (with some noise, via act_noise). 
          """
          if t > start_steps:
              a = get_action(o, act_noise)
          else:
              a = env.action_space.sample()
  
          # Step the env
          o2, r, d, _ = env.step(a)
          ep_ret += r
          ep_len += 1
  
          # Ignore the "done" signal if it comes from hitting the time
          # horizon (that is, when it's an artificial terminal signal
          # that isn't based on the agent's state)
          d = False if ep_len==max_ep_len else d
  
          # Store experience to replay buffer
          replay_buffer.store(o, a, r, o2, d)
  
          # Super critical, easy to overlook step: make sure to update 
          # most recent observation!
          o = o2
  
          if d or (ep_len == max_ep_len):
              """
              Perform all TD3 updates at the end of the trajectory
              (in accordance with source code of TD3 published by
              original authors).
              """
              for j in range(ep_len):
                  batch = replay_buffer.sample_batch(batch_size)
                  feed_dict = {x_ph: batch['obs1'],
                               x2_ph: batch['obs2'],
                               a_ph: batch['acts'],
                               r_ph: batch['rews'],
                               d_ph: batch['done']
                              }
                  q_step_ops = [q_loss, q1, q2, train_q_op]
                  outs = sess.run(q_step_ops, feed_dict)
                  logger.store(LossQ=outs[0], Q1Vals=outs[1], Q2Vals=outs[2])
  
                  if j % policy_delay == 0:
                      # Delayed policy update
                      outs = sess.run([pi_loss, train_pi_op, target_update], feed_dict)
                      logger.store(LossPi=outs[0])
  
              logger.store(EpRet=ep_ret, EpLen=ep_len)
              o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
  
          # End of epoch wrap-up
          if t > 0 and t % steps_per_epoch == 0:
              epoch = t // steps_per_epoch
  
              # Save model
              if (epoch % save_freq == 0) or (epoch == epochs-1):
                  logger.save_state({'env': env}, None)
  
              # Test the performance of the deterministic version of the agent.
              test_agent()
  
              # Log info about epoch
              logger.log_tabular('Epoch', epoch)
              logger.log_tabular('EpRet', with_min_and_max=True)
              logger.log_tabular('TestEpRet', with_min_and_max=True)
              logger.log_tabular('EpLen', average_only=True)
              logger.log_tabular('TestEpLen', average_only=True)
              logger.log_tabular('TotalEnvInteracts', t)
              logger.log_tabular('Q1Vals', with_min_and_max=True)
              logger.log_tabular('Q2Vals', with_min_and_max=True)
              logger.log_tabular('LossPi', average_only=True)
              logger.log_tabular('LossQ', average_only=True)
              logger.log_tabular('Time', time.time()-start_time)
              logger.dump_tabular()
  
  if __name__ == '__main__':
      import argparse
      parser = argparse.ArgumentParser()
      parser.add_argument('--env', type=str, default='HalfCheetah-v2')
      parser.add_argument('--seed', '-s', type=int, default=0)
      parser.add_argument('--exp_name', type=str, default='ex13-td3')
      parser.add_argument('--use_soln', action='store_true')
      args = parser.parse_args()
  
      from spinup.utils.run_utils import setup_logger_kwargs
      logger_kwargs = setup_logger_kwargs(args.exp_name + '-' + args.env.lower(), args.seed)
  
      all_kwargs = dict(
          env_fn=lambda : gym.make(args.env), 
          actor_critic=core.mlp_actor_critic,
          ac_kwargs=dict(hidden_sizes=[128,128]), 
          max_ep_len=150,
          seed=args.seed, 
          logger_kwargs=logger_kwargs,
          epochs=10
          )
      
      if args.use_soln:
          true_td3(**all_kwargs)
      else:
          td3(**all_kwargs)
  