code: import time
  
  import gym
  import numpy as np
  import torch
  import torch.nn.functional as F
  
  from fireup.algos.ddpg import core
  from fireup.utils.logx import EpochLogger
  
  
  class ReplayBuffer:
      """
      A simple FIFO experience replay buffer for DDPG agents.
      """
  
      def __init__(self, obs_dim, act_dim, size):
          self.obs1_buf = np.zeros([size, obs_dim], dtype=np.float32)
          self.obs2_buf = np.zeros([size, obs_dim], dtype=np.float32)
          self.acts_buf = np.zeros([size, act_dim], dtype=np.float32)
          self.rews_buf = np.zeros(size, dtype=np.float32)
          self.done_buf = np.zeros(size, dtype=np.float32)
          self.ptr, self.size, self.max_size = 0, 0, size
  
      def store(self, obs, act, rew, next_obs, done):
          self.obs1_buf[self.ptr] = obs
          self.obs2_buf[self.ptr] = next_obs
          self.acts_buf[self.ptr] = act
          self.rews_buf[self.ptr] = rew
          self.done_buf[self.ptr] = done
          self.ptr = (self.ptr + 1) % self.max_size
          self.size = min(self.size + 1, self.max_size)
  
      def sample_batch(self, batch_size=32):
          idxs = np.random.randint(0, self.size, size=batch_size)
          return dict(
              obs1=self.obs1_buf[idxs],
              obs2=self.obs2_buf[idxs],
              acts=self.acts_buf[idxs],
              rews=self.rews_buf[idxs],
              done=self.done_buf[idxs],
          )
  
  
  """
  
  Deep Deterministic Policy Gradient (DDPG)
  
  """
  
  
  def ddpg(
      env_fn,
      actor_critic=core.ActorCritic,
      ac_kwargs=dict(),
      seed=0,
      steps_per_epoch=5000,
      epochs=100,
      replay_size=int(1e6),
      gamma=0.99,
      polyak=0.995,
      pi_lr=1e-3,
      q_lr=1e-3,
      batch_size=100,
      start_steps=10000,
      act_noise=0.1,
      max_ep_len=1000,
      logger_kwargs=dict(),
      save_freq=1,
  ):
      """
  
      Args:
          env_fn : A function which creates a copy of the environment.
              The environment must satisfy the OpenAI Gym API.
  
          actor_critic: The agent's main model which takes some states ``x`` and
              and actions ``a`` and returns a tuple of:
  
              ===========  ================  ======================================
              Symbol       Shape             Description
              ===========  ================  ======================================
              ``pi``       (batch, act_dim)  | Deterministically computes actions
                                             | from policy given states.
              ``q``        (batch,)          | Gives the current estimate of Q* for
                                             | states ``x`` and actions in
                                             | ``a``.
              ``q_pi``     (batch,)          | Gives the composition of ``q`` and
                                             | ``pi`` for states in ``x``:
                                             | q(x, pi(x)).
              ===========  ================  ======================================
  
          ac_kwargs (dict): Any kwargs appropriate for the actor_critic
              class you provided to DDPG.
  
          seed (int): Seed for random number generators.
  
          steps_per_epoch (int): Number of steps of interaction (state-action pairs)
              for the agent and the environment in each epoch.
  
          epochs (int): Number of epochs to run and train agent.
  
          replay_size (int): Maximum length of replay buffer.
  
          gamma (float): Discount factor. (Always between 0 and 1.)
  
          polyak (float): Interpolation factor in polyak averaging for target
              networks. Target networks are updated towards main networks
              according to:
  
              .. math:: \\theta_{\\text{targ}} \\leftarrow
                  \\rho \\theta_{\\text{targ}} + (1-\\rho) \\theta
  
              where :math:`\\rho` is polyak. (Always between 0 and 1, usually
              close to 1.)
  
          pi_lr (float): Learning rate for policy.
  
          q_lr (float): Learning rate for Q-networks.
  
          batch_size (int): Minibatch size for SGD.
  
          start_steps (int): Number of steps for uniform-random action selection,
              before running real policy. Helps exploration.
  
          act_noise (float): Stddev for Gaussian exploration noise added to
              policy at training time. (At test time, no noise is added.)
  
          max_ep_len (int): Maximum length of trajectory / episode / rollout.
  
          logger_kwargs (dict): Keyword args for EpochLogger.
  
          save_freq (int): How often (in terms of gap between epochs) to save
              the current policy and value function.
  
      """
  
      logger = EpochLogger(**logger_kwargs)
      logger.save_config(locals())
  
      torch.manual_seed(seed)
      np.random.seed(seed)
  
      env, test_env = env_fn(), env_fn()
      obs_dim = env.observation_space.shape[0]
      act_dim = env.action_space.shape[0]
  
      # Action limit for clamping: critically, assumes all dimensions share the same bound!
      act_limit = env.action_space.high[0]
  
      # Share information about action space with policy architecture
      ac_kwargs["action_space"] = env.action_space
  
      # Main outputs from computation graph
      main = actor_critic(in_features=obs_dim, **ac_kwargs)
  
      # Target networks
      target = actor_critic(in_features=obs_dim, **ac_kwargs)
      target.eval()
  
      # Experience buffer
      replay_buffer = ReplayBuffer(obs_dim=obs_dim, act_dim=act_dim, size=replay_size)
  
      # Count variables
      var_counts = tuple(
          core.count_vars(module) for module in [main.policy, main.q, main]
      )
      print("\nNumber of parameters: \t pi: %d, \t q: %d, \t total: %d\n" % var_counts)
  
      # Separate train ops for pi, q
      pi_optimizer = torch.optim.Adam(main.policy.parameters(), lr=pi_lr)
      q_optimizer = torch.optim.Adam(main.q.parameters(), lr=q_lr)
  
      # Initializing targets to match main variables
      target.load_state_dict(main.state_dict())
  
      def get_action(o, noise_scale):
          pi = main.policy(torch.Tensor(o.reshape(1, -1)))
          a = pi.detach().numpy()[0] + noise_scale * np.random.randn(act_dim)
          return np.clip(a, -act_limit, act_limit)
  
      def test_agent(n=10):
          for _ in range(n):
              o, r, d, ep_ret, ep_len = test_env.reset(), 0, False, 0, 0
              while not (d or (ep_len == max_ep_len)):
                  # Take deterministic actions at test time (noise_scale=0)
                  o, r, d, _ = test_env.step(get_action(o, 0))
                  ep_ret += r
                  ep_len += 1
              logger.store(TestEpRet=ep_ret, TestEpLen=ep_len)
  
      start_time = time.time()
      o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
      total_steps = steps_per_epoch * epochs
  
      # Main loop: collect experience in env and update/log each epoch
      for t in range(total_steps):
          main.eval()
          """
          Until start_steps have elapsed, randomly sample actions
          from a uniform distribution for better exploration. Afterwards,
          use the learned policy (with some noise, via act_noise).
          """
          if t > start_steps:
              a = get_action(o, act_noise)
          else:
              a = env.action_space.sample()
  
          # Step the env
          o2, r, d, _ = env.step(a)
          ep_ret += r
          ep_len += 1
  
          # Ignore the "done" signal if it comes from hitting the time
          # horizon (that is, when it's an artificial terminal signal
          # that isn't based on the agent's state)
          d = False if ep_len == max_ep_len else d
  
          # Store experience to replay buffer
          replay_buffer.store(o, a, r, o2, d)
  
          # Super critical, easy to overlook step: make sure to update
          # most recent observation!
          o = o2
  
          if d or (ep_len == max_ep_len):
              main.train()
              """
              Perform all DDPG updates at the end of the trajectory,
              in accordance with tuning done by TD3 paper authors.
              """
              for _ in range(ep_len):
                  batch = replay_buffer.sample_batch(batch_size)
                  (obs1, obs2, acts, rews, done) = (
                      torch.Tensor(batch["obs1"]),
                      torch.Tensor(batch["obs2"]),
                      torch.Tensor(batch["acts"]),
                      torch.Tensor(batch["rews"]),
                      torch.Tensor(batch["done"]),
                  )
                  _, _, q_pi_targ = target(obs2, acts)
  
                  # Bellman backup for Q function
                  backup = (rews + gamma * (1 - done) * q_pi_targ).detach()
  
                  # DDPG Q loss
                  _, q, _ = main(obs1, acts)
                  q_loss = F.mse_loss(q, backup)
  
                  # Q-learning update
                  q_optimizer.zero_grad()
                  q_loss.backward()
                  q_optimizer.step()
                  logger.store(LossQ=q_loss.item(), QVals=q.data.numpy())
  
                  # DDPG Policy loss
                  _, _, q_pi = main(obs1, acts)
                  pi_loss = -q_pi.mean()
  
                  # Policy update
                  pi_optimizer.zero_grad()
                  pi_loss.backward()
                  pi_optimizer.step()
                  logger.store(LossPi=pi_loss.item())
  
                  # Polyak averaging for target parameters
                  for p_main, p_target in zip(main.parameters(), target.parameters()):
                      p_target.data.copy_(
                          polyak * p_target.data + (1 - polyak) * p_main.data
                      )
  
              logger.store(EpRet=ep_ret, EpLen=ep_len)
              o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
  
          # End of epoch wrap-up
          if t > 0 and t % steps_per_epoch == 0:
              epoch = t // steps_per_epoch
  
              # Save model
              if (epoch % save_freq == 0) or (epoch == epochs - 1):
                  logger.save_state({"env": env}, main, None)
  
              # Test the performance of the deterministic version of the agent.
              test_agent()
  
              # Log info about epoch
              logger.log_tabular("Epoch", epoch)
              logger.log_tabular("EpRet", with_min_and_max=True)
              logger.log_tabular("TestEpRet", with_min_and_max=True)
              logger.log_tabular("EpLen", average_only=True)
              logger.log_tabular("TestEpLen", average_only=True)
              logger.log_tabular("TotalEnvInteracts", t)
              logger.log_tabular("QVals", with_min_and_max=True)
              logger.log_tabular("LossPi", average_only=True)
              logger.log_tabular("LossQ", average_only=True)
              logger.log_tabular("Time", time.time() - start_time)
              logger.dump_tabular()
  
  
  if __name__ == "__main__":
      import argparse
  
      parser = argparse.ArgumentParser()
      parser.add_argument("--env", type=str, default="HalfCheetah-v2")
      parser.add_argument("--hid", type=int, default=300)
      parser.add_argument("--l", type=int, default=1)
      parser.add_argument("--gamma", type=float, default=0.99)
      parser.add_argument("--seed", "-s", type=int, default=0)
      parser.add_argument("--epochs", type=int, default=50)
      parser.add_argument("--exp_name", type=str, default="ddpg")
      args = parser.parse_args()
  
      from fireup.utils.run_utils import setup_logger_kwargs
  
      logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed)
  
      ddpg(
          lambda: gym.make(args.env),
          actor_critic=core.ActorCritic,
          ac_kwargs=dict(hidden_sizes=[args.hid] * args.l),
          gamma=args.gamma,
          seed=args.seed,
          epochs=args.epochs,
          logger_kwargs=logger_kwargs,
      )
  
apis: torch.nn.functional.mse_loss numpy.zeros torch.manual_seed numpy.random.seed numpy.random.randn numpy.clip numpy.random.randint torch.Tensor