code: import copy
  import glob
  import os
  import time
  import operator
  from functools import reduce
  
  import gym
  import numpy as np
  import torch
  import torch.nn as nn
  import torch.nn.functional as F
  import torch.optim as optim
  from torch.autograd import Variable
  
  from arguments import get_args
  from vec_env.dummy_vec_env import DummyVecEnv
  from vec_env.subproc_vec_env import SubprocVecEnv
  from envs import make_env
  from kfac import KFACOptimizer
  from model import CNNPolicy, MLPPolicy
  from storage import RolloutStorage
  from visualize import visdom_plot
  
  args = get_args()
  
  assert args.algo in ['a2c', 'ppo', 'acktr']
  if args.recurrent_policy:
      assert args.algo in ['a2c', 'ppo'], \
          'Recurrent policy is not implemented for ACKTR'
  
  num_updates = int(args.num_frames) // args.num_steps // args.num_processes
  
  torch.manual_seed(args.seed)
  if args.cuda:
      torch.cuda.manual_seed(args.seed)
  
  try:
      os.makedirs(args.log_dir)
  except OSError:
      files = glob.glob(os.path.join(args.log_dir, '*.monitor.csv'))
      for f in files:
          os.remove(f)
  
  def main():
      os.environ['OMP_NUM_THREADS'] = '1'
  
      if args.vis:
          from visdom import Visdom
          viz = Visdom()
          win = None
  
      envs = [make_env(args.env_name, args.seed, i, args.log_dir, args.start_container)
                  for i in range(args.num_processes)]
  
      if args.num_processes > 1:
          envs = SubprocVecEnv(envs)
      else:
          envs = DummyVecEnv(envs)
  
      obs_shape = envs.observation_space.shape
      obs_shape = (obs_shape[0] * args.num_stack, *obs_shape[1:])
      obs_numel = reduce(operator.mul, obs_shape, 1)
  
      if len(obs_shape) == 3 and obs_numel > 1024:
          actor_critic = CNNPolicy(obs_shape[0], envs.action_space, args.recurrent_policy)
      else:
          assert not args.recurrent_policy, \
              "Recurrent policy is not implemented for the MLP controller"
          actor_critic = MLPPolicy(obs_numel, envs.action_space)
  
      modelSize = 0
      for p in actor_critic.parameters():
          pSize = reduce(operator.mul, p.size(), 1)
          modelSize += pSize
      print(str(actor_critic))
      print('Total model size: %d' % modelSize)
  
      if envs.action_space.__class__.__name__ == "Discrete":
          action_shape = 1
      else:
          action_shape = envs.action_space.shape[0]
  
      if args.cuda:
          actor_critic.cuda()
  
      if args.algo == 'a2c':
          optimizer = optim.RMSprop(actor_critic.parameters(), args.lr, eps=args.eps, alpha=args.alpha)
      elif args.algo == 'ppo':
          optimizer = optim.Adam(actor_critic.parameters(), args.lr, eps=args.eps)
      elif args.algo == 'acktr':
          optimizer = KFACOptimizer(actor_critic)
  
      rollouts = RolloutStorage(args.num_steps, args.num_processes, obs_shape, envs.action_space, actor_critic.state_size)
      current_obs = torch.zeros(args.num_processes, *obs_shape)
  
      def update_current_obs(obs):
          shape_dim0 = envs.observation_space.shape[0]
          obs = torch.from_numpy(obs).float()
          if args.num_stack > 1:
              current_obs[:, :-shape_dim0] = current_obs[:, shape_dim0:]
          current_obs[:, -shape_dim0:] = obs
  
      obs = envs.reset()
      update_current_obs(obs)
  
      rollouts.observations[0].copy_(current_obs)
  
      # These variables are used to compute average rewards for all processes.
      episode_rewards = torch.zeros([args.num_processes, 1])
      final_rewards = torch.zeros([args.num_processes, 1])
      reward_avg = 0
  
      if args.cuda:
          current_obs = current_obs.cuda()
          rollouts.cuda()
  
      start = time.time()
      for j in range(num_updates):
          for step in range(args.num_steps):
              # Sample actions
              value, action, action_log_prob, states = actor_critic.act(
                  Variable(rollouts.observations[step]),
                  Variable(rollouts.states[step]),
                  Variable(rollouts.masks[step])
              )
              cpu_actions = action.data.squeeze(1).cpu().numpy()
  
              # Observation, reward and next obs
              obs, reward, done, info = envs.step(cpu_actions)
  
              # Maxime: clip the reward within [0,1] for more reliable training
              # This code deals poorly with large reward values
              reward = np.clip(reward, a_min=0, a_max=None) / 400
  
              reward = torch.from_numpy(np.expand_dims(np.stack(reward), 1)).float()
              episode_rewards += reward
  
              # If done then clean the history of observations.
              masks = torch.FloatTensor([[0.0] if done_ else [1.0] for done_ in done])
              final_rewards *= masks
              final_rewards += (1 - masks) * episode_rewards
              episode_rewards *= masks
  
              if args.cuda:
                  masks = masks.cuda()
  
              if current_obs.dim() == 4:
                  current_obs *= masks.unsqueeze(2).unsqueeze(2)
              else:
                  current_obs *= masks
  
              update_current_obs(obs)
              rollouts.insert(step, current_obs, states.data, action.data, action_log_prob.data, value.data, reward, masks)
  
          next_value = actor_critic(
              Variable(rollouts.observations[-1]),
              Variable(rollouts.states[-1]),
              Variable(rollouts.masks[-1])
          )[0].data
  
          rollouts.compute_returns(next_value, args.use_gae, args.gamma, args.tau)
  
          if args.algo in ['a2c', 'acktr']:
              values, action_log_probs, dist_entropy, states = actor_critic.evaluate_actions(Variable(rollouts.observations[:-1].view(-1, *obs_shape)),
                                                                                             Variable(rollouts.states[0].view(-1, actor_critic.state_size)),
                                                                                             Variable(rollouts.masks[:-1].view(-1, 1)),
                                                                                             Variable(rollouts.actions.view(-1, action_shape)))
  
              values = values.view(args.num_steps, args.num_processes, 1)
              action_log_probs = action_log_probs.view(args.num_steps, args.num_processes, 1)
  
              advantages = Variable(rollouts.returns[:-1]) - values
              value_loss = advantages.pow(2).mean()
  
              action_loss = -(Variable(advantages.data) * action_log_probs).mean()
  
              if args.algo == 'acktr' and optimizer.steps % optimizer.Ts == 0:
                  # Sampled fisher, see Martens 2014
                  actor_critic.zero_grad()
                  pg_fisher_loss = -action_log_probs.mean()
  
                  value_noise = Variable(torch.randn(values.size()))
                  if args.cuda:
                      value_noise = value_noise.cuda()
  
                  sample_values = values + value_noise
                  vf_fisher_loss = -(values - Variable(sample_values.data)).pow(2).mean()
  
                  fisher_loss = pg_fisher_loss + vf_fisher_loss
                  optimizer.acc_stats = True
                  fisher_loss.backward(retain_graph=True)
                  optimizer.acc_stats = False
  
              optimizer.zero_grad()
              (value_loss * args.value_loss_coef + action_loss - dist_entropy * args.entropy_coef).backward()
  
              if args.algo == 'a2c':
                  nn.utils.clip_grad_norm(actor_critic.parameters(), args.max_grad_norm)
  
              optimizer.step()
  
          elif args.algo == 'ppo':
              advantages = rollouts.returns[:-1] - rollouts.value_preds[:-1]
              advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-5)
  
              for e in range(args.ppo_epoch):
                  if args.recurrent_policy:
                      data_generator = rollouts.recurrent_generator(advantages,
                                                              args.num_mini_batch)
                  else:
                      data_generator = rollouts.feed_forward_generator(advantages,
                                                              args.num_mini_batch)
  
                  for sample in data_generator:
                      observations_batch, states_batch, actions_batch, \
                         return_batch, masks_batch, old_action_log_probs_batch, \
                              adv_targ = sample
  
                      # Reshape to do in a single forward pass for all steps
                      values, action_log_probs, dist_entropy, states = actor_critic.evaluate_actions(Variable(observations_batch),
                                                                                                     Variable(states_batch),
                                                                                                     Variable(masks_batch),
                                                                                                     Variable(actions_batch))
  
                      adv_targ = Variable(adv_targ)
                      ratio = torch.exp(action_log_probs - Variable(old_action_log_probs_batch))
                      surr1 = ratio * adv_targ
                      surr2 = torch.clamp(ratio, 1.0 - args.clip_param, 1.0 + args.clip_param) * adv_targ
                      action_loss = -torch.min(surr1, surr2).mean() # PPO's pessimistic surrogate (L^CLIP)
  
                      value_loss = (Variable(return_batch) - values).pow(2).mean()
  
                      optimizer.zero_grad()
                      (value_loss + action_loss - dist_entropy * args.entropy_coef).backward()
                      nn.utils.clip_grad_norm(actor_critic.parameters(), args.max_grad_norm)
                      optimizer.step()
  
          rollouts.after_update()
  
          if j % args.save_interval == 0 and args.save_dir != "":
              save_path = os.path.join(args.save_dir, args.algo)
              try:
                  os.makedirs(save_path)
              except OSError:
                  pass
  
              # A really ugly way to save a model to CPU
              save_model = actor_critic
              if args.cuda:
                  save_model = copy.deepcopy(actor_critic).cpu()
  
              save_model = [save_model,
                              hasattr(envs, 'ob_rms') and envs.ob_rms or None]
  
              torch.save(save_model, os.path.join(save_path, args.env_name + ".pt"))
  
          if j % args.log_interval == 0:
              reward_avg = 0.99 * reward_avg + 0.01 * final_rewards.mean()
              end = time.time()
              total_num_steps = (j + 1) * args.num_processes * args.num_steps
  
              print(
                  "Updates {}, num timesteps {}, FPS {}, running avg reward {:.3f}, entropy {:.5f}, value loss {:.5f}, policy loss {:.5f}".
                  format(
                      j,
                      total_num_steps,
                      int(total_num_steps / (end - start)),
                      reward_avg,
                      dist_entropy.data[0],
                      value_loss.data[0],
                      action_loss.data[0]
                  )
              )
  
              """
              print("Updates {}, num timesteps {}, FPS {}, mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}, entropy {:.5f}, value loss {:.5f}, policy loss {:.5f}".
                  format(
                      j,
                      total_num_steps,
                      int(total_num_steps / (end - start)),
                      final_rewards.mean(),
                      final_rewards.median(),
                      final_rewards.min(),
                      final_rewards.max(), dist_entropy.data[0],
                      value_loss.data[0], action_loss.data[0])
                  )
              """
  
          if args.vis and j % args.vis_interval == 0:
              try:
                  # Sometimes monitor doesn't properly flush the outputs
                  win = visdom_plot(viz, win, args.log_dir, args.env_name, args.algo)
              except IOError:
                  pass
  
  if __name__ == "__main__":
      main()
   #!/usr/bin/env python3
  
  import sys
  import argparse
  
  import pyglet
  import numpy as np
  
  import gym
  import gym_duckietown
  from gym_duckietown.envs import SimpleSimEnv
  
  parser = argparse.ArgumentParser()
  parser.add_argument('--env-name', default='SimpleSim-v0')
  parser.add_argument('--map-name', default='udem1')
  parser.add_argument('--draw-curve', action='store_true', help='draw the lane following curve')
  parser.add_argument('--no-random', action='store_true', help='disable domain randomization')
  
  args = parser.parse_args()
  
  if args.env_name == 'SimpleSim-v0':
      env = SimpleSimEnv(
          map_name = args.map_name,
          draw_curve = args.draw_curve,
          domain_rand = not args.no_random
      )
  else:
      env = gym.make(args.env_name)
  
  env.reset()
  env.render()
  
  def save_numpy_img(file_name, img):
      img = np.ascontiguousarray(img)
      img = (img * 255).astype(np.uint8)
      img = np.flip(img, 0)
  
      from skimage import io
      io.imsave(file_name, img)
  
  lastImgNo = 0
  def save_img(img):
      global lastImgNo
      save_numpy_img('img_%03d.png' % lastImgNo, img)
      lastImgNo += 1
  
  @env.window.event
  def on_key_press(symbol, modifiers):
      from pyglet.window import key
  
      action = None
      if symbol == key.LEFT:
          print('left')
          action = np.array([0.00, 0.40])
      elif symbol == key.RIGHT:
          print('right')
          action = np.array([0.40, 0.00])
      elif symbol == key.UP:
          print('forward')
          action = np.array([0.40, 0.40])
      elif symbol == key.BACKSPACE or symbol == key.SLASH:
          print('RESET')
          action = None
          env.reset()
          env.render()
      elif symbol == key.SPACE:
          action = np.array([0, 0])
      elif symbol == key.ESCAPE:
          env.close()
          sys.exit(0)
      else:
          return
  
      if action is not None:
          print('stepping')
          obs, reward, done, info = env.step(action)
          print('step_count = %s, reward=%.3f' % (env.step_count, reward))
  
          env.render()
  
          #save_img(obs)
  
          if done:
              print('done!')
              env.reset()
              env.render()
  
  # Enter main event loop
  pyglet.app.run()
  
  env.close()
  
apis: torch.FloatTensor torch.min torch.cuda.manual_seed numpy.stack torch.manual_seed torch.autograd.Variable torch.from_numpy numpy.clip torch.zeros torch.clamp
apis: numpy.array numpy.ascontiguousarray numpy.flip