code: from osim.env import L2M2019Env
  
  '''
      TD3
  '''
  import copy
  import numpy as np
  import pandas as pd
  
  import argparse
  import os
  import os.path
  from os import path
  import utils
  from collections import Iterable
  
  import torch
  import torch.nn as nn
  import torch.nn.functional as F
  
  
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
  
  # Implementation of Twin Delayed Deep Deterministic Policy Gradients (TD3)
  # Paper: https://arxiv.org/abs/1802.09477
  
  def flatten_list(lis):
      for item in lis:
          if isinstance(item, Iterable) and not isinstance(item, str):
              for x in flatten(item):
                  yield x
          else:        
              yield item
  
  def flatten(mydict):
      new_dict = {}
      for key, value in mydict.items():
          if type(value) == list:
              print("key: {}, value: {}".format(key, value))
              new_dict[key] = flatten_list(value)
          else:
              new_dict[key] = value
      return new_dict
  
  def FF(ss):
      state = flatten(ss)
      # print(state)
  
      new_state = []
  
      for v in state.values():       
          # print(type(v))
          if type(v) == dict:
              temp = pd.json_normalize(v, sep = '_')
              temp = list(temp.values)
              for item in temp:
                  if type(item) == np.ndarray:
                      for val in item:
                          if type(val) == list:
                              for t in val:
                                  new_state.append(float(t))
                          else:
                              new_state.append(float(val))
                  else:
                      new_state.append(item)
          else:
              # print("asdf")
              for arr in v:
                  for a in arr:
                      for val in a:
                          new_state.append(float(val))
  
      return new_state
  
  class Actor(nn.Module):
      def __init__(self, state_dim, action_dim, max_action):
          super(Actor, self).__init__()
  
          self.l1 = nn.Linear(state_dim, 256)
          self.l2 = nn.Linear(256, 256)
          self.l3 = nn.Linear(256, action_dim)
          
          self.max_action = max_action
          
  
      def forward(self, state):
          a = F.relu(self.l1(state))
          a = F.relu(self.l2(a))
          return self.max_action * torch.tanh(self.l3(a))
  
  
  class Critic(nn.Module):
      def __init__(self, state_dim, action_dim):
          super(Critic, self).__init__()
  
          # Q1 architecture
          self.l1 = nn.Linear(state_dim + action_dim, 256)
          self.l2 = nn.Linear(256, 256)
          self.l3 = nn.Linear(256, 1)
  
          # Q2 architecture
          self.l4 = nn.Linear(state_dim + action_dim, 256)
          self.l5 = nn.Linear(256, 256)
          self.l6 = nn.Linear(256, 1)
  
  
      def forward(self, state, action):
          sa = torch.cat([state, action], 1)
  
          q1 = F.relu(self.l1(sa))
          q1 = F.relu(self.l2(q1))
          q1 = self.l3(q1)
  
          q2 = F.relu(self.l4(sa))
          q2 = F.relu(self.l5(q2))
          q2 = self.l6(q2)
          return q1, q2
  
  
      def Q1(self, state, action):
          sa = torch.cat([state, action], 1)
  
          q1 = F.relu(self.l1(sa))
          q1 = F.relu(self.l2(q1))
          q1 = self.l3(q1)
          return q1
  
  
  class TD3(object):
      def __init__(
          self,
          state_dim,
          action_dim,
          max_action,
          discount=0.99,
          tau=0.005,
          policy_noise=0.2,
          noise_clip=0.5,
          policy_freq=2
      ):
  
          self.actor = Actor(state_dim, action_dim, max_action).to(device)
          self.actor_target = copy.deepcopy(self.actor)
          self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=3e-4)
  
          self.critic = Critic(state_dim, action_dim).to(device)
          self.critic_target = copy.deepcopy(self.critic)
          self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=3e-4)
  
          self.max_action = max_action
          self.discount = discount
          self.tau = tau
          self.policy_noise = policy_noise
          self.noise_clip = noise_clip
          self.policy_freq = policy_freq
  
          self.total_it = 0
  
      def select_action(self, state):
  
          new_state = FF(state)
  
          new_state = np.array(new_state)
          # print(new_state)
  
          new_state = torch.FloatTensor(new_state.reshape(1, -1)).to(device)
          return self.actor(new_state).cpu().data.numpy().flatten()
  
      def train(self, replay_buffer, batch_size=256):
          self.total_it += 1
  
          # Sample replay buffer 
          state, action, next_state, reward, not_done = replay_buffer.sample(batch_size)
  
          with torch.no_grad():
              # Select action according to policy and add clipped noise
              noise = (
                  torch.randn_like(action) * self.policy_noise
              ).clamp(-self.noise_clip, self.noise_clip)
              
              next_action = (
                  self.actor_target(next_state) + noise
              ).clamp(-self.max_action, self.max_action)
  
              # Compute the target Q value
              target_Q1, target_Q2 = self.critic_target(next_state, next_action)
              target_Q = torch.min(target_Q1, target_Q2)
              target_Q = reward + not_done * self.discount * target_Q
  
          # Get current Q estimates
          current_Q1, current_Q2 = self.critic(state, action)
  
          # Compute critic loss
          critic_loss = F.mse_loss(current_Q1, target_Q) + F.mse_loss(current_Q2, target_Q)
  
          # Optimize the critic
          self.critic_optimizer.zero_grad()
          critic_loss.backward()
          self.critic_optimizer.step()
  
          # Delayed policy updates
          if self.total_it % self.policy_freq == 0:
  
              # Compute actor losse
              actor_loss = -self.critic.Q1(state, self.actor(state)).mean()
              
              # Optimize the actor 
              self.actor_optimizer.zero_grad()
              actor_loss.backward()
              self.actor_optimizer.step()
  
              # Update the frozen target models
              for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):
                  target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data)
  
              for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()):
                  target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data)
  
  
      def save(self, filename):
          torch.save(self.critic.state_dict(), filename + "_critic")
          torch.save(self.critic_optimizer.state_dict(), filename + "_critic_optimizer")
          
          torch.save(self.actor.state_dict(), filename + "_actor")
          torch.save(self.actor_optimizer.state_dict(), filename + "_actor_optimizer")
  
  
      def load(self, filename):
          self.critic.load_state_dict(torch.load(filename + "_critic"))
          self.critic_optimizer.load_state_dict(torch.load(filename + "_critic_optimizer"))
          self.critic_target = copy.deepcopy(self.critic)
  
          self.actor.load_state_dict(torch.load(filename + "_actor"))
          self.actor_optimizer.load_state_dict(torch.load(filename + "_actor_optimizer"))
          self.actor_target = copy.deepcopy(self.actor)
  
      def check_existed_files(self, filename):
          print ("File exists:"+str(path.exists(filename + "_critic")))
          print ("File exists:" + str(path.exists(filename + "_critic_optimizer")))
          print ("File exists:"+str(path.exists(filename + "_actor")))
          print ("File exists:" + str(path.exists(filename + "_actor_optimizer")))
  
  '''
      REPLAYBUFFER
  '''
  
  class ReplayBuffer(object):
      def __init__(self, state_dim, action_dim, max_size=int(1e6)):
          self.max_size = max_size
          self.ptr = 0
          self.size = 0
  
          self.state = np.zeros((max_size, state_dim))
          self.action = np.zeros((max_size, action_dim))
          self.next_state = np.zeros((max_size, state_dim))
          self.reward = np.zeros((max_size, 1))
          self.not_done = np.zeros((max_size, 1))
  
          self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
  
  
      def add(self, state, action, next_state, reward, done):        
          new_state = FF(state)
          new_next_state = FF(next_state)
          self.state[self.ptr] = new_state
          self.action[self.ptr] = action
          self.next_state[self.ptr] = new_next_state
          self.reward[self.ptr] = reward
          self.not_done[self.ptr] = 1. - done
  
          self.ptr = (self.ptr + 1) % self.max_size
          self.size = min(self.size + 1, self.max_size)
  
  
      def sample(self, batch_size):
          ind = np.random.randint(0, self.size, size=batch_size)
  
          return (
              torch.FloatTensor(self.state[ind]).to(self.device),
              torch.FloatTensor(self.action[ind]).to(self.device),
              torch.FloatTensor(self.next_state[ind]).to(self.device),
              torch.FloatTensor(self.reward[ind]).to(self.device),
              torch.FloatTensor(self.not_done[ind]).to(self.device)
          )
  
  
  '''
      MAIN
  '''
  
  # Runs policy for X episodes and returns average reward
  # A fixed seed is used for the eval environment
  def eval_policy(policy, env_name, seed, eval_episodes=5):
      eval_env = L2M2019Env(visualize=True)
      eval_env.seed(seed + 100)
  
      avg_reward = 0.
      for _ in range(eval_episodes):
          state, done = eval_env.reset(), False
          while not done:
              action = policy.select_action(state)
  
              for i in range(len(action)):
                  if action[i] > 1:
                      action[i] = 1
                  elif action[i] < 0: 
                      action[i] = 0
  
              # print(action)
              state, reward, done, _ = eval_env.step(action)
              avg_reward += reward
  
      avg_reward /= eval_episodes
  
      print("---------------------------------------")
      print(f"Evaluation over {eval_episodes} episodes: {avg_reward:.3f}")
      print("---------------------------------------")
      return avg_reward
  
  
  if __name__ == "__main__":
      
      parser = argparse.ArgumentParser()
      parser.add_argument("--policy", default="TD3")                  # Policy name (TD3, DDPG or OurDDPG)
      parser.add_argument("--env", default="HalfCheetah-v2")          # OpenAI gym environment name
      parser.add_argument("--seed", default=0, type=int)              # Sets Gym, PyTorch and Numpy seeds
      parser.add_argument("--start_timesteps", default=10e2, type=int)# Time steps initial random policy is used
      parser.add_argument("--eval_freq", default=1e4, type=int)       # How often (time steps) we evaluate
      parser.add_argument("--max_timesteps", default=5e7, type=int)   # Max time steps to run environment
      parser.add_argument("--expl_noise", default=0.1)                # Std of Gaussian exploration noise
      parser.add_argument("--batch_size", default=256, type=int)      # Batch size for both actor and critic
      parser.add_argument("--discount", default=0.99)                 # Discount factor
      parser.add_argument("--tau", default=0.005)                     # Target network update rate
      parser.add_argument("--policy_noise", default=0.2)              # Noise added to target policy during critic update
      parser.add_argument("--noise_clip", default=0.5)                # Range to clip target policy noise
      parser.add_argument("--policy_freq", default=2, type=int)       # Frequency of delayed policy updates
      parser.add_argument("--save_model", action="store_true")        # Save model and optimizer parameters
      parser.add_argument("--load_model", default="true")                 # Model load file name, "" doesn't load, "default" uses file_name
      args = parser.parse_args()
  
      file_name = f"{args.policy}_{args.env}_{args.seed}"
      print("---------------------------------------")
      print(f"Policy: {args.policy}, Env: {args.env}, Seed: {args.seed}")
      print("---------------------------------------")
  
      if not os.path.exists("./results"):
          os.makedirs("./results")
  
      if args.save_model and not os.path.exists("./models"):
          os.makedirs("./models")
  
      env = L2M2019Env(visualize=True)
  
      # Set seeds
      env.seed(args.seed)
      # env.action_space.seed(seed)
      torch.manual_seed(args.seed)
      np.random.seed(args.seed)
      
      state_dim = env.observation_space.shape[0]
      action_dim = env.action_space.shape[0] 
      max_action = float(env.action_space.high[0])
  
      kwargs = {
          "state_dim": state_dim,
          "action_dim": action_dim,
          "max_action": max_action,
          "discount": args.discount,
          "tau": args.tau,
      }
  
      # Initialize policy
      if args.policy == "TD3":
          # Target policy smoothing is scaled wrt the action scale
          kwargs["policy_noise"] = args.policy_noise * max_action
          kwargs["noise_clip"] = args.noise_clip * max_action
          kwargs["policy_freq"] = args.policy_freq
          policy = TD3(**kwargs)
  
      if args.load_model != "":
          policy.check_existed_files("models/107062240_HW4_data")
          policy_file = file_name if args.load_model == "default" else args.load_model
          policy.load("models/107062240_HW4_data")
  
      replay_buffer = ReplayBuffer(state_dim, action_dim)
      
      # Evaluate untrained policy
      evaluations = [eval_policy(policy, args.env, args.seed)]
  
      state, done = env.reset(), False
      episode_reward = 0
      episode_timesteps = 0
      episode_num = 0
  
      for t in range(int(args.max_timesteps)):
          episode_timesteps += 1
  
          # Select action randomly or according to policy
          if t < args.start_timesteps:
              action = env.action_space.sample()
          else:
              action = (
                  policy.select_action(state)
                  + np.random.normal(0, max_action * args.expl_noise, size=action_dim)
              ).clip(-max_action, max_action)
  
          for i in range(len(action)):
              if action[i] > 1:
                  action[i] = 1
              elif action[i] < 0: 
                  action[i] = 0
  
          # print(action)
  
          # Perform action
          next_state, reward, done, _ = env.step(action) 
          done_bool = float(done) if episode_timesteps < 500 else 0
  
          # Store data in replay buffer
          replay_buffer.add(state, action, next_state, reward, done_bool)
  
          state = next_state
          episode_reward += reward 
  
          # Train agent after collecting sufficient data
          if t >= args.start_timesteps:
              # print("Start training ........ ")
              policy.train(replay_buffer, args.batch_size)
  
          if done: 
              # +1 to account for 0 indexing. +0 on ep_timesteps since it will increment +1 even if done=True
              print(f"Total T: {t+1} Episode Num: {episode_num+1} Episode T: {episode_timesteps} Reward: {episode_reward:.3f}")
              # Reset environment
              state, done = env.reset(), False
              episode_reward = 0
              episode_timesteps = 0
              episode_num += 1 
  
          # Evaluate episode
          if (t + 1) % args.eval_freq == 0:
              print("save new model!")
              evaluations.append(eval_policy(policy, args.env, args.seed))
              np.save(f"./results/{file_name}", evaluations)
              policy.save("models/107062240_HW4_data")
  
apis: numpy.save torch.min torch.nn.functional.mse_loss torch.nn.Linear torch.load numpy.zeros torch.FloatTensor torch.randn_like torch.manual_seed torch.no_grad numpy.random.seed pandas.json_normalize numpy.random.normal torch.cuda.is_available numpy.array numpy.random.randint torch.cat