text: import multiprocessing as mp
  import numpy as np
  import logging
  import os
  import sys
  from abr import ABREnv
  import network
  import tensorflow as tf
  import network
  
  os.environ['CUDA_VISIBLE_DEVICES'] = '3'
  os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
  
  S_DIM = [7, 8]
  A_DIM = 6
  ACTOR_LR_RATE =1e-4
  CRITIC_LR_RATE = 1e-3
  NUM_AGENTS = 25
  TRAIN_SEQ_LEN = 300  # take as a train batch
  TRAIN_EPOCH = 1000000
  MODEL_SAVE_INTERVAL = 500
  RANDOM_SEED = 42
  RAND_RANGE = 10000
  SUMMARY_DIR = './results'
  MODEL_DIR = './models'
  TRAIN_TRACES = './cooked_traces/'
  TEST_LOG_FOLDER = './test_results/'
  LOG_FILE = './results/log'
  
  # create result directory
  if not os.path.exists(SUMMARY_DIR):
      os.makedirs(SUMMARY_DIR)
      
  NN_MODEL = None
  
  def testing(epoch, nn_model, log_file):
      # clean up the test results folder
      os.system('rm -r ' + TEST_LOG_FOLDER)
      #os.system('mkdir ' + TEST_LOG_FOLDER)
  
      if not os.path.exists(TEST_LOG_FOLDER):
          os.makedirs(TEST_LOG_FOLDER)
      # run test script
      os.system('python rl_test.py ' + nn_model)
  
      # append test performance to the log
      rewards = []
      test_log_files = os.listdir(TEST_LOG_FOLDER)
      for test_log_file in test_log_files:
          reward = []
          with open(TEST_LOG_FOLDER + test_log_file, 'rb') as f:
              for line in f:
                  parse = line.split()
                  try:
                      reward.append(float(parse[-1]))
                  except IndexError:
                      break
          rewards.append(np.sum(reward[1:]))
  
      rewards = np.array(rewards)
  
      rewards_min = np.min(rewards)
      rewards_5per = np.percentile(rewards, 5)
      rewards_mean = np.mean(rewards)
      rewards_median = np.percentile(rewards, 50)
      rewards_95per = np.percentile(rewards, 95)
      rewards_max = np.max(rewards)
  
      log_file.write(str(epoch) + '\t' +
                     str(rewards_min) + '\t' +
                     str(rewards_5per) + '\t' +
                     str(rewards_mean) + '\t' +
                     str(rewards_median) + '\t' +
                     str(rewards_95per) + '\t' +
                     str(rewards_max) + '\n')
      log_file.flush()
  
      return rewards_mean
  
  def central_agent(net_params_queues, exp_queues):
  
      assert len(net_params_queues) == NUM_AGENTS
      assert len(exp_queues) == NUM_AGENTS
      with tf.Session() as sess, open(LOG_FILE + '_test.txt', 'w') as test_log_file:
  
          actor = network.Network(sess,
                  state_dim=S_DIM, action_dim=A_DIM,
                  learning_rate=ACTOR_LR_RATE)
  
          sess.run(tf.global_variables_initializer())
          saver = tf.train.Saver(max_to_keep=1000)  # save neural net parameters
  
          # restore neural net parameters
          nn_model = NN_MODEL
          if nn_model is not None:  # nn_model is the path to file
              saver.restore(sess, nn_model)
              print("Model restored.")
  
          # while True:  # assemble experiences from agents, compute the gradients
          for epoch in range(TRAIN_EPOCH):
              # synchronize the network parameters of work agent
              actor_net_params = actor.get_network_params()
              for i in range(NUM_AGENTS):
                  net_params_queues[i].put(actor_net_params)
  
              s, a, g = [], [], []
              for i in range(NUM_AGENTS):
                  s_, a_, g_ = exp_queues[i].get()
                  s += s_
                  a += a_
                  g += g_
              s_batch=np.stack(s, axis=0)
              a_batch=np.vstack(a)
              v_batch=np.vstack(g)
              actor.train(s_batch, a_batch, v_batch, epoch)
  
              if epoch % MODEL_SAVE_INTERVAL == 0:
                  # Save the neural net parameters to disk.
                  save_path = saver.save(sess, SUMMARY_DIR + "/nn_model_ep_" +
                                         str(epoch) + ".ckpt")
                  testing(epoch,
                      SUMMARY_DIR + "/nn_model_ep_" + str(epoch) + ".ckpt", 
                      test_log_file)
  
  def agent(agent_id, net_params_queue, exp_queue):
      env = ABREnv(agent_id)
      with tf.Session() as sess, open(SUMMARY_DIR + '/log_agent_' + str(agent_id), 'w') as log_file:
          actor = network.Network(sess,
                                  state_dim=S_DIM, action_dim=A_DIM,
                                  learning_rate=ACTOR_LR_RATE)
  
          # initial synchronization of the network parameters from the coordinator
          actor_net_params = net_params_queue.get()
          actor.set_network_params(actor_net_params)
  
          time_stamp = 0
  
          for epoch in range(TRAIN_EPOCH):
              obs = env.reset()
              s_batch, a_batch, r_batch = [], [], []
              for step in range(TRAIN_SEQ_LEN):
                  s_batch.append(obs)
  
                  buffer_bound = actor.predict(
                      np.reshape(obs, (1, S_DIM[0], S_DIM[1])))     
                  obs, rew, done, info = env.step(buffer_bound)
                  
                  a_batch.append([buffer_bound])
                  r_batch.append(rew)
  
                  if done:
                      break
              v_batch = actor.compute_v(s_batch, a_batch, r_batch, done)
              exp_queue.put([s_batch, a_batch, v_batch])
  
              actor_net_params = net_params_queue.get()
              actor.set_network_params(actor_net_params)
  
  def main():
  
      np.random.seed(RANDOM_SEED)
  
      # inter-process communication queues
      net_params_queues = []
      exp_queues = []
      for i in range(NUM_AGENTS):
          net_params_queues.append(mp.Queue(1))
          exp_queues.append(mp.Queue(1))
  
      # create a coordinator and multiple agent processes
      # (note: threading is not desirable due to python GIL)
      coordinator = mp.Process(target=central_agent,
                               args=(net_params_queues, exp_queues))
      coordinator.start()
  
      agents = []
      for i in range(NUM_AGENTS):
          agents.append(mp.Process(target=agent,
                                   args=(i,
                                         net_params_queues[i],
                                         exp_queues[i])))
      for i in range(NUM_AGENTS):
          agents[i].start()
  
      # wait unit training is done
      coordinator.join()
  
  
  if __name__ == '__main__':
      main()
  