Modify hiperparameters and output

a3c/discrete_A3C.py

@@ -15,7 +15,7 @@ import numpy as np
 
 UPDATE_GLOBAL_ITER = 5
 GAMMA = 0.9
-MAX_EP = 100000
+MAX_EP = 500000
 
 class Net(nn.Module):
     def __init__(self, s_dim, a_dim, word_list, words_width):
@@ -81,10 +81,10 @@ class Net(nn.Module):
 
 
 class Worker(mp.Process):
-    def __init__(self, gnet, opt, global_ep, global_ep_r, res_queue, name, env, N_S, N_A, words_list, word_width):
+    def __init__(self, gnet, opt, global_ep, global_ep_r, res_queue, name, env, N_S, N_A, words_list, word_width, winning_ep):
         super(Worker, self).__init__()
         self.name = 'w%02i' % name
-        self.g_ep, self.g_ep_r, self.res_queue = global_ep, global_ep_r, res_queue
+        self.g_ep, self.g_ep_r, self.res_queue, self.winning_ep = global_ep, global_ep_r, res_queue, winning_ep
         self.gnet, self.opt = gnet, opt
         self.word_list = words_list
         self.lnet = Net(N_S, N_A, words_list, word_width)           # local network
@@ -109,12 +109,12 @@ class Worker(mp.Process):
                 if total_step % UPDATE_GLOBAL_ITER == 0 or done:  # update global and assign to local net
                     # sync
                     push_and_pull(self.opt, self.lnet, self.gnet, done, s_, buffer_s, buffer_a, buffer_r, GAMMA)
-                    buffer_s, buffer_a, buffer_r = [], [], []
 
                     if done:  # done and print information
                         goal_word = self.env.decode_word(self.env.goal_word)
-                        record(self.g_ep, self.g_ep_r, ep_r, self.res_queue, self.name, goal_word, self.word_list[a])
+                        record(self.g_ep, self.g_ep_r, ep_r, self.res_queue, self.name, goal_word, self.word_list[a], len(buffer_a), self.winning_ep)
                         break
+                    buffer_s, buffer_a, buffer_r = [], [], []
 
                 s = s_
                 total_step += 1

a3c/utils.py

@@ -47,7 +47,7 @@ def push_and_pull(opt, lnet, gnet, done, s_, bs, ba, br, gamma):
     lnet.load_state_dict(gnet.state_dict())
 
 
-def record(global_ep, global_ep_r, ep_r, res_queue, name, goal_word, action):
+def record(global_ep, global_ep_r, ep_r, res_queue, name, goal_word, action, action_number, winning_ep):
     with global_ep.get_lock():
         global_ep.value += 1
     with global_ep_r.get_lock():
@@ -57,10 +57,12 @@ def record(global_ep, global_ep_r, ep_r, res_queue, name, goal_word, action):
             global_ep_r.value = global_ep_r.value * 0.99 + ep_r * 0.01
     res_queue.put(global_ep_r.value)
     if goal_word == action:
+        winning_ep.value += 1
         print(
             name,
             "Ep:", global_ep.value,
             "| Ep_r: %.0f" % global_ep_r.value,
             "| Goal :", goal_word,
-            "| Action: ", action
+            "| Action: ", action,
+            "| Actions: ", action_number
         )

main.py

@@ -20,10 +20,10 @@ if __name__ == "__main__":
     gnet = Net(N_S, N_A, words_list, word_width)        # global network
     gnet.share_memory()         # share the global parameters in multiprocessing
     opt = SharedAdam(gnet.parameters(), lr=1e-4, betas=(0.92, 0.999))      # global optimizer
-    global_ep, global_ep_r, res_queue = mp.Value('i', 0), mp.Value('d', 0.), mp.Queue()
+    global_ep, global_ep_r, res_queue, win_ep = mp.Value('i', 0), mp.Value('d', 0.), mp.Queue(), mp.Value('i', 0)
 
     # parallel training
-    workers = [Worker(gnet, opt, global_ep, global_ep_r, res_queue, i, env, N_S = N_S, N_A=N_A, words_list=words_list, word_width=word_width) for i in range(mp.cpu_count())]
+    workers = [Worker(gnet, opt, global_ep, global_ep_r, res_queue, i, env, N_S, N_A, words_list, word_width, win_ep) for i in range(mp.cpu_count())]
     [w.start() for w in workers]
     res = []                    # record episode reward to plot
     while True:
@@ -33,7 +33,8 @@ if __name__ == "__main__":
         else:
             break
     [w.join() for w in workers]
-
+    print("Jugadas:", global_ep.value)
+    print("Ganadas:", win_ep.value)
     plt.plot(res)
     plt.ylabel('Moving average ep reward')
     plt.xlabel('Step')