'Author: Aimore Resende Riquetti Dutra'
'''email: aimorerrd@hotmail.com'''
# -------------------------------------------------------------------------------------------------- #
# This code can run 4 different models of Reinforcement Learning:
# Q-Learning (QL), DQN, SRL (DSRL), SRL+CS(DSRL_object_near) and some other variations of SRL
# The setting for each run can be set at the end of the code
# It can load and save the models in Excel form
# There are some pre-defined environments, but you can create your own
# Press G to get intermediate Graphs and P to stop
# -------------------------------------------------------------------------------------------------- #

import Class
import pprint
import random
import sys
import numpy as np
import pygame
# from pyglet import clock
import pandas as pd
import time
import json
from time import sleep
import math
import matplotlib.pyplot as plt
import os
import glob

## Comment this part if not using DQN model:
# import keras
# from keras.models import Sequential
# from keras.layers import Dense, Activation, Flatten
# from keras.models import model_from_json
# from keras.optimizers import sgd
# from keras.utils import plot_model
# import tensorflow as tf
# from keras.backend.tensorflow_backend import set_session
# config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.3
# set_session(tf.Session(config=config))


# ------ environments ------
# region COLOR DEFINITION
explore_set = set()
explore_dict = dict()

white = (255, 255, 255)
black = (0, 0, 0)
grey = (80, 80, 80)
red = (255, 0, 0)
blue = (0, 0, 255)
green = (0, 255, 0)
yellow = (250, 250, 0)
pink = (250, 105, 180)
# endregion

# region PANDAS DEFINITION
pd.set_option('display.max_columns', None)
pd.set_option('display.large_repr', 'info')
desired_width = 180
pd.set_option('display.width', desired_width)
pd.set_option('precision', 4)
# endregion

np.random.seed(123)  # For reproducibility
pygame.init()  # Pygame initialialization
pp = pprint.PrettyPrinter(indent=4)
actions = ['up', 'down', 'right', 'left']
actions_dict = {'up':0, 'down':1, 'right':2, 'left':3}
p_keys = [pygame.K_w, pygame.K_a, pygame.K_s, pygame.K_d]
# clock.tick(20)

def pop(self):
    '''Removes a layer instance on top of the layer stack.
    '''
    while self.outputs:
        self.layers.pop()
        if not self.layers:
            self.outputs = []
            self.inbound_nodes = []
            self.outbound_nodes = []
        else:
            self.layers[-1].outbound_nodes = []
            self.outputs = [self.layers[-1].output]
        self.built = False

# region REWARDS
negative_reward = 5  # Negative Reward
positive_reward = 1  # Positive Reward
step_reward = 0  # Reward received by each step
# endregion

# ------ environments configuration (till line 640) ------

# region TEXT FONTS DEFINITION
smallfont = pygame.font.SysFont('comicsansms', 13)
smallfont_act = pygame.font.SysFont('arial', 13)
mediumfont_act = pygame.font.SysFont('arial', 18, bold=True)
pygame.font.init()
# endregion

# region DISPLAY FUNCTIONS
def show_Alg(alg, screen):
    text = smallfont.render("Alg: " + alg, True, black)
    screen.blit(text, [5 + 90 * 0, 0])

def show_Samples(sample, screen):
    text = smallfont.render("Sample: " + str(sample), True, black)
    screen.blit(text, [60+100*1, 0])

def show_Level(level, screen):
    text = smallfont.render("Episode: " + str(level), True, black)
    screen.blit(text, [50+100*2, 0])

def show_Score(score, screen):
    text = smallfont.render("Score: " + str(score), True, black)
    screen.blit(text, [50+100*3, 0])

def show_Steps(steps, screen):
    text = smallfont.render("Steps: " + str(steps), True, black)
    screen.blit(text, [50+100*4, 0])

def show_Percent(percent, screen):
    text = smallfont.render("Percent: " + str(['%.2f' % elem for elem in percent]), True, black)
    screen.blit(text, [5, 30 * 4])

def show_Steps_list(steps_list, screen):
    text = smallfont.render("Steps_list: " + str(steps_list), True, black)
    screen.blit(text, [5, 30 * 1])

def show_Act_List(act_list, screen):
    text = smallfont_act.render("act_list: " + str(act_list), True, black)
    screen.blit(text, [5, 30 * 2])

def show_Action(act, screen):
    text = smallfont_act.render("Chosen Action: " + act, True, black)
    screen.blit(text, [5, 30 * 3])

def show_Env(env, screen):
    text = mediumfont_act.render("Environment:  " + str(env), True, black)
    screen.blit(text, [50, 30 * 5])
# endregion

# region CREATE OBJ_LIST FROM STATE AND RELATIONSHIP LIST BETWEEN AGENT AND OBJECTS
''' CREATE obj_list - FROM env '''
def create_obj_list(env):
    obj_list_fun = []
    tp_list = []
    loc_list = []
    env = env.transpose()
    h_max = env.shape[0]
    # print("h_max", h_max)
    v_max = env.shape[1]
    # print("v_max",v_max)
    for h in range(1, (h_max - 1)):
        for v in range(1, (v_max - 1)):
            if env[h][v] != 0:
                tp_list.append(env[h][v])
                loc_list.append((h, v))
    for i in range(len(loc_list)):
        tp = tp_list[i]
        loc = loc_list[i]
        obj = Class.Obj(tp, loc)
        obj_list_fun.append(obj)
    return obj_list_fun

''' CREATE A RELATIONSHIP LIST BETWEEN AGENT AND OBJECTS - FROM obj_list '''
def relation_obj_list(obj_list, agent_pos):
    rel_list = []
    xA = agent_pos[0]
    yA = agent_pos[1]
    # print("xA", xA)
    # print("yA", yA)
    for obj in obj_list:
        xB = obj.loc[0]
        yB = obj.loc[1]
        x = xA - xB
        y = yA - yB
        loc_dif = (x, y)
        # loc_dif = (x[0], y[0])
        tp = obj.tp
        obj = Class.Obj(tp, loc_dif)
        rel_list.append(obj)
    return rel_list
# endregion

# region DRAW OBJECTS
x_zero_screen = 50
y_zero_screen = 180
size_obj = 37
def draw_objects(agent, positivo_list, negativo_list, wall_list, screen):
    # Class.Grid.draw_grid(screen) # Uncomment to display a Grid
    for i in positivo_list:  # POSITIVO
        screen.blit(i.icon, (i.pos[0] * size_obj + x_zero_screen, y_zero_screen + i.pos[1] * size_obj))
    for i in negativo_list:  # NEGATIVO
        screen.blit(i.icon, (i.pos[0] * size_obj + x_zero_screen, y_zero_screen + i.pos[1] * size_obj))
    screen.blit(agent.icon, (agent.pos[0] * size_obj + x_zero_screen, y_zero_screen + agent.pos[1] * size_obj))  # AGENT
    for i in wall_list:  # WALL
        screen.blit(i.icon, (i.pos[0] * size_obj + x_zero_screen, y_zero_screen + i.pos[1] * size_obj))
# endregion

# region CREATE THE STATE FROM THE ENVIRONMENT
def update_state(h_max, v_max, agent, positivo_list, negativo_list, wall_list):
    # state is defined in terms of symbols not pixels...
    state = np.zeros((v_max, h_max)).astype(np.int16)
    for i in positivo_list:
        state[i.pos[1]][i.pos[0]] = 60  # SYMBOL 60 POSITIVE
    for i in negativo_list:
        state[i.pos[1]][i.pos[0]] = 180  # SYMBOL 180 NEGATIVE
    for i in wall_list:
        state[i.pos[1]][i.pos[0]] = 255  # SYMBOL 255
    # state[agent.pos[1]][agent.pos[0]] = 120  # SYMBOL 60
    return state
    # TODO I have to check if this v_max and h_max have to be declared eveytime
# endregion

# region ENVIRONMENT CONFIGURATION
def environment_conf(s_env):
    if s_env == 1:
        v_max = 4
        h_max = 5
        x_agent = 1
        y_agent = 2
        m_nega = np.matrix([[0, 0, 0],
                            [0, 1, 0]])
        m_posi = np.matrix([[0, 1, 0],
                            [0, 0, 0]])

    elif s_env == 2:
        v_max = 4
        h_max = 5
        x_agent = 1
        y_agent = 2
        m_nega = np.matrix([[0, 0, 0],
                            [0, 0, 1]])
        m_posi = np.matrix([[0, 0, 1],
                            [0, 0, 0]])

    elif s_env == 3:
        v_max = 4
        h_max = 5
        x_agent = 1
        y_agent = 2
        m_nega = np.matrix([[1, 0, 0],
                            [0, 0, 0]])
        m_posi = np.matrix([[0, 1, 0],
                            [0, 0, 0]])

    elif s_env == 4:
        v_max = 4
        h_max = 4
        x_agent = 1
        y_agent = 1
        m_nega = np.matrix([[0, 0],
                            [0, 0]])
        m_posi = np.matrix([[0, 0],
                            [0, 1]])

    elif s_env == 5:
        v_max = 5
        h_max = 5
        x_agent = 2
        y_agent = 2
        m_nega = np.zeros(shape=(v_max - 2, h_max - 2))
        m_posi = np.zeros(shape=(v_max - 2, h_max - 2))
        while (True):
            x = random.randrange(0, h_max - 2)
            y = random.randrange(0, v_max - 2)
            if x != x_agent-1 or y != y_agent-1:
                element = (x, y)
                break
        m_posi[element] = 1

    elif s_env == 6:
        v_max = 7
        h_max = 7
        x_agent = 3
        y_agent = 3
        m_nega = np.zeros(shape=(v_max - 2, h_max - 2))
        m_posi = np.zeros(shape=(v_max - 2, h_max - 2))
        while (True):
            x = random.randrange(0, h_max - 2)
            y = random.randrange(0, v_max - 2)
            if x != x_agent - 1 or y != y_agent - 1:
                element = (x, y)
                break
        m_posi[element] = 1

    elif s_env == 7:
        v_max = 9
        h_max = 9
        x_agent = 4
        y_agent = 4
        m_nega = np.zeros(shape=(v_max - 2, h_max - 2))
        m_posi = np.zeros(shape=(v_max - 2, h_max - 2))
        while (True):
            x = random.randrange(0, h_max - 2)
            y = random.randrange(0, v_max - 2)
            if x != x_agent - 1 or y != y_agent - 1:
                element = (x, y)
                break
        m_posi[element] = 1

    elif s_env == 8:
        v_max = 5
        h_max = 5
        x_agent = 2
        y_agent = 2
        m_nega = np.matrix([[0, 0, 0],
                            [0, 0, 0],
                            [1, 0, 1]])
        m_posi = np.matrix([[1, 0, 1],
                            [0, 0, 0],
                            [0, 0, 0]])

    elif s_env == 9:
        v_max = 5
        h_max = 5
        x_agent = 2
        y_agent = 2
        m_nega = np.matrix([[1, 0, 0],
                            [0, 0, 0],
                            [0, 0, 1]])
        m_posi = np.matrix([[0, 0, 1],
                            [0, 0, 0],
                            [1, 0, 0]])

    elif s_env == 10:
        v_max = 9
        h_max = 9
        x_agent = 4
        y_agent = 4
        m_nega = np.matrix([[1, 0, 0, 0, 1, 0, 0],
                            [0, 0, 0, 0, 0, 0, 0],
                            [0, 0, 1, 0, 0, 0, 1],
                            [0, 0, 0, 0, 0, 0, 0],
                            [1, 0, 0, 0, 1, 0, 0],
                            [0, 0, 0, 0, 0, 0, 0],
                            [0, 0, 1, 0, 0, 0, 1]])
        m_posi = np.matrix([[0, 0, 1, 0, 0, 0, 1],
                            [0, 0, 0, 0, 0, 0, 0],
                            [1, 0, 0, 0, 1, 0, 0],
                            [0, 0, 0, 0, 0, 0, 0],
                            [0, 0, 1, 0, 0, 0, 1],
                            [0, 0, 0, 0, 0, 0, 0],
                            [1, 0, 0, 0, 1, 0, 0]])

    elif s_env == 11:
        v_max = 9
        h_max = 9
        x_agent = 4
        y_agent = 4
        element_list = []
        for n in range(14):
            while(True):
                x = random.randrange(0,7)
                y = random.randrange(0,7)
                if x != 3 and y != 3 and (x,y) not in element_list:
                    element = (x, y)
                    break
            element_list.append(element)

        m_nega = np.zeros(shape=(v_max-2, h_max-2))
        m_posi = np.zeros(shape=(v_max-2, h_max-2))
        half = len(element_list) / 2
        nega_list = element_list[:int(half)]
        posi_list = element_list[int(half):]
        for ele in nega_list:
            m_nega[ele] = 1
        for ele in posi_list:
            m_posi[ele] = 1

    elif s_env == 12:
        v_max = 3
        h_max = 5
        x_agent = 2
        y_agent = 1
        m_nega = np.matrix([1, 0, 0])
        m_posi = np.matrix([0, 0, 1])

    elif s_env == 13:
        v_max = 3
        h_max = 5
        x_agent = 2
        y_agent = 1
        m_nega = np.matrix([0, 0, 0])
        m_posi = np.matrix([1, 0, 1])

    elif s_env == 14:
        v_max = 3
        h_max = 6
        x_agent = 2
        y_agent = 1
        m_nega = np.matrix([1, 0, 0, 0])
        m_posi = np.matrix([0, 0, 0, 1])

    elif s_env == 15:
        v_max = 3
        h_max = 6
        x_agent = 2
        y_agent = 1
        m_nega = np.matrix([0, 0, 0, 0])
        m_posi = np.matrix([1, 0, 0, 1])

    elif s_env == 16:
        v_max = 3
        h_max = 7
        x_agent = 3
        y_agent = 1
        m_nega = np.matrix([1, 0, 0, 0, 0])
        m_posi = np.matrix([0, 0, 0, 0, 1])

    elif s_env == 17:
        v_max = 3
        h_max = 7
        x_agent = 3
        y_agent = 1
        m_nega = np.matrix([0, 0, 0, 0, 0])
        m_posi = np.matrix([1, 0, 0, 0, 1])

    elif s_env == 18:
        v_max = 3
        h_max = 9
        x_agent = 4
        y_agent = 1
        m_nega = np.matrix([1, 0, 0, 0, 0, 0, 0])
        m_posi = np.matrix([0, 0, 0, 0, 0, 0, 1])

    elif s_env == 19:
        v_max = 3
        h_max = 9
        x_agent = 4
        y_agent = 1
        m_nega = np.matrix([0, 0, 0, 0, 0, 0, 0])
        m_posi = np.matrix([1, 0, 0, 0, 0, 0, 1])

    elif s_env == 20:
        v_max = 5
        h_max = 5
        x_agent = 2
        y_agent = 2
        m_nega = np.matrix([[0, 0, 0],
                            [0, 0, 0],
                            [0, 0, 0]])
        m_posi = np.matrix([[1, 0, 1],
                            [0, 0, 0],
                            [0, 1, 0]])

    elif s_env == 21:
        v_max = 5
        h_max = 5
        x_agent = 2
        y_agent = 2
        m_nega = np.matrix([[0, 1, 0],
                            [0, 0, 0],
                            [1, 0, 1]])
        m_posi = np.matrix([[1, 0, 1],
                            [0, 0, 0],
                            [0, 1, 0]])

    elif s_env == 22:
        v_max = 5
        h_max = 5
        x_agent = 2
        y_agent = 2
        m_nega = np.matrix([[0, 0, 0],
                            [0, 0, 0],
                            [0, 0, 0]])
        m_posi = np.matrix([[1, 0, 1],
                            [0, 0, 0],
                            [1, 0, 1]])

    if s_env == 31:
        v_max = 5
        h_max = 5
        x_agent = 1
        y_agent = 2
        m_nega = np.matrix([[0, 0, 0],
                            [0, 1, 0],
                            [0, 0, 0]])
        m_posi = np.matrix([[0, 1, 0],
                            [0, 0, 0],
                            [0, 0, 0]])

    elif s_env == 32:
        v_max = 5
        h_max = 5
        x_agent = 1
        y_agent = 2
        m_nega = np.matrix([[0, 0, 0],
                            [0, 0, 1],
                            [0, 0, 0]])
        m_posi = np.matrix([[0, 0, 1],
                            [0, 0, 0],
                            [0, 0, 0]])

    elif s_env == 33:
        v_max = 5
        h_max = 5
        x_agent = 1
        y_agent = 2
        m_nega = np.matrix([[1, 0, 0],
                            [0, 0, 0],
                            [0, 0, 0]])
        m_posi = np.matrix([[0, 1, 0],
                            [0, 0, 0],
                            [0, 0, 0]])

    else:
        pass

    "INSTANCE THE wall_list"
    wall_list = []
    for y in range(v_max):
        for x in range(h_max):
            if y == v_max - 1 or y == 0 or x == h_max - 1 or x == 0:
                wall = Class.Wall('wall', x, y)
                wall_list.append(wall)
    "INSTANCE THE AGENT"
    agent = Class.Agent('agent', x_agent, y_agent)

    "INSTANCE POSITIVE OBJECTS"
    positivo_list = []
    for x in range(m_posi.shape[0]):
        for y in range(m_posi.shape[1]):
            if m_posi[x, y] == 1:
                positivo = Class.Positivo('positivo', y + 1, x + 1)
                positivo_list.append(positivo)

    "INSTANCE NEGATIVE OBJECTS"
    negativo_list = []
    for x in range(m_nega.shape[0]):
        for y in range(m_nega.shape[1]):
            if m_nega[x, y] == 1:
                negativo = Class.Negativo('negativo', y + 1, x + 1)
                negativo_list.append(negativo)

    return negativo_list, positivo_list, agent, wall_list, h_max, v_max
# endregion

# region SAVE - LOAD - CREATE
def save_model(model, path):
    model.save_weights(path + ".h5", overwrite=True)
    with open(path + ".json", "w") as outfile:
        json.dump(model.to_json(), outfile)

def load_model(s_alg, path):
    optimizer_config = []
    print(path)
    if s_alg == "QL":
        path = path + ".xlsx"
        model = pd.read_excel(path, sheetname="model")

    elif s_alg == "DSRL":
        path = path + ".xlsx"
        model = pd.read_excel(path, sheetname="model", header=[0], index_col=[0,1])

    elif s_alg == "DSRL_dist":
        path = path + ".xlsx"
        model = pd.read_excel(path, sheetname="model", header=[0], index_col=[0,1])

    elif s_alg == "DSRL_dist_type":
        path = path + ".xlsx"
        model = pd.read_excel(path, sheetname="model", header=[0], index_col=[0,1])

    elif s_alg == "DSRL_dist_type_near":
        path = path + ".xlsx"
        model = pd.read_excel(path, sheetname="model", header=[0], index_col=[0,1])

    elif s_alg == "DSRL_dist_type_near_propNeg":
        path = path + ".xlsx"
        model = pd.read_excel(path, sheetname="model", header=[0], index_col=[0,1])

    elif s_alg == "DSRL_object_near":
        path = path + ".xlsx"
        model = pd.read_excel(path, sheetname="model", header=[0], index_col=[0,1])

    elif s_alg == "DSRL_object":
        path = path + ".xlsx"
        model = pd.read_excel(path, sheetname="model", header=[0], index_col=[0, 1])

    elif s_alg == "DQN":
        with open(path + ".json", "r") as jfile:
            model = model_from_json(json.load(jfile))
        model.load_weights(path + ".h5")
        conf = pd.read_excel(path + ".xlsx", sheetname="Run_Conf", header=[0])
        # net_conf = conf.loc[[16:20],:]
        # print("net_conf", net_conf)
        optimizer = conf.loc[19, "A"]
        print("op_conf ", optimizer)
        # pd.Series({'N_actions': net_conf["N_actions"]}),
        # pd.Series({'Max_memory': net_conf["Max_memory"]}),
        # pd.Series({'Hidden_size': net_conf["Hidden_size"]}),
        # pd.Series({'Batch_size': net_conf["Batch_size"]}),
        # pd.Series({'Optimizer': net_conf["Optimizer"]}),
        # pd.Series({'lr': op_conf[0]}),
        # pd.Series({'beta_1': op_conf[1]}),
        # pd.Series({'beta_2': op_conf[2]}),
        # pd.Series({'epsilon': op_conf[3]}),
        # pd.Series({'decay': op_conf[4]}),
        # pd.Series({'rho': op_conf[5]})

        use_optimizer, optimizer_config = define_optimizer(optimizer)
        model.compile(loss='mse', optimizer=use_optimizer)
        model.summary()
        # pass
    return model, optimizer_config

def create_model(s_alg, state_shape, net_conf):
    optimizer_config = []
    if s_alg == "QL":
        model = pd.DataFrame()
        model.index.name = ["States", "Action"]

    elif s_alg == "DSRL" or s_alg == "DSRL_dist" or s_alg == "DSRL_dist_type" or s_alg == "DSRL_dist_type_near" or s_alg == "DSRL_dist_type_near_propNeg" or s_alg == "DSRL_object_near" or s_alg == "DSRL_object":
        m_index = pd.MultiIndex(levels=[[''], [""]],
                                labels=[[], []],
                                names=['state', 'actions'])
        model = pd.DataFrame(index=m_index)

    elif s_alg == "DQN":
        model = Sequential()
        pop(model)
        model = Sequential()
        model.add(Dense(net_conf["Hidden_size"],
                        input_dim=state_shape[0]*state_shape[1],
                        activation="relu",
                        name="DENSE_1"))

        model.add(Dense(net_conf["Hidden_size"],
                        activation='relu',
                        name="DENSE_2"))

        model.add(Dense(net_conf["N_actions"],
                        name="DENSE_3"))

        use_optimizer, optimizer_config = define_optimizer(net_conf["Optimizer"])
        model.compile(loss='mse', optimizer=use_optimizer)
        print(model.summary())
        # plot_model(model, to_file='model.png')
        # d3v.d3viz(model.get_output(), 'test.html')
    return model, optimizer_config

# endregion


# ------ RL algorithms (till line 1030) ------

# region DQN - CONFIGURATIONS
class ExperienceReplay(object):
    """
    During gameplay all the experiences < s, a, r, s’ > are stored in a replay memory.
    In training, batches of randomly drawn experiences are used to generate the input and target for training.
    """

    def __init__(self, max_memory=100, discount=.9):
        """
        Setup
        max_memory: the maximum number of experiences we want to store
        memory: a list of experiences
        discount: the discount factor for future experience

        In the memory the information whether the game ended at the state is stored seperately in a nested array
        [...
        [experience, game_over]
        [experience, game_over]
        ...]
        """
        self.max_memory = max_memory
        self.memory = list()
        self.discount = discount

    def remember(self, states, game_over):
        # Save a state to memory
        self.memory.append([states, game_over])
        # We don't want to store infinite memories, so if we have too many, we just delete the oldest one
        if len(self.memory) > self.max_memory:
            del self.memory[0]

        # print(">>> states:", states)

    def get_batch(self, model, batch_size=10):
        # How many experiences do we have?
        len_memory = len(self.memory)

        # Calculate the number of actions that can possibly be taken in the game
        num_actions = model.output_shape[-1]

        # Dimensions of the game field
        env_dim = self.memory[0][0][0].shape[1]

        # We want to return an input and target vector with inputs from an observed state...
        inputs = np.zeros((min(len_memory, batch_size), env_dim))

        # ...and the target r + gamma * max Q(s’,a’)
        # Note that our target is a matrix, with possible fields not only for the action taken but also for the other possible actions.
        # The actions not take the same value as the prediction to not affect them
        targets = np.zeros((inputs.shape[0], num_actions))

        # We draw states to learn from randomly
        for i, idx in enumerate(np.random.randint(0, len_memory, size=inputs.shape[0])):
            """
            Here we load one transition <s, a, r, s’> from memory
            state_t: initial state s
            action_t: action taken a
            reward_t: reward earned r
            state_tp1: the state that followed s’
            """
            state_t, action_t, reward_t, state_tp1 = self.memory[idx][0]
            # We also need to know whether the game ended at this state
            game_over = self.memory[idx][1]

            inputs[i:i + 1] = state_t

            # First we fill the target values with the predictions of the model.
            # They will not be affected by training (since the training loss for them is 0)
            targets[i] = model.predict(state_t)[0]
            # print("targets\n", targets)
            # print("action_t", action_t)
            """
            If the game ended, the expected reward Q(s,a) should be the final reward r.
            Otherwise the target value is r + gamma * max Q(s’,a’)
            """
            #  Here Q_sa is max_a'Q(s', a')
            Q_sa = np.max(model.predict(state_tp1)[0])

            # if the game ended, the reward is the final reward
            if game_over:  # if game_over is True
                targets[i, action_t] = reward_t
            else:
                # r + gamma * max Q(s’,a’)
                targets[i, action_t] = reward_t + self.discount * Q_sa
        return inputs, targets

def define_optimizer(s_optimizer):
    lr = 0
    beta_1 = 0
    beta_2 = 0
    epsilon = 0
    decay = 0
    rho = 0
    if s_optimizer == "adam":
        lr = 0.001  # 0.001
        beta_1 = 0.9  # 0.9
        beta_2 = 0.999  # 0.999
        epsilon = 1e-08  # 1e-08
        decay = 0.0  # 0.0
        optimizer_selected = keras.optimizers.Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, decay=decay)
    elif s_optimizer == "rms_opt":
        lr = 0.001  # 0.001
        rho = 0.9  # 0.9
        epsilon = 1e-08  # e-08
        decay = 0.0  # 0.0
        optimizer_selected = keras.optimizers.RMSprop(lr=lr, rho=rho, epsilon=epsilon, decay=decay)
    optimizer_config = [lr, beta_1, beta_2, epsilon, decay, rho]
    return optimizer_selected, optimizer_config
#

def choose_action(s_alg, state, agent_pos, model, s_prob,step):
    # print("\nPREVIOUS MODEL - CHOOSE ACTION\n", model)
    zero = False
    if s_alg == "QL":
        state[agent_pos[1]][agent_pos[0]] = 120
        s = str(state)
        if s not in model.index:
            indices = [np.array([s, s, s, s]), np.array(['up', 'down', 'right', 'left'])]
            df_zero = pd.DataFrame(np.zeros([4, 1]), index=indices)
            model = model.append(df_zero)
            model = model.fillna(0)
        n_action = np.argmax(model.loc[s][0])  # Choose the max argument
        if max(model.loc[s][0]) == 0: zero = True

    elif s_alg == "DSRL" or s_alg == "DSRL_dist" or s_alg == "DSRL_dist_type" or s_alg == "DSRL_dist_type_near" or s_alg == "DSRL_dist_type_near_propNeg" or s_alg == "DSRL_object_near" or s_alg == "DSRL_object":
        a_v_list = []
        d = {}
        obj_list = create_obj_list(state)
        rel_list = relation_obj_list(obj_list, agent_pos)
        new_state = rel_list

        for obj in new_state: # FOR ALL OBJECTS SEEN
            tp_n_c = str(obj.tp) # GET THE TYPE FROM THE NEW STATE
            s_n_c = str(obj.loc) # GET THE LOCATION FROM THE NEW STATE
            if tp_n_c not in model.columns:
                # print("tp_n_c not in model.columns", tp_n_c)
                model[tp_n_c] = 0
            if s_n_c not in model.index:
                # print("s_n_c not in model.index", s_n_c)
                m_index = pd.MultiIndex(levels=[[s_n_c], actions],
                                        labels=[[0, 0, 0, 0], [0, 1, 2, 3]],
                                        names=['state', 'actions'])
                df_zero = pd.DataFrame(index=m_index)
                model = model.append(df_zero)
                model = model.fillna(0)
            Qts_a = model[tp_n_c].loc[s_n_c]
            # print("Qts_a - ", Qts_a)
            global explore_dict
            if s_alg == "DSRL_dist_type_near" or s_alg == "DSRL_dist_type_near_propNeg" or s_alg == "DSRL_object_near": # Calculate the distance
                s_n_c_abs = [int(s) for s in s_n_c if s.isdigit()]  # s_n_c_abs = state_new_absolute_distance
                distance = np.sqrt(s_n_c_abs[0]**2 + s_n_c_abs[1]**2)
                # print("distance",distance)
                Qts_a = Qts_a.divide(distance*distance, axis=0)
            a_v = []
            for action, value in  Qts_a.items():
                pos_x = agent_pos[0]
                pos_y = agent_pos[1]
                if action == 'up':
                    pos_y-=1
                elif  action =="down":
                    pos_y+=1
                elif action =="right":
                    pos_x +=1
                else:
                    pos_x -=1
                if (pos_x, pos_y) in explore_dict:
                    a_v.append((action, value-0.1*explore_dict[(pos_x, pos_y)]))
                else:
                    a_v.append((action, value))
                
            # a_v = [(value, key) for value, key in Qts_a.items()]
            # print("Qts_a - NEW", Qts_a)
            a_v_list.append(a_v) # Append Q-value
        # print(a_v_list)

        # Sum the values of all Qs into a single Q
        for element in a_v_list:
            for a in element:
                act = a[0] # Action
                val = a[1] # Value
                d[act] = d.get(act, 0) + val # Sum values for each Q

        # print('a_v_list: (List of the action values for each object in the scene): ')
        # print('{0}'.format(a_v_list))
        # print('\nd: (The sum of all object`s action values )')
        # pp.pprint(d)

        if d != {}: # BE CAREFUL THIS IS A DICT (argmax does not work as usual)
            inverse = [(value, key) for key, value in d.items()] # CALCULATE ALL KEYS
            n_action = max(inverse)[1] # Choose the max argument

            if max(d.values()) == 0: zero = True
        else:
            # n_action = "down"
            n_action = random.choice(actions)

    elif s_alg == "DQN":
        state[agent_pos[1]][agent_pos[0]] = 120
        state = state.reshape((1, -1))
        q = model.predict(state)
        n_act = np.argmax(q[0])
        n_action = actions[n_act]
        if max(q[0]) == 0: zero = True

    x = random.random()  # E greedy exploration
    # if x < s_prob:
    if step < 5 or x < s_prob:
        n_action = random.choice(actions)
        print_action = 'Random Act (Prob):'
    elif zero == True:
        # n_action = random.choice(actions)
        print_action = 'NOT Random Act (Zero):'
        pass
    else:
        print_action = 'Chosen Act:'
    # print("\nNEW MODEL - CHOOSE ACTION\n", model)
    # explore_set.add(tuple(agent_pos))

    return n_action, model, print_action

alfa = 1 # Learning Rate
gamma = 0.9 # Temporal Discount Factor
def learn(s_alg, model, state_t, state_t1, agent_t_pos, agent_t1_pos, reward, action_t, end_game, net_conf, exp_replay):
    # print("\nPREVIOUS MODEL - LEARN\n", model)
    batch_loss = 0
    if s_alg == "QL":
        state_t[agent_t_pos[1]][agent_t_pos[0]] = 120
        state_t1[agent_t1_pos[1]][agent_t1_pos[0]] = 120
        s_t = str(state_t)
        s_t1 = str(state_t1)
        if s_t1 not in model.index:
            indices = [np.array([s_t1, s_t1, s_t1, s_t1]), np.array(['up', 'down', 'right', 'left'])]
            df_zero = pd.DataFrame(np.zeros([4, 1]), index=indices)
            model = model.append(df_zero)
        if s_t not in model.index:
            indices = [np.array([s_t, s_t, s_t, s_t]), np.array(['up', 'down', 'right', 'left'])]
            df_zero = pd.DataFrame(np.zeros([4, 1]), index=indices)
            model = model.append(df_zero)
        model = model.fillna(0)

        if end_game == False:
            max_value = max(model.loc[s_t1][0])  # max(df.loc[new_state][0])
            Q_value = model.loc[s_t, action_t][0]
            updated_model = Q_value + alfa * (reward + (gamma * (max_value)) - Q_value)
        else:
            updated_model = reward
        model.loc[s_t, action_t] = updated_model

    elif s_alg == "DSRL" or s_alg == "DSRL_dist" or s_alg == "DSRL_dist_type" or s_alg == "DSRL_dist_type_near" or s_alg == "DSRL_dist_type_near_propNeg" or s_alg == "DSRL_object_near" or s_alg == "DSRL_object":
        max_value = 0

        obj_list = create_obj_list(state_t)
        rel_list = relation_obj_list(obj_list, agent_t_pos)
        old_state = rel_list

        obj_list = create_obj_list(state_t1)
        rel_list = relation_obj_list(obj_list, agent_t1_pos)
        new_state = rel_list

        for i in range(len(old_state)):
            # Check all items in old state
            obj_prev = old_state[i]
            tp_prev = str(obj_prev.tp)
            s_prev = str(obj_prev.loc)
            # Check all items in new state
            obj_new = new_state[i]
            tp_new = str(obj_new.tp)
            s_new = str(obj_new.loc)

            if tp_new not in model.columns: # If type is new, then add type
                model[tp_new] = 0
            if s_new not in model.index: # If state is new, then add state
                m_index = pd.MultiIndex(levels=[[s_new], actions],
                                        labels=[[0, 0, 0, 0], [0, 1, 2, 3]],
                                        names=['state', 'actions'])
                df_zero = pd.DataFrame(index=m_index)
                model = model.append(df_zero)
                model = model.fillna(0)

            max_value = max(model[tp_new].loc[s_new])
            if s_alg == "DSRL": # THEY STILL HAVE THE PROBLEM OF NOT PROPAGATING THE NEGATIVE SIGNAL
                if end_game == False:
                    Q_v = model[tp_prev].loc[s_prev, action_t]
                    model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value) - Q_v)
                else:
                    model[tp_prev].loc[s_prev, action_t] = reward

            elif s_alg == "DSRL_dist": # THEY STILL HAVE THE PROBLEM OF NOT PROPAGATING THE NEGATIVE SIGNAL
                if reward != 0:
                    s_p_c = [int(s) for s in s_prev if s.isdigit()]
                    if s_p_c[0] < 2 and s_p_c[1] < 2:
                        # EDITIONG DELETE
                        if end_game == False:
                            Q_v = model[tp_prev].loc[s_prev, action_t]
                            model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value) - Q_v)
                        else:
                            model[tp_prev].loc[s_prev, action_t] = reward
                else:
                    if end_game == False:
                        Q_v = model[tp_prev].loc[s_prev, action_t]
                        model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value) - Q_v)
                    else:
                        model[tp_prev].loc[s_prev, action_t] = reward

            elif s_alg == "DSRL_dist_type" or s_alg == "DSRL_dist_type_near": # THEY STILL HAVE THE PROBLEM OF NOT PROPAGATING THE NEGATIVE SIGNAL
                max_value_positive = max(model[tp_new].loc[s_new])
                if reward != 0:
                    s_p_c = [int(s) for s in s_prev if s.isdigit()]  # s_p_c = state_previous_absolute_distance
                    if s_p_c[0] < 2 and s_p_c[1] < 2: # IF IT IS CLOSE BY, THEN UPDATE ONLY THE CLOSE ONE:
                        if reward < 0 and tp_new == "180": # IF REWARD IS NEGATIVE and NEW OBJECT IS NEGATIVE UPDATE ONLY NEGATIVE TYPE:
                            if end_game == False:
                                Q_v = model[tp_prev].loc[s_prev, action_t]
                                model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value_positive) - Q_v)
                            else:
                                model[tp_prev].loc[s_prev, action_t] = reward
                        elif reward > 0 and tp_new == "60":  # IF REWARD IS POSITIVE and NEW OBJECT IS POSITIVE UPDATE ONLY POSITIVE TYPE:
                            if end_game == False:
                                Q_v = model[tp_prev].loc[s_prev, action_t]
                                model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value_positive) - Q_v)
                            else:
                                model[tp_prev].loc[s_prev, action_t] = reward
                # IF reward is zero
                else:
                    if end_game == False:
                        Q_v = model[tp_prev].loc[s_prev, action_t]
                        if tp_prev == "180": # IF THE PREVIOUS OBJECT WAS NEGATIVE
                            model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value_positive) - Q_v)
                        elif tp_prev == "60": # IF THE PREVIOUS OBJECT WAS POSITIVE
                            model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value_positive) - Q_v)
                    else:
                        model[tp_prev].loc[s_prev, action_t] = reward

            elif s_alg == "DSRL_dist_type_near_propNeg": # I try to solve this with max and min, but it did not work very well(THEY STILL HAVE THE PROBLEM OF NOT PROPAGATING THE NEGATIVE SIGNAL)
                max_value_positive = max(model[tp_new].loc[s_new])
                min_value_negative = min(model[tp_new].loc[s_new])
                if reward != 0:
                    s_p_c = [int(s) for s in s_prev if s.isdigit()]  # s_p_c = state_previous_absolute_distance
                    if s_p_c[0] < 2 and s_p_c[1] < 2: # IF IT IS CLOSE BY, THEN UPDATE ONLY THE CLOSE ONE:
                        if reward < 0 and tp_new == "180": # IF REWARD IS NEGATIVE and NEW OBJECT IS NEGATIVE UPDATE ONLY NEGATIVE TYPE:
                            if end_game == False:
                                Q_v = model[tp_prev].loc[s_prev, action_t]
                                model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * min_value_negative) - Q_v)
                            else:
                                model[tp_prev].loc[s_prev, action_t] = reward
                        elif reward > 0 and tp_new == "60":  # IF REWARD IS POSITIVE and NEW OBJECT IS POSITIVE UPDATE ONLY POSITIVE TYPE:
                            if end_game == False:
                                Q_v = model[tp_prev].loc[s_prev, action_t]
                                model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value_positive) - Q_v)
                            else:
                                model[tp_prev].loc[s_prev, action_t] = reward
                # IF reward is zero
                else:
                    if end_game == False:
                        Q_v = model[tp_prev].loc[s_prev, action_t]
                        if tp_prev == "180": # IF THE PREVIOUS OBJECT WAS NEGATIVE
                            model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * min_value_negative) - Q_v)
                        elif tp_prev == "60": # IF THE PREVIOUS OBJECT WAS POSITIVE
                            model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value_positive) - Q_v)
                    else:
                        model[tp_prev].loc[s_prev, action_t] = reward

            elif s_alg == "DSRL_object_near" or s_alg == "DSRL_object":
                max_value_positive = max(model[tp_new].loc[s_new])

                # Find the object that the agent interacted with:
                # This means that the agents has to know that the object which interacted with
                # After finding it, he has to assign the value to that object.
                # This means that I have to find the type and the state of this object that has now x=zero y=zero

                # print("obj_new.loc[0]\n", obj_new.loc[0])
                # print("obj_new.loc[1]\n", obj_new.loc[1])
                # print("action_t\n", action_t)
                # print("s_prev\n", s_prev)

                if obj_new.loc[0] == 0 and obj_new.loc[1] == 0:
                    tp_to_update = tp_new
                    # print("tp_new\n", tp_new)
                    if action_t == "up":
                        s_prev_to_update = str((0,1))
                    elif action_t == "down":
                        s_prev_to_update = str((0,-1))
                    elif action_t == "right":
                        s_prev_to_update = str((-1,0))
                    elif action_t == "left":
                        s_prev_to_update = str((1,0))
                    # print("s_prev_to_update\n", s_prev_to_update)
                    if end_game == False:
                        Q_v = model[tp_to_update].loc[s_prev_to_update, action_t]
                        model[tp_to_update].loc[s_prev_to_update, action_t] = Q_v + alfa * (reward + (gamma * max_value_positive) - Q_v)
                    else:
                        model[tp_to_update].loc[s_prev_to_update, action_t] = reward

                if reward == 0:
                    if end_game == False:
                        Q_v = model[tp_prev].loc[s_prev, action_t]
                        model[tp_prev].loc[s_prev, action_t] = Q_v + alfa * (reward + (gamma * max_value_positive) - Q_v)
                    else:
                        model[tp_prev].loc[s_prev, action_t] = reward

    elif s_alg == "DQN":
        state_t[agent_t_pos[1]][agent_t_pos[0]] = 120
        state_t1[agent_t1_pos[1]][agent_t1_pos[0]] = 120
        state_t = state_t.reshape((1, -1))
        state_t1 = state_t1.reshape((1, -1))
        action_t = actions_dict[action_t]
        exp_replay.remember([state_t, action_t, reward, state_t1], end_game) # [old_state, old_action, reward, new_state]
        inputs, targets = exp_replay.get_batch(model, batch_size=net_conf["Batch_size"])
        batch_loss = model.train_on_batch(inputs, targets)

    # print("\nNEW MODEL - LEARN\n", model)
    return model, batch_loss, exp_replay

''' PROGRAM START '''
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
def run(s_env, s_alg, s_learn, s_load, s_print, s_auto, s_episode, s_cond_to_end, s_server, s_net_comb_param, s_load_path, s_prob, s_sample, s_save):
    net_conf = {"N_actions": n_actions,
                "Max_memory": max_memory_list[s_net_comb_param],
                "Hidden_size": hidden_size_list[s_net_comb_param],
                "Batch_size": batch_size_list[s_net_comb_param],
                "Optimizer": optimizer_list[0]}
    exp_replay = ExperienceReplay(max_memory=net_conf["Max_memory"])
    begin = time.time()
    begin_time = time.strftime('%X %x')
    print("\n\n --- BEGINING --- s_sample: %s \n begin_time: %s \n" % (s_sample, begin_time))

    df_score = pd.DataFrame()
    df_percent_list = pd.DataFrame()
    df_loss_list = pd.DataFrame()
    df_time_sample = pd.DataFrame()
    avg_last_score_list = []

    if s_server == False: screen = pygame.display.set_mode((400 + 37 * 5, 330 + 37 * 5))

    score_list_best = [0]
    for sample in list(range(1, s_sample+1)):
        experiment_configurations = (sample, s_env, s_alg, s_episode, s_learn, s_load, s_print, s_auto, s_cond_to_end, s_server, s_net_comb_param, s_prob)
        print("\n - START - "
              "\n sample: %s"
              "\n s_env: %s"
              "\n s_alg: %s"
              "\n s_episode: %s"
              "\n s_learn: %s"
              "\n s_load: %s"
              "\n s_print: %s"
              "\n s_auto: %s"
              "\n s_cond_to_end: %s"
              "\n s_server: %s"
              "\n s_net_comb_param: %s"
              "\n s_prob: %s" % experiment_configurations)

        start = time.time()
        start_time = time.strftime('%X %x')
        print("\nStart time: ", start_time)
        negativo_list, positivo_list, agent, wall_list, h_max, v_max = environment_conf(s_env)

        env_dim = [h_max, v_max]
        # load file for transfer learning
        if s_load == True:
            try:
                model, op_conf = load_model(s_alg, __location__ + s_load_path)
            except Exception as e:
                print("DID NOT FIND THE FILE", __location__ + s_load_path, str(e))
        else:
            model, op_conf = create_model(s_alg, env_dim, net_conf)

        # region INITIALIZE VARIABLES 1
        percent_list = []
        score = 0
        score_list = []
        episodes = 0
        episodes_list = []
        steps = 0
        steps_list = []
        batch_loss = 0
        loss_list = []
        # endregion

        # main component to an episode
        while (episodes < s_episode):  # max_episodes
            negativo_list, positivo_list, agent, wall_list, h_max, v_max = environment_conf(s_env)
            # region INITIALIZE VARIABLES 2
            episodes += 1
            episodes_list.append(episodes)
            max_steps = 100
            steps_list.append(steps)
            steps = 0
            act_list = []
            last_move = False
            action_chosen = ""
            encountered = 0
            pos_collected = 0
            prob = s_prob
            # endregion

            if s_server == False:
                # region DRAW SCREEN
                screen.fill(white)
                show_Alg(s_alg, screen)
                show_Samples(sample, screen)
                show_Level(episodes, screen)
                show_Score(score, screen)
                show_Steps(steps, screen)
                show_Percent(percent_list[-10:], screen)
                show_Steps_list(steps_list[-30:], screen)
                show_Act_List(act_list[-20:], screen)
                show_Action(action_chosen, screen)
                show_Env(s_env, screen)
                draw_objects(agent, positivo_list, negativo_list, wall_list, screen)
                pygame.display.flip()
                # endregion

            # main reinforcement learning part
            while (True):  # max_steps or condition to finish
                sleep(speed)
                ''' EVENT HANDLE '''
                key_pressed = False
                set_action = False
                while (s_server == False):
                    for event in pygame.event.get():
                        # QUIT GAME
                        if event.type == pygame.QUIT:
                            pygame.quit()
                            sys.exit()
                        # ADD OR DELETE WALL
                        if event.type == pygame.MOUSEBUTTONDOWN:
                            pass
                            # if (pygame.mouse.get_pressed() == (1, 0, 0)):  # LEFT BUTTON (add wall)
                            #     pos = pygame.mouse.get_pos()
                            #     x = (pos[0] - x_g) / (m + w)
                            #     y = (pos[1] - y_g) / (m + h)
                            #     x = math.trunc(x)
                            #     y = math.trunc(y)
                            #     w_has = False
                            #     for item in wall_list:
                            #         if math.trunc((item[0] - x_g) / (m + w)) == x and math.trunc(
                            #                         (item[1] - y_g) / (m + h)) == y:
                            #             w_has = True
                            #     if w_has == False:
                            #         wall = Class.Wall('wall', x, y)
                            #         print('wall ', wall, 'added')
                            #         wall_list.append(wall)

                            # if (pygame.mouse.get_pressed() == (0, 0, 1)):  # RIGHTBUTTON (delete wall)
                            #     pos = pygame.mouse.get_pos()
                            #     x = (pos[0] - x_g) / (m + w)
                            #     y = (pos[1] - y_g) / (m + h)
                            #     x = math.trunc(x)
                            #     y = math.trunc(y)
                            #     wall = Class.Wall('wall', x, y)
                            #     for i in wall_list:
                            #         if i == wall:
                            #             wall_list.remove(wall)
                            #             print('wall ', wall, 'removed')

                            # EVENT - ANY PRESSED KEY
                        # PRESS A KEY
                        if event.type == pygame.KEYDOWN:
                            # SAVE AND QUIT - KEY P
                            if event.key == pygame.K_p:
                                pygame.quit()
                                sys.exit()
                            # PLOT AGENT`S PERFORMENCE - KEY G
                            if event.key == pygame.K_g:
                                plt.plot(score_list)
                                plt.ylabel('Score')
                                plt.xlabel('Total Steps')
                                plt.title('Performance of the Agent')
                                plt.show()

                                plt.plot(percent_list)
                                plt.ylabel('Percentage of objects +')
                                plt.xlabel('Total Steps')
                                plt.title('Episode over 100 times step each')
                                plt.show()
                                if s_alg == "DQN":
                                    plt.plot(loss_list)
                                    plt.ylabel('loss')
                                    plt.xlabel('Total Steps')
                                    plt.title('batch_loss')
                                    plt.show()
                            # MOVE - SPACE BAR
                            if event.key == pygame.K_SPACE:
                                key_pressed = True
                                break
                            # MOVE - ARROW KEYS
                            if event.key in p_keys:
                                key_pressed = True
                                set_action = True
                                if event.key == pygame.K_w:  # North # add_act('↑') ⇦ ⇨ ⇧ ⇩
                                    key_action = "up"
                                if event.key == pygame.K_s:  # South # add_act('↓') ⬅  ➡  ⬆ ⬇
                                    key_action = "down"
                                if event.key == pygame.K_d:  # West # add_act('→')
                                    key_action = "right"
                                if event.key == pygame.K_a:  # East # add_act('←')
                                    key_action = "left"
                                break
                    # Run game if key is preseed or automatic is selected
                    if key_pressed or s_auto:
                        break
                # BREAK IF IT WAS THE LAST MOVE
                if last_move == True:
                    break
                # RUN_GAME
                steps += 1
                ''' OLD STATE - S 1 - 1'''
                state_t = update_state(h_max, v_max, agent, positivo_list, negativo_list, wall_list)
                agent_t = agent.pos
                ''' CHOOSE ACTION - AGENT ACT - 2'''
                action_chosen, model, print_action = choose_action(s_alg, state_t, agent_t, model, prob,steps)
                

                if set_action: action_chosen = key_action

                ''' CHANGE THE WORLD - UP_ENV - 3'''
                agent.try_move(action_chosen, wall_list)
                act_list.append(action_chosen)
                # if s_print: print(print_action, action_chosen)

                ''' NEW STATE - S2 - 4'''
                state_t1 = update_state(h_max, v_max, agent, positivo_list, negativo_list, wall_list)
                agent_t1 = agent.pos
                global explore_set
                global explore_dict

                if s_print:
                    # print('\n>>>>   Level: ' + str(episodes) + ' |  Step: ' + str(
                    #     steps) + ' |  New_agent_pos: ' + str(agent.pos) + '  <<<<')
                    pos_tuple = tuple(agent.pos)
                    explore_set.add(pos_tuple)
                    if pos_tuple not in explore_dict:
                        explore_dict[pos_tuple] = 1
                    else:
                        explore_dict[pos_tuple] += 1
                if steps==max_steps:
                    print("Number of explore node: "+str(len(explore_set)))
                    print("Explored Node postion: "+str(explore_dict))
                    explore_set = set()
                    explore_dict = dict()

                ''' GET REWARD - 5 '''
                # region GET REWARD AND DELETE COLLECTED OBJECT
                prev_score = score
                score += step_reward

                for positivo in positivo_list:
                    if agent.pos == positivo.pos:
                        encountered += 1
                        pos_collected += 1
                        score += positive_reward
                        positivo = Class.Positivo('positivo', agent.pos[0], agent.pos[1])
                        positivo_list.remove(positivo)
                        # if s_print == True and s_server == False:
                            # print('                                 Hit the Positivo')
                for negativo in negativo_list:
                    if agent.pos == negativo.pos:
                        encountered += 1
                        score -= negative_reward
                        negativo = Class.Negativo('negativo', agent.pos[0], agent.pos[1])
                        negativo_list.remove(negativo)
                        # if s_print == True and s_server == False:
                            # print('                                 Hit the Negativo')

                new_score = score
                score_list.append(score)
                reward = new_score - prev_score
                # endregion

                ''' LEARN - 6 '''
                # CONDITION TO FINISH THE Episode
                if s_cond_to_end == 'max_steps':
                    if steps == max_steps:
                        last_move = True
                elif s_cond_to_end == 'coll_all' or steps > max_steps:
                    if len(positivo_list) == 0 and len(negativo_list) == 0 or steps > max_steps:
                        last_move = True
                elif s_cond_to_end == 'only_positive' or steps > max_steps:
                    if len(positivo_list) == 0 or steps > max_steps:
                        last_move = True
                elif s_cond_to_end == 'only_negative' or steps > max_steps:
                    if len(negativo_list) == 0 or steps > max_steps:
                        last_move = True

                

                # LEARN
                if s_learn == True:
                    action_t = action_chosen
                    if last_move == False:
                        ''' LEARN '''
                        model, batch_loss, exp_replay = learn(s_alg, model, state_t, state_t1, agent_t, agent_t1, reward, action_t, False, net_conf, exp_replay)
                    else:
                        ''' LEARN FINAL '''
                        model, batch_loss, exp_replay = learn(s_alg, model, state_t, state_t1, agent_t, agent_t1, reward, action_t, True, net_conf, exp_replay)

                if s_server == False:
                    # region DRAW SCREEN
                    screen.fill(white)
                    show_Alg(s_alg, screen)
                    show_Samples(sample, screen)
                    show_Level(episodes, screen)
                    show_Score(score, screen)
                    show_Steps(steps, screen)
                    show_Percent(percent_list[-10:], screen)
                    show_Steps_list(steps_list[-30:], screen)
                    show_Act_List(act_list[-20:], screen)
                    show_Action(action_chosen, screen)
                    show_Env(s_env, screen)

                    draw_objects(agent, positivo_list, negativo_list, wall_list, screen)
                    pygame.display.flip()
                    # endregion

            try:
                percent = pos_collected / encountered
            except ZeroDivisionError:
                percent = 0
            percent_list.append(percent)
            loss_list.append(batch_loss)
            print("Episode: ", episodes)

        # region TIME 1
        print("Start time: ", start_time)
        end = time.time()
        end_time = time.strftime('%X %x')
        print("End time: ", end_time)
        time_elapsed = end - start
        print("Time elapsed: ", time_elapsed)
        # endregion

        '''GET THE BEST MODEL'''
        if max(score_list) > max(score_list_best):
            best_model = model
            score_list_best = score_list

        # region MAKE LIST OF THE RESULTS
        avg_last_score_list.append(score_list[-1])

        score_list_df = pd.DataFrame({'Score': score_list})
        percent_list_df = pd.DataFrame({'Percent': percent_list})
        loss_list_df = pd.DataFrame({'Batch_loss': loss_list})
        time_sample_df = pd.DataFrame({'Time': [time_elapsed]})

        df_score = pd.concat([df_score, score_list_df], ignore_index=True, axis=1)
        df_percent_list = pd.concat([df_percent_list, percent_list_df], ignore_index=True, axis=1)
        df_loss_list = pd.concat([df_loss_list, loss_list_df], ignore_index=True, axis=1)
        df_time_sample = pd.concat([df_time_sample, time_sample_df], ignore_index=True, axis=1)
        # endregion

    if s_save == True:
        # region PATH TO SAVE
        save_path_core = __location__ + "/Results/"
        if s_learn == True: save_path = save_path_core + "Train/Env_" + str(s_env) + "/Train_Env_" + str(s_env) + "_" + s_alg
        else: save_path = save_path_core + "Test/Env_" + str(s_env) + "/Test_Env_" + str(s_env) + "_" + s_alg
        if s_alg == "DQN": save_path += "_" + str(s_net_comb_param)

        # convert begin_time to string and format it
        time_path = begin_time.replace(" ", "   ")
        time_path = time_path.replace(":", " ")
        time_path = time_path.replace("/", "-")
        # append to the save path
        save_path = save_path + "   " + time_path

        if s_load == True:
            load_path = " loaded_with " + s_load_path.replace("/", "_")
            save_path = save_path + load_path

        # If it doesnt find the path, then create a new path
        if not os.path.exists(os.path.dirname(save_path)):
            try:
                os.makedirs(os.path.dirname(save_path))
            except OSError as exc:  # Guard against race condition
                print("ERROR when saving the File")
        # endregion
        print("save_path: ", save_path)

        # region SAVE ALL
        # IF IT IS NOT DQN NULL NET CONF. VALUES
        if s_alg != "DQN":
            op_conf = [0, 0, 0, 0, 0, 0]
            net_conf = {"N_actions":0, "Max_memory":0, "Hidden_size":0, "Batch_size":0, "Optimizer":"none"}

        avg_last_score = np.average(avg_last_score_list)
        config_list = pd.concat([pd.Series({'Run_Conf': "A"}),
                                 pd.Series({'Env_conf': s_env}),
                                 pd.Series({'Algort': s_alg}),
                                 pd.Series({'Learn': s_learn}),
                                 pd.Series({'Load': s_load}),
                                 pd.Series({'Samples': s_sample}),
                                 pd.Series({'Episode': s_episode}),
                                 pd.Series({'Max_steps': max_steps}),
                                 pd.Series({'s_cond_to_end': s_cond_to_end}),
                                 pd.Series({'Auto': s_auto}),
                                 pd.Series({'Server': s_server}),
                                 pd.Series({'Print': s_print}),
                                 pd.Series({'MODEL CONF': ""}),
                                 pd.Series({'alfa': alfa}),
                                 pd.Series({'gamma': gamma}),
                                 pd.Series({'Prob': Prob}),
                                 pd.Series({'N_actions': net_conf["N_actions"]}),
                                 pd.Series({'Max_memory': net_conf["Max_memory"]}),
                                 pd.Series({'Hidden_size': net_conf["Hidden_size"]}),
                                 pd.Series({'Batch_size': net_conf["Batch_size"]}),
                                 pd.Series({'Optimizer': net_conf["Optimizer"]}),
                                 pd.Series({'lr': op_conf[0]}),
                                 pd.Series({'beta_1': op_conf[1]}),
                                 pd.Series({'beta_2': op_conf[2]}),
                                 pd.Series({'epsilon': op_conf[3]}),
                                 pd.Series({'decay': op_conf[4]}),
                                 pd.Series({'rho': op_conf[5]}),
                                 pd.Series({'': ""}),
                                 pd.Series({'AVG SCORE': avg_last_score})])
        config_list = config_list.to_frame()

        if s_print: print("\nconfig_list:\n", config_list)

        # Create a Pandas Excel writer using XlsxWriter as the engine.
        writer = pd.ExcelWriter(save_path + ".xlsx", engine='xlsxwriter')

        # SAVING CONFIG:
        config_list.to_excel(writer, sheet_name='Run_Conf', header=False)
        worksheet = writer.sheets['Run_Conf']
        worksheet.set_column('A:B', 15)
        # SAVING SCORE:
        df_score_mean = df_score.mean(axis=1)
        df_score.insert(0, "Avg " + str(s_sample), df_score_mean)
        df_score.to_excel(writer, sheet_name='Score')
        worksheet = writer.sheets['Score']
        worksheet.write(0, 0, "Score")
        # SAVING PERCENT:
        df_percent_list_mean = df_percent_list.mean(axis=1)
        df_percent_list.insert(0, "Avg " + str(s_sample), df_percent_list_mean)
        df_percent_list.to_excel(writer, sheet_name='Percent')
        worksheet = writer.sheets['Percent']
        worksheet.write(0, 0, "Percent")
        # SAVING LOSS:
        df_loss_list.to_excel(writer, sheet_name='Loss')
        worksheet = writer.sheets['Loss']
        worksheet.write(0, 0, "Loss")
        # SAVING TIME:
        df_time_sample.to_excel(writer, sheet_name='Time')
        worksheet = writer.sheets['Time']
        worksheet.write(0, 0, "Time")
        # region CELL SIZE

        # worksheet = writer.sheets['Score']
        # worksheet.set_column('A:B', 15)
        # worksheet = writer.sheets['Time']
        # worksheet.set_column('A:B', 15)
        # endregion

        # SAVING BEST MODEL (out of # Samples):
        if s_alg == "DSRL" or s_alg == "QL" or s_alg == "DSRL_dist" or s_alg == "DSRL_dist_type" or s_alg == "DSRL_dist_type_near" or s_alg == "DSRL_dist_type_near_propNeg" or s_alg == "DSRL_object_near" or s_alg == "DSRL_object":
            # SAVING MODEL CONFIGURATIONS:
            best_model.to_excel(writer, sheet_name='model')
            # CONDITIONAL COLOR
            worksheet = writer.sheets['model']
            for x in range(2, 700, 4):
                cell = "C" + str(x) + ":D" + str(x + 3)
                worksheet.conditional_format(cell, {'type': '3_color_scale'})
            # CELL SIZE
            worksheet = writer.sheets['model']
            worksheet.set_column('A:A', 50)

            # region ADD PLOTS
        # worksheet = writer.sheets['results']
        # workbook = writer.book
        # chart = workbook.add_chart({'type': 'line'})
        # chart2 = workbook.add_chart({'type': 'line'})
        # chart.add_series({'values': '=results!$B$2:$B$100'})
        # chart2.add_series({'values': '=results!$C$2:$C$10'})
        # worksheet.insert_chart('F3', chart)
        # worksheet.insert_chart('N3', chart2)

        # SAVE DQN MODEL
        if s_learn == True and s_alg == "DQN":
            save_model(best_model, save_path)

        writer.save()
        # endregion

    print("\n - END - "
          "\n sample: %s"
          "\n s_env: %s"
          "\n s_alg: %s"
          "\n s_episode: %s"
          "\n s_learn: %s"
          "\n s_load: %s"
          "\n s_print: %s"
          "\n s_auto: %s"
          "\n s_cond_to_end: %s"
          "\n s_server: %s"
          "\n s_net_comb_param: %s"
          "\n s_prob: %s" % experiment_configurations)

    # region TIME 2
    print("\n\nBegin time: ", begin_time)
    finish = time.time()
    finish_time = time.strftime('%X %x')
    print("Final time: ", finish_time)
    total_time = finish - begin
    print("Total time: ", total_time)
    # endregion

    return





# -------------------------------------------------------------------------------------------------- #
''' SELECT PARAMETERS TO RUN THE SOFTWARE '''
# environment configuration
Env = 11
Alg_list = ["QL",
            "DSRL",
            "DSRL_object_near",
            "DQN",
            "DSRL_dist",
            "DSRL_dist_type",
            "DSRL_dist_type_near",
            "DSRL_dist_type_near_propNeg",
            "DSRL_object"]
Alg = Alg_list[2] # Select the algorithm to be used
Learn = False # To update its knowledge
Load = True # To load a learned model
Load_path = "/Results/Train/Env_11/Train_Env_11_DSRL   02 41 20   05-05-21"

# algorithm configuration
Samples = 2 # Usually 10 samples (repeat 100 episodes for 10 times)
Print = True # Print some info in the terminal
Auto = True # Agent moves Automatic or if False it moves by pressing the Spacebar key
Server = False # If running in the server since

# change Prob to 1 for probe training??
Prob = 0.3 # Probability to make a random move (exploration rate)
Cond_to_end = "max_steps" # Choose from below (there are 4)
Save = False # Save the model
speed = 0.05 # seconds per frame

# Cond_to_end = "max_steps"
# Cond_to_end = "coll_all"
# Cond_to_end = "only_negative"
Episodes = 500 # Usually 1000 or 100

# region DQN Model Configurations:
# max_memory_list =  [5, 5,  5,   30,  30, 30,  100, 100, 100]
# hidden_size_list = [5, 30, 270, 5,   30, 270, 5,   30,  270]
# batch_size_list =  [1, 1,  1,   10,  10, 10,  32,  32,  32]
max_memory_list =  [100,    100,    100,    300, 300,   300,    900, 900, 900]
hidden_size_list = [5,      10,     15,     5,   10,    15,     5,   10,  15]
batch_size_list =  [32,     32,     32,     32,  32,    32,     32,  32,  32]
optimizer_list = ["adam", "rms_opt"]
n_actions = 4  # [move_up, move_down, move_left, move_right]
# endregion
Net_comb_param = 4


# ------------------------------------------------------------------------------------------- #
run(Env, Alg, Learn, Load, Print, Auto, Episodes, Cond_to_end, Server, Net_comb_param, Load_path, Prob, Samples, Save)
# ------------------------------------------------------------------------------------------- #

'''                 REPEAT DQN Net_Comb_Param                  '''
# for i in range(9):
#     Net_comb_param = i
#     run(Env, Alg, Learn, Load, Print, Auto, Episodes, Cond_to_end, Server, Net_comb_param, Load_path, Prob, Samples, Save)


'''                 REPEAT Alg for a list of Env                  '''
# env_list = [2,3]
# for Env in env_list:
#     run(Env, Alg, Learn, Load, Print, Auto, Episodes, Cond_to_end, Server, Net_comb_param, Load_path, Prob, Samples, Save)


'''                 Alg_list for Env_list                  '''
# env_list = [2,3]
# alg_list = ["QL", "DSRL", "DSRL_object_near", "DQN"]
# for Env in env_list:
#     for Alg in alg_list:
#         run(Env, Alg, Learn, Load, Print, Auto, Episodes, Cond_to_end, Server, Net_comb_param, Load_path, Prob, Samples, Save)



[((1761, 1803), 'pandas.set_option', 'pd.set_option', (['"""display.max_columns"""', 'None'], {}), "('display.max_columns', None)\n", (1774, 1803), True, 'import pandas as pd\n'), ((1805, 1848), 'pandas.set_option', 'pd.set_option', (['"""display.large_repr"""', '"""info"""'], {}), "('display.large_repr', 'info')\n", (1818, 1848), True, 'import pandas as pd\n'), ((1871, 1916), 'pandas.set_option', 'pd.set_option', (['"""display.width"""', 'desired_width'], {}), "('display.width', desired_width)\n", (1884, 1916), True, 'import pandas as pd\n'), ((1918, 1947), 'pandas.set_option', 'pd.set_option', (['"""precision"""', '(4)'], {}), "('precision', 4)\n", (1931, 1947), True, 'import pandas as pd\n'), ((1964, 1983), 'numpy.random.seed', 'np.random.seed', (['(123)'], {}), '(123)\n', (1978, 1983), True, 'import numpy as np\n'), ((2008, 2021), 'pygame.init', 'pygame.init', ([], {}), '()\n', (2019, 2021), False, 'import pygame\n'), ((2056, 2086), 'pprint.PrettyPrinter', 'pprint.PrettyPrinter', ([], {'indent': '(4)'}), '(indent=4)\n', (2076, 2086), False, 'import pprint\n'), ((2953, 2991), 'pygame.font.SysFont', 'pygame.font.SysFont', (['"""comicsansms"""', '(13)'], {}), "('comicsansms', 13)\n", (2972, 2991), False, 'import pygame\n'), ((3009, 3041), 'pygame.font.SysFont', 'pygame.font.SysFont', (['"""arial"""', '(13)'], {}), "('arial', 13)\n", (3028, 3041), False, 'import pygame\n'), ((3060, 3103), 'pygame.font.SysFont', 'pygame.font.SysFont', (['"""arial"""', '(18)'], {'bold': '(True)'}), "('arial', 18, bold=True)\n", (3079, 3103), False, 'import pygame\n'), ((3105, 3123), 'pygame.font.init', 'pygame.font.init', ([], {}), '()\n', (3121, 3123), False, 'import pygame\n'), ((16194, 16232), 'Class.Agent', 'Class.Agent', (['"""agent"""', 'x_agent', 'y_agent'], {}), "('agent', x_agent, y_agent)\n", (16205, 16232), False, 'import Class\n'), ((29918, 29933), 'random.random', 'random.random', ([], {}), '()\n', (29931, 29933), False, 'import random\n'), ((42499, 42510), 'time.time', 'time.time', ([], {}), '()\n', (42508, 42510), False, 'import time\n'), ((42529, 42551), 'time.strftime', 'time.strftime', (['"""%X %x"""'], {}), "('%X %x')\n", (42542, 42551), False, 'import time\n'), ((42665, 42679), 'pandas.DataFrame', 'pd.DataFrame', ([], {}), '()\n', (42677, 42679), True, 'import pandas as pd\n'), ((42703, 42717), 'pandas.DataFrame', 'pd.DataFrame', ([], {}), '()\n', (42715, 42717), True, 'import pandas as pd\n'), ((42738, 42752), 'pandas.DataFrame', 'pd.DataFrame', ([], {}), '()\n', (42750, 42752), True, 'import pandas as pd\n'), ((42775, 42789), 'pandas.DataFrame', 'pd.DataFrame', ([], {}), '()\n', (42787, 42789), True, 'import pandas as pd\n'), ((64818, 64829), 'time.time', 'time.time', ([], {}), '()\n', (64827, 64829), False, 'import time\n'), ((64849, 64871), 'time.strftime', 'time.strftime', (['"""%X %x"""'], {}), "('%X %x')\n", (64862, 64871), False, 'import time\n'), ((5270, 5288), 'Class.Obj', 'Class.Obj', (['tp', 'loc'], {}), '(tp, loc)\n', (5279, 5288), False, 'import Class\n'), ((5798, 5820), 'Class.Obj', 'Class.Obj', (['tp', 'loc_dif'], {}), '(tp, loc_dif)\n', (5807, 5820), False, 'import Class\n'), ((7538, 7571), 'numpy.matrix', 'np.matrix', (['[[0, 0, 0], [0, 1, 0]]'], {}), '([[0, 0, 0], [0, 1, 0]])\n', (7547, 7571), True, 'import numpy as np\n'), ((7619, 7652), 'numpy.matrix', 'np.matrix', (['[[0, 1, 0], [0, 0, 0]]'], {}), '([[0, 1, 0], [0, 0, 0]])\n', (7628, 7652), True, 'import numpy as np\n'), ((14939, 14983), 'numpy.matrix', 'np.matrix', (['[[0, 0, 0], [0, 1, 0], [0, 0, 0]]'], {}), '([[0, 0, 0], [0, 1, 0], [0, 0, 0]])\n', (14948, 14983), True, 'import numpy as np\n'), ((15060, 15104), 'numpy.matrix', 'np.matrix', (['[[0, 1, 0], [0, 0, 0], [0, 0, 0]]'], {}), '([[0, 1, 0], [0, 0, 0], [0, 0, 0]])\n', (15069, 15104), True, 'import numpy as np\n'), ((17260, 17298), 'pandas.read_excel', 'pd.read_excel', (['path'], {'sheetname': '"""model"""'}), "(path, sheetname='model')\n", (17273, 17298), True, 'import pandas as pd\n'), ((19704, 19718), 'pandas.DataFrame', 'pd.DataFrame', ([], {}), '()\n', (19716, 19718), True, 'import pandas as pd\n'), ((23153, 23193), 'numpy.zeros', 'np.zeros', (['(inputs.shape[0], num_actions)'], {}), '((inputs.shape[0], num_actions))\n', (23161, 23193), True, 'import numpy as np\n'), ((26110, 26136), 'numpy.argmax', 'np.argmax', (['model.loc[s][0]'], {}), '(model.loc[s][0])\n', (26119, 26136), True, 'import numpy as np\n'), ((30032, 30054), 'random.choice', 'random.choice', (['actions'], {}), '(actions)\n', (30045, 30054), False, 'import random\n'), ((41934, 41945), 'os.getcwd', 'os.getcwd', ([], {}), '()\n', (41943, 41945), False, 'import os\n'), ((41947, 41972), 'os.path.dirname', 'os.path.dirname', (['__file__'], {}), '(__file__)\n', (41962, 41972), False, 'import os\n'), ((42858, 42911), 'pygame.display.set_mode', 'pygame.display.set_mode', (['(400 + 37 * 5, 330 + 37 * 5)'], {}), '((400 + 37 * 5, 330 + 37 * 5))\n', (42881, 42911), False, 'import pygame\n'), ((43617, 43628), 'time.time', 'time.time', ([], {}), '()\n', (43626, 43628), False, 'import time\n'), ((43651, 43673), 'time.strftime', 'time.strftime', (['"""%X %x"""'], {}), "('%X %x')\n", (43664, 43673), False, 'import time\n'), ((56661, 56672), 'time.time', 'time.time', ([], {}), '()\n', (56670, 56672), False, 'import time\n'), ((56693, 56715), 'time.strftime', 'time.strftime', (['"""%X %x"""'], {}), "('%X %x')\n", (56706, 56715), False, 'import time\n'), ((57145, 57180), 'pandas.DataFrame', 'pd.DataFrame', (["{'Score': score_list}"], {}), "({'Score': score_list})\n", (57157, 57180), True, 'import pandas as pd\n'), ((57208, 57247), 'pandas.DataFrame', 'pd.DataFrame', (["{'Percent': percent_list}"], {}), "({'Percent': percent_list})\n", (57220, 57247), True, 'import pandas as pd\n'), ((57272, 57311), 'pandas.DataFrame', 'pd.DataFrame', (["{'Batch_loss': loss_list}"], {}), "({'Batch_loss': loss_list})\n", (57284, 57311), True, 'import pandas as pd\n'), ((57338, 57376), 'pandas.DataFrame', 'pd.DataFrame', (["{'Time': [time_elapsed]}"], {}), "({'Time': [time_elapsed]})\n", (57350, 57376), True, 'import pandas as pd\n'), ((57399, 57462), 'pandas.concat', 'pd.concat', (['[df_score, score_list_df]'], {'ignore_index': '(True)', 'axis': '(1)'}), '([df_score, score_list_df], ignore_index=True, axis=1)\n', (57408, 57462), True, 'import pandas as pd\n'), ((57490, 57562), 'pandas.concat', 'pd.concat', (['[df_percent_list, percent_list_df]'], {'ignore_index': '(True)', 'axis': '(1)'}), '([df_percent_list, percent_list_df], ignore_index=True, axis=1)\n', (57499, 57562), True, 'import pandas as pd\n'), ((57587, 57653), 'pandas.concat', 'pd.concat', (['[df_loss_list, loss_list_df]'], {'ignore_index': '(True)', 'axis': '(1)'}), '([df_loss_list, loss_list_df], ignore_index=True, axis=1)\n', (57596, 57653), True, 'import pandas as pd\n'), ((57680, 57750), 'pandas.concat', 'pd.concat', (['[df_time_sample, time_sample_df]'], {'ignore_index': '(True)', 'axis': '(1)'}), '([df_time_sample, time_sample_df], ignore_index=True, axis=1)\n', (57689, 57750), True, 'import pandas as pd\n'), ((59299, 59330), 'numpy.average', 'np.average', (['avg_last_score_list'], {}), '(avg_last_score_list)\n', (59309, 59330), True, 'import numpy as np\n'), ((61547, 61603), 'pandas.ExcelWriter', 'pd.ExcelWriter', (["(save_path + '.xlsx')"], {'engine': '"""xlsxwriter"""'}), "(save_path + '.xlsx', engine='xlsxwriter')\n", (61561, 61603), True, 'import pandas as pd\n'), ((6874, 6898), 'numpy.zeros', 'np.zeros', (['(v_max, h_max)'], {}), '((v_max, h_max))\n', (6882, 6898), True, 'import numpy as np\n'), ((7804, 7837), 'numpy.matrix', 'np.matrix', (['[[0, 0, 0], [0, 0, 1]]'], {}), '([[0, 0, 0], [0, 0, 1]])\n', (7813, 7837), True, 'import numpy as np\n'), ((7885, 7918), 'numpy.matrix', 'np.matrix', (['[[0, 0, 1], [0, 0, 0]]'], {}), '([[0, 0, 1], [0, 0, 0]])\n', (7894, 7918), True, 'import numpy as np\n'), ((15286, 15330), 'numpy.matrix', 'np.matrix', (['[[0, 0, 0], [0, 0, 1], [0, 0, 0]]'], {}), '([[0, 0, 0], [0, 0, 1], [0, 0, 0]])\n', (15295, 15330), True, 'import numpy as np\n'), ((15407, 15451), 'numpy.matrix', 'np.matrix', (['[[0, 0, 1], [0, 0, 0], [0, 0, 0]]'], {}), '([[0, 0, 1], [0, 0, 0], [0, 0, 0]])\n', (15416, 15451), True, 'import numpy as np\n'), ((17376, 17444), 'pandas.read_excel', 'pd.read_excel', (['path'], {'sheetname': '"""model"""', 'header': '[0]', 'index_col': '[0, 1]'}), "(path, sheetname='model', header=[0], index_col=[0, 1])\n", (17389, 17444), True, 'import pandas as pd\n'), ((20002, 20081), 'pandas.MultiIndex', 'pd.MultiIndex', ([], {'levels': "[[''], ['']]", 'labels': '[[], []]', 'names': "['state', 'actions']"}), "(levels=[[''], ['']], labels=[[], []], names=['state', 'actions'])\n", (20015, 20081), True, 'import pandas as pd\n'), ((20165, 20192), 'pandas.DataFrame', 'pd.DataFrame', ([], {'index': 'm_index'}), '(index=m_index)\n', (20177, 20192), True, 'import pandas as pd\n'), ((23278, 23332), 'numpy.random.randint', 'np.random.randint', (['(0)', 'len_memory'], {'size': 'inputs.shape[0]'}), '(0, len_memory, size=inputs.shape[0])\n', (23295, 23332), True, 'import numpy as np\n'), ((8070, 8103), 'numpy.matrix', 'np.matrix', (['[[1, 0, 0], [0, 0, 0]]'], {}), '([[1, 0, 0], [0, 0, 0]])\n', (8079, 8103), True, 'import numpy as np\n'), ((8151, 8184), 'numpy.matrix', 'np.matrix', (['[[0, 1, 0], [0, 0, 0]]'], {}), '([[0, 1, 0], [0, 0, 0]])\n', (8160, 8184), True, 'import numpy as np\n'), ((15633, 15677), 'numpy.matrix', 'np.matrix', (['[[1, 0, 0], [0, 0, 0], [0, 0, 0]]'], {}), '([[1, 0, 0], [0, 0, 0], [0, 0, 0]])\n', (15642, 15677), True, 'import numpy as np\n'), ((15754, 15798), 'numpy.matrix', 'np.matrix', (['[[0, 1, 0], [0, 0, 0], [0, 0, 0]]'], {}), '([[0, 1, 0], [0, 0, 0], [0, 0, 0]])\n', (15763, 15798), True, 'import numpy as np\n'), ((16090, 16114), 'Class.Wall', 'Class.Wall', (['"""wall"""', 'x', 'y'], {}), "('wall', x, y)\n", (16100, 16114), False, 'import Class\n'), ((16435, 16475), 'Class.Positivo', 'Class.Positivo', (['"""positivo"""', '(y + 1)', '(x + 1)'], {}), "('positivo', y + 1, x + 1)\n", (16449, 16475), False, 'import Class\n'), ((16726, 16766), 'Class.Negativo', 'Class.Negativo', (['"""negativo"""', '(y + 1)', '(x + 1)'], {}), "('negativo', y + 1, x + 1)\n", (16740, 16766), False, 'import Class\n'), ((17526, 17594), 'pandas.read_excel', 'pd.read_excel', (['path'], {'sheetname': '"""model"""', 'header': '[0]', 'index_col': '[0, 1]'}), "(path, sheetname='model', header=[0], index_col=[0, 1])\n", (17539, 17594), True, 'import pandas as pd\n'), ((25874, 25896), 'numpy.array', 'np.array', (['[s, s, s, s]'], {}), '([s, s, s, s])\n', (25882, 25896), True, 'import numpy as np\n'), ((25898, 25939), 'numpy.array', 'np.array', (["['up', 'down', 'right', 'left']"], {}), "(['up', 'down', 'right', 'left'])\n", (25906, 25939), True, 'import numpy as np\n'), ((25977, 25993), 'numpy.zeros', 'np.zeros', (['[4, 1]'], {}), '([4, 1])\n', (25985, 25993), True, 'import numpy as np\n'), ((29625, 29647), 'random.choice', 'random.choice', (['actions'], {}), '(actions)\n', (29638, 29647), False, 'import random\n'), ((29816, 29831), 'numpy.argmax', 'np.argmax', (['q[0]'], {}), '(q[0])\n', (29825, 29831), True, 'import numpy as np\n'), ((30934, 30968), 'numpy.array', 'np.array', (['[s_t1, s_t1, s_t1, s_t1]'], {}), '([s_t1, s_t1, s_t1, s_t1])\n', (30942, 30968), True, 'import numpy as np\n'), ((30970, 31011), 'numpy.array', 'np.array', (["['up', 'down', 'right', 'left']"], {}), "(['up', 'down', 'right', 'left'])\n", (30978, 31011), True, 'import numpy as np\n'), ((31049, 31065), 'numpy.zeros', 'np.zeros', (['[4, 1]'], {}), '([4, 1])\n', (31057, 31065), True, 'import numpy as np\n'), ((31185, 31215), 'numpy.array', 'np.array', (['[s_t, s_t, s_t, s_t]'], {}), '([s_t, s_t, s_t, s_t])\n', (31193, 31215), True, 'import numpy as np\n'), ((31217, 31258), 'numpy.array', 'np.array', (["['up', 'down', 'right', 'left']"], {}), "(['up', 'down', 'right', 'left'])\n", (31225, 31258), True, 'import numpy as np\n'), ((31296, 31312), 'numpy.zeros', 'np.zeros', (['[4, 1]'], {}), '([4, 1])\n', (31304, 31312), True, 'import numpy as np\n'), ((45804, 45825), 'pygame.display.flip', 'pygame.display.flip', ([], {}), '()\n', (45823, 45825), False, 'import pygame\n'), ((45985, 45997), 'time.sleep', 'sleep', (['speed'], {}), '(speed)\n', (45990, 45997), False, 'from time import sleep\n'), ((58726, 58752), 'os.path.dirname', 'os.path.dirname', (['save_path'], {}), '(save_path)\n', (58741, 58752), False, 'import os\n'), ((59365, 59393), 'pandas.Series', 'pd.Series', (["{'Run_Conf': 'A'}"], {}), "({'Run_Conf': 'A'})\n", (59374, 59393), True, 'import pandas as pd\n'), ((59429, 59459), 'pandas.Series', 'pd.Series', (["{'Env_conf': s_env}"], {}), "({'Env_conf': s_env})\n", (59438, 59459), True, 'import pandas as pd\n'), ((59495, 59523), 'pandas.Series', 'pd.Series', (["{'Algort': s_alg}"], {}), "({'Algort': s_alg})\n", (59504, 59523), True, 'import pandas as pd\n'), ((59559, 59588), 'pandas.Series', 'pd.Series', (["{'Learn': s_learn}"], {}), "({'Learn': s_learn})\n", (59568, 59588), True, 'import pandas as pd\n'), ((59624, 59651), 'pandas.Series', 'pd.Series', (["{'Load': s_load}"], {}), "({'Load': s_load})\n", (59633, 59651), True, 'import pandas as pd\n'), ((59687, 59719), 'pandas.Series', 'pd.Series', (["{'Samples': s_sample}"], {}), "({'Samples': s_sample})\n", (59696, 59719), True, 'import pandas as pd\n'), ((59755, 59788), 'pandas.Series', 'pd.Series', (["{'Episode': s_episode}"], {}), "({'Episode': s_episode})\n", (59764, 59788), True, 'import pandas as pd\n'), ((59824, 59859), 'pandas.Series', 'pd.Series', (["{'Max_steps': max_steps}"], {}), "({'Max_steps': max_steps})\n", (59833, 59859), True, 'import pandas as pd\n'), ((59895, 59938), 'pandas.Series', 'pd.Series', (["{'s_cond_to_end': s_cond_to_end}"], {}), "({'s_cond_to_end': s_cond_to_end})\n", (59904, 59938), True, 'import pandas as pd\n'), ((59974, 60001), 'pandas.Series', 'pd.Series', (["{'Auto': s_auto}"], {}), "({'Auto': s_auto})\n", (59983, 60001), True, 'import pandas as pd\n'), ((60037, 60068), 'pandas.Series', 'pd.Series', (["{'Server': s_server}"], {}), "({'Server': s_server})\n", (60046, 60068), True, 'import pandas as pd\n'), ((60104, 60133), 'pandas.Series', 'pd.Series', (["{'Print': s_print}"], {}), "({'Print': s_print})\n", (60113, 60133), True, 'import pandas as pd\n'), ((60169, 60198), 'pandas.Series', 'pd.Series', (["{'MODEL CONF': ''}"], {}), "({'MODEL CONF': ''})\n", (60178, 60198), True, 'import pandas as pd\n'), ((60234, 60259), 'pandas.Series', 'pd.Series', (["{'alfa': alfa}"], {}), "({'alfa': alfa})\n", (60243, 60259), True, 'import pandas as pd\n'), ((60295, 60322), 'pandas.Series', 'pd.Series', (["{'gamma': gamma}"], {}), "({'gamma': gamma})\n", (60304, 60322), True, 'import pandas as pd\n'), ((60358, 60383), 'pandas.Series', 'pd.Series', (["{'Prob': Prob}"], {}), "({'Prob': Prob})\n", (60367, 60383), True, 'import pandas as pd\n'), ((60419, 60466), 'pandas.Series', 'pd.Series', (["{'N_actions': net_conf['N_actions']}"], {}), "({'N_actions': net_conf['N_actions']})\n", (60428, 60466), True, 'import pandas as pd\n'), ((60502, 60551), 'pandas.Series', 'pd.Series', (["{'Max_memory': net_conf['Max_memory']}"], {}), "({'Max_memory': net_conf['Max_memory']})\n", (60511, 60551), True, 'import pandas as pd\n'), ((60587, 60638), 'pandas.Series', 'pd.Series', (["{'Hidden_size': net_conf['Hidden_size']}"], {}), "({'Hidden_size': net_conf['Hidden_size']})\n", (60596, 60638), True, 'import pandas as pd\n'), ((60674, 60723), 'pandas.Series', 'pd.Series', (["{'Batch_size': net_conf['Batch_size']}"], {}), "({'Batch_size': net_conf['Batch_size']})\n", (60683, 60723), True, 'import pandas as pd\n'), ((60759, 60806), 'pandas.Series', 'pd.Series', (["{'Optimizer': net_conf['Optimizer']}"], {}), "({'Optimizer': net_conf['Optimizer']})\n", (60768, 60806), True, 'import pandas as pd\n'), ((60842, 60871), 'pandas.Series', 'pd.Series', (["{'lr': op_conf[0]}"], {}), "({'lr': op_conf[0]})\n", (60851, 60871), True, 'import pandas as pd\n'), ((60907, 60940), 'pandas.Series', 'pd.Series', (["{'beta_1': op_conf[1]}"], {}), "({'beta_1': op_conf[1]})\n", (60916, 60940), True, 'import pandas as pd\n'), ((60976, 61009), 'pandas.Series', 'pd.Series', (["{'beta_2': op_conf[2]}"], {}), "({'beta_2': op_conf[2]})\n", (60985, 61009), True, 'import pandas as pd\n'), ((61045, 61079), 'pandas.Series', 'pd.Series', (["{'epsilon': op_conf[3]}"], {}), "({'epsilon': op_conf[3]})\n", (61054, 61079), True, 'import pandas as pd\n'), ((61115, 61147), 'pandas.Series', 'pd.Series', (["{'decay': op_conf[4]}"], {}), "({'decay': op_conf[4]})\n", (61124, 61147), True, 'import pandas as pd\n'), ((61183, 61213), 'pandas.Series', 'pd.Series', (["{'rho': op_conf[5]}"], {}), "({'rho': op_conf[5]})\n", (61192, 61213), True, 'import pandas as pd\n'), ((61249, 61268), 'pandas.Series', 'pd.Series', (["{'': ''}"], {}), "({'': ''})\n", (61258, 61268), True, 'import pandas as pd\n'), ((61304, 61344), 'pandas.Series', 'pd.Series', (["{'AVG SCORE': avg_last_score}"], {}), "({'AVG SCORE': avg_last_score})\n", (61313, 61344), True, 'import pandas as pd\n'), ((8336, 8363), 'numpy.matrix', 'np.matrix', (['[[0, 0], [0, 0]]'], {}), '([[0, 0], [0, 0]])\n', (8345, 8363), True, 'import numpy as np\n'), ((8411, 8438), 'numpy.matrix', 'np.matrix', (['[[0, 0], [0, 1]]'], {}), '([[0, 0], [0, 1]])\n', (8420, 8438), True, 'import numpy as np\n'), ((17681, 17749), 'pandas.read_excel', 'pd.read_excel', (['path'], {'sheetname': '"""model"""', 'header': '[0]', 'index_col': '[0, 1]'}), "(path, sheetname='model', header=[0], index_col=[0, 1])\n", (17694, 17749), True, 'import pandas as pd\n'), ((27070, 27180), 'pandas.MultiIndex', 'pd.MultiIndex', ([], {'levels': '[[s_n_c], actions]', 'labels': '[[0, 0, 0, 0], [0, 1, 2, 3]]', 'names': "['state', 'actions']"}), "(levels=[[s_n_c], actions], labels=[[0, 0, 0, 0], [0, 1, 2, 3]\n    ], names=['state', 'actions'])\n", (27083, 27180), True, 'import pandas as pd\n'), ((27285, 27312), 'pandas.DataFrame', 'pd.DataFrame', ([], {'index': 'm_index'}), '(index=m_index)\n', (27297, 27312), True, 'import pandas as pd\n'), ((27805, 27851), 'numpy.sqrt', 'np.sqrt', (['(s_n_c_abs[0] ** 2 + s_n_c_abs[1] ** 2)'], {}), '(s_n_c_abs[0] ** 2 + s_n_c_abs[1] ** 2)\n', (27812, 27851), True, 'import numpy as np\n'), ((32850, 32960), 'pandas.MultiIndex', 'pd.MultiIndex', ([], {'levels': '[[s_new], actions]', 'labels': '[[0, 0, 0, 0], [0, 1, 2, 3]]', 'names': "['state', 'actions']"}), "(levels=[[s_new], actions], labels=[[0, 0, 0, 0], [0, 1, 2, 3]\n    ], names=['state', 'actions'])\n", (32863, 32960), True, 'import pandas as pd\n'), ((33065, 33092), 'pandas.DataFrame', 'pd.DataFrame', ([], {'index': 'm_index'}), '(index=m_index)\n', (33077, 33092), True, 'import pandas as pd\n'), ((46187, 46205), 'pygame.event.get', 'pygame.event.get', ([], {}), '()\n', (46203, 46205), False, 'import pygame\n'), ((56252, 56273), 'pygame.display.flip', 'pygame.display.flip', ([], {}), '()\n', (56271, 56273), False, 'import pygame\n'), ((58802, 58828), 'os.path.dirname', 'os.path.dirname', (['save_path'], {}), '(save_path)\n', (58817, 58828), False, 'import os\n'), ((8590, 8628), 'numpy.zeros', 'np.zeros', ([], {'shape': '(v_max - 2, h_max - 2)'}), '(shape=(v_max - 2, h_max - 2))\n', (8598, 8628), True, 'import numpy as np\n'), ((8647, 8685), 'numpy.zeros', 'np.zeros', ([], {'shape': '(v_max - 2, h_max - 2)'}), '(shape=(v_max - 2, h_max - 2))\n', (8655, 8685), True, 'import numpy as np\n'), ((17841, 17909), 'pandas.read_excel', 'pd.read_excel', (['path'], {'sheetname': '"""model"""', 'header': '[0]', 'index_col': '[0, 1]'}), "(path, sheetname='model', header=[0], index_col=[0, 1])\n", (17854, 17909), True, 'import pandas as pd\n'), ((53146, 53200), 'Class.Positivo', 'Class.Positivo', (['"""positivo"""', 'agent.pos[0]', 'agent.pos[1]'], {}), "('positivo', agent.pos[0], agent.pos[1])\n", (53160, 53200), False, 'import Class\n'), ((53643, 53697), 'Class.Negativo', 'Class.Negativo', (['"""negativo"""', 'agent.pos[0]', 'agent.pos[1]'], {}), "('negativo', agent.pos[0], agent.pos[1])\n", (53657, 53697), False, 'import Class\n'), ((8726, 8756), 'random.randrange', 'random.randrange', (['(0)', '(h_max - 2)'], {}), '(0, h_max - 2)\n', (8742, 8756), False, 'import random\n'), ((8774, 8804), 'random.randrange', 'random.randrange', (['(0)', '(v_max - 2)'], {}), '(0, v_max - 2)\n', (8790, 8804), False, 'import random\n'), ((9063, 9101), 'numpy.zeros', 'np.zeros', ([], {'shape': '(v_max - 2, h_max - 2)'}), '(shape=(v_max - 2, h_max - 2))\n', (9071, 9101), True, 'import numpy as np\n'), ((9120, 9158), 'numpy.zeros', 'np.zeros', ([], {'shape': '(v_max - 2, h_max - 2)'}), '(shape=(v_max - 2, h_max - 2))\n', (9128, 9158), True, 'import numpy as np\n'), ((18009, 18077), 'pandas.read_excel', 'pd.read_excel', (['path'], {'sheetname': '"""model"""', 'header': '[0]', 'index_col': '[0, 1]'}), "(path, sheetname='model', header=[0], index_col=[0, 1])\n", (18022, 18077), True, 'import pandas as pd\n'), ((46328, 46341), 'pygame.quit', 'pygame.quit', ([], {}), '()\n', (46339, 46341), False, 'import pygame\n'), ((46371, 46381), 'sys.exit', 'sys.exit', ([], {}), '()\n', (46379, 46381), False, 'import sys\n'), ((9199, 9229), 'random.randrange', 'random.randrange', (['(0)', '(h_max - 2)'], {}), '(0, h_max - 2)\n', (9215, 9229), False, 'import random\n'), ((9247, 9277), 'random.randrange', 'random.randrange', (['(0)', '(v_max - 2)'], {}), '(0, v_max - 2)\n', (9263, 9277), False, 'import random\n'), ((9540, 9578), 'numpy.zeros', 'np.zeros', ([], {'shape': '(v_max - 2, h_max - 2)'}), '(shape=(v_max - 2, h_max - 2))\n', (9548, 9578), True, 'import numpy as np\n'), ((9597, 9635), 'numpy.zeros', 'np.zeros', ([], {'shape': '(v_max - 2, h_max - 2)'}), '(shape=(v_max - 2, h_max - 2))\n', (9605, 9635), True, 'import numpy as np\n'), ((18166, 18234), 'pandas.read_excel', 'pd.read_excel', (['path'], {'sheetname': '"""model"""', 'header': '[0]', 'index_col': '[0, 1]'}), "(path, sheetname='model', header=[0], index_col=[0, 1])\n", (18179, 18234), True, 'import pandas as pd\n'), ((48556, 48569), 'pygame.quit', 'pygame.quit', ([], {}), '()\n', (48567, 48569), False, 'import pygame\n'), ((48603, 48613), 'sys.exit', 'sys.exit', ([], {}), '()\n', (48611, 48613), False, 'import sys\n'), ((48768, 48788), 'matplotlib.pyplot.plot', 'plt.plot', (['score_list'], {}), '(score_list)\n', (48776, 48788), True, 'import matplotlib.pyplot as plt\n'), ((48822, 48841), 'matplotlib.pyplot.ylabel', 'plt.ylabel', (['"""Score"""'], {}), "('Score')\n", (48832, 48841), True, 'import matplotlib.pyplot as plt\n'), ((48875, 48900), 'matplotlib.pyplot.xlabel', 'plt.xlabel', (['"""Total Steps"""'], {}), "('Total Steps')\n", (48885, 48900), True, 'import matplotlib.pyplot as plt\n'), ((48934, 48971), 'matplotlib.pyplot.title', 'plt.title', (['"""Performance of the Agent"""'], {}), "('Performance of the Agent')\n", (48943, 48971), True, 'import matplotlib.pyplot as plt\n'), ((49005, 49015), 'matplotlib.pyplot.show', 'plt.show', ([], {}), '()\n', (49013, 49015), True, 'import matplotlib.pyplot as plt\n'), ((49051, 49073), 'matplotlib.pyplot.plot', 'plt.plot', (['percent_list'], {}), '(percent_list)\n', (49059, 49073), True, 'import matplotlib.pyplot as plt\n'), ((49107, 49144), 'matplotlib.pyplot.ylabel', 'plt.ylabel', (['"""Percentage of objects +"""'], {}), "('Percentage of objects +')\n", (49117, 49144), True, 'import matplotlib.pyplot as plt\n'), ((49178, 49203), 'matplotlib.pyplot.xlabel', 'plt.xlabel', (['"""Total Steps"""'], {}), "('Total Steps')\n", (49188, 49203), True, 'import matplotlib.pyplot as plt\n'), ((49237, 49282), 'matplotlib.pyplot.title', 'plt.title', (['"""Episode over 100 times step each"""'], {}), "('Episode over 100 times step each')\n", (49246, 49282), True, 'import matplotlib.pyplot as plt\n'), ((49316, 49326), 'matplotlib.pyplot.show', 'plt.show', ([], {}), '()\n', (49324, 49326), True, 'import matplotlib.pyplot as plt\n'), ((9676, 9706), 'random.randrange', 'random.randrange', (['(0)', '(h_max - 2)'], {}), '(0, h_max - 2)\n', (9692, 9706), False, 'import random\n'), ((9724, 9754), 'random.randrange', 'random.randrange', (['(0)', '(v_max - 2)'], {}), '(0, v_max - 2)\n', (9740, 9754), False, 'import random\n'), ((10017, 10061), 'numpy.matrix', 'np.matrix', (['[[0, 0, 0], [0, 0, 0], [1, 0, 1]]'], {}), '([[0, 0, 0], [0, 0, 0], [1, 0, 1]])\n', (10026, 10061), True, 'import numpy as np\n'), ((10138, 10182), 'numpy.matrix', 'np.matrix', (['[[1, 0, 1], [0, 0, 0], [0, 0, 0]]'], {}), '([[1, 0, 1], [0, 0, 0], [0, 0, 0]])\n', (10147, 10182), True, 'import numpy as np\n'), ((18318, 18386), 'pandas.read_excel', 'pd.read_excel', (['path'], {'sheetname': '"""model"""', 'header': '[0]', 'index_col': '[0, 1]'}), "(path, sheetname='model', header=[0], index_col=[0, 1])\n", (18331, 18386), True, 'import pandas as pd\n'), ((49416, 49435), 'matplotlib.pyplot.plot', 'plt.plot', (['loss_list'], {}), '(loss_list)\n', (49424, 49435), True, 'import matplotlib.pyplot as plt\n'), ((49473, 49491), 'matplotlib.pyplot.ylabel', 'plt.ylabel', (['"""loss"""'], {}), "('loss')\n", (49483, 49491), True, 'import matplotlib.pyplot as plt\n'), ((49529, 49554), 'matplotlib.pyplot.xlabel', 'plt.xlabel', (['"""Total Steps"""'], {}), "('Total Steps')\n", (49539, 49554), True, 'import matplotlib.pyplot as plt\n'), ((49592, 49615), 'matplotlib.pyplot.title', 'plt.title', (['"""batch_loss"""'], {}), "('batch_loss')\n", (49601, 49615), True, 'import matplotlib.pyplot as plt\n'), ((49653, 49663), 'matplotlib.pyplot.show', 'plt.show', ([], {}), '()\n', (49661, 49663), True, 'import matplotlib.pyplot as plt\n'), ((10363, 10407), 'numpy.matrix', 'np.matrix', (['[[1, 0, 0], [0, 0, 0], [0, 0, 1]]'], {}), '([[1, 0, 0], [0, 0, 0], [0, 0, 1]])\n', (10372, 10407), True, 'import numpy as np\n'), ((10484, 10528), 'numpy.matrix', 'np.matrix', (['[[0, 0, 1], [0, 0, 0], [1, 0, 0]]'], {}), '([[0, 0, 1], [0, 0, 0], [1, 0, 0]])\n', (10493, 10528), True, 'import numpy as np\n'), ((18578, 18641), 'pandas.read_excel', 'pd.read_excel', (["(path + '.xlsx')"], {'sheetname': '"""Run_Conf"""', 'header': '[0]'}), "(path + '.xlsx', sheetname='Run_Conf', header=[0])\n", (18591, 18641), True, 'import pandas as pd\n'), ((10710, 10890), 'numpy.matrix', 'np.matrix', (['[[1, 0, 0, 0, 1, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0, 1], [0, 0,\n    0, 0, 0, 0, 0], [1, 0, 0, 0, 1, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 1,\n    0, 0, 0, 1]]'], {}), '([[1, 0, 0, 0, 1, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0,\n    1], [0, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 1, 0, 0], [0, 0, 0, 0, 0, 0, 0],\n    [0, 0, 1, 0, 0, 0, 1]])\n', (10719, 10890), True, 'import numpy as np\n'), ((11075, 11255), 'numpy.matrix', 'np.matrix', (['[[0, 0, 1, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 1, 0, 0], [0, 0,\n    0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0], [1, 0, 0,\n    0, 1, 0, 0]]'], {}), '([[0, 0, 1, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 1, 0,\n    0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0],\n    [1, 0, 0, 0, 1, 0, 0]])\n', (11084, 11255), True, 'import numpy as np\n'), ((11891, 11929), 'numpy.zeros', 'np.zeros', ([], {'shape': '(v_max - 2, h_max - 2)'}), '(shape=(v_max - 2, h_max - 2))\n', (11899, 11929), True, 'import numpy as np\n'), ((11944, 11982), 'numpy.zeros', 'np.zeros', ([], {'shape': '(v_max - 2, h_max - 2)'}), '(shape=(v_max - 2, h_max - 2))\n', (11952, 11982), True, 'import numpy as np\n'), ((18502, 18518), 'json.load', 'json.load', (['jfile'], {}), '(jfile)\n', (18511, 18518), False, 'import json\n'), ((12352, 12372), 'numpy.matrix', 'np.matrix', (['[1, 0, 0]'], {}), '([1, 0, 0])\n', (12361, 12372), True, 'import numpy as np\n'), ((12391, 12411), 'numpy.matrix', 'np.matrix', (['[0, 0, 1]'], {}), '([0, 0, 1])\n', (12400, 12411), True, 'import numpy as np\n'), ((11630, 11652), 'random.randrange', 'random.randrange', (['(0)', '(7)'], {}), '(0, 7)\n', (11646, 11652), False, 'import random\n'), ((11673, 11695), 'random.randrange', 'random.randrange', (['(0)', '(7)'], {}), '(0, 7)\n', (11689, 11695), False, 'import random\n'), ((12535, 12555), 'numpy.matrix', 'np.matrix', (['[0, 0, 0]'], {}), '([0, 0, 0])\n', (12544, 12555), True, 'import numpy as np\n'), ((12574, 12594), 'numpy.matrix', 'np.matrix', (['[1, 0, 1]'], {}), '([1, 0, 1])\n', (12583, 12594), True, 'import numpy as np\n'), ((12718, 12741), 'numpy.matrix', 'np.matrix', (['[1, 0, 0, 0]'], {}), '([1, 0, 0, 0])\n', (12727, 12741), True, 'import numpy as np\n'), ((12760, 12783), 'numpy.matrix', 'np.matrix', (['[0, 0, 0, 1]'], {}), '([0, 0, 0, 1])\n', (12769, 12783), True, 'import numpy as np\n'), ((12907, 12930), 'numpy.matrix', 'np.matrix', (['[0, 0, 0, 0]'], {}), '([0, 0, 0, 0])\n', (12916, 12930), True, 'import numpy as np\n'), ((12949, 12972), 'numpy.matrix', 'np.matrix', (['[1, 0, 0, 1]'], {}), '([1, 0, 0, 1])\n', (12958, 12972), True, 'import numpy as np\n'), ((13096, 13122), 'numpy.matrix', 'np.matrix', (['[1, 0, 0, 0, 0]'], {}), '([1, 0, 0, 0, 0])\n', (13105, 13122), True, 'import numpy as np\n'), ((13141, 13167), 'numpy.matrix', 'np.matrix', (['[0, 0, 0, 0, 1]'], {}), '([0, 0, 0, 0, 1])\n', (13150, 13167), True, 'import numpy as np\n'), ((13291, 13317), 'numpy.matrix', 'np.matrix', (['[0, 0, 0, 0, 0]'], {}), '([0, 0, 0, 0, 0])\n', (13300, 13317), True, 'import numpy as np\n'), ((13336, 13362), 'numpy.matrix', 'np.matrix', (['[1, 0, 0, 0, 1]'], {}), '([1, 0, 0, 0, 1])\n', (13345, 13362), True, 'import numpy as np\n'), ((13486, 13518), 'numpy.matrix', 'np.matrix', (['[1, 0, 0, 0, 0, 0, 0]'], {}), '([1, 0, 0, 0, 0, 0, 0])\n', (13495, 13518), True, 'import numpy as np\n'), ((13537, 13569), 'numpy.matrix', 'np.matrix', (['[0, 0, 0, 0, 0, 0, 1]'], {}), '([0, 0, 0, 0, 0, 0, 1])\n', (13546, 13569), True, 'import numpy as np\n'), ((13693, 13725), 'numpy.matrix', 'np.matrix', (['[0, 0, 0, 0, 0, 0, 0]'], {}), '([0, 0, 0, 0, 0, 0, 0])\n', (13702, 13725), True, 'import numpy as np\n'), ((13744, 13776), 'numpy.matrix', 'np.matrix', (['[1, 0, 0, 0, 0, 0, 1]'], {}), '([1, 0, 0, 0, 0, 0, 1])\n', (13753, 13776), True, 'import numpy as np\n'), ((13900, 13944), 'numpy.matrix', 'np.matrix', (['[[0, 0, 0], [0, 0, 0], [0, 0, 0]]'], {}), '([[0, 0, 0], [0, 0, 0], [0, 0, 0]])\n', (13909, 13944), True, 'import numpy as np\n'), ((14021, 14065), 'numpy.matrix', 'np.matrix', (['[[1, 0, 1], [0, 0, 0], [0, 1, 0]]'], {}), '([[1, 0, 1], [0, 0, 0], [0, 1, 0]])\n', (14030, 14065), True, 'import numpy as np\n'), ((14247, 14291), 'numpy.matrix', 'np.matrix', (['[[0, 1, 0], [0, 0, 0], [1, 0, 1]]'], {}), '([[0, 1, 0], [0, 0, 0], [1, 0, 1]])\n', (14256, 14291), True, 'import numpy as np\n'), ((14368, 14412), 'numpy.matrix', 'np.matrix', (['[[1, 0, 1], [0, 0, 0], [0, 1, 0]]'], {}), '([[1, 0, 1], [0, 0, 0], [0, 1, 0]])\n', (14377, 14412), True, 'import numpy as np\n'), ((14594, 14638), 'numpy.matrix', 'np.matrix', (['[[0, 0, 0], [0, 0, 0], [0, 0, 0]]'], {}), '([[0, 0, 0], [0, 0, 0], [0, 0, 0]])\n', (14603, 14638), True, 'import numpy as np\n'), ((14715, 14759), 'numpy.matrix', 'np.matrix', (['[[1, 0, 1], [0, 0, 0], [1, 0, 1]]'], {}), '([[1, 0, 1], [0, 0, 0], [1, 0, 1]])\n', (14724, 14759), True, 'import numpy as np\n')]