<filename>connect4/q_learning_agent.py
from basis_function import basis_vector
from basis_function import basis_vector
from connect4 import Connect4Board
from connect4 import GameState
from connect4 import Player

import numpy as np
import random

class QLearningAgent:
  name = None
  use_offline_params = None
  theta_online = None
  theta_offline = None

  def __init__(self, name, use_offline_params = False):
    self.name = name
    self.use_offline_params = use_offline_params
    self.theta_online = {
      "player_num_possible_wins_in_col_3": 8139.292023795227,
      "opponent_3_out_of_5": 2561.271758401387,
      "opponent_num_consecutive_possible_wins_in_col_0": 46.73571428571428,
      "opponent_num_consecutive_possible_wins_in_col_1": 475.7211703185538,
      "player_num_possible_wins_in_col_1": 8777.713527430345,
      "opponent_num_consecutive_possible_wins_in_col_3": 1285.4441669256018,
      "opponent_num_consecutive_possible_wins_in_col_2": 2525.2296120645165,
      "opponent_num_possible_wins_in_col_2": 14622.372716140242,
      "opponent_num_possible_wins_in_col_1": 8791.161880415817,
      "player_num_consecutive_possible_wins_in_col_4": 2545.6364189051965,
      "player_num_consecutive_possible_wins_in_col_1": 419.86472573265627,
      "player_num_consecutive_possible_wins_in_col_0": 12.05,
      "opponent_num_consecutive_possible_wins_in_col_5": 405.6517231182447,
      "opponent_num_possible_wins_in_col_4": 15419.644292408922,
      "player_num_possible_wins_in_col_5": 8716.604582743756,
      "player_num_possible_wins_in_col_6": 4590.894549704054,
      "opponent_num_consecutive_possible_wins_in_col_6": 33.67738095238096,
      "player_num_possible_wins_in_col_0": 4541.45043239415,
      "player_num_consecutive_possible_wins_in_col_5": 389.90075993694416,
      "opponent_num_consecutive_possible_wins_in_col_4": 2637.5778891690657,
      "player_num_consecutive_possible_wins_in_col_3": 1365.0624284672979,
      "player_num_possible_wins_in_col_4": 15563.058470657974,
      "player_3_out_of_5": 2121.6246593828373,
      "player_2_out_of_4": 523419.5014114607,
      "opponent_win": -3428.3721526893664,
      "player_num_consecutive_possible_wins_in_col_2": 2814.3064544596305,
      "opponent_2_out_of_4": 517775.44510431204,
      "opponent_num_possible_wins_in_col_5": 8580.171730788232,
      "opponent_num_possible_wins_in_col_3": 8345.846012715356,
      "player_num_consecutive_possible_wins_in_col_6": 17.133333333333333,
      "opponent_3_out_of_4": 70835.96875824689,
      "player_3_out_of_4": 71692.55094079571,
      "opponent_num_possible_wins_in_col_6": 4344.458997589873,
      "opponent_num_possible_wins_in_col_0": 4355.954989445178,
      "player_win": 3434.4884802072806,
      "player_num_possible_wins_in_col_2": 14947.595671627998
    }
    self.theta_offline = {
      "player_num_possible_wins_in_col_3": 0.14359636760651984,
      "player_num_consecutive_possible_wins_in_col_5": 0.06811800950809425,
      "player_num_consecutive_possible_wins_in_col_4": -0.14309882661794826,
      "opponent_3_out_of_5": 0.04125866449222762,
      "opponent_num_consecutive_possible_wins_in_col_2": -0.07155909478062643,
      "player_3_out_of_5": -0.041258664492228495,
      "opponent_num_possible_wins_in_col_5": -0.12706819548969345,
      "player_num_possible_wins_in_col_1": 0.13821540837263072,
      "opponent_win": -0.729016518959015,
      "opponent_num_consecutive_possible_wins_in_col_3": -0.6886344031521995,
      "opponent_3_out_of_4": -0.0732572816754046,
      "player_num_consecutive_possible_wins_in_col_3": 0.6886344031521776,
      "player_num_consecutive_possible_wins_in_col_2": 0.07155909478062634,
      "player_num_possible_wins_in_col_4": 0.10757042551191198,
      "opponent_num_possible_wins_in_col_1": -0.13821540837263496,
      "opponent_num_possible_wins_in_col_6": -0.1413009532163512,
      "player_num_consecutive_possible_wins_in_col_1": -0.2971123695163536,
      "player_num_consecutive_possible_wins_in_col_0": 0.02840868572463406,
      "opponent_num_consecutive_possible_wins_in_col_5": -0.06811800950809999,
      "player_2_out_of_4": 0.018104917640001928,
      "player_num_possible_wins_in_col_0": 0.10060372086326475,
      "player_num_consecutive_possible_wins_in_col_6": -0.0610932311092139,
      "opponent_num_consecutive_possible_wins_in_col_0": -0.028408685724633287,
      "opponent_num_possible_wins_in_col_2": -0.0587029001387138,
      "player_num_possible_wins_in_col_5": 0.12706819548969484,
      "player_win": 0.7290165189590107,
      "player_3_out_of_4": 0.07325728167540471,
      "player_num_possible_wins_in_col_6": 0.14130095321635477,
      "opponent_num_possible_wins_in_col_3": -0.14359636760651828,
      "opponent_num_consecutive_possible_wins_in_col_6": 0.0610932311092044,
      "opponent_num_possible_wins_in_col_4": -0.10757042551191245,
      "opponent_2_out_of_4": -0.018104917640002962,
      "opponent_num_consecutive_possible_wins_in_col_1": 0.29711236951635067,
      "player_num_possible_wins_in_col_2": 0.05870290013871207,
      "opponent_num_possible_wins_in_col_0": -0.10060372086326627,
      "opponent_num_consecutive_possible_wins_in_col_4": 0.14309882661794818
    }

  def get_name(self):
    return self.name

  def get_action(self, player, game):
    def q_value(game, player):
      basis = basis_vector(game, player)
      q = 0
      for key in basis.keys():
        if self.use_offline_params:
          q += self.theta_offline[key]*basis[key] #seems ok
        else:
          q += self.theta_online[key]*basis[key] #seems not great
      return q

    valid_actions = [action for action in range(game.NUM_COLS) if game.valid_action(action)]
    random.shuffle(valid_actions)
    best_score = float("-inf")
    best_action = None
    for action in valid_actions:
      score = q_value(game.add_piece(player, action), player)
      if score > best_score:
        best_score = score
        best_action = action
    return best_action
