"""
Target Problem:
---------------
* A classifier for the diagnosis of Autism Spectrum Disorder (ASD)

Proposed Solution (Machine Learning Pipeline):
----------------------------------------------
* Min-max Scaling -> Elimination of Highly Coorelated Features -> SelectKBest algorithm -> XGBoost

Input to Proposed Solution:
---------------------------
* Directories of training and testing data in csv file format
* These two types of data should be stored in n x m pattern in csv file format.

  Typical Example:
  ----------------
  n x m samples in training csv file (n number of samples, m - 1 number of features, ground truth labels at last column)
  k x s samples in testing csv file (k number of samples, s number of features)

* These data set files are ready by load_data() function.
* For comprehensive information about input format, please check the section
  "Data Sets and Usage Format of Source Codes" in README.md file on github.

Output of Proposed Solution:
----------------------------
* Predictions generated by learning model for testing set
* They are stored in "submission.csv" file.

Code Owner:
-----------
* Copyright © Team 5. All rights reserved.
* Copyright © Istanbul Technical University, Learning From Data Spring 2019. All rights reserved. """

import numpy as np
import pandas as pd

from numpy import genfromtxt
from xgboost import XGBClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.feature_selection import SelectKBest, chi2


def load_data(train_path, test_path):

    """
    The method reads train and test data from their files.
    Then, it deletes feature names by using numpy delete() method.
    Finally, it splits train data into two parts: train features (train_x) and train labels (train_y).

    Parameters
    ----------
    train_path: directory name of the file in which training data exists
    test_path: directory name of the file in which testing data exists

    """

    train = genfromtxt(train_path, delimiter=',')
    train = np.delete(train, 0, 0)
    test = genfromtxt(test_path, delimiter=',')
    test = np.delete(test, 0, 0)

    train_x = train[:, 0:595]
    train_y = train[:, 595]

    return train_x, train_y, test


def preprocessing(train_x, train_y, test):

    """
    * The method at first performs min-max scaling on train and test data.
    * Then, it computes correlation matrix to find out which features are mostly-correlated to each other.
      Two features which are very correlated to each other have almost same impact on classification and labels.
      Hence, one of these two features is discarded. We do not need to use both of them.
    * Finally, selectKBest() algorithm is used with chi square value to choose top 100 features.
    * Totally, two feature selection methods are used to reduce the dimension of training and testing data.

    Parameters
    ----------
    train_x: features of train data samples
    train_y: labels of train data samples
    test: features of test data samples

    """

    scaler = MinMaxScaler()
    scaler.fit(train_x)
    scaled_test = scaler.transform(test)
    scaled_train_x = scaler.transform(train_x)

    df_test = pd.DataFrame(data=scaled_test)
    df_train_x = pd.DataFrame(data=scaled_train_x)

    corr_matrix = df_train_x.corr().abs()
    upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))
    drop = [column for column in upper.columns if any(upper[column] > 0.9)]  # finding highly-correlated features

    df_train_x = df_train_x.drop(df_train_x.columns[drop], axis=1)  # discarding those features
    df_test = df_test.drop(df_test.columns[drop], axis=1)

    # Select features
    selector = SelectKBest(chi2, k=100)
    selector.fit(df_train_x, train_y)
    reduced_train_x = selector.transform(df_train_x)
    reduced_test = selector.transform(df_test)

    return reduced_train_x, reduced_test


def train_model(train_x, train_y):

    """
    The method trains XGB Classifier model by using training samples and their ground truth labels.

    Parameters
    ----------
    train_x: features of training samples
    train_y: labels of training samples

    """

    s_clf = XGBClassifier(learning_rate=0.1, n_estimators=140, max_depth=5, min_child_weight=3, gamma=0.2,
                          subsample=0.6, colsample_bytree=1.0, objective='binary:logistic', nthread=4,
                          scale_pos_weight=1, seed=27, silent=1)

    s_clf.fit(train_x, train_y)
    return s_clf


def predict(model, test):

    """
    The method make predictions by using passed learning model "model" and testing data "scaled_test".

    Parameters
    ----------
    model: Learning model object trained by training data
    scaled_test: features of testing samples

    """

    predictions = model.predict(test)
    return predictions


def write_output(prediction):
    # Write output to csv file
    f = open("submission.csv","w+")
    f.write("ID,Predicted\n")
    for i, item in enumerate(prediction):
        f.write(str(1+i) + "," + str(int(item)) + "\n")
    f.close()


# ********** MAIN PROGRAM ********** #

train_x, train_y, test = load_data('train.csv', 'test.csv')
scaled_train_x, scaled_test = preprocessing(train_x, train_y, test)

model = train_model(scaled_train_x, train_y)
predictions = predict(model, scaled_test)
write_output(predictions)
