# coding: utf-8
from __future__ import print_function
# In[1]:
import warnings
warnings.filterwarnings('ignore')
warnings.simplefilter("ignore")
import os
import numpy as np
import tensorflow as tf
import random as rn
os.environ['KMP_DUPLICATE_LIB_OK']='True'

from keras import regularizers
from keras.datasets import mnist
from keras.layers import Dense, Dropout
from keras.models import Sequential
from keras.optimizers import RMSprop
from pandas.plotting import scatter_matrix
from pprint import pprint
from sklearn import linear_model
from sklearn import preprocessing
from sklearn import svm
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import f_regression
from sklearn.feature_selection import SelectFromModel
from sklearn.feature_selection import SelectKBest
from sklearn.gaussian_process.kernels import WhiteKernel, ExpSineSquared
from sklearn.isotonic import IsotonicRegression
from sklearn.kernel_ridge import KernelRidge
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import RidgeClassifier
from sklearn.linear_model import SGDRegressor
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import r2_score
from sklearn.model_selection import cross_val_predict
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RandomizedSearchCV
from sklearn.neighbors import KNeighborsRegressor
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Normalizer
from sklearn.preprocessing import StandardScaler
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
import configparser
import csv
import datetime
import glob
import keras
import math
import matplotlib
matplotlib.use("TkAgg")
from matplotlib import pyplot as plt
import os
import pandas as pd
import pickle
import random
import sys
import time
import traceback
from minepy import MINE
from sklearn.linear_model import LogisticRegression
from os import path
from pathlib import PurePath
from tensorflow.python.util import deprecation
import tensorflow as tf

deprecation._PRINT_DEPRECATION_WARNINGS = False

config = tf.ConfigProto()
#config.gpu_options.per_process_gpu_memory_fraction = 0.5 # maximun alloc gpu50% of MEM
#config.gpu_options.allow_growth = True #allocate dynamically

#config = tf.ConfigProto( device_count = {'GPU': 2 , 'CPU':4 } ) 
sess = tf.Session(config=config)
keras.backend.set_session(sess)
#os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '2'

def save_test_data(predictions, actual_values, filename):
    df_predictions = pd.DataFrame(predictions, columns=['predictions'])
    df_actual_values = pd.DataFrame(actual_values, columns=['actual_values'])
    df = df_predictions.join(df_actual_values)
    df.to_csv(filename)
    
# In[3]:

def load_header(csv_file, print_out = False):
    
    # Loading headers
    
    headers = pd.read_csv(csv_file,header=None, nrows=1).values[0]
    idx_to_key = {}
    key_to_idx = {}
    
    for i in range(0, len(headers)):
        idx = i
        key = headers[i]
        key_to_idx[key] = idx
        idx_to_key[idx] = key
    
    if print_out is True:
        print(idx_to_key)
        
    return headers, idx_to_key, key_to_idx

def model_name(model_abbr):
    model_name = None
    if(model_abbr =='RF'):
        model_name = 'Random Forest'
    elif(model_abbr =='NET'):
        model_name = 'Neural Network'
    elif(model_abbr =='LR'):
        model_name = 'Linear Regression'
    elif(model_abbr =='LRC'):
        model_name = 'Logistic Regression'
    elif(model_abbr =='RG'):
        model_name = 'Ridge'
    elif(model_abbr =='KR'):
        model_name = 'Kernel Ridge'
    elif(model_abbr =='BR'):
        model_name = 'Bayesian Ridge'
    elif(model_abbr =='SVM'):
        model_name = 'Support Vector Machine'
    elif(model_abbr =='NN'):
        model_name = 'k-Nearest Neighbor'
    return model_name

# In[4]:

def data_load_shuffle(csv_file, input_col, cols_to_remove, target_col, random_state=0, delimiter = ',', map_all = None):
    data = pd.read_csv(csv_file, delimiter = delimiter)
    
    data_df = data[data[target_col].notnull()]
    if cols_to_remove is not None:
        for col in cols_to_remove:
            del data_df[col]
    
    data_df_shuffle = data_df.sample(frac=1, random_state=random_state)
    
    if input_col is not None:
        data_df_shuffle = data_df_shuffle[input_col+[target_col]]
        data_df = data_df[input_col+[target_col]]
    y_train = pd.DataFrame(data_df_shuffle[target_col])
    
    # training set is without target column
    del data_df_shuffle[target_col]
    x_train = data_df_shuffle.copy()
    headers, idx_to_key, key_to_idx = load_header(csv_file)
    
    cols_to_remove_ = []
    headers = list(headers)
    header_y = target_col
    
    if cols_to_remove is not None:
        for col in cols_to_remove:
            headers.remove(col)
    
    headers.remove(target_col)
    # dataframe to numpy array
    x_train_original = x_train
    y_train = y_train.values
    x_train = x_train.values
    
    # reshaping target values
    y_train = y_train.reshape(y_train.shape[0],1)
    
    if input_col is not None:
        header_x = np.array(input_col)
    else:
        header_x = np.array(headers)

    key_idx = -1
    if map_all is not None:
        for key in map_all.keys():
            for i in range(0, len(header_x)):
                if header_x[i]==key:
                    key_idx = i
                    break
            if key_idx!=-1:
                for item in x_train:  
                    item[key_idx] = map_all[key][item[key_idx]]
    
    if map_all is not None:
        for key in map_all.keys():
            if header_y==key:
                new_y_train = []
                for item in y_train:
                    item = [map_all[key][item[0]]]
                    new_y_train.append(item)
                y_train = np.array(new_y_train)
                
    
        for key in map_all.keys():
            data_df[key] = data_df[key].map(map_all[key])

    x_train = x_train.astype('float32')
    y_train = y_train.astype('float32')
    
    return data_df, x_train, y_train, header_x, header_y

def correlation_analysis_all(data_df, target_col, top_k=10, file_to_save = None, save_chart = None, only_pcc= False, feature_selection_file = None):
    
    if feature_selection_file == None:
        print("* correlation_analysis_all")
        pcc = data_df.corr()[target_col]

        if(len(pcc)<top_k):
            top_k=len(pcc)
        print("Computing PCC, PCC_SQRT ..")
        pcc = pcc.sort_values(ascending = False).dropna()
        pcc = pcc.rename("PCC")
        try:
            del pcc[target_col]
        except:
            pass
        pcc_sqrt = pcc.apply(lambda x: np.sqrt(x* x))
        pcc_sqrt = pcc_sqrt.sort_values(ascending = False).dropna()
        pcc_sqrt = pcc_sqrt.rename("PCC_SQRT")
        MICs = []
        MASs = []
        MEVs = []
        MCNs = []
        MCN_generals = []
        GMICs = []
        TICs = []
        print("Computing all other metrics ..")
        if only_pcc==False or only_pcc=='False':
            for col in data_df.columns:
                print(" - computing for ", col, "...")
                if col!=target_col:
                    x = data_df[col].values
                    y = data_df[target_col].values
                    mine = MINE()
                    mine.compute_score(x,y)
                    MICs.append((col,mine.mic()))
                    MASs.append((col,mine.mas()))
                    MEVs.append((col,mine.mev()))
                    MCNs.append((col,mine.mcn(0)))
                    MCN_generals.append((col,mine.mcn_general()))
                    GMICs.append((col,mine.gmic()))
                    TICs.append((col,mine.tic())) 
                
        top_k_pcc = list(pcc.keys())[:top_k]
        top_k_pcc_sqrt = list(pcc_sqrt.keys())[:top_k]
        top_k_mic = [tup[0] for tup in sorted(MICs, key=lambda tup: tup[1], reverse = True)[:top_k]]
        top_k_mas = [tup[0] for tup in sorted(MASs, key=lambda tup: tup[1], reverse = True)[:top_k]]
        top_k_mev = [tup[0] for tup in sorted(MEVs, key=lambda tup: tup[1], reverse = True)[:top_k]]
        top_k_mcn = [tup[0] for tup in sorted(MCNs, key=lambda tup: tup[1], reverse = True)[:top_k]]
        top_k_mcn_general = [tup[0] for tup in sorted(MCN_generals, key=lambda tup: tup[1], reverse = True)[:top_k]]
        top_k_gmic = [tup[0] for tup in sorted(GMICs, key=lambda tup: tup[1], reverse = True)[:top_k]]
        top_k_tic = [tup[0] for tup in sorted(TICs, key=lambda tup: tup[1], reverse = True)[:top_k]]
        
        mic_df = pd.DataFrame([tup[1] for tup in MICs],columns=['MIC'],index=[tup[0] for tup in MICs])
        mas_df = pd.DataFrame([tup[1] for tup in MASs],columns=['MAS'],index=[tup[0] for tup in MASs])
        mev_df = pd.DataFrame([tup[1] for tup in MEVs],columns=['MEV'],index=[tup[0] for tup in MEVs])
        mcn_df = pd.DataFrame([tup[1] for tup in MCNs],columns=['MCN'],index=[tup[0] for tup in MCNs])
        mcn_general_df = pd.DataFrame([tup[1] for tup in MCN_generals],columns=['MCN_general'],index=[tup[0] for tup in MCN_generals])
        gmic_df = pd.DataFrame([tup[1] for tup in GMICs],columns=['GMIC'],index=[tup[0] for tup in GMICs])
        tic_df = pd.DataFrame([tup[1] for tup in TICs],columns=['TIC'],index=[tup[0] for tup in TICs])       
        
        if only_pcc==False or only_pcc=='False':
            final_report = mic_df.join(mas_df).join(mev_df).join(mcn_df).join(mcn_general_df).join(gmic_df).join(tic_df).sort_index().join(pcc_sqrt).join(pcc)
        else:
            pcc_sqrt = pd.DataFrame(pcc_sqrt)
            final_report = pcc_sqrt.join(pcc)
        
        if file_to_save is not None:
            # save to correlation report
            final_report.to_csv(file_to_save)

        if save_chart is not None:
            for col in final_report.keys():
                ax = final_report[col].sort_values(ascending=False).plot(kind='bar',alpha=0.8)
                ax.set_ylabel(col+" (target_col = '"+target_col+"')", fontsize=12)
                plt.axhline(0, color='k')
                plt.savefig(save_chart)
                plt.close()

        fs_dict = {'PCC':top_k_pcc,'PCC_SQRT':top_k_pcc_sqrt,'MIC':top_k_mic,'MAS':top_k_mas,'MEV':top_k_mev,'MCN':top_k_mcn,'MCN_general':top_k_mcn_general,'GMIC':top_k_gmic,'TIC':top_k_tic}
    
    else:
        final_report = pd.read_csv(feature_selection_file)
        top_k_pcc = list(final_report.sort_values(by=['PCC'], ascending=False).T.values[0])[:top_k]
        top_k_pcc_sqrt = list(final_report.sort_values(by=['PCC_SQRT'], ascending=False).T.values[0])[:top_k]
        top_k_mic = list(final_report.sort_values(by=['MIC'], ascending=False).T.values[0])[:top_k]
        top_k_mas = list(final_report.sort_values(by=['MAS'], ascending=False).T.values[0])[:top_k]
        top_k_mev = list(final_report.sort_values(by=['MEV'], ascending=False).T.values[0])[:top_k]
        top_k_mcn = list(final_report.sort_values(by=['MCN'], ascending=False).T.values[0])[:top_k]
        top_k_mcn_general = list(final_report.sort_values(by=['MCN_general'], ascending=False).T.values[0])[:top_k]
        top_k_gmic = list(final_report.sort_values(by=['GMIC'], ascending=False).T.values[0])[:top_k]
        top_k_tic = list(final_report.sort_values(by=['TIC'], ascending=False).T.values[0])[:top_k]
        fs_dict = {'PCC':top_k_pcc,'PCC_SQRT':top_k_pcc_sqrt,'MIC':top_k_mic,'MAS':top_k_mas,'MEV':top_k_mev,'MCN':top_k_mcn,'MCN_general':top_k_mcn_general,'GMIC':top_k_gmic,'TIC':top_k_tic}
        
        if file_to_save is not None:
            # save to correlation report
            final_report.to_csv(file_to_save)

        if save_chart is not None:
            for col in final_report.keys()[1:]:
                ax = final_report[col].sort_values(ascending=False).plot(kind='bar',alpha=0.8)
                ax.set_ylabel(col+" (target_col = '"+target_col+"')", fontsize=12)
                plt.axhline(0, color='k')
                plt.savefig(save_chart)
                plt.close()

    return fs_dict, final_report

# In[6]:


def default_model_parameters_classifier():
    
    model_parameters = {
    'scaler_option':'StandardScaler', \
    'rf_n_estimators': '100', 'rf_max_features': 'auto', 'rf_max_depth': 'None', \
    'rf_min_samples_split': '2', 'rf_min_samples_leaf': '1', 'rf_bootstrap': 'True', \
    'rf_criterion':'gini','rf_min_weight_fraction_leaf':'0.','rf_max_leaf_nodes':'None',\
    'rf_min_impurity_decrease':'0.',\
    'nn_n_neighbors': '5', 'nn_weights': 'uniform', 'nn_algorithm': 'auto', 'nn_leaf_size': '30', 'nn_p': '2',\
    'nn_metric':'minkowski','nn_metric_params':'None',
    'rg_alpha':'1','rg_fit_intercept':'True','rg_normalize':'False','rg_max_iter':'None','rg_tol':'0.001','rg_class_weight':'None','rg_solver':'auto','svm_kernel': 'rbf', \
    'svm_degree': '3', 'svm_coef0': '0.0', 'svm_tol': '1e-3', 'svm_c': '1.0', \
    'svm_gamma': 'auto', \
    'svm_decision_function_shape':'ovr', \
    
    'net_structure':'16 16 16',\
    'net_layer_n':'3',\
    'net_dropout': '0.0',\
    'net_l_2': '0.01',\
    'net_learning_rate': '0.01',\
    'net_epochs': '100',\
    'net_batch_size': '2',\

    }

    return model_parameters 

def default_model_parameters():
    
    model_parameters = {
    'scaler_option':'StandardScaler', \
    'rf_n_estimators': '100', 'rf_max_features': 'auto', 'rf_max_depth': 'None', \
    'rf_min_samples_split': '2', 'rf_min_samples_leaf': '1', 'rf_bootstrap': 'True', \
    'rf_criterion':'mse','rf_min_weight_fraction_leaf':'0.','rf_max_leaf_nodes':'None',\
    'rf_min_impurity_decrease':'0.',\
    'nn_n_neighbors': '5', 'nn_weights': 'uniform', 'nn_algorithm': 'auto', 'nn_leaf_size': '30', 'nn_p': '2',\
    'nn_metric':'minkowski','nn_metric_params':'None',\
   
    'kr_alpha': '1', 'kr_kernel': 'linear', 'kr_gamma': 'None', 'kr_degree': '3', 'kr_coef0': '1', \
    
    'br_n_iter': '300', 'br_alpha_1': '1.2e-6', 'br_alpha_2': '1.e-6', 'br_tol': '1.e-3', \
    'br_lambda_1': '1.e-6', 'br_lambda_2': '1.e-6', 'br_compute_score': 'False', 'br_fit_intercept': 'True', \
    'br_normalize':'False',\
    
    'svm_kernel': 'rbf', \
    'svm_degree': '3', 'svm_coef0': '0.0', 'svm_tol': '1e-3', 'svm_c': '1.0', \
    'svm_epsilon': '0.1', 'svm_shrinking': 'True', 'svm_gamma': 'auto', \
    
    
    'net_structure':'16 16 16',\
    'net_layer_n':'3',\
    'net_dropout': '0.0',\
    'net_l_2': '0.01',\
    'net_learning_rate': '0.01',\
    'net_epochs': '100',\
    'net_batch_size': '2',\

    }

    return model_parameters 

def load_model_parameter_from_file(filename):

    config = configparser.RawConfigParser()
    config.read(filename)
    model_parameters = {}
    for key in config['HYPERPARAMETERS']:  
        model_parameters[key] = config["HYPERPARAMETERS"][key]
    
    return model_parameters

# In[7]:

def fix_value(val, val_type):
    if val is None or val=='None':
        return None
    elif val=='auto':
        return val
    else:
        if(val_type=='float'):
            return float(val)
        elif(val_type=='str'):
            return str(val)
        elif(val_type=='int'):
            return int(val)
        elif(val_type=='bool'):
            return str2bool(val)
        elif val_type=='PurePath':
            return purePath(val)
        else:
            return val

def define_model_classifier(model_type, model_parameters, x_header_size, random_state = None):

    if model_type == "LRC":
        model = Pipeline([
          ('classification', LogisticRegression())
        ])
    elif model_type == "RF":
        model = Pipeline([
          ('classification', RandomForestClassifier(n_estimators = int(model_parameters['rf_n_estimators']), 
                                                   max_features = fix_value(model_parameters['rf_max_features'],'int'), 
                                                   max_depth = fix_value(model_parameters['rf_max_depth'],'int'), 
                                                   min_samples_split = int(model_parameters['rf_min_samples_split']), 
                                                   min_samples_leaf = int(model_parameters['rf_min_samples_leaf']), 
                                                   bootstrap = str2bool(model_parameters['rf_bootstrap']), 
                                                   criterion = model_parameters['rf_criterion'], 
                                                   random_state = random_state,
                                                   min_weight_fraction_leaf = float(model_parameters['rf_min_weight_fraction_leaf']), 
                                                   max_leaf_nodes = fix_value(model_parameters['rf_max_leaf_nodes'],'int'),
                                                   min_impurity_decrease = float(model_parameters['rf_min_impurity_decrease']), 
                                                  ))])
    elif model_type == "NN":
        model = Pipeline([
          ('classification', KNeighborsClassifier(n_neighbors = int(model_parameters['nn_n_neighbors']),
                                                 weights = model_parameters['nn_weights'],
                                                 algorithm = model_parameters['nn_algorithm'],
                                                 leaf_size = int(model_parameters['nn_leaf_size']),
                                                 metric = model_parameters['nn_metric'],
                                                 metric_params = fix_value(model_parameters['nn_metric_params'],'str'),
                                                 p = int(model_parameters['nn_p'])))
        ])

    elif model_type == "RG":
        model = Pipeline([
          ('classification', RidgeClassifier(alpha = float(model_parameters['rg_alpha']),
                                            fit_intercept = fix_value(model_parameters['rg_fit_intercept'],'bool'),
                                            normalize = fix_value(model_parameters['rg_normalize'],'bool'),
                                            max_iter = fix_value(model_parameters['rg_max_iter'],'int'),
                                            tol = float(model_parameters['rg_tol']),
                                            class_weight = fix_value(model_parameters['rg_class_weight'],'str'),
                                            solver = fix_value(model_parameters['rg_solver'],'str')))                                                                              
        ])
    elif model_type == "SVM":
        model = Pipeline([
          ('classification', svm.SVC(kernel = model_parameters['svm_kernel'],
                                     degree = int(model_parameters['svm_degree']),
                                     coef0 = float(model_parameters['svm_coef0']),
                                     tol = float(model_parameters['svm_tol']),
                                     C = float(model_parameters['svm_c']),
                                     gamma = fix_value(model_parameters['svm_gamma'],'float'),
                                     decision_function_shape = model_parameters['svm_decision_function_shape']
                                     ))
        ])
        
    return model

def define_model_regression(model_type, model_parameters, x_header_size, random_state = None):

    if model_type == "LR":
        model = Pipeline([
          ('regression', LinearRegression())
        ])
    elif model_type == "RF":
        model = Pipeline([
          ('regression', RandomForestRegressor(n_estimators = int(model_parameters['rf_n_estimators']), 
                                                   max_features = fix_value(model_parameters['rf_max_features'],'int'), 
                                                   max_depth = fix_value(model_parameters['rf_max_depth'],'int'), 
                                                   min_samples_split = int(model_parameters['rf_min_samples_split']), 
                                                   min_samples_leaf = int(model_parameters['rf_min_samples_leaf']), 
                                                   bootstrap = str2bool(model_parameters['rf_bootstrap']), 
                                                   criterion = model_parameters['rf_criterion'], random_state = random_state,
                                                   min_weight_fraction_leaf = float(model_parameters['rf_min_weight_fraction_leaf']), 
                                                   max_leaf_nodes = fix_value(model_parameters['rf_max_leaf_nodes'],'int'),
                                                   min_impurity_decrease = float(model_parameters['rf_min_impurity_decrease']), 
                                                  ))])
    elif model_type == "NN":
        model = Pipeline([
          ('regression', KNeighborsRegressor(n_neighbors = int(model_parameters['nn_n_neighbors']),
                                                 weights = model_parameters['nn_weights'],
                                                 algorithm = model_parameters['nn_algorithm'],
                                                 leaf_size = int(model_parameters['nn_leaf_size']),
                                                 metric = model_parameters['nn_metric'],
                                                 metric_params = fix_value(model_parameters['nn_metric_params'],'str'),
                                                 p = int(model_parameters['nn_p'])))
        ])

    elif model_type == "BR":
        model = Pipeline([
          ('regression', linear_model.BayesianRidge(n_iter = int(model_parameters['br_n_iter']),
                                                        alpha_1 = float(model_parameters['br_alpha_1']),
                                                        alpha_2 = float(model_parameters['br_alpha_2']),
                                                        tol = float(model_parameters['br_tol']),
                                                        lambda_1 = float(model_parameters['br_lambda_1']),
                                                        lambda_2 = float(model_parameters['br_lambda_2']),
                                                        compute_score = fix_value(model_parameters['br_compute_score'],'bool'),
                                                        normalize = fix_value(model_parameters['br_normalize'],'bool'),
                                                        fit_intercept = fix_value(model_parameters['br_fit_intercept'],'bool')))
        ])
    elif model_type == "SVM":
        model = Pipeline([
          ('regression', svm.SVR(kernel = model_parameters['svm_kernel'],
                                     degree = int(model_parameters['svm_degree']),
                                     coef0 = float(model_parameters['svm_coef0']),
                                     tol = float(model_parameters['svm_tol']),
                                     C = float(model_parameters['svm_c']),
                                     gamma = fix_value(model_parameters['svm_gamma'],'float'),
                                     epsilon = float(model_parameters['svm_epsilon']),
                                     ))
        ])
        # 'kr_alpha': '1', 'kr_kernel': 'linear', 'kr_gamma': 'None', 'kr_degree': '3', 'kr_coef0': '1', \
    elif model_type == "KR":
        model = Pipeline([
        ('regression', KernelRidge(alpha = int(model_parameters['kr_alpha']),
                                        kernel = fix_value(model_parameters['kr_kernel'],'str'),
                                        gamma = fix_value(model_parameters['kr_gamma'],'str'),
                                        degree = int(model_parameters['kr_degree']),
                                        coef0 = int(model_parameters['kr_coef0']),))  
        ])
        
    return model


# In[8]:

def rescale_x(scaler_option, x_train):
    scale = None
    if scaler_option=='False':
        x_train_ = x_train
    elif scaler_option == "MinMaxScaler":
        scale = preprocessing.MinMaxScaler()
        x_train_ = scale.fit_transform(x_train)
    elif scaler_option == "MaxAbsScaler":
        scale = preprocessing.MaxAbsScaler()
        x_train_ = scale.fit_transform(x_train)
    elif scaler_option == "RobustScaler":
        scale = preprocessing.RobustScaler()
        x_train_ = scale.fit_transform(x_train)
    elif scaler_option == "QuantileTransformer":
        scale = preprocessing.QuantileTransformer()
        x_train_ = scale.fit_transform(x_train)
    elif scaler_option == "Normalizer":
        scale = preprocessing.Normalizer()
        x_train_ = scale.fit_transform(x_train)
    else:
        scale = preprocessing.StandardScaler()
        x_train_ = scale.fit_transform(x_train)
    return x_train_, scale

# In[9]:

def str2bool(v):
  return v.lower() in ("yes", "true", "t", "1")


def cross_val_predict_net_classifier(model, x_train, y_train, epochs=1000, batch_size=8, verbose = 0, scaler_option='StandardScaler', num_of_folds = 5, num_of_class = 2, force_to_proceed = False, accuracy_threshold = 0.5, fast_tune = True):
    
    x_trains, y_trains, x_tests, y_tests = split_data(x_train, y_train, num_of_folds=num_of_folds)
    
    predictions_total = []
    actual_values_total = []

    for j in range(0, num_of_folds):
        print(" Evaluating fold(%d) ..."%(j))
        start_time = time.time()
        
        x_train_, scale = rescale_x(scaler_option, x_trains[j]) 
        
        # This is the change
        if scale is not None:
            x_test_ = scale.transform(x_tests[j])
        else:
            x_test_ = x_tests[j]

        dummy_y = keras.utils.to_categorical(y_trains[j], num_classes=num_of_class, dtype='float32')
        
        history = model.fit(x_train_, dummy_y,
                            batch_size=batch_size,
                            epochs=epochs,
                            verbose=verbose)
        
        predictions  = model.predict_classes(x_test_)
        actual_values = y_tests[j]
        actual_values = actual_values.reshape(actual_values.shape[0],)
        
        accuracy= evaluate_classifier(predictions, actual_values)
        
        if force_to_proceed == False:
            if accuracy<accuracy_threshold:
                return [],[]

        print(" accuracy = %8.3f "%(accuracy))

        predictions_total+=list(predictions)
        actual_values_total+=list(actual_values)
        
        if fast_tune==True:
            print("* Fast tuning enabled. so we only test 1 fold, and move on ..")
            break

    return np.array(predictions_total), np.array(actual_values_total)


def cross_val_predict_net(model, x_train, y_train, epochs=1000, batch_size=8, verbose = 0, scaler_option='StandardScaler', num_of_folds = 5, force_to_proceed= False, fast_tune=True):
    
    x_trains, y_trains, x_tests, y_tests = split_data(x_train, y_train, num_of_folds=num_of_folds)
    
    predictions_total = []
    actual_values_total = []

    for j in range(0, num_of_folds):
        print(" Evaluating fold(%d) ..."%(j))
        start_time = time.time()
        
        x_train_, scale = rescale_x(scaler_option, x_trains[j]) 

        # This is the change
        if scale is not None:
            x_test_ = scale.transform(x_tests[j])
        else:
            x_test_ = x_tests[j]
        
        history = model.fit(x_train_, y_trains[j],
                            batch_size=batch_size,
                            epochs=epochs,
                            verbose=verbose)

        predictions  = model.predict(x_test_)
        actual_values = y_tests[j]
        
        MAE, R2 = evaluate(predictions, actual_values)
        print(" MAE = %8.3f R2 = %8.3f ..."%(MAE, R2))

        if force_to_proceed == False:
            if R2<0:
                return [],[]

        predictions_total+=list(predictions)
        actual_values_total+=list(actual_values)
        if fast_tune==True:
            print("* Fast tuning enabled. so we only test 1 fold, and move on ..")
            break

    return np.array(predictions_total), np.array(actual_values_total)

def save_parameters(model_parameters, filename):
    f = open(filename,'w')
    f.write("[HYPERPARAMETERS]\n\n")
    for key in model_parameters.keys():
        f.write(str(key)+" = "+str(model_parameters[key])+"\n")
    f.close()

def save_args(model_args,filename):
    f = open(filename,'w')
    f.write("[ARGUMENTS]\n\n")
    for key in model_args.keys():
        f.write(str(key)+" = "+str(model_args[key])+"\n")
    f.close()

def save_metadata(model_args, model_stats,filepath):
    if path.exists(filepath):
        meta=pd.read_csv(filepath,index_col=0)
    else:
        meta=pd.DataFrame(columns=["session","model_args","model_stats"])
    size=len(meta.index)+1
    meta=meta.append(pd.DataFrame(data=[[size,model_args,model_stats]],columns=["session","model_args","model_stats"]))
    meta.to_csv(filepath)
    return size
        

def train_and_predict(model, x_train, y_train, scaler_option, num_of_folds=5):
    
    x_train_, scale = rescale_x(scaler_option, x_train)
    y_train_ = y_train.reshape(y_train.shape[0],)
    predictions = cross_val_predict(model, x_train_, y_train_, cv=num_of_folds)
    actual_values = y_train_
    
    return predictions, actual_values

def get_session(project_file):
    if path.exists(project_file / "metadata.csv"):
        meta=pd.read_csv(project_file / "metadata.csv")
    else:
        return 1
    return len(meta.index)+1

# In[10]:

def train_and_save_net_classifier(model, tag, input_cols, target_col, x_train, y_train, scaler_option, accuracy=None, path_to_save = '.', num_of_folds=5, epochs=100, batch_size=2, num_of_class = 2):
        
    if accuracy is None:
        
        print('* Model has not been evaluated. Evaluation initiated via %d-fold cross validation'%(num_of_folds))
        
        predictions, actual_values = cross_val_predict_net_classifier(model, epochs=epochs, batch_size=batch_size, x_train = x_train, y_train = y_train, verbose = 0, scaler_option = scaler_option, num_of_folds = num_of_folds, num_of_class = num_of_class, fast_tune = False)
        accuracy = evaluate_classifier(predictions, actual_values)

    x_train_, scale = rescale_x(scaler_option, x_train)
    
    dummy_y = keras.utils.to_categorical(y_train, num_classes=num_of_class, dtype='float32')

    print('* Training initiated ..')
    
    model.fit(x_train_, dummy_y, epochs=epochs, batch_size=batch_size)
    print('* Training done.')
    
    model_dict = {}
    model_dict['tag'] = tag
    model_dict['model'] = model
    model_dict['model_abbr'] = 'NET'
    model_dict['input_cols'] = input_cols
    model_dict['target_col'] = target_col
    model_dict['accuracy'] = accuracy
    model_dict['fitted_scaler_x'] = scale
    output_file = PurePath(path_to_save) / (tag)
    #print(model_dict)
    
    print("* Trained model saved to file:", str(output_file))
    
    output = open(output_file, 'wb')
    pickle.dump(model_dict, output)

def train_and_save_net(model, tag, input_cols, target_col, x_train, y_train, scaler_option, MAE=None, R2=None, path_to_save = '.', num_of_folds=5, epochs=100, batch_size=2):
        
    if MAE is None or R2 is None:
        
        print('* Model has not been evaluated. Evaluation initiated via %d-fold cross validation'%(num_of_folds))
        
        predictions, actual_values = cross_val_predict_net(model, epochs=epochs, batch_size=batch_size, x_train = x_train, y_train = y_train, verbose = 0, scaler_option = scaler_option, fast_tune = False)
        MAE, R2 = evaluate(predictions, actual_values)

    x_train_, scale = rescale_x(scaler_option, x_train)
    print('* Training initiated ..')
    model.fit(x_train_, y_train, epochs=epochs, batch_size=batch_size)
    print('* Training done.')
    
    model_dict = {}
    model_dict['tag'] = tag
    model_dict['model'] = model
    model_dict['model_abbr'] = 'NET'
    model_dict['input_cols'] = input_cols
    model_dict['target_col'] = target_col
    model_dict['MAE'] = MAE
    model_dict['R2'] = R2
    model_dict['fitted_scaler_x'] = scale
    output_file = PurePath(path_to_save) / (tag)
    #print(model_dict)
    
    print("* Trained model saved to file:", str(output_file))
    
    output = open(output_file, 'wb')
    pickle.dump(model_dict, output)

def train_and_save_classifier(model, tag, model_abbr, input_cols, target_col, x_train, y_train, scaler_option, accuracy=None, path_to_save = '.', num_of_folds=5):
    
    x_train_, scale = rescale_x(scaler_option, x_train)
    y_train_ = y_train.reshape(y_train.shape[0],)
    
    if accuracy is None:
        print('* Model has not been evaluated. Evaluation initiated via %d-fold cross validation'%(num_of_folds))
        
        predictions = cross_val_predict(model, x_train_, y_train_, cv=num_of_folds)
        actual_values = y_train_

        accuracy = evaluate_classifier(predictions, actual_values)

    print('* Training initiated ..')
    model.fit(x_train_, y_train_)
    print('* Training done.')
    actual_values = y_train_
    
    model_dict = {}
    model_dict['tag'] = tag
    model_dict['model'] = model
    model_dict['model_abbr'] = model_abbr
    model_dict['input_cols'] = input_cols
    model_dict['target_col'] = target_col
    model_dict['accuracy'] = accuracy
    model_dict['fitted_scaler_x'] = scale
    output_file = PurePath(path_to_save) / (tag)
    #print(model_dict)
    
    print("* Trained model saved to file:", str(output_file))
    
    output = open(output_file, 'wb')
    pickle.dump(model_dict, output)

def train_and_save(model, tag, model_abbr, input_cols, target_col, x_train, y_train, scaler_option, MAE=None, R2=None, path_to_save = '.', num_of_folds=5):
    
    x_train_, scale = rescale_x(scaler_option, x_train)
    y_train_ = y_train.reshape(y_train.shape[0],)
    
    if MAE is None or R2 is None:
        print('* Model has not been evaluated. Evaluation initiated via %d-fold cross validation'%(num_of_folds))
        
        predictions = cross_val_predict(model, x_train_, y_train_, cv=num_of_folds)
        actual_values = y_train_

        MAE, R2 = evaluate(predictions, actual_values)

    print('* Training initiated ..')
    model.fit(x_train_, y_train_)
    print('* Training done.')
    actual_values = y_train_
    
    model_dict = {}
    model_dict['tag'] = tag
    model_dict['model'] = model
    model_dict['model_abbr'] = model_abbr
    model_dict['input_cols'] = input_cols
    model_dict['target_col'] = target_col
    model_dict['MAE'] = MAE
    model_dict['R2'] = R2
    model_dict['fitted_scaler_x'] = scale
    output_file = PurePath(path_to_save) / ((tag))
    #print(model_dict)
    
    print("* Trained model saved to file:", str(output_file))
    
    output = open(output_file, 'wb')
    pickle.dump(model_dict, output)

def evaluate_classifier(predictions, actual_values):

    correct = 0
    wrong = 0
    for i in range(0,len(predictions)):
        if predictions[i]==actual_values[i]:
            correct+=1
        else:
            wrong+=1
    
    accuracy = float(correct)/(float(correct)+float(wrong))

    return accuracy

def evaluate(predictions, actual_values):

    try:
        MAE = mean_absolute_error(predictions,actual_values)
        R2 = r2_score(actual_values, predictions, multioutput='variance_weighted')
    except Exception as e:
        MAE = -1
        R2 = -1

    return MAE, R2


def save_comparison_chart(predictions, actual_values, filename):
    plt.close()
    min_val = min(predictions)
    max_val = max(actual_values)

    plt.xlabel('Predicted Value')
    plt.ylabel('Actual Value')
    plt.ylim([min_val, max_val])
    plt.xlim([min_val, max_val])
    plt.grid(True)

    plt.scatter(predictions,actual_values)
    t = np.arange(min_val, max_val, 0.01)
    line, = plt.plot(t, t, lw=1)
    
    if type(filename)==str:
        filename = PurePath(filename)

    if not os.path.exists(filename.parent): os.makedirs(filename.parent)
    plt.savefig(filename)
    plt.close()


# In[13]:

def add_key_to_params(tag, params):
    tag = tag.lower()
    parameters = {}
    for key in params.keys():
        parameters[(tag+'_'+key).lower()] = params[key]
    return parameters


# In[14]:

def hyperparameter_tuning_classifier(tag, x_train, y_train, num_of_folds, scaler_option, n_iter=100, random_state=0, verbose=1):
    
    rf_n_estimators = [int(x) for x in np.linspace(start = 10, stop = 1000, num = 20)]
    rf_max_features = list(range(1,x_train.shape[1]))
    rf_max_depth = [int(x) for x in np.linspace(1, 32, 32)]
    rf_max_depth.append(None)
    rf_min_samples_split = [int(x) for x in np.linspace(start = 2, stop = 15, num = 20)]
    rf_min_samples_leaf = [int(x) for x in np.linspace(start = 2, stop = 15, num = 20)]
    rf_bootstrap = ['True', 'False']
    rf_criterion = ['gini']
    rf_min_weight_fraction_leaf = [float(x) for x in np.linspace(start = 0, stop = 1.e-5, num = 10)]
    rf_max_leaf_nodes = [2, 5, 10, 50, 100]
    rf_min_impurity_decrease = [float(x) for x in np.linspace(start = 0, stop = 1.e-5, num = 10)]
    
    rf_random_grid = {'n_estimators': rf_n_estimators,
                            'max_features': rf_max_features,
                            'max_depth': rf_max_depth,
                            'min_samples_split': rf_min_samples_split,
                            'min_samples_leaf': rf_min_samples_leaf,
                            'bootstrap': rf_bootstrap,
                            'criterion': rf_criterion,
                            'min_weight_fraction_leaf': rf_min_weight_fraction_leaf,
                            'max_leaf_nodes': rf_max_leaf_nodes,
                            'min_impurity_decrease': rf_min_impurity_decrease}

    nn_n_neighbors = [int(x) for x in np.linspace(start = 2, stop = 15, num = 10)]
    nn_weights = ['uniform', 'distance']
    nn_algorithm = ['auto','ball_tree','kd_tree','brute']
    nn_leaf_size = [1,2,3,4,5]
    nn_p = [int(x) for x in np.linspace(start = 1, stop = 5, num = 5)]
    nn_metric = ['minkowski']
    nn_metric_params = [None]

    nn_random_grid = {'n_neighbors': nn_n_neighbors,
                'weights': nn_weights,
                'algorithm': nn_algorithm,
                'leaf_size': nn_leaf_size,
                'metric': nn_metric,
                'metric_params': nn_metric_params,
                'p': nn_p}

    rg_alpha = [float(x) for x in np.linspace(start = 0, stop = 10, num = 10)]
    rg_fit_intercept = ['True', 'False']
    rg_max_iter = [100, 500, 1000, None]
    rg_tol = [float(x) for x in np.linspace(start = 1.e-5, stop = 1.e-2, num = 20)]
    rg_class_weight = [None,'balanced']
    rg_normalize = ['True', 'False']
    rg_solver = ['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga']
    
    rg_random_grid = {'max_iter': rg_max_iter,
                'alpha': rg_alpha,
                'fit_intercept': rg_fit_intercept,
                'tol': rg_tol,
                'class_weight': rg_class_weight,
                'normalize': rg_normalize,
                'solver': rg_solver
                }

    svm_kernel = ['rbf', 'poly','linear','sigmoid']           
    svm_gamma = ['auto', 0.001, 0.01, 0.1, 1]
    svm_degree = [1, 2, 3]
    svm_coef0 = [0, 1, 2, 3]
    svm_tol = [float(x) for x in np.linspace(start = 1.e-4, stop = 1.e-2, num = 20)]
    #svm_C = [float(x) for x in np.linspace(start = 0.001, stop = 3000, num = 100)]
    svm_C = [0.001, 0.01, 0.1, 1, 10]
    svm_decision_function_shape = ['ovr','ovo']    
    svm_random_grid = {'kernel': svm_kernel,
                'degree': svm_degree,
                'gamma' : svm_gamma,
                'coef0': svm_coef0,
                'tol': svm_tol,
                'C' : svm_C,
                'decision_function_shape':svm_decision_function_shape}

    if tag=='RF':
        estimator = RandomForestClassifier()
        random_grid = rf_random_grid
    elif tag=='NN':
        estimator = KNeighborsClassifier()
        random_grid = nn_random_grid
    elif tag=='RG':
        estimator = RidgeClassifier()
        random_grid = rg_random_grid
    elif tag=='SVM':
        estimator = svm.SVC()
        random_grid = svm_random_grid
    else:
        estimator = None
    
    tuned_parameters = None
    if estimator is not None:
        model = RandomizedSearchCV(
                estimator = estimator, 
                param_distributions = random_grid, 
                n_iter = n_iter, 
                cv = num_of_folds, 
                verbose=verbose, 
                random_state=random_state, 
                n_jobs = -1)
        x_train_, scale = rescale_x(scaler_option, x_train)
        y_train_ = y_train.reshape(y_train.shape[0],)

        model.fit(x_train_, y_train_)

        tuned_parameters = add_key_to_params(tag, model.best_params_)
        tuned_parameters['scaler_option'] = scaler_option

    return tuned_parameters

def hyperparameter_tuning(tag, x_train, y_train, num_of_folds, scaler_option, n_iter=100, random_state=0, verbose=1):
    
    rf_n_estimators = [int(x) for x in np.linspace(start = 10, stop = 1000, num = 20)]
    rf_max_features = list(range(1,x_train.shape[1]))
    rf_max_depth = [int(x) for x in np.linspace(1, 32, 32)]
    rf_max_depth.append(None)
    rf_min_samples_split = [int(x) for x in np.linspace(start = 2, stop = 15, num = 20)]
    rf_min_samples_leaf = [int(x) for x in np.linspace(start = 2, stop = 15, num = 20)]
    rf_bootstrap = ['True', 'False']
    rf_criterion = ['mse']
    rf_min_weight_fraction_leaf = [float(x) for x in np.linspace(start = 0, stop = 1.e-5, num = 10)]
    rf_max_leaf_nodes = [2, 5, 10, 50, 100]
    rf_min_impurity_decrease = [float(x) for x in np.linspace(start = 0, stop = 1.e-5, num = 10)]
    
    rf_random_grid = {'n_estimators': rf_n_estimators,
                            'max_features': rf_max_features,
                            'max_depth': rf_max_depth,
                            'min_samples_split': rf_min_samples_split,
                            'min_samples_leaf': rf_min_samples_leaf,
                            'bootstrap': rf_bootstrap,
                            'criterion': rf_criterion,
                            'min_weight_fraction_leaf': rf_min_weight_fraction_leaf,
                            'max_leaf_nodes': rf_max_leaf_nodes,
                            'min_impurity_decrease': rf_min_impurity_decrease}

    nn_n_neighbors = [int(x) for x in np.linspace(start = 2, stop = 15, num = 10)]
    nn_weights = ['uniform', 'distance']
    nn_algorithm = ['auto','ball_tree','kd_tree','brute']
    nn_leaf_size = [1,2,3,4,5]
    nn_p = [int(x) for x in np.linspace(start = 1, stop = 5, num = 5)]
    nn_metric = ['minkowski']
    nn_metric_params = [None]

    nn_random_grid = {'n_neighbors': nn_n_neighbors,
                'weights': nn_weights,
                'algorithm': nn_algorithm,
                'leaf_size': nn_leaf_size,
                'metric': nn_metric,
                'metric_params': nn_metric_params,
                'p': nn_p}

    kr_alpha = [float(x) for x in np.linspace(start = 0, stop = 10, num = 50)]
    kr_gamma = [None, 'RBF', 'laplacian','polynomial','chi2','exponential','sigmoid']
    kr_degree = [1,2,3]
    kr_coef0 = [0,1,2,3,4,5]
    kr_kernel =  ['linear']
        
    kr_random_grid = {'alpha': kr_alpha,
                      'kernel': kr_kernel,
                'gamma': kr_gamma,
                'degree': kr_degree,
                'coef0': kr_coef0}

    br_n_iter = [int(x) for x in np.linspace(start = 100, stop = 1000, num = 50)]
    br_alpha_1 = [float(x) for x in np.linspace(start = 1.e-7, stop = 1.e-4, num = 50)]
    br_alpha_2 = [float(x) for x in np.linspace(start = 1.e-7, stop = 1.e-4, num = 50)]
    br_tol = [float(x) for x in np.linspace(start = 1.e-2, stop = 1.e-4, num = 50)]
    br_lambda_1 = [float(x) for x in np.linspace(start = 1.e-7, stop = 1.e-4, num = 50)]
    br_lambda_2 = [float(x) for x in np.linspace(start = 1.e-7, stop = 1.e-4, num = 50)]
    br_compute_score = ['True', 'False']
    br_fit_intercept = ['True', 'False']
    br_normalize = ['True', 'False']
    
    br_random_grid = {'n_iter': br_n_iter,
                'alpha_1': br_alpha_1,
                'alpha_2': br_alpha_2,
                'tol': br_tol,
                'lambda_1': br_lambda_1,
                'lambda_2': br_lambda_2,
                'compute_score': br_compute_score,
                'fit_intercept': br_fit_intercept,
                'normalize':br_normalize}

    svm_kernel = ['rbf', 'poly','linear','sigmoid']           
    svm_gamma = ['auto', 0.001, 0.01, 0.1, 1]
    svm_degree = [1, 2, 3]
    svm_coef0 = [0, 1, 2, 3]
    svm_tol = [float(x) for x in np.linspace(start = 1.e-4, stop = 1.e-2, num = 50)]
    #svm_C = [float(x) for x in np.linspace(start = 0.001, stop = 3000, num = 100)]
    #svm_C+=[0.001, 0.01, 0.1, 1]
    svm_C = [0.001, 0.01, 0.1, 1, 10]
    svm_epsilon = [0.01, 0.1, 0.2, 0.3]
    
    svm_random_grid = {'kernel': svm_kernel,
                'degree': svm_degree,
                'gamma' : svm_gamma,
                'coef0': svm_coef0,
                'tol': svm_tol,
                'C': svm_C,
                'epsilon': svm_epsilon}

    if tag=='RF':
        estimator = RandomForestRegressor()
        random_grid = rf_random_grid
    elif tag=='NN':
        estimator = KNeighborsRegressor()
        random_grid = nn_random_grid
    elif tag=='KR':
        estimator = KernelRidge()
        random_grid = kr_random_grid
    elif tag=='BR':
        estimator = linear_model.BayesianRidge()
        random_grid = br_random_grid
    elif tag=='SVM':
        estimator = svm.SVR()
        random_grid = svm_random_grid
    else:
        estimator = None
    
    tuned_parameters = None
    if estimator is not None:
        model = RandomizedSearchCV(
                estimator = estimator, 
                param_distributions = random_grid, 
                n_iter = n_iter, 
                cv = num_of_folds, 
                verbose=verbose, 
                random_state=random_state, 
                n_jobs = -1)
        x_train_, scale = rescale_x(scaler_option, x_train)
        y_train_ = y_train.reshape(y_train.shape[0],)

        model.fit(x_train_, y_train_)

        tuned_parameters = add_key_to_params(tag, model.best_params_)
        tuned_parameters['scaler_option'] = scaler_option

    return tuned_parameters

def split_data(x_train, y_train, num_of_folds=5):
    
    num = x_train.shape[0]/int(num_of_folds)
    x_train_parts = []
    y_train_parts = []
    for i in range(0,int(num_of_folds)):
        start_idx = int(i*num+1) -1
        end_idx = int(num*(i+1))
        if i==int(num_of_folds)-1:
            x_train_parts.append(x_train[start_idx:])
            y_train_parts.append(y_train[start_idx:])
        else:
            x_train_parts.append(x_train[start_idx:end_idx])
            y_train_parts.append(y_train[start_idx:end_idx])

    x_trains = []
    y_trains = []
    x_tests = []
    y_tests = []
    
    for i in range(0, num_of_folds):
        x_test = x_train_parts[i]
        y_test = y_train_parts[i]
        x_train = []
        y_train = []
        for j in range(0, num_of_folds):
            if j!=i:
                x_train+=list(x_train_parts[j])
                y_train+=list(y_train_parts[j])
        
        x_trains.append(np.array(x_train))
        y_trains.append(np.array(y_train))
        x_tests.append(np.array(x_test))
        y_tests.append(np.array(y_test))
        
    return x_trains, y_trains, x_tests, y_tests

def net_define(params = [8, 8, 8], layer_n = 3, input_size = 16, dropout=0, l_2=0.01, optimizer='adam', random_state = 0):
    
    if len(params)!=layer_n or layer_n<1:
        return None
    
    model = Sequential()
    model.add(Dense(params[0], kernel_initializer='normal', activation='relu', input_dim=input_size, kernel_regularizer=regularizers.l2(l_2)))
    
    for i in range(1, layer_n):
        if dropout!=0:
            model.add(Dropout(dropout))
        model.add(Dense(params[i], kernel_initializer='normal', activation='relu', kernel_regularizer=regularizers.l2(l_2)))
    
    model.add(Dense(1, activation = 'linear'))

    model.compile(loss='mse',
                  optimizer=optimizer, metrics=['mape'])
    
    print(params, layer_n, dropout, l_2, optimizer)
    
    return model

def net_define_classifier(params = [8, 8, 8], layer_n = 3, num_of_class = 2, input_size = 16, dropout=0, l_2=0.01, optimizer='adam', random_state = 0):
    
    if len(params)!=layer_n or layer_n<1:
        return None
    
    model = Sequential()
    model.add(Dense(params[0], kernel_initializer='normal', activation='relu', input_dim=input_size, kernel_regularizer=regularizers.l2(l_2)))
    
    for i in range(1, layer_n):
        if dropout!=0:
            model.add(Dropout(dropout))
        model.add(Dense(params[i], kernel_initializer='normal', activation='relu', kernel_regularizer=regularizers.l2(l_2)))
    
    model.add(Dense(num_of_class, activation = 'softmax'))

    model.compile(loss='categorical_crossentropy',
                  optimizer=optimizer, metrics=['accuracy'])
    
    return model

def evaluate_net(model, x_train, y_train, x_test, y_test, epochs=1000, batch_size=8, verbose = 0):
    if optimizer is None:
        optimizer = 'adam'
    
    history = model.fit(x_train, y_train,
                        batch_size=batch_size,
                        epochs=epochs,
                        verbose=verbose,
                        validation_data=(x_test, y_test))

    score = model.evaluate(x_test, y_test, verbose=0)
    
    return score, history

def net_tuning_classifier(x_train, y_train, num_of_class = 2, tries = 10, lr = None, layer = None, params=None, epochs=None, batch_size=None, dropout=None, l_2 = None, neuron_max=[64, 64, 64], batch_size_max=32, layer_min=1, layer_max=3, dropout_max=0.2, scaler_option='StandardScaler', default_neuron_max=32, checkpoint = None, num_of_folds=5, fast_tune = True, random_state = 0):
    tuned_parameters = {}

    if layer is not None:
        
        if layer<1:
            print("Error: layer must be >=1")
            sys.exit()
    
    # Trying to tune hyperparameters
    
    best_score = 0
    best_params = None

    _layer = layer
    _params = params
    _epochs = epochs
    _batch_size = batch_size
    _dropout = dropout
    _l_2 = l_2
    _neuron_max = neuron_max
    _batch_size_max = batch_size_max
    _dropout_max = dropout_max
    _lr = lr
    
    for i in range(0, tries):

        optimizer = 'adam'
        
        if lr is None:
            lr = 10**np.random.uniform(-4,-3)
            optimizer = keras.optimizers.Adam(lr=lr)
        else:
            optimizer = keras.optimizers.Adam(lr=lr)

        if layer is None:
            layer = random.sample(range(layer_min, layer_max+1), 1)[0]

        if params is None:
            params = []
            for j in range(0, layer):
                try:
                    params.append(random.sample(range(1, neuron_max[j]), 1)[0])
                except:
                    params.append(random.sample(range(1, default_neuron_max), 1)[0])

        if epochs is None:
            epochs = int(10**np.random.uniform(2,3)) # 100 - 1000

        if batch_size is None:
            batch_size = random.sample(range(1, batch_size_max), 1)[0]

        if dropout is None:
            dropout = 0
        elif dropout is True:
            dropout = random.uniform(0,dropout_max)
        else:
            dropout = dropout

        if l_2 is None:
            l_2 = 10**np.random.uniform(-3,-1)

        model = net_define_classifier(params=params, layer_n = layer, input_size = x_train.shape[1], dropout=dropout, l_2=l_2, optimizer=optimizer, num_of_class = num_of_class, random_state = random_state)  
        
        print("\n Cross-validation (iteration=%d): [layer=%d, structure=[%s], epochs=%d, dropout=%8.4f, l_2=%8.7f, batch_size=%d, lr=%8.7f]"%(i, layer, params, epochs, dropout, l_2, batch_size, lr))
        start_time = time.time()
        predictions, actual_values = cross_val_predict_net_classifier(model, epochs=epochs, batch_size=batch_size, x_train = x_train, y_train = y_train, verbose = 0, scaler_option = scaler_option, num_of_folds = num_of_folds, num_of_class = num_of_class, fast_tune = fast_tune)
        
        if predictions == []:
            print(" Validation stopped early with the setting:","[layer=%d, structure=[%s], epochs=%d, dropout=%8.4f, l_2=%8.7f, batch_size=%d, lr=%8.7f]"%(layer, params, epochs, dropout, l_2, batch_size, lr))
            print(' Keep trying to find best settings .., took %ss'%(time.time()-start_time))
            accuracy= -1
        else:
            accuracy = evaluate_classifier(predictions, actual_values)
        
        print("Cross validation result - accuracy = %8.3f, took %ss "%(accuracy, time.time()-start_time))

        if(accuracy>best_score):          
            
            best_score = accuracy
            best_params = (layer, params, epochs, dropout, l_2, batch_size, lr)
            tuned_parameters = {"net_layer_n":best_params[0], \
            "net_structure":str(best_params[1])[1:-1].replace(",",""), \
            "net_epochs":best_params[2], \
            "net_dropout":best_params[3], \
            "net_l_2":best_params[4], \
            "net_batch_size":best_params[5], \
            "net_learning_rate": best_params[6]}

            if checkpoint is not None:
                print("Best so far parameters stored :", str(checkpoint)+",Model=NET,Scaler="+str(scaler_option)+",accuracy="+str(accuracy)+".tuned.checkpoint.prop")
                save_parameters(tuned_parameters, str(checkpoint)+",Model=NET,Scaler="+str(scaler_option)+",accuracy="+str(accuracy)+".tuned.checkpoint.prop")
        
        if(best_score!=0):
            print("Best accuracy = %8.3f"%(best_score),"[layer=%d, structure=[%s], epochs=%d, dropout=%8.4f, l_2=%8.7f, batch_size=%d, lr=%8.7f]"%best_params)

        # set to original values
        layer = _layer
        params = _params
        epochs = _epochs
        batch_size = _batch_size
        dropout = _dropout
        l_2 = _l_2
        neuron_max = _neuron_max
        batch_size_max = _batch_size_max
        dropout_max = _dropout_max
        lr = _lr
    
    tuned_parameters['scaler_option'] = scaler_option
    
    return tuned_parameters

    print(" -- DONE --")

def net_tuning(x_train, y_train, tries = 10, lr = None, layer = None, params=None, epochs=None, batch_size=None, dropout=None, l_2 = None, neuron_max=[64, 64, 64], batch_size_max=32, layer_min=1, layer_max=3, dropout_max=0.2, scaler_option='StandardScaler', default_neuron_max=32, checkpoint = None, num_of_folds=5, fast_tune = True, random_state = 0):
    
    tuned_parameters = {}
    if layer is not None:
        
        if layer<1:
            print("Error: layer must be >=1")
            sys.exit()
    
    # Trying to tune hyperparameters
    
    best_score = 0
    best_params = None
    best_mae = 0

    _layer = layer
    _params = params
    _epochs = epochs
    _batch_size = batch_size
    _dropout = dropout
    _l_2 = l_2
    _neuron_max = neuron_max
    _batch_size_max = batch_size_max
    _dropout_max = dropout_max
    _lr = lr
    
    for i in range(0, tries):

        optimizer = 'adam'
        
        if lr is None:
            lr = 10**np.random.uniform(-4,-3)
            optimizer = keras.optimizers.Adam(lr=lr)
        else:
            optimizer = keras.optimizers.Adam(lr=lr)

        if layer is None:
            layer = random.sample(range(layer_min, layer_max+1), 1)[0]
        
        params = _params
        if params is None:
            params = []
            for j in range(0, layer):
                try:
                    params.append(random.sample(range(1, neuron_max[j]), 1)[0])
                except:
                    params.append(random.sample(range(1, default_neuron_max), 1)[0])

        if epochs is None:
            epochs = int(10**np.random.uniform(2,3)) # 100 - 1000

        if batch_size is None:
            batch_size = random.sample(range(1, batch_size_max), 1)[0]

        if dropout is None:
            dropout = 0
        elif dropout is True:
            dropout = random.uniform(0,dropout_max)
        else:
            dropout = dropout

        if l_2 is None:
            l_2 = 10**np.random.uniform(-3,-1)

        model = net_define(params=params, layer_n = layer, input_size = x_train.shape[1], dropout=dropout, l_2=l_2, optimizer=optimizer, random_state = random_state)
        
        print("\n Cross-validation (iteration=%d): [layer=%d, structure=[%s], epochs=%d, dropout=%8.4f, l_2=%8.7f, batch_size=%d, lr=%8.7f]"%(i, layer, params, epochs, dropout, l_2, batch_size, lr))
        start_time = time.time()
        predictions, actual_values = cross_val_predict_net(model, epochs=epochs, batch_size=batch_size, x_train = x_train, y_train = y_train, verbose = 0, scaler_option = scaler_option, num_of_folds = num_of_folds, fast_tune = fast_tune)
        
        if predictions == []:
            print(" Validation stopped early with the setting:","[layer=%d, structure=[%s], epochs=%d, dropout=%8.4f, l_2=%8.7f, batch_size=%d, lr=%8.7f]"%(layer, params, epochs, dropout, l_2, batch_size, lr))
            print(' Keep trying to find best settings .., took %ss'%(time.time()-start_time))

            MAE= -1
            R2 = -1
        else:
            MAE, R2 = evaluate(predictions, actual_values)

        print("Cross validation result - MAE = %8.3f R2 = %8.3f, took %ss "%(MAE, R2, time.time()-start_time))

        if(R2>best_score and R2!=-1) or (best_score==0 and R2!=-1):          
        
            best_score = R2
            best_mae = MAE
            best_params = (layer, params, epochs, dropout, l_2, batch_size, lr)
            tuned_parameters = {"net_layer_n":best_params[0], \
            "net_structure":str(best_params[1])[1:-1].replace(",",""), \
            "net_epochs":best_params[2], \
            "net_dropout":best_params[3], \
            "net_l_2":best_params[4], \
            "net_batch_size":best_params[5], \
            "net_learning_rate": best_params[6]}

            if checkpoint is not None:
                print("Best so far parameters stored :", str(checkpoint)+",Model=NET,Scaler="+str(scaler_option)+",MAE="+str(MAE)+",R2="+str(R2)+".tuned.checkpoint.prop")
                save_parameters(tuned_parameters, str(checkpoint)+",Model=NET,Scaler="+str(scaler_option)+",MAE="+str(MAE)+",R2="+str(R2)+".tuned.checkpoint.prop")
                
                print(" Saving test charts to : ", str(checkpoint)+",Model=NET,Scaler="+str(scaler_option)+",MAE="+str(MAE)+",R2="+str(R2)+".tuned.checkpoint.png")
                #try:    
                save_comparison_chart(predictions, actual_values, str(checkpoint)+",Model=NET,Scaler="+str(scaler_option)+",MAE="+str(MAE)+",R2="+str(R2)+".tuned.checkpoint.png")
                #except:
                #    print(" * Warning: couldn't generate a chart - please make sure the model is properly trained .. ")
        
        if(best_score!=0):
            print("Best R2 = %8.3f"%(best_score),"MAE=",best_mae, "[layer=%d, structure=[%s], epochs=%d, dropout=%8.4f, l_2=%8.7f, batch_size=%d, lr=%8.7f]"%best_params)

        # set to original values
        layer = _layer
        params = _params
        epochs = _epochs
        batch_size = _batch_size
        dropout = _dropout
        l_2 = _l_2
        neuron_max = _neuron_max
        batch_size_max = _batch_size_max
        dropout_max = _dropout_max
        lr = _lr
    
    tuned_parameters['scaler_option'] = scaler_option
    
    return tuned_parameters

    print(" -- DONE --")
    
