# Reload modules 
# It is an optional step. It is useful to run when external Python modules are being modified
# It is reloading all modules (except those excluded by %aimport) every time before executing the Python code typed.
# Note: It may conflict with serialisation, when external modules are being modified

# %load_ext autoreload 
# %autoreload 2

"""
Explanation: Temporal-Comorbidity Adjusted Risk of Emergency Readmission (TCARER)
<font style="font-weight:bold;color:gray">Basic Models</font>
1. Initialise
<br>
2. Generate Features
<br>
3. Read Data
<br>
4. Filter Features
<br>
5. Set Samples &amp; Target Features
<br>
6. Recategorise &amp; Transform
<br>
7. Rank &amp; Select Features
<br>
8. Model
<br>
This Jupyter IPython Notebook applies the Temporal-Comorbidity Adjusted Risk of Emergency Readmission (TCARER).
This Jupyter IPython Notebook extract aggregated features from the MySQL database, &amp; then pre-process, configure &amp; apply several modelling approaches. 
The pre-processing framework &amp; modelling algorithms in this Notebook are developed as part of the Integrated Care project at the <a href="http://www.healthcareanalytics.co.uk/">Health &amp; Social Care Modelling Group (HSCMG)</a>, The <a href="http://www.westminster.ac.uk">University of Westminster</a>.
Note that some of the scripts are optional or subject to some pre-configurations. Please refer to the comments &amp; the project documentations for further details.
<hr>
<font size="1" color="gray">Copyright 2017 The Project Authors. All Rights Reserved.
It is licensed under the Apache License, Version 2.0. you may not use this file except in compliance with the License. You may obtain a copy of the License at
<a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.</font>
<hr>
1. Initialise
Reload modules
End of explanation
"""


# Import Python libraries
import logging
import os
import sys
import gc
import pandas as pd
from IPython.display import display, HTML
from collections import OrderedDict
import numpy as np
import statistics
from scipy.stats import stats

# Import local Python modules
from Configs.CONSTANTS import CONSTANTS
from Configs.Logger import Logger
from Features.Variables import Variables
from ReadersWriters.ReadersWriters import ReadersWriters
from Stats.PreProcess import PreProcess
from Stats.FeatureSelection import FeatureSelection
from Stats.TrainingMethod import TrainingMethod
from Stats.Plots import Plots

# Check the interpreter
print("\nMake sure the correct Python interpreter is used!")
print(sys.version)
print("\nMake sure sys.path of the Python interpreter is correct!")
print(os.getcwd())

"""
Explanation: Import libraries
End of explanation
"""


config_path = os.path.abspath("ConfigInputs/CONFIGURATIONS.ini")
io_path = os.path.abspath("../../tmp/TCARER/Basic_prototype")
app_name = "T-CARER"
submodel_name = "hesIp"
submodel_input_name = "tcarer_model_features_ip"

print("\n The full path of the configuration file: \n\t", config_path,
      "\n The full path of the output folder: \n\t", io_path,
      "\n The application name (the suffix of the outputs file name): \n\t", app_name,
      "\n The sub-model name, to locate the related feature configuration: \n\t", submodel_name,
      "\n The the sub-model's the file name of the input: \n\t", submodel_input_name)

"""
Explanation: <br/><br/>
1.1.  Initialise General Settings
<font style="font-weight:bold;color:red">Main configuration Settings: </font>
- Specify the full path of the configuration file 
<br/>&#9; &#8594; config_path
- Specify the full path of the output folder 
<br/>&#9; &#8594; io_path
- Specify the application name (the suffix of the outputs file name) 
<br/>&#9; &#8594; app_name
- Specify the sub-model name, to locate the related feature configuration, based on the "Table_Reference_Name" column in the configuration file
<br/>&#9; &#8594; submodel_name
- Specify the sub-model's the file name of the input (excluding the CSV extension)
<br/>&#9; &#8594; submodel_input_name
<br/>
<br/>
<font style="font-weight:bold;color:red">External Configration Files: </font>
- The MySQL database configuration setting &amp; other configration metadata
<br/>&#9; &#8594; <i>Inputs/CONFIGURATIONS_1.ini</i>
- The input features' confugration file (Note: only the CSV export of the XLSX will be used by this Notebook)
<br/>&#9; &#8594; <i>Inputs/config_features_path.xlsx</i>
<br/>&#9; &#8594; <i>Inputs/config_features_path.csv</i>
End of explanation
"""


if not os.path.exists(io_path):
    os.makedirs(io_path, exist_ok=True)

logger = Logger(path=io_path, app_name=app_name, ext="log")
logger = logging.getLogger(app_name)

"""
Explanation: <br/><br/>
Initialise logs
End of explanation
"""


# Initialise constants        
CONSTANTS.set(io_path, app_name)

# Initialise other classes
readers_writers = ReadersWriters()
preprocess = PreProcess(io_path)
feature_selection = FeatureSelection()
plts = Plots()

# Set print settings
pd.set_option('display.width', 1600, 'display.max_colwidth', 800)

"""
Explanation: Initialise constants and some of classes
End of explanation
"""


# variables settings
features_metadata = dict()

features_metadata_all = readers_writers.load_csv(path=CONSTANTS.io_path, title=CONSTANTS.config_features_path, dataframing=True)
features_metadata = features_metadata_all.loc[(features_metadata_all["Selected"] == 1) & 
                                              (features_metadata_all["Table_Reference_Name"] == submodel_name)]
features_metadata.reset_index()
    
# print
display(features_metadata)

"""
Explanation: 1.2.  Initialise Features Metadata
Read the input features' confugration file &amp; store the features metadata
End of explanation
"""


# Dictionary of features types, dtypes, & max-states
features_types = dict()
features_dtypes = dict()
features_states_values = dict()
features_names_group = dict()

for _, row in features_metadata.iterrows():
    if not pd.isnull(row["Variable_Max_States"]):
        states_values = str(row["Variable_Max_States"]).split(',') 
        states_values = list(map(int, states_values))
    else: 
        states_values = None
        
    if not pd.isnull(row["Variable_Aggregation"]):
        postfixes = row["Variable_Aggregation"].replace(' ', '').split(',')
        f_types = row["Variable_Type"].replace(' ', '').split(',')
        f_dtypes = row["Variable_dType"].replace(' ', '').split(',')
        for p in range(len(postfixes)):
            features_types[row["Variable_Name"] + "_" + postfixes[p]] = f_types[p]
            features_dtypes[row["Variable_Name"] + "_" + postfixes[p]] = pd.Series(dtype=f_dtypes[p])
            features_states_values[row["Variable_Name"] + "_" + postfixes[p]] = states_values
            features_names_group[row["Variable_Name"] + "_" + postfixes[p]] = row["Variable_Name"] + "_" + postfixes[p]
    else:
        features_types[row["Variable_Name"]] = row["Variable_Type"]
        features_dtypes[row["Variable_Name"]] = row["Variable_dType"]
        features_states_values[row["Variable_Name"]] = states_values
        features_names_group[row["Variable_Name"]] = row["Variable_Name"]
        if states_values is not None:
            for postfix in states_values:
                features_names_group[row["Variable_Name"] + "_" + str(postfix)] = row["Variable_Name"]
            
features_dtypes = pd.DataFrame(features_dtypes).dtypes

# Dictionary of features groups
features_types_group = OrderedDict()

f_types = set([f_type for f_type in features_types.values()])
features_types_group = OrderedDict(zip(list(f_types), [set() for _ in range(len(f_types))]))
for f_name, f_type in features_types.items():
    features_types_group[f_type].add(f_name)
    
print("Available features types: " + ','.join(f_types))

"""
Explanation: Set input features' metadata dictionaries
End of explanation
"""


skip = True

# settings
csv_schema = ["my_db_schema"]
csv_input_tables = ["tcarer_features"]
csv_history_tables = ["hesIp"]
csv_column_index = "localID"
csv_output_table = "tcarer_model_features_ip"
csv_query_batch_size =  100000

if skip is False:
    # generate the csv file
    variables = Variables(submodel_name,
                          CONSTANTS.io_path,
                          CONSTANTS.io_path,
                          CONSTANTS.config_features_path,
                          csv_output_table)
    variables.set(csv_schema, csv_input_tables, csv_history_tables, csv_column_index, csv_query_batch_size)

"""
Explanation: <br/><br/>
<font style="font-weight:bold;color:red">2. Generate Features</font>
<font style="font-weight:bold;color:red">Notes:</font>
- It generates the final spell-wise &amp; temporal features from the MySQL table(s), &amp; converts it into CSV(s);
- It generates the CSV(s) based on the configuration file of the features (Note: only the CSV export of the XLSX will be used by this Notebook)
<br/>&#9; &#8594; <i>Inputs/config_features_path.xlsx</i>
<br/>&#9; &#8594; <i>Inputs/config_features_path.csv</i>
End of explanation
"""


features_input = readers_writers.load_csv(path=CONSTANTS.io_path, title=submodel_input_name, dataframing=True)
features_input.astype(dtype=features_dtypes)

print("Number of columns: ", len(features_input.columns), "; Total records: ", len(features_input.index))

"""
Explanation: <br/><br/>
3. Read Data
Read the input features from the CSV input file
End of explanation
"""


display(features_input.head())

"""
Explanation: Verify features visually
End of explanation
"""


file_name = "Step_04_Data_ColumnNames"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, data=list(features_input.columns.values), append=False)
file_name = "Step_04_Stats_Categorical"
o_stats = preprocess.stats_discrete_df(df=features_input, includes=features_types_group["CATEGORICAL"],
                                       file_name=file_name)
file_name = "Step_04_Stats_Continuous"
o_stats = preprocess.stats_continuous_df(df=features_input, includes=features_types_group["CONTINUOUS"], 
                                         file_name=file_name)
file_name = "Step_04_Stats_Target"
o_stats = preprocess.stats_discrete_df(df=features_input, includes=features_types_group["TARGET"], 
                                       file_name=file_name)

"""
Explanation: <br/><br/>
4. Filter Features
4.1. Descriptive Statsistics
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
End of explanation
"""


# Exclusion of unused features
# excluded = [name for name in features_input.columns if name not in features_names_group.keys()]
# features_input = features_input.drop(excluded, axis=1)

# print("Number of columns: ", len(features_input.columns), "; Total records: ", len(features_input.index))

"""
Explanation: 4.2. Selected Population
4.2.1. Remove Excluded Population, Remove Unused Features
<i>Nothing to do!<i/> 
<br/>
<font style="font-weight:bold;color:red">Notes: </font> 
- Ideally the features must be configured before generating the CSV feature file, as it is very inefficient to derive new features at this stage
- This step is not necessary, if all the features are generated in prior to the generatiion of the CSV feature file
End of explanation
"""


frac_train = 0.50
replace = False
random_state = 100

nrows = len(features_input.index)
features = {"train": dict(), "test": dict()}
features["train"] = features_input.sample(frac=frac_train, replace=False, random_state=100)
features["test"] = features_input.drop(features["train"].index)

features["train"] = features["train"].reset_index(drop=True)
features["test"] = features["test"].reset_index(drop=True)

"""
Explanation: <br/><br/>
5. Set Samples &amp; Target Features
5.1. Set Features
5.1.1. Train & Test Samples
Set the samples
End of explanation
"""


display(features_input.head())

"""
Explanation: Verify features visually
End of explanation
"""


features_input = None
gc.collect()

"""
Explanation: <font style="font-weight:bold;color:red">Clean-Up</font>
End of explanation
"""


target_labels = list(features_types_group["TARGET"])
target_id = ["patientID"]

features["train_indep"] = dict()
features["train_target"] = dict()
features["train_id"] = dict()
features["test_indep"] = dict()
features["test_target"] = dict()
features["test_id"] = dict()

# Independent and target features
def set_features_indep_target(df):
    df_targets = pd.DataFrame(dict(zip(target_labels, [[]] * len(target_labels))))
    for i in range(len(target_labels)):
        df_targets[target_labels[i]] = df[target_labels[i]]
        
    df_indep = df.drop(target_labels + target_id, axis=1)
    df_id = pd.DataFrame({target_id[0]: df[target_id[0]]})
    
    return df_indep, df_targets, df_id

# train & test sets
features["train_indep"], features["train_target"], features["train_id"] = set_features_indep_target(features["train"])
features["test_indep"], features["test_target"], features["test_id"] = set_features_indep_target(features["test"])

# print    
print("Number of columns: ", len(features["train_indep"].columns)) 
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")

"""
Explanation: 5.1.2. Independent & Target variable¶
Set independent, target &amp; ID features
End of explanation
"""


display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))

"""
Explanation: Verify features visually
End of explanation
"""


del features["train"]
del features["test"]
gc.collect()

"""
Explanation: <font style="font-weight:bold;color:red">Clean-Up</font>
End of explanation
"""


file_name = "Step_05_Features"
readers_writers.save_serialised_compressed(path=CONSTANTS.io_path, title=file_name, objects=features)

# print
print("Number of columns: ", len(features["train_indep"].columns), 
      "features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")

"""
Explanation: 5.5. Save Samples
Serialise &amp; save the samples before any feature transformation. 
<br/>This snapshot of the samples may be used for the population profiling
End of explanation
"""


thresh_unique_cut = 100
thresh_freq_cut = 1000

excludes = []
file_name = "Step_05_Preprocess_NZV_config"
features["train_indep"], o_summaries = preprocess.near_zero_var_df(df=features["train_indep"], 
                                                             excludes=excludes, 
                                                             file_name=file_name, 
                                                             thresh_unique_cut=thresh_unique_cut, 
                                                             thresh_freq_cut=thresh_freq_cut,
                                                             to_search=True)

file_name = "Step_05_Preprocess_NZV"
readers_writers.save_text(path=CONSTANTS.io_path, title=file_name, data=o_summaries, append=False, ext="log")

file_name = "Step_05_Preprocess_NZV_config"
features["test_indep"], o_summaries = preprocess.near_zero_var_df(df=features["test_indep"], 
                                                            excludes=excludes, 
                                                            file_name=file_name, 
                                                            thresh_unique_cut=thresh_unique_cut, 
                                                            thresh_freq_cut=thresh_freq_cut,
                                                            to_search=False)

# print
print("Number of columns: ", len(features["train_indep"].columns)) 
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")

"""
Explanation: 5.2. Remove - Near Zero Variance
In order to reduce sparseness and invalid features, highly stationary ones were withdrawn. The features that had constant counts less than or equal a threshold were ltered out, to exclude highly constants and near-zero variances.
The near zero variance rules are presented in below:
- Frequency ratio: The frequency of the most prevalent value over the second most frequent value to be greater than a threshold;
- Percent of unique values: The number of unique values divided by the total number of samples to be greater than the threshold
<font style="font-weight:bold;color:red">Configure:</font> the function
- The cutoff for the percentage of distinct values out of the number of total samples (upper limit). e.g. 10 * 100 / 100
<br/>&#9; &#8594; thresh_unique_cut
- The cutoff for the ratio of the most common value to the second most common value (lower limit). eg. 95/5
<br/>&#9; &#8594; thresh_freq_cut
End of explanation
"""


thresh_corr_cut = 0.95

excludes = list(features_types_group["CATEGORICAL"])
file_name = "Step_05_Preprocess_Corr_config"
features["train_indep"], o_summaries = preprocess.high_linear_correlation_df(df=features["train_indep"], 
                                                                       excludes=excludes, 
                                                                       file_name=file_name, 
                                                                       thresh_corr_cut=thresh_corr_cut,
                                                                       to_search=True)

file_name = "Step_05_Preprocess_Corr"
readers_writers.save_text(path=CONSTANTS.io_path, title=file_name, data=o_summaries, append=False, ext="log")

file_name = "Step_05_Preprocess_Corr_config"
features["test_indep"], o_summaries = preprocess.high_linear_correlation_df(df=features["test_indep"], 
                                                                      excludes=excludes, 
                                                                      file_name=file_name, 
                                                                      thresh_corr_cut=thresh_corr_cut,
                                                                      to_search=False)

# print
print("Number of columns: ", len(features["train_indep"].columns)) 
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")

"""
Explanation: 5.3. Remove Highly Linearly Correlated
In this step, features that were highly linearly correlated were excluded. 
<font style="font-weight:bold;color:red">Configure:</font> the function
- A numeric value for the pair-wise absolute correlation cutoff. e.g. 0.95
<br/>&#9; &#8594; thresh_corr_cut
End of explanation
"""


# columns
file_name = "Step_05_Data_ColumnNames_Train"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, 
                         data=list(features["train_indep"].columns.values), append=False)

# Sample - Train
file_name = "Step_05_Stats_Categorical_Train"
o_stats = preprocess.stats_discrete_df(df=features["train_indep"], includes=features_types_group["CATEGORICAL"], 
                                       file_name=file_name)
file_name = "Step_05_Stats_Continuous_Train"
o_stats = preprocess.stats_continuous_df(df=features["train_indep"], includes=features_types_group["CONTINUOUS"], 
                                         file_name=file_name)

# Sample - Test
file_name = "Step_05_Stats_Categorical_Test"
o_stats = preprocess.stats_discrete_df(df=features["test_indep"], includes=features_types_group["CATEGORICAL"],
                                       file_name=file_name)
file_name = "Step_05_Stats_Continuous_Test"
o_stats = preprocess.stats_continuous_df(df=features["test_indep"], includes=features_types_group["CONTINUOUS"], 
                                         file_name=file_name)

"""
Explanation: 5.4. Descriptive Statistics
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
End of explanation
"""


display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))

"""
Explanation: <br/><br/>
6. Recategorise &amp; Transform
Verify features visually
End of explanation
"""


def factorise_settings(max_categories_frac, min_categories_num, exclude_zero):
    categories_dic = dict()
    labels_dic = dict()
    dtypes_dic = dict()
    dummies = []
    
    for f_name in features_types_group["CATEGORICAL"]:
        if f_name in features["train_indep"]:
            # find top & valid states
            summaries = stats.itemfreq(features["train_indep"][f_name])
            summaries = pd.DataFrame({"value": summaries[:, 0], "freq": summaries[:, 1]})
            summaries["value"] = list(map(int, summaries["value"]))
            summaries = summaries.sort_values("freq", ascending=False)
            summaries = list(summaries["value"])

            # exclude zero state
            if exclude_zero is True and len(summaries) > 1:
                summaries = [s for s in summaries if s != 0]
                
            # if included in the states
            summaries = [v for v in summaries if v in set(features_states_values[f_name])]

            # limit number of states
            max_cnt = max(int(len(summaries) * max_categories_frac), min_categories_num)

            # set states
            categories_dic[f_name] = summaries[0:max_cnt]
            labels_dic[f_name] = [f_name + "_" + str(c) for c in categories_dic[f_name]]
            dtypes_dic = {**dtypes_dic,
                          **dict(zip(labels_dic[f_name], [pd.Series(dtype='i') for _ in range(len(categories_dic[f_name]))]))}
            dummies += labels_dic[f_name] 
                
    dtypes_dic = pd.DataFrame(dtypes_dic).dtypes

    # print        
    print("Total Categorical Variables : ", len(categories_dic.keys()), 
          "; Total Number of Dummy Variables: ", sum([len(categories_dic[f_name]) for f_name in categories_dic.keys()]))
    return categories_dic, labels_dic, dtypes_dic, features_types

"""
Explanation: 6.1. Recategorise
Define the factorisation function to generate dummy features for the categorical features.
End of explanation
"""


max_categories_frac = 0.90
min_categories_num = 1
exclude_zero = False # if possible remove state zero

categories_dic, labels_dic, dtypes_dic, features_types_group["DUMMIES"] = \
    factorise_settings(max_categories_frac, min_categories_num, exclude_zero)

"""
Explanation: Select categories: by order of freq., max_categories_frac, & max_categories_num
<br/><font style="font-weight:bold;color:red">Configure:</font> The input arguments are:
- Specify the maximum number of categories a feature can have
<br/>&#9; &#8594; max_categories_frac
- Specify the minimum number of categories a feature can have
<br/>&#9; &#8594; min_categories_num
- Specify to exclude the state '0' (zero). State zero in our features represents 'any other state', including NULL
<br/>&#9; &#8594; exclude_zero = False
End of explanation
"""


features["train_indep_temp"] = preprocess.factoring_feature_wise(features["train_indep"], categories_dic, labels_dic, dtypes_dic, threaded=False)
features["test_indep_temp"] = preprocess.factoring_feature_wise(features["test_indep"], categories_dic, labels_dic, dtypes_dic, threaded=False)

# print
print("Number of columns: ", len(features["train_indep"].columns)) 
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")

"""
Explanation: Manually add dummy variables to the dataframe &amp; remove the original Categorical variables
End of explanation
"""


display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep_temp"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep_temp"].head()], axis=1))

"""
Explanation: Verify features visually
End of explanation
"""


features["train_indep"] = features["train_indep_temp"].copy(True)
features["test_indep"] = features["test_indep_temp"].copy(True)

"""
Explanation: Set
End of explanation
"""


del features["train_indep_temp"]
del features["test_indep_temp"]
gc.collect()

"""
Explanation: <font style="font-weight:bold;color:red">Clean-Up</font>
End of explanation
"""


# the cutoff for the percentage of distinct values out of the number of total samples (upper limit). e.g. 10 * 100 / 100
thresh_unique_cut = 100
# the cutoff for the ratio of the most common value to the second most common value (lower limit). eg. 95/5
thresh_freq_cut = 1000

excludes = []
file_name = "Step_06_Preprocess_NZV_config"
features["train_indep"], o_summaries = preprocess.near_zero_var_df(df=features["train_indep"], 
                                                             excludes=excludes, 
                                                             file_name=file_name, 
                                                             thresh_unique_cut=thresh_unique_cut, 
                                                             thresh_freq_cut=thresh_freq_cut,
                                                             to_search=True)

file_name = "Step_06_Preprocess_NZV"
readers_writers.save_text(path=CONSTANTS.io_path, title=file_name, data=o_summaries, append=False, ext="log")

file_name = "Step_06_Preprocess_NZV_config"
features["test_indep"], o_summaries = preprocess.near_zero_var_df(df=features["test_indep"], 
                                                            excludes=excludes, 
                                                            file_name=file_name, 
                                                            thresh_unique_cut=thresh_unique_cut, 
                                                            thresh_freq_cut=thresh_freq_cut,
                                                            to_search=False)

# print
print("Number of columns: ", len(features["train_indep"].columns)) 
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")

"""
Explanation: 6.2. Remove - Near Zero Variance
Optional: Remove more features with near zero variance, after the factorisation step.
<font style="font-weight:bold;color:red">Configure:</font> the function
End of explanation
"""


# A numeric value for the pair-wise absolute correlation cutoff. e.g. 0.95
thresh_corr_cut = 0.95

excludes = []
file_name = "Step_06_Preprocess_Corr_config"
features["train_indep"], o_summaries = preprocess.high_linear_correlation_df(df=features["train_indep"], 
                                                                       excludes=excludes, 
                                                                       file_name=file_name, 
                                                                       thresh_corr_cut=thresh_corr_cut,
                                                                       to_search=True)

file_name = "Step_06_Preprocess_Corr"
readers_writers.save_text(path=CONSTANTS.io_path, title=file_name, data=o_summaries, append=False, ext="log")

file_name = "Step_06_Preprocess_Corr_config"
features["test_indep"], o_summaries = preprocess.high_linear_correlation_df(df=features["test_indep"], 
                                                                      excludes=excludes, 
                                                                      file_name=file_name, 
                                                                      thresh_corr_cut=thresh_corr_cut,
                                                                      to_search=False)

# print
print("Number of columns: ", len(features["train_indep"].columns)) 
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")

"""
Explanation: 6.3. Remove Highly Linearly Correlated
Optional: Remove more features with highly linearly correlated, after the factorisation step.
<font style="font-weight:bold;color:red">Configure:</font> the function
End of explanation
"""


# columns
file_name = "Step_06_4_Data_ColumnNames_Train"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, 
                         data=list(features["train_indep"].columns.values), append=False)

# Sample - Train
file_name = "Step_06_4_Stats_Categorical_Train"
o_stats = preprocess.stats_discrete_df(df=features["train_indep"], includes=features_types_group["CATEGORICAL"], 
                                       file_name=file_name)
file_name = "Step_06_4_Stats_Continuous_Train"
o_stats = preprocess.stats_continuous_df(df=features["train_indep"], includes=features_types_group["CONTINUOUS"], 
                                         file_name=file_name)

# Sample - Test
file_name = "Step_06_4_Stats_Categorical_Test"
o_stats = preprocess.stats_discrete_df(df=features["test_indep"], includes=features_types_group["CATEGORICAL"],
                                       file_name=file_name)
file_name = "Step_06_4_Stats_Continuous_Test"
o_stats = preprocess.stats_continuous_df(df=features["test_indep"], includes=features_types_group["CONTINUOUS"], 
                                         file_name=file_name)

"""
Explanation: 6.4. Descriptive Statsistics
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
End of explanation
"""


display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))

"""
Explanation: 6.5. Transformations
Verify features visually
End of explanation
"""


transform_type = "scale"
kwargs = {"with_mean": True}
method_args = dict()
excludes = list(features_types_group["CATEGORICAL"]) + list(features_types_group["DUMMIES"])

features["train_indep"], method_args = preprocess.transform_df(df=features["train_indep"], excludes=excludes, 
                                                               transform_type=transform_type, threaded=False, 
                                                               method_args=method_args, **kwargs)
features["test_indep"], _ = preprocess.transform_df(df=features["test_indep"], excludes=excludes, 
                                                    transform_type=transform_type, threaded=False, 
                                                    method_args=method_args, **kwargs)

# print("Metod arguments:", method_args)

"""
Explanation: <font style="font-weight:bold;color:blue">Tranformation:</font> scale
<font style="font-weight:bold;color:brown">Note:</font>: It is highly resource intensive
End of explanation
"""


transform_type = "yeo_johnson"
kwargs = {"lmbda": -0.5, "derivative": 0, "epsilon": np.finfo(np.float).eps, "inverse": False}
method_args = dict()
excludes = list(features_types_group["CATEGORICAL"]) + list(features_types_group["DUMMIES"])

features["train_indep"], method_args = preprocess.transform_df(df=features["train_indep"], excludes=excludes, 
                                                               transform_type=transform_type, threaded=False, 
                                                               method_args=method_args, **kwargs)
features["test_indep"], _ = preprocess.transform_df(df=features["test_indep"], excludes=excludes, 
                                                    transform_type=transform_type, threaded=False, 
                                                    method_args=method_args, **kwargs)

# print("Metod arguments:", method_args)

"""
Explanation: <font style="font-weight:bold;color:blue">Tranformation:</font> Yeo-Johnson
<font style="font-weight:bold;color:brown">Note:</font>: It is highly resource intensive
End of explanation
"""


display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))

"""
Explanation: Visual verification
End of explanation
"""


# Statsistics report for 'Categorical', 'Continuous', & 'TARGET' variables
# columns
file_name = "Step_06_6_Data_ColumnNames_Train"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, 
                         data=list(features["train_indep"].columns.values), append=False)

# Sample - Train
file_name = "Step_06_6_Stats_Categorical_Train"
o_stats = preprocess.stats_discrete_df(df=features["train_indep"], includes=features_types_group["CATEGORICAL"], 
                                       file_name=file_name)
file_name = "Step_06_6_Stats_Continuous_Train"
o_stats = preprocess.stats_continuous_df(df=features["train_indep"], includes=features_types_group["CONTINUOUS"], 
                                         file_name=file_name)

# Sample - Test
file_name = "Step_06_6_Stats_Categorical_Test"
o_stats = preprocess.stats_discrete_df(df=features["test_indep"], includes=features_types_group["CATEGORICAL"],
                                       file_name=file_name)
file_name = "Step_06_6_Stats_Continuous_Test"
o_stats = preprocess.stats_continuous_df(df=features["test_indep"], includes=features_types_group["CONTINUOUS"], 
                                         file_name=file_name)

"""
Explanation: 6.6. Summary Statistics
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
End of explanation
"""


# select the target variable
target_feature = "label365" # "label30", "label365"

# number of trials
num_trials = 1

model_rank = dict()
o_summaries_df = dict()

"""
Explanation: <br/><br/>
7. Rank &amp; Select Features
<font style="font-weight:bold;color:red">Configure:</font> the general settings
End of explanation
"""


def rank_random_forest_brieman(features_indep_arg, features_target_arg, num_trials):
    num_settings = 3
    o_summaries_df = [pd.DataFrame({'Name': list(features_indep_arg.columns.values)}) for _ in range(num_trials * num_settings)]
    model_rank = [None] * (num_trials * num_settings)

    # trials 
    for i in range(num_trials):   
        print("Trial: " + str(i))
        # setting-1
        s_i = i
        model_rank[s_i] = feature_selection.rank_random_forest_breiman(
            features_indep_arg.values, features_target_arg.values,
            **{"n_estimators": 10, "criterion": 'gini', "max_depth": None, "min_samples_split": 2, "min_samples_leaf": 1,
            "min_weight_fraction_leaf": 0.0, "max_features": 'auto', "max_leaf_nodes": None, "bootstrap": True,
            "oob_score": False, "n_jobs": -1, "random_state": None, "verbose": 0, "warm_start": False, "class_weight": None})

        # setting-2
        s_i = num_trials + i
        model_rank[s_i] = feature_selection.rank_random_forest_breiman(
            features_indep_arg.values, features_target_arg.values,
            **{"n_estimators": 10, "criterion": 'gini', "max_depth": None, "min_samples_split": 50, "min_samples_leaf": 25,
            "min_weight_fraction_leaf": 0.0, "max_features": 'auto', "max_leaf_nodes": None, "bootstrap": True,
            "oob_score": False, "n_jobs": -1, "random_state": None, "verbose": 0, "warm_start": False, "class_weight": None})

        # setting-3
        s_i = (num_trials * 2) + i
        model_rank[s_i] = feature_selection.rank_random_forest_breiman(
            features_indep_arg.values, features_target_arg.values,
            **{"n_estimators": 10, "criterion": 'gini', "max_depth": None, "min_samples_split": 40, "min_samples_leaf": 20,
            "min_weight_fraction_leaf": 0.0, "max_features": 'auto', "max_leaf_nodes": None, "bootstrap": True,
            "oob_score": False, "n_jobs": -1, "random_state": None, "verbose": 0, "warm_start": True, "class_weight": None})

    for i in range((num_trials * num_settings)):
        o_summaries_df[i]['Importance'] = list(model_rank[i].feature_importances_)
        o_summaries_df[i] = o_summaries_df[i].sort_values(['Importance'], ascending = [0])
        o_summaries_df[i] = o_summaries_df[i].reset_index(drop = True)
        o_summaries_df[i]['Order'] = range(1, len(o_summaries_df[i]['Importance']) + 1)
    return model_rank, o_summaries_df

"""
Explanation: 7.1. Define
<font style="font-weight:bold;color:blue">Ranking Method:</font> Random forest classifier (Brieman)
<br/>Define a set of classifiers with different settings, to be used in feature ranking trials.
End of explanation
"""


def rank_gbrt(features_indep_arg, features_target_arg, num_trials):
    num_settings = 3
    o_summaries_df = [pd.DataFrame({'Name': list(features_indep_arg.columns.values)}) for _ in range(num_trials * num_settings)]
    model_rank = [None] * (num_trials * num_settings)

    # trials 
    for i in range(num_trials):   
        print("Trial: " + str(i))
        # setting-1
        s_i = i
        model_rank[s_i] = feature_selection.rank_tree_gbrt(
            features_indep_arg.values, features_target_arg.values, 
            **{"loss": 'ls', "learning_rate": 0.1, "n_estimators": 100, "subsample": 1.0, "min_samples_split": 2, "min_samples_leaf": 1,
            "min_weight_fraction_leaf": 0.0, "max_depth": 10, "init": None, "random_state": None, "max_features": None, "alpha": 0.9,
            "verbose": 0, "max_leaf_nodes": None, "warm_start": False, "presort": True})
        
        # setting-2
        s_i = num_trials + i
        model_rank[s_i] = feature_selection.rank_tree_gbrt(
            features_indep_arg.values, features_target_arg.values,
            **{"loss": 'ls', "learning_rate": 0.1, "n_estimators": 100, "subsample": 1.0, "min_samples_split": 2, "min_samples_leaf": 1,
            "min_weight_fraction_leaf": 0.0, "max_depth": 5, "init": None, "random_state": None, "max_features": None, "alpha": 0.9,
            "verbose": 0, "max_leaf_nodes": None, "warm_start": False, "presort": True})

        # setting-3
        s_i = (num_trials * 2) + i
        model_rank[s_i] = feature_selection.rank_tree_gbrt(
            features_indep_arg.values, features_target_arg.values,
            **{"loss": 'ls', "learning_rate": 0.1, "n_estimators": 100, "subsample": 1.0, "min_samples_split": 2, "min_samples_leaf": 1,
            "min_weight_fraction_leaf": 0.0, "max_depth": 3, "init": None, "random_state": None, "max_features": None, "alpha": 0.9,
            "verbose": 0, "max_leaf_nodes": None, "warm_start": False, "presort": True})

    for i in range((num_trials * num_settings)):
        o_summaries_df[i]['Importance'] = list(model_rank[i].feature_importances_)
        o_summaries_df[i] = o_summaries_df[i].sort_values(['Importance'], ascending = [0])
        o_summaries_df[i] = o_summaries_df[i].reset_index(drop = True)
        o_summaries_df[i]['Order'] = range(1, len(o_summaries_df[i]['Importance']) + 1)
    return model_rank, o_summaries_df

"""
Explanation: <font style="font-weight:bold;color:blue">Ranking Method:</font> Gradient Boosted Regression Trees (GBRT) 
<br/>Define a set of classifiers with different settings, to be used in feature ranking trials.
End of explanation
"""


def rank_randLogit(features_indep_arg, features_target_arg, num_trials):
    num_settings = 3
    o_summaries_df = [pd.DataFrame({'Name': list(features_indep_arg.columns.values)}) for _ in range(num_trials * num_settings)]
    model_rank = [None] * (num_trials * num_settings)

    # trials 
    for i in range(num_trials):   
        print("Trial: " + str(i))
        # setting-1
        s_i = i
        model_rank[s_i] = feature_selection.rank_random_logistic_regression(
            features_indep_arg.values, features_target_arg.values,
            **{"C": 1, "scaling": 0.5, "sample_fraction": 0.75, "n_resampling": 200, "selection_threshold": 0.25, "tol": 0.001,
            "fit_intercept": True, "verbose": False, "normalize": True, "random_state": None, "n_jobs": 1, "pre_dispatch": '3*n_jobs'})

        # setting-2
        s_i = num_trials + i
        model_rank[s_i] = feature_selection.rank_random_logistic_regression(
            features_indep_arg.values, features_target_arg.values,
            **{"C": 1, "scaling": 0.5, "sample_fraction": 0.50, "n_resampling": 200, "selection_threshold": 0.25, "tol": 0.001,
            "fit_intercept": True, "verbose": False, "normalize": True, "random_state": None, "n_jobs": 1, "pre_dispatch": '3*n_jobs'})

        # setting-3
        s_i = (num_trials * 2) + i
        model_rank[s_i] = feature_selection.rank_random_logistic_regression(
            features_indep_arg.values, features_target_arg.values,
            **{"C": 1, "scaling": 0.5, "sample_fraction": 0.90, "n_resampling": 200, "selection_threshold": 0.25, "tol": 0.001,
            "fit_intercept": True, "verbose": False, "normalize": True, "random_state": None, "n_jobs": 1, "pre_dispatch": '3*n_jobs'})
                
    for i in range((num_trials * num_settings)):
        o_summaries_df[i]['Importance'] = list(model_rank[i].scores_)
        o_summaries_df[i] = o_summaries_df[i].sort_values(['Importance'], ascending = [0])
        o_summaries_df[i] = o_summaries_df[i].reset_index(drop = True)
        o_summaries_df[i]['Order'] = range(1, len(o_summaries_df[i]['Importance']) + 1)
    return model_rank, o_summaries_df

"""
Explanation: <font style="font-weight:bold;color:blue">Ranking Method:</font> Randomized Logistic Regression
<br/>Define a set of classifiers with different settings, to be used in feature ranking trials.
End of explanation
"""


rank_model = "rfc"
model_rank[rank_model] = dict() 
o_summaries_df[rank_model] = dict() 
model_rank[rank_model], o_summaries_df[rank_model] = rank_random_forest_brieman(
    features["train_indep"], features["train_target"][target_feature], num_trials)

"""
Explanation: 7.2. Run
Run one or more feature ranking methods and trials
<font style="font-weight:bold;color:blue">Ranking Method:</font> Random forest classifier (Brieman)
<font style="font-weight:bold;color:brown">Note:</font>: It is moderately resource intensive
End of explanation
"""


rank_model = "gbrt"
model_rank[rank_model] = dict() 
o_summaries_df[rank_model] = dict() 
model_rank[rank_model], o_summaries_df[rank_model] = rank_gbrt(
    features["train_indep"], features["train_target"][target_feature], num_trials)

"""
Explanation: <font style="font-weight:bold;color:blue">Ranking Method:</font> Gradient Boosted Regression Trees (GBRT)
<font style="font-weight:bold;color:brown">Note:</font>: It is moderately resource intensive
End of explanation
"""


rank_model = "randLogit"
model_rank[rank_model] = dict() 
o_summaries_df[rank_model] = dict() 
model_rank[rank_model], o_summaries_df[rank_model] = rank_randLogit(
    features["train_indep"], features["train_target"][target_feature], num_trials)

"""
Explanation: <font style="font-weight:bold;color:blue">Ranking Method</font>: Randomized Logistic Regression
<font style="font-weight:bold;color:brown">Note:</font>: It is moderately resource intensive
End of explanation
"""


# combine scores
def rank_summarise (features_arg, o_summaries_df_arg):
    summaries_temp = {'Order_avg': [], 'Order_max': [],  'Order_min': [], 'Importance_avg': []}
    summary_order = []
    summary_importance = []
    
    for f_name in list(features_arg.columns.values):
        for i in range(len(o_summaries_df_arg)):
            summary_order.append(o_summaries_df_arg[i][o_summaries_df_arg[i]['Name'] == f_name]['Order'].values)
            summary_importance.append(o_summaries_df_arg[i][o_summaries_df_arg[i]['Name'] == f_name]['Importance'].values)

        summaries_temp['Order_avg'].append(statistics.mean(np.concatenate(summary_order)))
        summaries_temp['Order_max'].append(max(np.concatenate(summary_order)))
        summaries_temp['Order_min'].append(min(np.concatenate(summary_order)))
        summaries_temp['Importance_avg'].append(statistics.mean(np.concatenate(summary_importance)))

    summaries_df = pd.DataFrame({'Name': list(features_arg.columns.values)})
    summaries_df['Order_avg'] = summaries_temp['Order_avg']
    summaries_df['Order_max'] = summaries_temp['Order_max']
    summaries_df['Order_min'] = summaries_temp['Order_min']
    summaries_df['Importance_avg'] = summaries_temp['Importance_avg']
    summaries_df = summaries_df.sort_values(['Order_avg'], ascending = [1])
    return summaries_df

# combine scores
summaries_df = dict()

for rank_model in o_summaries_df.keys():
    summaries_df[rank_model] = dict()
    summaries_df[rank_model] = rank_summarise(features["train_indep"], o_summaries_df[rank_model])

"""
Explanation: 7.3. Summaries
End of explanation
"""


for rank_model in model_rank.keys():
    file_name = "Step_07_Model_Train_model_rank_" + rank_model
    readers_writers.save_serialised_compressed(path=CONSTANTS.io_path, title=file_name, objects=model_rank[rank_model])
    
    file_name = "Step_07_Model_Train_model_rank_summaries_" + rank_model
    readers_writers.save_serialised_compressed(path=CONSTANTS.io_path, title=file_name, objects=o_summaries_df[rank_model])

"""
Explanation: Save
End of explanation
"""


rank_model = "rfc"
file_name = "Step_07_Top_Features_" + rank_model
rank_top_features_max = 400
rank_top_features_score_min = 0.1 * (10 ^ -20)

# sort features
features_names_selected = summaries_df[rank_model]['Name'][summaries_df[rank_model]['Order_avg'] >= rank_top_features_score_min]
features_names_selected = (features_names_selected[0:rank_top_features_max]).tolist()

"""
Explanation: 7.4. Select Top Features
<font style="font-weight:bold;color:red">Configure:</font> the selection method
End of explanation
"""


# save to CSV
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, data=features_names_selected, append=False, header=False)

# print     
print("Number of columns: ", len(features["train_indep"].columns)) 
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
print("List of sorted features, which can be modified:\n  " + CONSTANTS.io_path + file_name + "csv")

"""
Explanation: Save
End of explanation
"""


file_name = "Step_07_Top_Features_rfc_adhoc" 

features_names_selected = readers_writers.load_csv(path=CONSTANTS.io_path, title=file_name, dataframing=False)[0]
features_names_selected = [f.replace("\n", "") for f in features_names_selected]
display(pd.DataFrame(features_names_selected))

"""
Explanation: <font style="font-weight:bold;color:red">Configure</font>: the selected feature manually if it isnecessary!
End of explanation
"""


# print     
print("Number of columns: ", len(features["train_indep"].columns), 
    ";\nNumber of top columns: ", len(features["train_indep"][features_names_selected].columns)) 
print("features: {train: ", len(features["train_indep"][features_names_selected]), ", test: ", len(features["test_indep"][features_names_selected]), "}")

"""
Explanation: Verify the top features visually
End of explanation
"""


# columns
file_name = "Step_07_Data_ColumnNames_Train"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, 
                         data=list(features["train_indep"][features_names_selected].columns.values), append=False)

# Sample - Train
file_name = "Step_07_Stats_Categorical_Train"
o_stats = preprocess.stats_discrete_df(df=features["train_indep"][features_names_selected], includes=features_types_group["CATEGORICAL"], 
                                       file_name=file_name)
file_name = "Step_07_Stats_Continuous_Train"
o_stats = preprocess.stats_continuous_df(df=features["train_indep"][features_names_selected], includes=features_types_group["CONTINUOUS"], 
                                         file_name=file_name)

# Sample - Test
file_name = "Step_07_Stats_Categorical_Test"
o_stats = preprocess.stats_discrete_df(df=features["test_indep"][features_names_selected], includes=features_types_group["CATEGORICAL"],
                                       file_name=file_name)
file_name = "Step_07_Stats_Continuous_Test"
o_stats = preprocess.stats_continuous_df(df=features["test_indep"][features_names_selected], includes=features_types_group["CONTINUOUS"], 
                                         file_name=file_name)

"""
Explanation: 7.5. Summary Statistics
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
End of explanation
"""


file_name = "Step_07_Features"
readers_writers.save_serialised_compressed(path=CONSTANTS.io_path, title=file_name, objects=features)

# print     
print("File size: ", os.stat(os.path.join(CONSTANTS.io_path, file_name + ".bz2")).st_size)
print("Number of columns: ", len(features["train_indep"].columns)) 
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")

"""
Explanation: 7.6. Save Features
End of explanation
"""


# open fetures
file_name = "Step_07_Features"
features = readers_writers.load_serialised_compressed(path=CONSTANTS.io_path, title=file_name)

# print     
print("File size: ", os.stat(os.path.join(CONSTANTS.io_path, file_name + ".bz2")).st_size)
print("Number of columns: ", len(features["train_indep"].columns)) 
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")

# open scoring model files
rank_models = ["rfc", "gbrt", "randLogit"]
model_rank = dict()
o_summaries_df = dict()

for rank_model in rank_models:
    file_name = "Step_07_Model_Train_model_rank_" + rank_model
    if not readers_writers.exists_serialised(path=CONSTANTS.io_path, title=file_name, ext="bz2"):
        continue

    file_name = "Step_07_Model_Train_model_rank_" + rank_model
    model_rank[rank_model] = readers_writers.load_serialised_compressed(path=CONSTANTS.io_path, title=file_name)

    file_name = "Step_07_Model_Train_model_rank_summaries_" + rank_model
    o_summaries_df[rank_model] = readers_writers.load_serialised_compressed(path=CONSTANTS.io_path, title=file_name)

"""
Explanation: <br/><br/>
<br/><br/>
8. Model
<font style="font-weight:bold;color:orange">Load a Saved Samples and Features Ranking:</font> 
<br/> It is an optional step. The step loads the serialised & compressed outputs of Step-7.
End of explanation
"""


display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))

"""
Explanation: Verify features visually
End of explanation
"""


method_name = "rfc"
kwargs = {"n_estimators": 20, "criterion": 'gini', "max_depth": None, "min_samples_split": 100,
    "min_samples_leaf": 50, "min_weight_fraction_leaf": 0.0, "max_features": 'auto',
    "max_leaf_nodes": None, "bootstrap": True, "oob_score": False, "n_jobs": -1, "random_state": None,
    "verbose": 0, "warm_start": False, "class_weight": "balanced_subsample"}

"""
Explanation: <br/><br/>
8.1.  Initialise
8.1.1. Algorithms
<font style="font-weight:bold;color:red">Configure:</font> the trianing algorithm
<font style="font-weight:bold;color:brown">Algorithm 1</font>: Random Forest
End of explanation
"""


method_name = "lr"
kwargs = {"penalty": 'l1', "dual": False, "tol": 0.0001, "C": 1, "fit_intercept": True, "intercept_scaling": 1,
          "class_weight": None, "random_state": None, "solver": 'liblinear', "max_iter": 100, "multi_class": 'ovr',
          "verbose": 0, "warm_start": False, "n_jobs": -1}

"""
Explanation: <font style="font-weight:bold;color:brown">Algorithm 2</font>: Logistic Regression
End of explanation
"""


method_name = "lr_cv"
kwargs = {"Cs": 10, "fit_intercept": True, "cv": None, "dual": False, "penalty": 'l2', "scoring": None, 
          "solver": 'lbfgs', "tol": 0.0001, "max_iter": 10, "class_weight": None, "n_jobs": -1, "verbose": 0, 
          "refit": True, "intercept_scaling": 1.0, "multi_class": "ovr", "random_state": None}

"""
Explanation: <font style="font-weight:bold;color:brown">Algorithm 3</font>: Logistic Cross-Validation
End of explanation
"""


method_name = "nn"
kwargs = {"solver": 'lbfgs', "alpha": 1e-5, "hidden_layer_sizes": (5, 2), "random_state": 1}

"""
Explanation: <font style="font-weight:bold;color:brown">Algorithm 4</font>: Neural Network
End of explanation
"""


method_name = "knc"
kwargs = {"n_neighbors": 5, "weights": 'distance', "algorithm": 'auto', "leaf_size": 30,
          "p": 2, "metric": 'minkowski', "metric_params": None, "n_jobs": -1}

"""
Explanation: <font style="font-weight:bold;color:brown">Algorithm 5</font>: k-Nearest Neighbourhood
End of explanation
"""


method_name = "dtc"
kwargs = {"criterion": 'gini', "splitter": 'best', "max_depth": None, "min_samples_split": 30,
        "min_samples_leaf": 30, "min_weight_fraction_leaf": 0.0, "max_features": None,
        "random_state": None, "max_leaf_nodes": None, "class_weight": None, "presort": False}

"""
Explanation: <font style="font-weight:bold;color:brown">Algorithm 6</font>: Decision Tree
End of explanation
"""


method_name = "gbc"
kwargs = {"loss": 'deviance', "learning_rate": 0.1, "n_estimators": 100, "subsample": 1.0, "min_samples_split": 30,
        "min_samples_leaf": 30, "min_weight_fraction_leaf": 0.0, "max_depth": 3, "init": None, "random_state": None,
        "max_features": None, "verbose": 0, "max_leaf_nodes": None, "warm_start": False, "presort": 'auto'}

"""
Explanation: <font style="font-weight:bold;color:brown">Algorithm 7</font>: Gradient Boosting Classifier
End of explanation
"""


method_name = "nb"
training_method = TrainingMethod(method_name)
kwargs = {"alpha": 1.0, "fit_prior": True, "class_prior": None}

"""
Explanation: <font style="font-weight:bold;color:brown">Algorithm 8</font>: Naive Bayes<br/>
Note: features must be positive
End of explanation
"""


# select the target variable
target_feature = "label365" # "label30" , "label365" 

# file name
file_name = "Step_09_Model_" + method_name + "_" + target_feature

# initialise
training_method = TrainingMethod(method_name)

"""
Explanation: <br/><br/>
8.1.2. Other Settings
<font style="font-weight:bold;color:red">Configure:</font> other modelling settings
End of explanation
"""


sample_train = features["train_indep"][features_names_selected] # features["train_indep"][features_names_selected], features["train_indep"]
sample_train_target = features["train_target"][target_feature] # features["train_target"][target_feature]
sample_test = features["test_indep"][features_names_selected] # features["test_indep"][features_names_selected], features["test_indep"]
sample_test_target = features["test_target"][target_feature] # features["test_target"][target_feature]

"""
Explanation: 8.1.3. Features
End of explanation
"""


o_summaries = dict()
# Fit
model = training_method.train(sample_train, sample_train_target, **kwargs)
training_method.save_model(path=CONSTANTS.io_path, title=file_name)

# load model
# training_method.load(path=CONSTANTS.io_path, title=file_name)

# short summary
o_summaries = training_method.train_summaries()

"""
Explanation: 8.3. Fit
Fit the model, using the train sample
End of explanation
"""


o_summaries = dict()
# predict
model = training_method.predict(sample_train, "train")

# short summary
o_summaries = training_method.predict_summaries(pd.Series(sample_train_target), "train")

# Print the main performance statistics
for k in o_summaries.keys():
    print(k,  o_summaries[k])

# Print the by risk-bands of a selection of statistics
o_summaries = training_method.predict_summaries_risk_bands(pd.Series(sample_train_target), "train", np.arange(0, 1.05, 0.05))
display(o_summaries)

"""
Explanation: Predict & report performance, using the train sample
End of explanation
"""


o_summaries = dict()
# predict
model = training_method.predict(sample_test, "test")

# short summary
o_summaries = training_method.predict_summaries(pd.Series(sample_test_target), "test")

# Print the main performance statistics
for k in o_summaries.keys():
    print(k,  o_summaries[k])

# Print the by risk-bands of a selection of statistics
o_summaries = training_method.predict_summaries_risk_bands(pd.Series(sample_test_target), "test", np.arange(0, 1.05, 0.05))
display(o_summaries)

"""
Explanation: 8.4. Predict
Predict & report performance, using the test sample
End of explanation
"""


o_summaries = dict()
score = training_method.cross_validate(sample_test, sample_test_target, scoring="neg_mean_squared_error", cv=10)

# short summary
o_summaries = training_method.cross_validate_summaries()
print("Scores: ", o_summaries)

"""
Explanation: 8.5. Cross-Validation
Perform k-fold cross-validation
End of explanation
"""


training_method.save_model(path=CONSTANTS.io_path, title=file_name)

"""
Explanation: 8.6. Save
Save the training model.
End of explanation
"""


