# Python Standard Library
import getopt
import os
import sys
import math
import time
import collections
import random

# IPython
from IPython.display import display

# pandas
import pandas as pd
pd.set_option("display.max_rows", 10000)
pd.set_option("display.max_columns", 10000)

# Matplotlib
%matplotlib inline
import matplotlib.pyplot as plt                       
import matplotlib.mlab as mlab
from matplotlib.ticker import MultipleLocator

# seaborn
import seaborn as sns
sns.set_style("whitegrid")
sns.despine()

# NumPy
import numpy as np                                    

# SciPy
import scipy as sp
from scipy.stats import gaussian_kde

# StatsModels
import statsmodels.api as sm

 # scikit-learn
import sklearn                                       
from sklearn import metrics
from sklearn import preprocessing

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import LeaveOneGroupOut

from sklearn.neighbors import KNeighborsRegressor
from sklearn.model_selection import cross_val_predict
from sklearn.pipeline import make_pipeline

#TensorFlow
import tensorflow as tf
#Keras
from tensorflow import keras
from tensorflow.keras import layers
from keras.wrappers.scikit_learn import KerasRegressor

"""
Explanation: Wi-Fi Fingerprinting Experiments (Alternative Version)
Import modules and set up the environment
End of explanation
"""


def experiment_plots(results, save_to=None, figsize=(8, 8)):
    fig, axarr = plt.subplots(2, 1, figsize=figsize)
    
    for key, result in results.items():
        max_error = math.ceil(result["error"].max())
        kde = gaussian_kde(result["error"].values)
        X_plot=np.linspace(0, max_error, 1000)
        axarr[0].plot(X_plot, kde.evaluate(X_plot), "-", label=key)
    
    axarr[0].set_xlabel("Error (e) in meters (m)")
    axarr[0].set_ylabel(r"$F_X(e)$")
    axarr[0].xaxis.set_major_locator(MultipleLocator(0.5))
    axarr[0].set_xlim(0, result["error"].quantile(q=0.9975))
    axarr[0].legend()

    for key, result in results.items():
        ecdf = sm.distributions.ECDF(result["error"])
        x = np.linspace(min(result["error"]), max(result["error"]))
        y = ecdf(x)
        axarr[1].plot(x, y, label=key)
    
    axarr[1].set_xlabel("Error (e) in meters (m)")
    axarr[1].set_ylabel(r"$f_X(e)$")
    axarr[1].xaxis.set_major_locator(MultipleLocator(0.5))
    axarr[1].yaxis.set_major_locator(MultipleLocator(0.1))
    axarr[1].set_xlim(0, result["error"].quantile(q=0.9975))
    axarr[1].set_ylim(0)
    axarr[1].legend()
    
    fig.tight_layout()
    if save_to is not None:
        fig.savefig(output_data_directory+"/"+save_to, dpi=300)
    plt.show()

def experiment_statistics(result):
    statistics = collections.OrderedDict([
        ("mae",  result["error"].abs().mean()),
        ("mse",  (result["error"]**2).mean()),
        ("rmse", np.sqrt((result["error"]**2).mean())),
        ("sd",   result["error"].std()),
        ("p50",  result["error"].quantile(q=0.50)),
        ("p75",  result["error"].quantile(q=0.75)),
        ("p90",  result["error"].quantile(q=0.90)),
        ("p95",  result["error"].quantile(q=0.95)),
        ("min",  result["error"].min()),
        ("max",  result["error"].max()),
    ])    
    return statistics

def regression_experiment(regressor, data, test_data, train_cols, coord_cols, scaler=None, leave_out=False):
    result = None
    if scaler is not None:
        estimator = make_pipeline(scaler, regressor)
    else:
        estimator = regressor
    locations = data.groupby(coord_cols).indices.keys()
    for coords in locations:
        if leave_out is True:
            train_data = data[(data[coord_cols[0]] != coords[0]) |
                                  (data[coord_cols[1]] != coords[1])].reset_index(drop=True)
        else:
            train_data = data.reset_index(drop=True)
        target_values = test_data[(test_data[coord_cols[0]] == coords[0]) &
                                      (test_data[coord_cols[1]] == coords[1])].reset_index(drop=True)
            
        estimator.fit(train_data[train_cols], train_data[coord_cols])
        predictions = pd.DataFrame(estimator.predict(target_values[train_cols]), columns=coord_cols)
        curr_result = target_values[coord_cols].join(predictions, rsuffix="_predicted")
        error = pd.DataFrame((predictions[coord_cols] - curr_result[coord_cols]).apply(np.linalg.norm, axis=1),
                             columns=["error"])
        curr_result = pd.concat([curr_result, error], axis=1)
        result = pd.concat([result, curr_result])
    return result

def knn_experiment(data, test_data, train_cols, coord_cols,
                   scaler=None, n_neighbors=5, weights="uniform",
                   algorithm="auto", leaf_size=30, p=2, metric="minkowski",
                   metric_params=None, n_jobs=1, leave_out=False):
    regressor = KNeighborsRegressor(n_neighbors=n_neighbors, weights=weights, algorithm=algorithm,
                              leaf_size=leaf_size, p=p, metric=metric,
                              metric_params=metric_params, n_jobs=n_jobs)
    return regression_experiment(regressor, data, test_data, train_cols, coord_cols, scaler, leave_out)

def keras_model():
    model = keras.Sequential([
        layers.Dense(128, activation='relu', input_shape=[len(mac_addresses)]),
        layers.Dense(128, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(2)
    ])
    model.compile(loss="mean_squared_error" , optimizer="adam", metrics=["mean_squared_error", "mean_absolute_error"])
    return model

def keras_experiment(data, test_data, train_cols, coord_cols, scaler=None, batch_size=10, epochs=25, verbose=0, leave_out=False):
    regressor = KerasRegressor(build_fn=keras_model, batch_size=batch_size, epochs=epochs, verbose=verbose)
    return regression_experiment(regressor, data, test_data, train_cols, coord_cols, scaler, leave_out)

def regression_experiment_cv(regressor, data, cross_validation, train_cols, coord_cols, scaler=None):
    result = None
    if scaler is not None:
        estimator = make_pipeline(scaler, regressor)
    else:
        estimator = regressor
    X = data[train_cols]
    y = data[coord_cols]
    predictions = pd.DataFrame(cross_val_predict(estimator, X, y, cv=cross_validation), columns=coord_cols)
    result = y.join(predictions, rsuffix="_predicted")
    error = pd.DataFrame((predictions[coord_cols] - result[coord_cols]).apply(np.linalg.norm, axis=1), columns=["error"])
    result = pd.concat([result, error], axis=1)    
    return result


def knn_experiment_cv(data, cross_validation, train_cols, coord_cols,    
                      scaler=None, n_neighbors=5, weights='uniform',
                      algorithm="auto", leaf_size=30, p=2, metric="minkowski",
                      metric_params=None, n_jobs=1):    
    regressor = KNeighborsRegressor(n_neighbors=n_neighbors, weights=weights, algorithm=algorithm,
                                    leaf_size=leaf_size, p=p, metric=metric,
                                    metric_params=metric_params, n_jobs=n_jobs)
    
    return regression_experiment_cv(regressor, data, cross_validation, train_cols, coord_cols, scaler)

def keras_experiment_cv(data, cross_validation, train_cols, coord_cols, scaler=None, batch_size=10, epochs=25, verbose=0):
    regressor = KerasRegressor(build_fn=keras_model, batch_size=batch_size, epochs=epochs, verbose=verbose)
    return regression_experiment_cv(regressor, data, cross_validation, train_cols, coord_cols, scaler)

"""
Explanation: Helper Functions
End of explanation
"""


from yanux.cruncher.model.loader import JsonLoader

"""
Explanation: Load the model classes
A class responsible for loading a JSON file (or all the JSON files in a given directory) into a Python dictionary
End of explanation
"""


from yanux.cruncher.model.wifi import WifiLogs

"""
Explanation: A class that takes a set of Python dictionaries containing Wi-Fi logging data loaded from JSON files collected by the YanuX Scavenger Android application
End of explanation
"""


input_data_directory = "data"
output_data_directory = "out"
statistics_excel_writer = pd.ExcelWriter(output_data_directory+"/alt-statistics.xlsx")

"""
Explanation: Initialize Input & Output Data Directories and other parameters
End of explanation
"""


if not os.path.exists(output_data_directory):
    os.makedirs(output_data_directory)

"""
Explanation: Create the output directory if it doesn't exist
End of explanation
"""


json_loader = JsonLoader(input_data_directory+"/wifi-fingerprints")
wifi_logs = WifiLogs(json_loader.json_data)

"""
Explanation: Load Data from the Input Data Directory
Load all files from the data folder.
The logs currently placed there were collected using the Yanux Scavenger Android application on April 28<sup>th</sup>, 2016 using an LG Nexus 5 running Androdid Marshmallow 6.0.1
End of explanation
"""


num_samples_per_location = int(len(wifi_logs.wifi_samples()) / len(wifi_logs.locations))
num_samples_per_location

"""
Explanation: Wi-Fi Readings
Number of Recorded Samples per Location
End of explanation
"""


wifi_results_columns = ["filename", "place", "floor", "x", "y", "orientation", "sample_id", "mac_address",
                        "timestamp", "signal_strength"]

wifi_results = pd.DataFrame(wifi_logs.wifi_results(), columns=wifi_results_columns)
wifi_results.to_csv(output_data_directory + "/alt-wifi_results.csv")

"""
Explanation: Store the data into a Pandas Dataframe, in which each Wi-Fi result reading is represented by a single line
End of explanation
"""


mac_addresses = wifi_results.mac_address.unique()

"""
Explanation: Identify the unique MAC Addresses present in the recorded data. Each one represents a single Wi-Fi Access Point.
End of explanation
"""


wifi_samples_columns = ["filename", "place", "floor", "x", "y", "orientation", "sample_id", "timestamp"]
wifi_samples_columns.extend(mac_addresses)

wifi_samples = pd.DataFrame(wifi_logs.wifi_samples(), columns=wifi_samples_columns)
wifi_samples = wifi_samples.sort_values(["filename", "x", "y", "floor", "sample_id"]).reset_index(drop=True)
wifi_samples.to_csv(output_data_directory + "/alt-wifi_samples.csv")

"""
Explanation: Similarly, store the data into a Pandas Dataframe in which each line represents a single sampling cycle with n different readings for each of the Access Points within range. Those readings are stored as columns along each sample.
End of explanation
"""


len(wifi_results)

"""
Explanation: Data Set Statistics
Number of Results
End of explanation
"""


len(wifi_results.mac_address.unique())

"""
Explanation: Number of Unique Mac Addresses
End of explanation
"""


wifi_results_mac_address_group = wifi_results.groupby("mac_address")
wifi_results_mac_address_group.size().plot(kind="bar")
wifi_results_mac_address_group.size()

wifi_results_mac_address_group.size().mean()

"""
Explanation: How often has each Access Point been detected
End of explanation
"""


wifi_results_coord_group = wifi_results.groupby(["x", "y"])
wifi_results_coord_group.size().plot(kind="bar")
wifi_results_coord_group.size()

wifi_results_coord_group.size().describe()

"""
Explanation: How many Wi-Fi results were gathered at each location
End of explanation
"""


wifi_ap_per_location = wifi_samples.groupby(["x","y"]).min()[wifi_results_mac_address_group.size().keys()].count(axis=1)
wifi_ap_per_location.plot(kind="bar")
wifi_ap_per_location

wifi_ap_per_location.describe()

"""
Explanation: How many APs were detected at each location
End of explanation
"""


coords = wifi_results[["x","y"]].drop_duplicates().sort_values(by=["x","y"]).reset_index(drop=True)
coords_plot_size = (min(coords["x"].min(),coords["y"].min()), max(coords["x"].max(),coords["y"].max()))
#TODO: If I end up using it in the document, then I should refactor the plot to use matplotlib directly to tweak a few things.
coords.plot(figsize=(16,5), x="x",y="y", style="o", grid=True, legend=False,
            xlim=coords_plot_size, ylim=coords_plot_size,
            xticks=np.arange(coords_plot_size[0]-1, coords_plot_size[1]+1, 1),
            yticks=np.arange(coords_plot_size[0]-1, coords_plot_size[1]+1, 1)).axis('equal')

"""
Explanation: The coordinates of the points where data was captured
End of explanation
"""


wifi_results.hist(column="signal_strength")

"""
Explanation: Signal Strength Distribution
End of explanation
"""


train_cols = mac_addresses
coord_cols = ["x","y"]

full_data_scenario = wifi_samples.copy()
full_data_scenario_groups = full_data_scenario["x"].map(str)+","+full_data_scenario["y"].map(str)+","+full_data_scenario["filename"].map(str)

default_data_scenario, test_data_scenario, default_data_scenario_groups, test_data_scenario_groups=train_test_split(full_data_scenario,
                                                                                                                    full_data_scenario_groups,
                                                                                                                    stratify=full_data_scenario_groups,
                                                                                                                    test_size=0.3,
                                                                                                                    random_state=42)

default_data_scenario = default_data_scenario.reset_index(drop=True)
test_data_scenario = test_data_scenario.reset_index(drop=True)
default_data_scenario_groups = default_data_scenario_groups.reset_index(drop=True)
test_data_scenario_groups = test_data_scenario_groups.reset_index(drop=True)

"""
Explanation: Set a train and test scenario to be used by default when testing.
End of explanation
"""


default_n_neighbors=5
default_weights="distance"
default_metric="manhattan"
default_nan_filler=default_data_scenario[mac_addresses].min().min()*1.010
default_scaler=preprocessing.MinMaxScaler()
default_leave_out=True

#DefaultCrossValidator = StratifiedKFold
DefaultCrossValidator = LeaveOneGroupOut

"""
Explanation: Playground
Defaults
End of explanation
"""


n_neighbors=default_n_neighbors
weights=default_weights
metric=default_metric
nan_filler=default_nan_filler
scaler=default_scaler
leave_out=default_leave_out

curr_data = default_data_scenario.fillna(nan_filler)
test_data = test_data_scenario.fillna(nan_filler)

curr_result = knn_experiment(curr_data,
                             test_data,
                             mac_addresses,
                             coord_cols,
                             scaler=scaler,
                             algorithm="brute",
                             n_neighbors=n_neighbors,
                             weights=weights,
                             metric=metric,
                             leave_out=leave_out)

curr_statistics = experiment_statistics(curr_result)    
curr_result.to_csv(output_data_directory+"/alt-results-base.csv")

statistics_table = pd.DataFrame([curr_statistics], columns=list(curr_statistics.keys()))
statistics_table.to_csv(output_data_directory+"/alt-statistics-base.csv")
statistics_table.to_excel(statistics_excel_writer, "base")

#show table
display(statistics_table)
#plots
experiment_plots({'Base KNN':curr_result})

"""
Explanation: Base KNN (k-nearest neighbors) Example
End of explanation
"""


n_neighbors=default_n_neighbors
weights=default_weights
metric=default_metric
nan_filler=default_nan_filler
scaler=default_scaler
leave_out=default_leave_out

curr_data = default_data_scenario.fillna(nan_filler)
cross_validation = DefaultCrossValidator()
y = default_data_scenario_groups

curr_result = knn_experiment_cv(curr_data,
                                cross_validation.split(curr_data[mac_addresses], y, groups=default_data_scenario_groups),
                                mac_addresses,
                                coord_cols,
                                scaler=scaler,
                                algorithm="brute",
                                n_neighbors=n_neighbors,
                                weights=weights,
                                metric=metric)

curr_statistics = experiment_statistics(curr_result)    
curr_result.to_csv(output_data_directory+"/alt-results-base-cv.csv")

statistics_table = pd.DataFrame([curr_statistics], columns=list(curr_statistics.keys()))
statistics_table.to_csv(output_data_directory+"/alt-statistics-base-cv.csv")
statistics_table.to_excel(statistics_excel_writer, "base-cv")

#show table
display(statistics_table)
#plots
experiment_plots({'Base KNN CV':curr_result})

"""
Explanation: Base KNN (k-nearest neighbors) Cross-validated Example
End of explanation
"""


default_batch_size=10
default_epochs=25
default_verbose=1

"""
Explanation: Keras
Keras Defaults
End of explanation
"""


model = keras_model()
model.summary()

X_train = default_data_scenario[mac_addresses].fillna(nan_filler)
y_train = default_data_scenario[coord_cols]

X_test = test_data_scenario[mac_addresses].fillna(nan_filler)
y_test = test_data_scenario[coord_cols]

model.fit(X_train, y_train, batch_size=default_batch_size, epochs=default_epochs, verbose=default_verbose)
model.evaluate(X_test, y_test, verbose=default_verbose)

"""
Explanation: Keras - Simple Example
End of explanation
"""


model = keras_model()
model.summary()

data_train, data_validate = train_test_split(default_data_scenario, stratify=default_data_scenario_groups, test_size=0.3, random_state=24)

data_train = data_train.reset_index(drop=True)
data_validate = data_validate.reset_index(drop=True)

X_train = data_train[mac_addresses].fillna(nan_filler)
y_train = data_train[coord_cols]

X_validate = data_validate[mac_addresses].fillna(nan_filler)
y_validate = data_validate[coord_cols]

X_test = test_data_scenario[mac_addresses].fillna(nan_filler)
y_test = test_data_scenario[coord_cols]

model.fit(X_train, y_train, validation_data=(X_validate, y_validate), batch_size=default_batch_size, epochs=default_epochs, verbose=default_verbose)
model.evaluate(X_test, y_test, verbose=default_verbose)

"""
Explanation: Keras - Simple Example with "validation_data"
End of explanation
"""


nan_filler=default_nan_filler
scaler=default_scaler
batch_size=default_batch_size
epochs=default_epochs
verbose=0 #default_verbose
leave_out=default_leave_out

curr_data = default_data_scenario.fillna(nan_filler)
test_data = test_data_scenario.fillna(nan_filler)

curr_result = keras_experiment(curr_data,
                               test_data,
                               mac_addresses,
                               coord_cols,
                               scaler=scaler,
                               batch_size=batch_size,
                               epochs=epochs,
                               verbose=verbose,
                               leave_out=leave_out)

curr_statistics = experiment_statistics(curr_result)    
curr_result.to_csv(output_data_directory+"/alt-results-keras.csv")

statistics_table = pd.DataFrame([curr_statistics], columns=list(curr_statistics.keys()))
statistics_table.to_csv(output_data_directory+"/alt-statistics-keras.csv")
statistics_table.to_excel(statistics_excel_writer, "keras")

#show table
display(statistics_table)
#plots
experiment_plots({'Base Keras':curr_result})

"""
Explanation: Base Keras Example
End of explanation
"""


nan_filler=default_nan_filler
scaler=default_scaler
batch_size=default_batch_size
epochs=default_epochs
verbose=0 #default_verbose

curr_data = default_data_scenario.fillna(nan_filler)
cross_validation = DefaultCrossValidator()
y = default_data_scenario_groups

curr_result = keras_experiment_cv(curr_data,
                                  cross_validation.split(curr_data[mac_addresses], y, groups=default_data_scenario_groups),
                                  mac_addresses,
                                  coord_cols,
                                  batch_size=batch_size,
                                  epochs=epochs,
                                  verbose=verbose,
                                  scaler=scaler)

curr_statistics = experiment_statistics(curr_result)    
curr_result.to_csv(output_data_directory+"/alt-results-keras-cv.csv")

statistics_table = pd.DataFrame([curr_statistics], columns=list(curr_statistics.keys()))
statistics_table.to_csv(output_data_directory+"/alt-statistics-keras-cv.csv")
statistics_table.to_excel(statistics_excel_writer, "base-cv")

#show table
display(statistics_table)
#plots
experiment_plots({'Base Keras CV':curr_result})

"""
Explanation: Base Keras Cross-validated Example
End of explanation
"""


metric=default_metric
nan_filler=default_nan_filler
scaler=default_scaler

n_neighbors=np.arange(1,11,1)
weights=["uniform", "distance"]

cross_validation = DefaultCrossValidator()
y = default_data_scenario_groups

curr_data  = default_data_scenario.fillna(nan_filler)

# Just a statistics accumulator
statistics = []
for k in n_neighbors:
    for w in weights:
        curr_result = knn_experiment_cv(curr_data,
                                        cross_validation.split(curr_data[mac_addresses],
                                                               y,
                                                               groups=default_data_scenario_groups),
                                        mac_addresses,
                                        coord_cols,
                                        scaler=scaler,
                                        algorithm="brute",
                                        n_neighbors=k,
                                        weights=w,
                                        metric=metric)
        
        curr_statistics = experiment_statistics(curr_result)
        curr_statistics["k"] = k
        curr_statistics["weights"] = w
        statistics.append(curr_statistics)
    
cols = ["k","weights"] + list(curr_statistics.keys())[:-2]
statistics_table = pd.DataFrame(statistics, columns=cols)
statistics_table.to_csv(output_data_directory + "/alt-statistics-neighbors-weights.csv")
statistics_table.to_excel(statistics_excel_writer, "neighbors-weights")

#show table
display(statistics_table.sort_values(cols[3:]))

# Plotting Error statistics
fig, ax = plt.subplots(figsize=(8, 5))
index = n_neighbors

ax.plot(index, statistics_table[statistics_table["weights"] == "uniform"]["mae"].tolist(),
         color="b", ls="-", label="Uniform (MAE)")

ax.plot(index, statistics_table[statistics_table["weights"] == "distance"]["mae"].tolist(),
         color="r", ls="-", label="Distance (MAE)")

ax.plot(index, statistics_table[statistics_table["weights"] == "uniform"]["rmse"].tolist(),
         color="b", ls="--", label="Uniform (RMSE)")

ax.plot(index, statistics_table[statistics_table["weights"] == "distance"]["rmse"].tolist(),
         color="r", ls="--", label="Distance (RMSE)")

ax.xaxis.set_major_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.05))

ax.set_xlabel("Number of Neighbours (k)")
ax.set_ylabel("Error (e) in meters (m)")

plt.legend()
plt.tight_layout()
plt.savefig(output_data_directory+"/alt-plot-neighbors_weights.pdf", dpi=300)
plt.show()

"""
Explanation: KNN - # Neighbors & Distance Weights
End of explanation
"""


n_neighbors=default_n_neighbors
weights=default_weights
nan_filler=default_nan_filler
scaler=default_scaler

distance_statistics=["euclidean", "manhattan", "canberra", "braycurtis"]

cross_validation = DefaultCrossValidator()
y = default_data_scenario_groups

curr_data = default_data_scenario.fillna(nan_filler)
# Results and statistics accumulators
results = {}
statistics = []
for metric in distance_statistics:
    curr_result = knn_experiment_cv(curr_data,
                                    cross_validation.split(curr_data[mac_addresses],
                                                           y,
                                                           groups=default_data_scenario_groups),
                                    mac_addresses,
                                    coord_cols,
                                    scaler=scaler,
                                    algorithm="brute",
                                    n_neighbors=n_neighbors,
                                    weights=weights,
                                    metric=metric)
    results[metric] = curr_result
    curr_statistics = experiment_statistics(curr_result)
    curr_statistics["metric"] = metric
    statistics.append(curr_statistics)
    
cols = ["metric"] + list(curr_statistics.keys())[:-1]
statistics_table = pd.DataFrame(statistics, columns=cols)
statistics_table.to_csv(output_data_directory + "/alt-statistics-metric.csv")
statistics_table.to_excel(statistics_excel_writer, "metric")
#show table
display(statistics_table.sort_values(cols[2:]))
#plots
experiment_plots(results, "alt-plot-metric.pdf")

"""
Explanation: KNN - Metric
Just test a few different distance statistics to assess if there is a better alternative than the plain old euclidean distance. The tested statistics include:
- Euclidean Distance
    - sqrt(sum((x - y)^2))
- Manhattan Distance
    - sum(|x - y|) 
- Chebyshev Distance
    - sum(max(|x - y|))
- Hamming Distance
    - N_unequal(x, y) / N_tot
- Canberra Distance
    - sum(|x - y| / (|x| + |y|))
- Braycurtis Similarity
    - sum(|x - y|) / (sum(|x|) + sum(|y|))
End of explanation
"""


n_neighbors=default_n_neighbors
weights=default_weights
metric=default_metric
nan_filler=default_nan_filler

cross_validation = DefaultCrossValidator()
y = default_data_scenario_groups

scalers = {"No Scaling": None,
           "Rescaling": preprocessing.MinMaxScaler(),
           "Standardization": preprocessing.StandardScaler()}
# Results and statistics accumulators
results = {}
statistics = []
for scaler_name, scaler in scalers.items():
    curr_data = default_data_scenario.fillna(nan_filler)
    curr_result = knn_experiment_cv(curr_data,
                                    cross_validation.split(curr_data[mac_addresses],
                                                           y,
                                                           groups=default_data_scenario_groups),
                                    mac_addresses,
                                    coord_cols,
                                    scaler=scaler,
                                    algorithm="brute",
                                    n_neighbors=n_neighbors,
                                    weights=weights,
                                    metric=metric)
    results[scaler_name] = curr_result
    curr_statistics = experiment_statistics(results[scaler_name])
    curr_statistics["scaler"] = scaler_name
    statistics.append(curr_statistics)

cols = ["scaler"] + list(curr_statistics.keys())[:-1]
statistics_table = pd.DataFrame(statistics, columns=cols)
statistics_table.to_csv(output_data_directory + "/alt-statistics-feature_scaling.csv")
statistics_table.to_excel(statistics_excel_writer, "feature_scaling")
#show table
display(statistics_table.sort_values(cols[2:]))
#plots
experiment_plots(results, "alt-plot-feature_scaling.pdf")

"""
Explanation: KNN - Feature Scaling
Test different data scaling and normalization approaches to find out if any of them provides a clear advantage over the others.
End of explanation
"""


n_neighbors=default_n_neighbors
weights=default_weights
metric=default_metric
scaler=default_scaler

min_rssi_value = default_data_scenario[mac_addresses].min().min()
mean_rssi_value = default_data_scenario[mac_addresses].stack().mean()
median_rssi_value = default_data_scenario[mac_addresses].stack().median()
max_rssi_value = default_data_scenario[mac_addresses].max().max()

nan_fillers = [min_rssi_value,
               min_rssi_value*1.001,
               min_rssi_value*1.010,
               min_rssi_value*1.100,
               min_rssi_value*1.500,
               mean_rssi_value,
               median_rssi_value,
               max_rssi_value]

cross_validation = DefaultCrossValidator()
y = default_data_scenario_groups

# Results and statistics accumulators
results = {}
statistics = []
for nf in nan_fillers:
    curr_data = default_data_scenario.fillna(nf)
    curr_result = knn_experiment_cv(curr_data,
                                    cross_validation.split(curr_data[mac_addresses],
                                                           y,
                                                           groups=default_data_scenario_groups),
                                    mac_addresses,
                                    coord_cols,
                                    scaler=scaler,
                                    algorithm="brute",
                                    n_neighbors=n_neighbors,
                                    weights=weights,
                                    metric=metric)
    results[nf] = curr_result
    curr_statistics = experiment_statistics(curr_result)
    curr_statistics["nan_filler"] = nf
    statistics.append(curr_statistics)

cols = ["nan_filler"] + list(curr_statistics.keys())[:-1]
statistics_table = pd.DataFrame(statistics, columns=cols)
statistics_table.to_csv(output_data_directory + "/alt-statistics-nan_filler.csv")
statistics_table.to_excel(statistics_excel_writer, "nan_filler")
#show table
display(statistics_table.sort_values(cols[2:]))
#plots
experiment_plots(results, "alt-plot-nan_filler.pdf")

"""
Explanation: KNN - NaN filler values
Test which is the signal strength value that should be considered for Access Points that are currently out of range. This is needed as part of the process of computing the distance/similarity between different fingerprints.
End of explanation
"""


k_neighbors_values = range(1,31,1)
weights_values = [
                    "uniform",
                    "distance"
                 ]
metric_values = [
                    "euclidean",
                    "manhattan",
                    "canberra", 
                    "braycurtis"
                ]
algorithm_values = ["brute"]

nan_filler=default_nan_filler

curr_data = default_data_scenario.fillna(nan_filler)

param_grid = {
                "kneighborsregressor__n_neighbors": list(k_neighbors_values),
                "kneighborsregressor__weights": weights_values,
                "kneighborsregressor__metric": metric_values,
                "kneighborsregressor__algorithm": algorithm_values,
              }

scaler = preprocessing.StandardScaler()

cross_validation = DefaultCrossValidator()
y = default_data_scenario_groups

estimator = make_pipeline(preprocessing.StandardScaler(), KNeighborsRegressor())

grid = GridSearchCV(estimator,
                    param_grid=param_grid,
                    cv=cross_validation.split(curr_data[mac_addresses], y, groups=default_data_scenario_groups),
                    n_jobs=-1,
                    scoring=sklearn.metrics.make_scorer(sklearn.metrics.mean_squared_error,
                                                        greater_is_better=False,
                                                        multioutput="uniform_average"))

grid.fit(curr_data[mac_addresses], curr_data[coord_cols], groups=default_data_scenario_groups)
print("Best parameters set found on development set:")
print(grid.best_params_)

print("Grid scores on development set:")
gridcv_results = pd.DataFrame(grid.cv_results_)
gridcv_results[['mean_test_score', 'std_test_score', 'params']]

"""
Explanation: KNN - Grid Search (Automatically searching for the best estimator parameters)
End of explanation
"""


filename_prefixes = ["left-to-right-point", "right-to-left-point"]
filename_prefix_data_scenarios = {}
filename_prefix_data_scenarios["all"] = default_data_scenario
for filename_prefix in filename_prefixes:
    filename_prefix_data_scenarios[filename_prefix] = default_data_scenario[default_data_scenario["filename"].str.startswith(filename_prefix)].reset_index(drop=True)

filename_prefix_test_data_scenarios = {}
filename_prefix_test_data_scenarios["all"] = test_data_scenario
for filename_prefix in filename_prefixes:
    filename_prefix_test_data_scenarios[filename_prefix] = test_data_scenario[test_data_scenario["filename"].str.startswith(filename_prefix)].reset_index(drop=True)

n_neighbors=default_n_neighbors
weights=default_weights
metric=default_metric
nan_filler=default_nan_filler
scaler=default_scaler

# Results and statistics accumulators
results = {}
statistics = []
for train_data_keys, train_data in filename_prefix_data_scenarios.items():
    for test_data_keys, test_data in filename_prefix_test_data_scenarios.items():
        curr_data = train_data.fillna(nan_filler)
        curr_test_data = test_data.fillna(nan_filler)
        curr_result = knn_experiment(curr_data,
                                     curr_test_data,
                                     mac_addresses,
                                     coord_cols,
                                     scaler=scaler,
                                     algorithm="brute",
                                     n_neighbors=n_neighbors,
                                     weights=weights,
                                     metric=metric)
        label = "Train: "+train_data_keys+" Test: "+test_data_keys
        results[label] = curr_result
        curr_statistics = experiment_statistics(curr_result)
        curr_statistics["orientation"] = label
        statistics.append(curr_statistics)

cols = ["orientation"] + list(curr_statistics.keys())[:-1]
statistics_table = pd.DataFrame(statistics, columns=cols)
statistics_table.to_csv(output_data_directory + "/alt-statistics-orientation.csv")
statistics_table.to_excel(statistics_excel_writer, "orientation")
#show table
display(statistics_table.sort_values(cols[2:]))
#plots
experiment_plots(results, "alt-plot-orientation.pdf")

"""
Explanation: KNN - Impact of orientation in the results
End of explanation
"""


subset_reference_points_scenarios = {}
coords_indices = default_data_scenario.groupby(coord_cols).indices

odd_coords_keys = list(coords_indices.keys())[0::2]
odd_ids = []
for key in odd_coords_keys:
    odd_ids.extend(coords_indices[key])

even_coords_keys = list(coords_indices.keys())[1::2]
even_ids = []
for key in even_coords_keys:
    even_ids.extend(coords_indices[key])

subset_reference_points_scenarios["odd"] = full_data_scenario.loc[odd_ids].reset_index(drop=True)
subset_reference_points_scenarios["even"] = full_data_scenario.loc[even_ids].reset_index(drop=True)
subset_reference_points_scenarios["all"] = full_data_scenario

n_neighbors=default_n_neighbors
weights=default_weights
metric=default_metric
nan_filler=default_nan_filler
scaler=default_scaler
# Results and statistics accumulators
results = {}
statistics = []
for train_data_keys, train_data in subset_reference_points_scenarios.items():
    curr_data = train_data.fillna(nan_filler)
    
    cross_validation = LeaveOneGroupOut()
    curr_data_scenario_groups = curr_data["x"].map(str)+","+curr_data["y"].map(str)
    y = curr_data_scenario_groups
    
    curr_result = knn_experiment_cv(curr_data,
                                    cross_validation.split(curr_data[mac_addresses],
                                                           y,
                                                           groups=curr_data_scenario_groups),
                                    mac_addresses,
                                    coord_cols,
                                    scaler=scaler,
                                    algorithm="brute",
                                    n_neighbors=n_neighbors,
                                    weights=weights,
                                    metric=metric)
    results[train_data_keys] = curr_result
    curr_statistics = experiment_statistics(curr_result)
    curr_statistics["reference_points_spacing"] = train_data_keys
    statistics.append(curr_statistics)

cols = ["reference_points_spacing"] + list(curr_statistics.keys())[:-1]
statistics_table = pd.DataFrame(statistics, columns=cols)
statistics_table.to_csv(output_data_directory + "/alt-statistics-reference_points_spacing.csv")
statistics_table.to_excel(statistics_excel_writer, "reference_points_spacing")
#show table
display(statistics_table.sort_values(cols[2:]))
#plots
experiment_plots(results, "alt-plot-reference_points_spacing.pdf")

"""
Explanation: KNN - Impact of the spacing between reference points in the results
End of explanation
"""


n_neighbors=default_n_neighbors
weights=default_weights
metric=default_metric
nan_filler=default_nan_filler
scaler=default_scaler

partial_data = [1.0, 0.9, 0.7, 0.5, 0.3, 0.25]
repetitions = 5
train_data = full_data_scenario[mac_addresses]
target_values = full_data_scenario[coord_cols].copy()
target_values["label"] = full_data_scenario_groups

# Results and statistics accumulators
results = {}
statistics = []
for partial in partial_data:
    curr_result = pd.DataFrame()
    for repetition in range(repetitions):
        if partial == 1.0:
            X_train = train_data
            y_train = target_values
        else:
            X_train, X_test, y_train, y_test = train_test_split(train_data,
                                                                target_values,
                                                                test_size=1-partial,
                                                                stratify=target_values["label"].values)

        train_split_data = pd.concat([y_train, X_train], axis=1).reset_index(drop=True)
        curr_data = train_split_data.fillna(nan_filler)
        
        cross_validation = DefaultCrossValidator()
        curr_data_scenario_groups = curr_data["x"].map(str)+","+curr_data["y"].map(str)
        y = curr_data_scenario_groups
        
        curr_result = curr_result.append(knn_experiment_cv(curr_data,
                                                           cross_validation.split(curr_data[mac_addresses],
                                                                                  y,
                                                                                  groups=curr_data_scenario_groups),
                                                            mac_addresses,
                                                            coord_cols,
                                                            scaler=scaler,
                                                            algorithm="brute",
                                                            n_neighbors=n_neighbors,
                                                            weights=weights,
                                                            metric=metric), ignore_index=True)
    results[partial] = curr_result
    curr_statistics = experiment_statistics(curr_result)
    curr_statistics["partial_data"] = partial
    statistics.append(curr_statistics)

cols = ["partial_data"] + list(curr_statistics.keys())[:-1]
statistics_table = pd.DataFrame(statistics, columns=cols)
statistics_table.to_csv(output_data_directory + "/alt-statistics-partial_data.csv")
statistics_table.to_excel(statistics_excel_writer, "partial_data")
#show table
display(statistics_table.sort_values(cols[2:]))
#plots
experiment_plots(results, "alt-plot-partial_data.pdf")

"""
Explanation: KNN - Impact of the amount of available data in the results
End of explanation
"""


statistics_excel_writer.save()

"""
Explanation: Save all the data that was collected into an Excel file
End of explanation
"""


