# -*- coding: utf-8 -*-
"""Classification approach for human activity detection using smart phone Dataset.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1k2eGF5VY4KfY8P-CDBSERVsf0XF2-LHb

# Data link
https://archive.ics.uci.edu/ml/datasets/human+activity+recognition+using+smartphones<br/>
[click](https://github.com/MadhavShashi/Human-Activity-Recognition-Using-Smartphones-Sensor-DataSet/blob/master/README.md#problem-statement)

# Data Description

Total classes for clustering<br/>
1. WALKING<br/>
2. WALKING_UPSTAIRS<br/>
3. WALKING_DOWNSTAIRS<br/>
4. SITTING<br/>
5. STANDING<br/>
6. LAYING<br/>
<br/>

# Importing Python Libraries
"""

# import pandas module
import pandas as pd
import numpy as np
from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.multiclass import OneVsRestClassifier
from sklearn.utils import shuffle
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, accuracy_score, classification_report
import warnings
warnings.filterwarnings('ignore')
import plotly.express as px

# load the dataset from the google drive
from google.colab import drive
drive.mount('/content/drive')

"""# Feature Description"""

# get the name of the features from the file (features.txt)
features = list()
with open('/content/drive/MyDrive/UCI HAR Dataset/features.txt') as f:
    features = [line.split()[1] for line in f.readlines()]

print(f'No of Features: {len(features)}')
print("Name of the features:\n",features)

"""**For train (for plotting)**"""

# get the data from txt files to pandas dataffame
X_train = pd.read_csv('/content/drive/MyDrive/UCI HAR Dataset/train/X_train.txt', delim_whitespace=True, header=None)
X_train.columns = features

y_train = pd.read_csv('/content/drive/MyDrive/UCI HAR Dataset/train/y_train.txt', names=['Activity'], squeeze=True)
y_train_labels = y_train.map({1: 'WALKING', 2:'WALKING_UPSTAIRS',3:'WALKING_DOWNSTAIRS', 4:'SITTING', 5:'STANDING',6:'LAYING'})

# put all columns in a single dataframe
train = X_train.copy()
train['subject'] = pd.read_csv('/content/drive/MyDrive/UCI HAR Dataset/train/subject_train.txt', header=None, squeeze=True)
train['Activity'] = y_train
train['ActivityName'] = y_train_labels

"""**For Test (for plotting)**"""

# get the data from txt files to pandas dataffame
X_test = pd.read_csv('/content/drive/MyDrive/UCI HAR Dataset/test/X_test.txt', delim_whitespace=True, header=None)

y_test = pd.read_csv('/content/drive/MyDrive/UCI HAR Dataset/test/y_test.txt', names=['Activity'], squeeze=True)
y_test_labels = y_test.map({1: 'WALKING', 2:'WALKING_UPSTAIRS',3:'WALKING_DOWNSTAIRS', 4:'SITTING', 5:'STANDING',6:'LAYING'})
# put all columns in a single dataframe
test = X_test.copy()
test['subject'] = pd.read_csv('/content/drive/MyDrive/UCI HAR Dataset/test/subject_test.txt', header=None, squeeze=True)
test['Activity'] = y_test
test['ActivityName'] = y_test_labels

# for plotting only
columns = train.columns

columns = list(map(lambda a: a.replace('()', ''), columns))
columns = list(map(lambda a: a.replace('-',''), columns))
columns = list(map(lambda a: a.replace(',',''), columns))

train.columns = columns
test.columns = columns

train.to_csv('/content/drive/MyDrive/UCI HAR Dataset/train.csv', index=False)
test.to_csv('/content/drive/MyDrive/UCI HAR Dataset/test.csv', index=False)

"""**For doing classification**<br/>
opening CSV File
"""

try:
  # creating a data frame
  X_train = pd.read_csv("/content/drive/MyDrive/X_train.csv", header = None)
  y_train = pd.read_csv("/content/drive/MyDrive/y_train.csv",  header = None)
  X_test = pd.read_csv("/content/drive/MyDrive/X_test.csv",  header = None)
  y_test = pd.read_csv("/content/drive/MyDrive/y_test.csv",  header = None)
  # add X_train and X_test in one dataframe for the K fold cross validation
  X = pd.concat([X_train, X_test], ignore_index=True)
  # add y_train and y_test in one CSV file for the K fold cross validation
  y = pd.concat([y_train, y_test], ignore_index=True)
except Exception as e:
  print("An exception occurred:")
  print(e)

"""# Data Visualization"""

# No of classes
print("Different Activity labels in dataset: ", y[0].unique())

"""**Is there any class Imbalance issue in the datset ?**"""

import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style('whitegrid')
plt.rcParams['font.family'] = 'Dejavu Sans'

plt.figure(figsize=(18,13))
plt.title('Data provided by each of 30 user', fontsize=20)
sns.countplot(x='subject',hue='ActivityName', data = train)
plt.savefig("Data provided by each of 30 user.pdf")
plt.show()

plt.figure(figsize=(12,8))
sns.set()
sns.countplot(train.ActivityName, palette ='hls')
plt.xticks(rotation=90)
plt.savefig("Activity label.pdf")
plt.show()

"""Class Imbalance: A issue in the dataset, it occures when our class distributions are highly imbalanced
                 i.e. when no of patterns present in one class is much higher/lower than no of 
                 patterns present in another class.
As the class distributions of differnent Activity labels are nearby equally distributed,
so there is no class imbalanced issue in the UCI HAR Data Set.

**Is there any Data Scarcity issue in the dataset ?**<br/>
Note: Data scarcity is when a) there is limited amount or a complete lack of labeled training data, or b) lack of data for a given label compared to the other labels (a.k.a data imbalance).
"""

print("No of labelled training patterns/samples (in one training fold as we are using 5 fold cross validation):\n", int((X.shape[0]*4)/5))
print("No of labelled testing patterns/samples (in one testing fold as we are using 5 fold cross validation):\n", int(X.shape[0]/5))

"""No of labelled training patterns/samples (in one training fold as we are using 5 fold cross validation): 8239
As there is no lack of labeled training data, so there is no Data Scarcity issue in the dataset.

# Feature Scaling (Min-Max Normalization)<br/>
$X'$ = $\frac{X - min(X)}{max(X) - min(X)}$
"""

# dataframe to array
X = np.array(X)
y = np.array(y)

# Noramalize the X
scaler = MinMaxScaler()
scaler.fit(X)
X = scaler.transform(X)

"""# Shuffel the Data Set"""

# shuffel the dataset
X, y = shuffle(X, y, random_state=100)

print("Shape of dataset(input features):", X.shape)
print("Shape of y(input features):", y.shape)

"""# K-Fold Cross Validation (K=5)
We have used 5-fold to increase the generalization of the model (in terms of more precise accuracy claiming) and also to increse the robustness of the model.<br/>
In the 5 fold cross validation every sample will go 1 time for testing and 4 times for training of the model.
"""

def fold(features, y_actual):
  """
  INPUT
    features: 2D array conatins the dataset or input of the model, shape = (no of patterns, no of features)
    y_actual: 1D array conatins the actual class label of the data sample, shape = (no of patterns, )
  OUTPUT
    all_x_train:  2D array conatains training fold (fold no, feature training data)
    all_x_test:   2D array conatains training fold (fold no, feature testing data)
    all_y_train:  2D array conatains training fold (fold no, labeled training data)
    all_y_test:   2D array conatains training fold (fold no, labeled tesing data)
  """
  kf = KFold(n_splits = 5, random_state = 1000, shuffle = True)
  kf.get_n_splits(features)
  all_x_train, all_x_test, all_y_train, all_y_test = [], [], [], []
  for train_index, test_index in kf.split(features):
    X_train, X_test = features[train_index], features[test_index]
    y_train, y_test = y_actual[train_index], y_actual[test_index]
    all_x_train.append(X_train)
    all_x_test.append(X_test)
    all_y_train.append(y_train)
    all_y_test.append(y_test)
  all_x_train, all_x_test, all_y_train, all_y_test  = np.array(all_x_train), np.array(all_x_test), np.array(all_y_train), np.array(all_y_test)
  for i in range(0, 5):
    all_y_train[i] = all_y_train[i].flatten()
  return all_x_train, all_x_test, all_y_train, all_y_test

all_x_train, all_x_test, all_y_train, all_y_test = fold(X, y)

"""# 1. Classification"""

"""
Creating the validation set to get the best hyperparameter.
We are dividing the first training fold into two segments
that are training and validation into an 80:20 ratio.
Best Hyperparameter will be chosen best on which
pair of hyperparameters has the maximum accuracy on the validation set.
"""
X_train, X_validation, y_train, y_validation = train_test_split(all_x_train[0], all_y_train[0], test_size=0.20, random_state=42)

"""# Using inbuild Logistic Regression (SGD) classifier<br/>
Using One vs ALL/REST
"""

# hyperparameter tuning for logistic accuracy
def logistic_hyperparameter_tuning(epoch, alpha, roh, n_iter_no_change, X_train, X_validation, y_train, y_validation):
  """
  INPUT
    epoch:            1D int arary contains different values of epochs (hyperparameter)
    alpha:            1D float array contains different values of alphas or learning rates (hyperparameter)
    roh:              1D float array contains different values of  tolerence or roh (hyperparameter)
    n_iter_no_change: 1D int array conatins different values of 
                          Number of iterations with no improvement to wait before stopping fitting (hyperparameter).
    X_train:          2D array of shape = (no of patterns, no of features)
    X_validation:     2D array of shape = (no of patterns, no of features)
    y_train:          2D array of shape = (no of patterns, )
    y_validation:     2D array of shape = (no of patterns, )
  OUTPUT
    best_hyperparameter: tuple (epoch, alpha, roh, n_iter_no_change) which has best accuracy on the validation set.
  """
  val_acc = []
  for i in range(0, epoch.shape[0]):
    # we are taking logloss function for error calculation
    logistic_reg_classifier = OneVsRestClassifier(SGDClassifier(loss = 'log', alpha = alpha[i], fit_intercept = True, max_iter = epoch[i], tol = roh[i], n_iter_no_change = n_iter_no_change[i])).fit(X_train, y_train)
    predicted = logistic_reg_classifier.predict(X_validation)
    val_acc.append(accuracy_score(y_validation, predicted)*100)
  # Get the maximum accuracy on validation
  max_value = max(val_acc)
  max_index = val_acc.index(max_value)
  best_hyperparameter = (epoch[max_index], alpha[max_index], roh[max_index], n_iter_no_change[max_index])
  print("Best Hyperparameter:")
  print("Epoch = ", epoch[max_index])
  print("Alpha = ", alpha[max_index])
  print("Roh = ", roh[max_index])
  print("n_iter_no_change (Number of iterations with no improvement) = ", n_iter_no_change[max_index])
  return best_hyperparameter

epoch = np.array([100, 150, 200, 250, 300, 350, 400, 450, 500, 550])
alpha = np.array([0.01, 0.001, 0.0001, 0.00001, 0.125, 0.15, 0.18, 0.2, 0.25, 0.3])
roh = np.array([0.00001, 0.00001, 0.000001, 0.0000001, 0.000001, 0.0001, 0.0001, 0.0001, 0.0001, 0.000001])
n_iter_no_change = np.array([8, 9, 10, 11, 12, 13, 14, 15, 16, 18])
epoch, alpha, roh, n_iter_no_change = logistic_hyperparameter_tuning(epoch, alpha, roh, n_iter_no_change, X_train, X_validation, y_train, y_validation)

for i in range(0, 5): # for 5 fold
  print("For fold no:", i+1)
  print("-"*100)
  logistic_regression = OneVsRestClassifier(SGDClassifier(loss = 'log', alpha = alpha, fit_intercept = True,\
                                                          max_iter = epoch, tol = roh, n_iter_no_change = n_iter_no_change,\
                                                          verbose= False))
  logistic_regression.fit(all_x_train[i], all_y_train[i])
  print("Accuracy on training data: " + str(logistic_regression.score(all_x_train[i], all_y_train[i])*100) + "%")
  predicted = logistic_regression.predict(all_x_test[i])
  print("Testing Accuracy Score: " + str(accuracy_score(all_y_test[i], predicted)*100))
  print("Confusion Matrix : \n" + str(confusion_matrix(all_y_test[i], predicted)))
  print("Classification Report for 6-classes: ")
  out_labels = [1, 2, 3, 4, 5, 6]
  print(classification_report(all_y_test[i], predicted, out_labels, digits=4))
  print("-"*100)

"""**Is there any overfit issue in Logistic Regrerssion?**<br/>
It's not getting overfit as we are getting good accuracy on training  set as well as in testing set.<br/>
So, to assure the overfit issue we are using validation set and making the plot in which we are plotting training and validation mse (both) vs epoch during training in which we are getting one saturation point (epoch) in which training mse is decreasing but validation mse in not fllowing the same trend of training mse (either validation mse is constant or increaing).
"""

logistic_regression = OneVsRestClassifier(SGDClassifier(loss = 'log', alpha = alpha, fit_intercept = True, max_iter = epoch, tol = roh, n_iter_no_change = n_iter_no_change))
logistic_regression.fit(X_train, y_train)
print("Accuracy on training data: " + str(logistic_regression.score(X_train, y_train)*100) + "%")
print("-"*100)
test_predicted = logistic_regression.predict(all_x_test[0])
print("Testing Accuracy Score: " + str(accuracy_score(all_y_test[0], test_predicted)*100))
print("Testing Confusion Matrix : \n" + str(confusion_matrix(all_y_test[0], test_predicted)))
print("-"*100)
valid_predicted = logistic_regression.predict(X_validation)
print("Validation Accuracy Score: " + str(accuracy_score(y_validation, valid_predicted)*100))
print("Validation Confusion Matrix : \n" + str(confusion_matrix(y_validation, valid_predicted)))
print("-"*100)

"""# Single Layer Perceptron"""

# it's slp
class SingleLayerPerceptron():
    def predict(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        return self.predict_(self.add_bias(X))
    
    def predict_(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        pre_vals = np.dot(X, self.weights.T).reshape(-1,len(self.classes))
        return self.softmax(pre_vals)
    
    def softmax(self, z):
        """
        z: 1D array of shape (no of patterns, )
        """
        return np.exp(z) / np.sum(np.exp(z), axis=1).reshape(-1,1)

    def predict_classes(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        self.probs_ = self.predict(X)
        return np.vectorize(lambda c: self.classes[c])(np.argmax(self.probs_, axis=1))
  
    def add_bias(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        return np.insert(X, 0, 1, axis=1)

    def one_hot(self, y):
        """
        y: 1D array of shape (no of patterns, )
        """
        return np.eye(len(self.classes))[np.vectorize(lambda c: self.class_labels[c])(y).reshape(-1)]
    
    def score(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        return np.mean(self.predict_classes(X) == y)
    
    def evaluate_(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        return np.mean(np.argmax(self.predict_(X), axis=1) == np.argmax(y, axis=1))

    def logloss(self, y, probs):
        """
        y:      1D array of shape (no of patterns, )
        probs:  1D array of shape (no of patterns, )
        """
        return np.mean(-y*np.log(probs) - (1-y)*np.log(1-probs))

    def cross_entropy(self, y, probs):
        """
        y:      1D array of shape (no of patterns, )
        probs:  1D array of shape (no of patterns, )
        """
        return -1 * np.mean(y * np.log(probs))

    def mse(self, y, probs):
        """
        y:      1D array of shape (no of patterns, )
        probs:  1D array of shape (no of patterns, )
        """
        return (((y - probs)**2).mean())/2
        
    def fit(self, X, y, epoch, roh, lr):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        epoch:  int, convergence criteria (hyperparameter)
        roh:    float, convergence criteria (hyperparameter)
        lr:     float, learning rate (hyperparameter)
        """
        self.epoch = epoch
        self.roh = roh
        self.lr = lr 
        self.classes = np.unique(y)
        self.class_labels = {c:i for i,c in enumerate(self.classes)}
        X = self.add_bias(X)
        y = self.one_hot(y)
        self.loss = []
        self.weights = np.zeros(shape=(len(self.classes),X.shape[1]))*0.1
        self.fit_data(X, y)
        return self
 
    def fit_data(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        itr = 0
        while (not self.epoch or itr < self.epoch):
            self.loss.append(self.mse(y, self.predict_(X)))
            # put the thershold function on the predicted value i.e. here self.predict_(X)
            temp = self.predict_(X)
            # threshold activation on the predicted value of all patterns
            for k in range(0, temp.shape[0]):
              for j in range(0, temp.shape[1]):
                if(temp[k][j]>=0.5):
                  temp[k][j] = 1
                else:
                  temp[k][j] = 0
            #print("Iteration: ", itr+1, " Mse: ", self.mse(y, self.predict_(X)))
            error = y - temp
            update = (self.lr * np.dot(error.T, X))
            self.weights += update
            if np.abs(update).max() < self.roh:
              print("Converged through roh criteria.")
              break
            itr +=1
        if(itr==self.epoch):
          print("Converged through maximum of Iteration:")

"""Hyperparameter Tuning for slp"""

def slp_hyperparameter_tuning(epoch, alpha, roh, X_train, X_validation, y_train, y_validation):
  """
  INPUT
    epoch:            1D int arary contains different values of epochs (hyperparameter)
    alpha:            1D float array contains different values of alphas or learning rates (hyperparameter)
    roh:              1D float array contains different values of  tolerence or roh (hyperparameter)
    X_train:          2D array of shape = (no of patterns, no of features)
    X_validation:     2D array of shape = (no of patterns, no of features)
    y_train:          2D array of shape = (no of patterns, 1)
    y_validation:     2D array of shape = (no of patterns, 1)
  OUTPUT
    best_hyperparameter: tuple (epoch, alpha, roh) which has best accuracy on the validation set.
  """
  val_acc = []
  for i in range(0, epoch.shape[0]):
    # we are taking logloss function for error calculation
    slp_classifier = SingleLayerPerceptron().fit(X_train, y_train, epoch = epoch[i], roh = roh[i], lr = alpha[i])
    predicted = slp_classifier.predict_classes(X_validation)
    val_acc.append(accuracy_score(y_validation, predicted)*100)
  # Get the maximum accuracy on validation
  max_value = max(val_acc)
  max_index = val_acc.index(max_value)
  best_hyperparameter = (epoch[max_index], alpha[max_index], roh[max_index])
  print("Best Hyperparameter:")
  print("Epoch = ", epoch[max_index])
  print("Alpha = ", alpha[max_index])
  print("Roh = ", roh[max_index])
  return best_hyperparameter

epoch = np.array([100, 150, 200, 250, 300, 350, 400, 450, 500, 550])
alpha = np.array([0.0001, 0.0001, 0.00001, 0.00001, 0.00001, 0.00001, 0.00001, 0.00001, 0.0001, 0.0001])
roh = np.array([0.00001, 0.00001, 0.000001, 0.0000001, 0.000001, 0.0001, 0.0001, 0.0001, 0.0001, 0.000001])
epoch, alpha, roh = slp_hyperparameter_tuning(epoch, alpha, roh, X_train, X_validation, y_train, y_validation)

for i in range(0, 5): # for 5 fold
  print("For fold no:", i+1)
  print("-"*100)
  slp = SingleLayerPerceptron().fit(all_x_train[i], all_y_train[i], epoch = epoch, roh = roh, lr = alpha)
  print("Accuracy on training data: " + str(slp.score(all_x_train[i], all_y_train[i])*100) + "%")
  predicted = slp.predict_classes(all_x_test[i])
  print("Testing Accuracy Score: " + str(accuracy_score(all_y_test[i], predicted)*100))
  print('Confusion Matrix : \n' + str(confusion_matrix(all_y_test[i], predicted)))
  out_labels = [1, 2, 3, 4, 5, 6]
  print(classification_report(all_y_test[i], predicted, out_labels, digits = 6))
  print("-"*100)

# Overfit Detection
import numpy as np
import matplotlib.pyplot as plt
# it's slp
class Overfit_SingleLayerPerceptron():
    def predict(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        return self.predict_(self.add_bias(X))
    
    def predict_(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        pre_vals = np.dot(X, self.weights.T).reshape(-1, len(np.unique(y)))
        return self.softmax(pre_vals)
    
    def softmax(self, z):
        """
        z: 1D array of shape (no of patterns, )
        """
        return np.exp(z) / np.sum(np.exp(z), axis=1).reshape(-1,1)

    def predict_classes(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        self.probs_ = self.predict(X)
        return np.vectorize(lambda c: self.classes[c])(np.argmax(self.probs_, axis=1))
  
    def add_bias(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        return np.insert(X, 0, 1, axis=1)

    def one_hot(self, y):
        """
        y: 1D array of shape (no of patterns, )
        """
        return np.eye(len(self.classes))[np.vectorize(lambda c: self.class_labels[c])(y).reshape(-1)]
    
    def score(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        return np.mean(self.predict_classes(X) == y)
    
    def evaluate_(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        return np.mean(np.argmax(self.predict_(X), axis=1) == np.argmax(y, axis=1))

    def logloss(self, y, probs):
        """
        y: 1D array of shape (no of patterns, )
        probs: 1D array of shape (no of patterns, )
        """
        return np.mean(-y*np.log(probs) - (1-y)*np.log(1-probs))

    def cross_entropy(self, y, probs):
        """
        y: 1D array of shape (no of patterns, )
        probs: 1D array of shape (no of patterns, )
        """
        return -1 * np.mean(y * np.log(probs))

    def mse(self, y, probs):
        """
        y: 1D array of shape (no of patterns, )
        probs: 1D array of shape (no of patterns, )
        """
        return (((y - probs)**2).mean())/2
        
    def fit(self, X, y, X_validation, y_validation, epoch, roh, lr):
        """
        X: 2D array of shape (no of patterns in training set, no of features)
        y: 1D array of shape (no of patterns in training set, )
        X_validation: 2D array of shape (no of patterns in validation set, no of features)
        y_validation: 1D array of shape (no of patterns in validation set, )
        epoch:  int, convergence criteria (hyperparameter)
        roh:    float, convergence criteria (hyperparameter)
        lr:     float, learning rate (hyperparameter)
        """
        self.epoch = epoch
        self.roh = roh
        self.lr = lr 
        self.classes = np.unique(y)
        self.class_labels = {c:i for i,c in enumerate(self.classes)}
        X = self.add_bias(X)
        X_validation  = self.add_bias(X_validation)
        y = self.one_hot(y)
        y_valid = self.one_hot(y_validation)
        self.loss_train = []
        self.loss_valid = []
        self.weights = np.zeros(shape=(len(self.classes),X.shape[1]))*0.1
        self.fit_data(X, y, X_validation, y_valid)
        return self
 
    def fit_data(self, X, y, X_validation, y_valid):
        """
        X: 2D array of shape (no of patterns in training set, no of features)
        y: 1D array of shape (no of patterns in training set, )
        X_validation: 2D array of shape (no of patterns in validation set, no of features)
        y_validation: 1D array of shape (no of patterns in validation set, )
        """
        itr = 0
        while (not self.epoch or itr < self.epoch):
            self.loss_train.append(self.mse(y, self.predict_(X)))
            self.loss_valid.append(self.mse(y_valid, self.predict_(X_validation)))
            # put the thershold function on the predicted value i.e. here self.predict_(X)
            temp = self.predict_(X)
            # threshold activation on the predicted value of all patterns
            for k in range(0, temp.shape[0]):
              for j in range(0, temp.shape[1]):
                if(temp[k][j]>=0.5):
                  temp[k][j] = 1
                else:
                  temp[k][j] = 0
            #print("Iteration: ", itr+1, " Mse: ", self.mse(y, self.predict_(X)))
            error = y - temp
            update = (self.lr * np.dot(error.T, X))
            self.weights += update
            if np.abs(update).max() < self.roh:
              print("Converged through roh criteria.")
              break
            itr +=1
        if(itr==self.epoch):
          print("Converged through maximum of Iteration:")

oslp = Overfit_SingleLayerPerceptron().fit(X_train, y_train, X_validation, y_validation, epoch, alpha, roh)
print("Accuracy on training data: " + str(oslp.score(X_train, y_train)*100) + "%")
print("-"*100)
test_predicted = oslp.predict_classes(all_x_test[0])
print("Testing Accuracy Score: " + str(accuracy_score(all_y_test[0], test_predicted)*100))
print("Testing Confusion Matrix : \n" + str(confusion_matrix(all_y_test[0], test_predicted)))
print("-"*100)
valid_predicted = oslp.predict_classes(X_validation)
print("Validation Accuracy Score: " + str(accuracy_score(y_validation, valid_predicted)*100))
print("Validation Confusion Matrix : \n" + str(confusion_matrix(y_validation, valid_predicted)))
print("-"*100)

loss_train_mse = oslp.loss_train
loss_valid_mse = oslp.loss_valid
def plotting(x, y_1, y_2, label_1, label_2, t):
  plt.plot(x, y_1, label = label_1)
  plt.plot(x, y_2, label = label_2)
  plt.xlabel("Epoch")
  plt.ylabel("Training and Validation MSE")
  plt.legend()
  plt.title(t)
  plt.savefig("slp_overfit.pdf")
  plt.show()
x = []
for i in range(0, len(loss_valid_mse)):
  x.append(i)
plotting(x, loss_train_mse, loss_valid_mse, "Training MSE", "Validation MSE", "Training and Validation MSE")

"""# Sigmoid Neuron"""

# it's Sigmoid neuron
class SigmoidNeuron():
    def predict(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        return self.predict_(self.add_bias(X))
    
    def predict_(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        pre_vals = np.dot(X, self.weights.T).reshape(-1,len(self.classes))
        return self.softmax(pre_vals)
    
    def softmax(self, z):
        """
        z: 1D array of shape (no of patterns, )
        """
        return np.exp(z) / np.sum(np.exp(z), axis=1).reshape(-1,1)

    def predict_classes(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        self.probs_ = self.predict(X)
        return np.vectorize(lambda c: self.classes[c])(np.argmax(self.probs_, axis=1))
  
    def add_bias(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        return np.insert(X, 0, 1, axis=1)

    def one_hot(self, y):
        """
        y: 1D array of shape (no of patterns, )
        """
        return np.eye(len(self.classes))[np.vectorize(lambda c: self.class_labels[c])(y).reshape(-1)]
    
    def score(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        return np.mean(self.predict_classes(X) == y)
    
    def evaluate_(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        return np.mean(np.argmax(self.predict_(X), axis=1) == np.argmax(y, axis=1))

    def logloss(self, y, probs):
        """
        y:      1D array of shape (no of patterns, )
        probs:  1D array of shape (no of patterns, )
        """
        return np.mean(-y*np.log(probs) - (1-y)*np.log(1-probs))

    def cross_entropy(self, y, probs):
        """
        y:      1D array of shape (no of patterns, )
        probs:  1D array of shape (no of patterns, )
        """
        return -1 * np.mean(y * np.log(probs))

    def mse(self, y, probs):
        """
        y:      1D array of shape (no of patterns, )
        probs:  1D array of shape (no of patterns, )
        """
        return (((y - probs)**2).mean())/2
        
    def fit(self, X, y, epoch, roh, lr):
        """
        X: 2D array of shape (no of patterns in training set, no of features)
        y: 1D array of shape (no of patterns in training set, )
        epoch:  int, convergence criteria (hyperparameter)
        roh:    float, convergence criteria (hyperparameter)
        lr:     float, learning rate (hyperparameter)
        """
        self.epoch = epoch
        self.roh = roh
        self.lr = lr 
        self.classes = np.unique(y)
        self.class_labels = {c:i for i,c in enumerate(self.classes)}
        X = self.add_bias(X)
        y = self.one_hot(y)
        self.loss = []
        self.weights = np.zeros(shape=(len(self.classes),X.shape[1]))*0.1
        self.fit_data(X, y)
        return self
 
    def fit_data(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        itr = 0
        while (not self.epoch or itr < self.epoch):
            self.loss.append(self.mse(y, self.predict_(X)))
            # put the thershold function on the predicted value i.e. here self.predict_(X)
            temp = self.predict_(X)
            #print("Iteration: ", itr+1, " Mse: ", self.mse(y, self.predict_(X)))
            error = y - temp
            update = (self.lr * np.dot(error.T, X))
            self.weights += update
            if np.abs(update).max() < self.roh:
              print("Converged through roh criteria.")
              break
            itr +=1
        if(itr==self.epoch):
          print("Converged through maximum of Iteration:")

# hyperparameter tuning
def sn_hyperparameter_tuning(epoch, alpha, roh, X_train, X_validation, y_train, y_validation):
  """
  INPUT
    epoch:            1D int arary contains different values of epochs (hyperparameter)
    alpha:            1D float array contains different values of alphas or learning rates (hyperparameter)
    roh:              1D float array contains different values of  tolerence or roh (hyperparameter)
    X_train:          2D array of shape = (no of patterns, no of features)
    X_validation:     2D array of shape = (no of patterns, no of features)
    y_train:          2D array of shape = (no of patterns, 1)
    y_validation:     2D array of shape = (no of patterns, 1)
  OUTPUT
    best_hyperparameter: tuple (epoch, alpha, roh) which has best accuracy on the validation set.
  """
  val_acc = []
  for i in range(0, epoch.shape[0]):
    # we are taking logloss function for error calculation
    sn_classifier = SigmoidNeuron().fit(X_train, y_train, epoch = epoch[i], roh = roh[i], lr = alpha[i])
    predicted = sn_classifier.predict_classes(X_validation)
    val_acc.append(accuracy_score(y_validation, predicted)*100)
  # Get the maximum accuracy on validation
  max_value = max(val_acc)
  max_index = val_acc.index(max_value)
  best_hyperparameter = (epoch[max_index], alpha[max_index], roh[max_index])
  print("Best Hyperparameter:")
  print("Epoch = ", epoch[max_index])
  print("Alpha = ", alpha[max_index])
  print("Roh = ", roh[max_index])
  return best_hyperparameter

epoch = np.array([100, 150, 200, 250, 300, 350, 400, 450, 500, 550])
alpha = np.array([0.0001, 0.0001, 0.00001, 0.00001, 0.00001, 0.00001, 0.00001, 0.00001, 0.0001, 0.0001])
roh = np.array([0.00001, 0.00001, 0.000001, 0.0000001, 0.000001, 0.0001, 0.0001, 0.0001, 0.0001, 0.000001])
epoch, alpha, roh = sn_hyperparameter_tuning(epoch, alpha, roh, X_train, X_validation, y_train, y_validation)

for i in range(0, 5): # for 5 fold
  print("For fold no:", i+1)
  print("-"*100)
  sn = SigmoidNeuron().fit(all_x_train[i], all_y_train[i], epoch = epoch, roh = roh, lr = alpha)
  print("Accuracy on training data: " + str(sn.score(all_x_train[i], all_y_train[i])*100) + "%")
  predicted = sn.predict_classes(all_x_test[i])
  print("Testing Accuracy Score: " + str(accuracy_score(all_y_test[i], predicted)*100))
  print('Confusion Matrix : \n' + str(confusion_matrix(all_y_test[i], predicted)))
  out_labels = [1, 2, 3, 4, 5, 6]
  print(classification_report(all_y_test[i], predicted, out_labels, digits = 6))
  print("-"*100)

# Overfit Detection
import numpy as np
import matplotlib.pyplot as plt
# it's slp
class Overfit_SigmoidNeuron():
    def predict(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        return self.predict_(self.add_bias(X))
    
    def predict_(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        pre_vals = np.dot(X, self.weights.T).reshape(-1, len(np.unique(y)))
        return self.softmax(pre_vals)
    
    def softmax(self, z):
        """
        z: 1D array of shape (no of patterns, )
        """
        return np.exp(z) / np.sum(np.exp(z), axis=1).reshape(-1,1)

    def predict_classes(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        self.probs_ = self.predict(X)
        return np.vectorize(lambda c: self.classes[c])(np.argmax(self.probs_, axis=1))
  
    def add_bias(self, X):
        """
        X: 2D array of shape (no of patterns, no of features)
        """
        return np.insert(X, 0, 1, axis=1)

    def one_hot(self, y):
        """
        y: 1D array of shape (no of patterns, )
        """
        return np.eye(len(self.classes))[np.vectorize(lambda c: self.class_labels[c])(y).reshape(-1)]
    
    def score(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        return np.mean(self.predict_classes(X) == y)
    
    def evaluate_(self, X, y):
        """
        X: 2D array of shape (no of patterns, no of features)
        y: 1D array of shape (no of patterns, )
        """
        return np.mean(np.argmax(self.predict_(X), axis=1) == np.argmax(y, axis=1))

    def logloss(self, y, probs):
        """
        y: 1D array of shape (no of patterns, )
        probs: 1D array of shape (no of patterns, )
        """
        return np.mean(-y*np.log(probs) - (1-y)*np.log(1-probs))

    def cross_entropy(self, y, probs):
        """
        y: 1D array of shape (no of patterns, )
        probs: 1D array of shape (no of patterns, )
        """
        return -1 * np.mean(y * np.log(probs))

    def mse(self, y, probs):
        """
        y: 1D array of shape (no of patterns, )
        probs: 1D array of shape (no of patterns, )
        """
        return (((y - probs)**2).mean())/2
        
    def fit(self, X, y, X_validation, y_validation, epoch, roh, lr):
        """
        X: 2D array of shape (no of patterns in training set, no of features)
        y: 1D array of shape (no of patterns in training set, )
        X_validation: 2D array of shape (no of patterns in validation set, no of features)
        y_validation: 1D array of shape (no of patterns in validation set, )
        epoch:  int, convergence criteria (hyperparameter)
        roh:    float, convergence criteria (hyperparameter)
        lr:     float, learning rate (hyperparameter)
        """
        self.epoch = epoch
        self.roh = roh
        self.lr = lr 
        self.classes = np.unique(y)
        self.class_labels = {c:i for i,c in enumerate(self.classes)}
        X = self.add_bias(X)
        X_validation  = self.add_bias(X_validation)
        y = self.one_hot(y)
        y_valid = self.one_hot(y_validation)
        self.loss_train = []
        self.loss_valid = []
        self.weights = np.zeros(shape=(len(self.classes),X.shape[1]))*0.1
        self.fit_data(X, y, X_validation, y_valid)
        return self
 
    def fit_data(self, X, y, X_validation, y_valid):
        """
        X: 2D array of shape (no of patterns in training set, no of features)
        y: 1D array of shape (no of patterns in training set, )
        X_validation: 2D array of shape (no of patterns in validation set, no of features)
        y_validation: 1D array of shape (no of patterns in validation set, )
        """
        itr = 0
        while (not self.epoch or itr < self.epoch):
            self.loss_train.append(self.mse(y, self.predict_(X)))
            self.loss_valid.append(self.mse(y_valid, self.predict_(X_validation)))
            # put the thershold function on the predicted value i.e. here self.predict_(X)
            temp = self.predict_(X)
            #print("Iteration: ", itr+1, " Mse: ", self.mse(y, self.predict_(X)))
            error = y - temp
            update = (self.lr * np.dot(error.T, X))
            self.weights += update
            if np.abs(update).max() < self.roh:
              print("Converged through roh criteria.")
              break
            itr +=1
        if(itr==self.epoch):
          print("Converged through maximum of Iteration:")

osn = Overfit_SigmoidNeuron().fit(X_train, y_train, X_validation, y_validation, epoch, alpha, roh)
print("Accuracy on training data: " + str(osn.score(X_train, y_train)*100) + "%")
print("-"*100)
test_predicted = osn.predict_classes(all_x_test[0])
print("Testing Accuracy Score: " + str(accuracy_score(all_y_test[0], test_predicted)*100))
print("Testing Confusion Matrix : \n" + str(confusion_matrix(all_y_test[0], test_predicted)))
print("-"*100)
valid_predicted = osn.predict_classes(X_validation)
print("Validation Accuracy Score: " + str(accuracy_score(y_validation, valid_predicted)*100))
print("Validation Confusion Matrix : \n" + str(confusion_matrix(y_validation, valid_predicted)))
print("-"*100)

loss_train_mse = osn.loss_train
loss_valid_mse = osn.loss_valid
def plotting(x, y_1, y_2, label_1, label_2, t):
  plt.plot(x, y_1, label = label_1)
  plt.plot(x, y_2, label = label_2)
  plt.xlabel("Epoch")
  plt.ylabel("Training and Validation MSE")
  plt.legend()
  plt.title(t)
  plt.savefig("sn_overfit.pdf")
  plt.show()
x = []
for i in range(0, len(loss_valid_mse)):
  x.append(i)
plotting(x, loss_train_mse, loss_valid_mse, "Training MSE", "Validation MSE", "Training and Validation MSE")

"""# Multi-layer Perceptron (with only one hidden layer)<br/>
[ref-1](https://www.kaggle.com/vitorgamalemos/multilayer-perceptron-from-scratch)<br/>
[ref-2](https://github.com/eriklindernoren/ML-From-Scratch/blob/master/mlfromscratch/supervised_learning/multilayer_perceptron.py)<br/>
[ref-3](https://github.com/bamtak/machine-learning-implemetation-python/blob/master/Multi%20Class%20Logistic%20Regression.ipynb)
"""

from sklearn.neural_network import MLPClassifier

# hyperparameter tuning for logistic accuracy
def mlp_hyperparameter_tuning(no_of_hidden_neurons, epoch, alpha, roh, n_iter_no_change, X_train, X_validation, y_train, y_validation):
  """
  INPUT
    no_of_hidden_neurons: 1D int arary contains different values of no of neurons
                            present in 1st hidden layer (hyperparameter)
    epoch:                1D int arary contains different values of epochs (hyperparameter)
    alpha:                1D float array contains different values of alphas or learning rates (hyperparameter)
    roh:                  1D float array contains different values of  tolerence or roh (hyperparameter)
    n_iter_no_change:     1D int array conatins different values of 
                            Number of iterations with no improvement to wait before stopping fitting (hyperparameter).
    X_train:              2D array of shape = (no of patterns, no of features)
    X_validation:         2D array of shape = (no of patterns, no of features)
    y_train:              2D array of shape = (no of patterns, )
    y_validation:         2D array of shape = (no of patterns, )
  OUTPUT
    best_hyperparameter: a tuple (epoch, alpha, roh, n_iter_no_change) which has best accuracy on the validation set.
  """
  val_acc = []
  for i in range(0, epoch.shape[0]):
    mlp_classifier = MLPClassifier(hidden_layer_sizes = (no_of_hidden_neurons[i],), activation = 'logistic', solver = 'sgd', learning_rate = 'constant',\
      learning_rate_init = alpha[i], max_iter = epoch[i], shuffle = True, random_state = 100, tol = roh[i],\
      verbose = False, early_stopping = True, n_iter_no_change = n_iter_no_change[i]).fit(X_train, y_train)
    # we are taking logloss function for error calculation
    predicted = mlp_classifier.predict(X_validation)
    val_acc.append(accuracy_score(y_validation, predicted)*100)
  # Get the maximum accuracy on validation
  max_value = max(val_acc)
  max_index = val_acc.index(max_value)
  best_hyperparameter = (no_of_hidden_neurons[max_index], epoch[max_index], alpha[max_index], roh[max_index], n_iter_no_change[max_index])
  print("Best Hyperparameter:")
  print("No of neurons in the 1st hidden layer = ", no_of_hidden_neurons[max_index])
  print("Epoch = ", epoch[max_index])
  print("Alpha = ", alpha[max_index])
  print("Roh = ", roh[max_index])
  print("n_iter_no_change (Number of iterations with no improvement) = ", n_iter_no_change[max_index])
  return best_hyperparameter

no_of_hidden_neurons = np.array([561, 581, 591, 600, 620, 650, 670, 700, 730, 750])
"""
No of neurons present in 1st hidden layer will be greater than the no of features
i.e. no of neurons present in the input layer to increase the dimention in which
we can solve the non linear problem also.
"""
epoch = np.array([100, 150, 200, 250, 300, 350, 400, 450, 500, 550])
alpha = np.array([0.01, 0.001, 0.0001, 0.00001, 0.125, 0.15, 0.18, 0.2, 0.25, 0.3])
roh = np.array([0.00001, 0.00001, 0.000001, 0.0000001, 0.000001, 0.0001, 0.0001, 0.0001, 0.0001, 0.000001])
n_iter_no_change = np.array([8, 9, 10, 11, 12, 13, 14, 15, 16, 18])
no_of_hidden_neurons, epoch, alpha, roh, n_iter_no_change = mlp_hyperparameter_tuning(no_of_hidden_neurons, epoch, alpha, roh, n_iter_no_change, X_train, X_validation, y_train, y_validation)

for i in range(0, 5): # for 5 fold
  print("For fold no:", i+1)
  print("-"*100)
  mlp_classifier = MLPClassifier(hidden_layer_sizes = (no_of_hidden_neurons,), activation = 'logistic',\
                                 solver = 'sgd', learning_rate = 'constant', learning_rate_init = alpha,\
                                 max_iter = epoch, shuffle = True, random_state = 100, tol = roh,\
                            verbose = False, early_stopping = True, n_iter_no_change = n_iter_no_change)
  mlp_classifier.fit(all_x_train[i], all_y_train[i])
  print("Accuracy on training data: " + str(mlp_classifier.score(all_x_train[i], all_y_train[i])*100) + "%")
  predicted = mlp_classifier.predict(all_x_test[i])
  print("Testing Accuracy Score: " + str(accuracy_score(all_y_test[i], predicted)*100))
  print("Confusion Matrix : \n" + str(confusion_matrix(all_y_test[i], predicted)))
  print("Classification Report for 6-classes: ")
  out_labels = [1, 2, 3, 4, 5, 6]
  print(classification_report(all_y_test[i], predicted, out_labels, digits=4))
  print("-"*100)

# Is there any overfit issue?
mlp_classifier = MLPClassifier(hidden_layer_sizes = (no_of_hidden_neurons,), activation = 'logistic',\
                                solver = 'sgd', learning_rate = 'constant', learning_rate_init = alpha,\
                                max_iter = epoch, shuffle = True, random_state = 100, tol = roh,\
                                verbose = False, early_stopping = True, n_iter_no_change = n_iter_no_change)
mlp_classifier.fit(X_train, y_train)
print("Accuracy on training data: " + str(mlp_classifier.score(X_train, y_train)*100) + "%")
print("-"*100)
test_predicted = mlp_classifier.predict(all_x_test[0])
print("Testing Accuracy Score: " + str(accuracy_score(all_y_test[0], test_predicted)*100))
print("Testing Confusion Matrix : \n" + str(confusion_matrix(all_y_test[0], test_predicted)))
print("-"*100)
valid_predicted = mlp_classifier.predict(X_validation)
print("Validation Accuracy Score: " + str(accuracy_score(y_validation, valid_predicted)*100))
print("Validation Confusion Matrix : \n" + str(confusion_matrix(y_validation, valid_predicted)))
print("-"*100)

"""**Feature Selection using PCA to get the optimal features**


"""

from sklearn.decomposition import PCA

# Let's say, components = 2
pca = PCA(n_components = 20)
pca.fit(X)
x_pca = pca.transform(X)
x_pca.shape

all_x_train, all_x_test, all_y_train, all_y_test = fold(x_pca, y)