"""
#### If you want to know more about how the data is collected, Cleaned and Preprocessed You can visit this repo on my github: https://github.com/eyadayman12/FC-Bayern-Munich-Face-Recognation
"""
"""
# Table of Contents
<a id="toc"></a>
- [1. Data Description](#1)
- [2. Import Necssaries Libraries](#2)
- [3. Explore the Data](#3)
    - [3.1 Information about Data](#3.1)
    - [3.2 Visualization](#3.2)
- [4. Assign Features and target Variable](#4)
- [5. Spliting the data into Training and Testing Data](#5)
- [6. Hyperparameter Tuning](#6)
- [7. Machine Learning Modeling](#7)
    - [7.1 SVM Model](#7.1)
        - [7.1.1 Cross Validation](#7.1.1)
        - [7.1.2 Accuarcy of the model](#7.1.2)
        - [7.1.3 Confusion Matrix](#7.1.3)
        - [7.1.4 Classification Report](#7.1.4)
        - [7.1.5 ROC Curve](#7.1.5)
    - [7.2 Logistic Regression Model](#7.2)
        - [7.2.1 Cross Validation](#7.2.1)
        - [7.2.2 Accuarcy of the model](#7.2.2)
        - [7.2.3 Confusion Matrix](#7.2.3)
        - [7.2.4 Classification Report](#7.2.4)
        - [7.2.5 ROC Curve](#7.2.5)
    - [7.3 Random Forest Model](#7.3)
        - [7.3.1 Cross Validation](#7.3.1)
        - [7.3.2 Accuarcy of the model](#7.3.2)
        - [7.3.3 Confusion Matrix](#7.3.3)
        - [7.3.4 Classification Report](#7.3.4)
    - [7.4 Decision Tree Model](#7.4)
        - [7.4.1 Cross Validation](#7.4.1)
        - [7.4.2 Accuarcy of the model](#7.4.2)
        - [7.4.3 Confusion Matrix](#7.4.3)
        - [7.4.4 Classification Report](#7.4.4)
    - [7.5 Bagging](#7.5)
        - [7.5.1 Cross Validation](#7.5.1)
        - [7.5.2 Accuarcy of the model](#7.5.2)
        - [7.5.3 Confusion Matrix](#7.5.3)
        - [7.5.4 Classification Report](#7.5.4)
    - [7.6 Boosting](#7.6)
        - [7.6.1 Cross Validation](#7.6.1)
        - [7.6.2 Accuarcy of the model](#7.6.2)
        - [7.6.3 Confusion Matrix](#7.6.3)
        - [7.6.4 Classification Report](#7.6.4)
    - [7.7 KNN](#7.7)
        - [7.7.1 Cross Validation](#7.7.1)
        - [7.7.2 Accuarcy of the model](#7.7.2)
        - [7.7.3 Confusion Matrix](#7.7.3)
        - [7.7.4 Classification Report](#7.7.4)
    - [7.8 ANN](#7.8)
        - [7.8.1 Build and Train the ANN](#7.8.1)
        - [7.8.2 Accuarcy of the model](#7.8.2)
        - [7.8.3 Loss curve](#7.8.3)
        - [7.8.4 Confusion Matrix](#7.8.4)
        - [7.8.5 Classification Report](#7.8.5)
"""
"""
<a id="1"></a>
# Data Description
"""
"""
***The data contains 5 Bayern Munich players and each player has about 100 random images collected from Google <br>
so our data have 5 classes:***
 - Kingsley Coman
 - Joshua Kimmich
 - Robert Lewandowski
 - Manuel Neuer
 - Leory Sane<br><br>

***Kingsley Coman Class 0***<br>
***Joshua Kimmich Class 1***<br>
***Robert Lewandowski Class 2***<br>
***Manuel Neuer Class 3***<br>
***Leory Sane Class 4***<br>
"""
"""
<a d='2'></a>
# Import Necssaries Libraries
"""
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
%matplotlib inline
import seaborn as sns
from math import sqrt

from sklearn.preprocessing import StandardScaler, robust_scale, MinMaxScaler
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.metrics import classification_report, confusion_matrix, f1_score, roc_curve, auc
from sklearn.preprocessing import LabelEncoder, label_binarize
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.multiclass import OneVsRestClassifier

import tensorflow as tf
from tensorflow.keras import layers 
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Flatten
from tensorflow.keras import callbacks
from tensorflow.keras.utils import to_categorical

import warnings
warnings.filterwarnings("ignore")
"""
<a id='3'></a>
# Explore the Data
"""
df = pd.read_csv("../input/fc-bayern-face-recognation/Bayern.csv") # read the data
df.head()
"""
<a id='3.1'></a>
**Information about data**
"""
print(f"The data contains {len(df)} images")
print(f"Each image is {int(sqrt(df.shape[1] - 1))} * {int(sqrt(df.shape[1]-1))} Pixels")
print(f"Data contains {len(df.Target.unique())} Classes")
df.Target.value_counts()
"""
<a id='3.2'></a>
**Visualization**
"""
plt.figure(figsize=(18,12))
plt.subplot(2,2,1)
sns.countplot(x='Target', data=df);

plt.subplot(2,2,2)
coman = df.Target[df.Target == 0].count()
kimmich = df.Target[df.Target == 1].count()
lewa = df.Target[df.Target == 2].count()
neuer = df.Target[df.Target == 3].count()
sane = df.Target[df.Target == 4].count()
weights = [coman, kimmich, lewa, neuer, sane]
labels = ['Coman', 'Kimmich', 'Lewandowski', 'Neuer', 'Sane']
plt.pie(weights, labels=labels, autopct='%.2f%%', explode=[0.01,0.01,0.01,0.01,0.01])
my_circle = plt.Circle( (0,0), 0.4, color='white')
plt.gcf().gca().add_artist(my_circle)
plt.legend(bbox_to_anchor=(1, 1))
plt.show()
"""
<a id='4'></a>
# Assign Feature and Target Variable
"""
X = df.drop("Target", axis=1)
y = df.Target
X.shape
y.shape
"""
<a id='5'></a>
# Splitting the data into Training and Testing Data
"""
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.25, random_state=150)
print(f"Number of Training data: {len(X_train)}")
print(f"Number of Testing data: {len(X_test)}")
X_train.shape
y_train.shape
"""
<a id='6'></a>
# Hyperparameter Tuning
"""
model_params = {
    'svm': {
        'model': SVC(gamma='auto'),
        'params' : {
            'C': [100, 10, 1.0, 0.1, 0.001],
            'kernel': ['rbf','linear', 'poly', 'sigmoid']
        }  
    },
    'random_forest': {
        'model': RandomForestClassifier(),
        'params' : {
            'max_features' : np.arange(1,21),
            'max_features' : ['sqrt', 'log2'],
            'n_estimators': [10,100,1000]
        }
    },
    'logistic_regression' : {
        'model': LogisticRegression(),
        'params': {
            'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
            'penalty': ['none', 'l1', 'l2', 'elasticnet'],
            'C' : [100,10,1.0,0.1,0.01]
        }
    },
    'KNN': {
        'model' : KNeighborsClassifier(),
        'params' : {
            'n_neighbors' : np.arange(1,22),
            'weights' : ['uniform', 'distance']
        }
    }
}
scores = []

for model_name, mp in model_params.items():
    clf =  GridSearchCV(mp['model'], mp['params'], cv=5, return_train_score=False)
    clf.fit(X, y)
    scores.append({
        'model': model_name,
        'best_score': clf.best_score_,
        'best_params': clf.best_params_
    })
    
df = pd.DataFrame(scores,columns=['model','best_score','best_params'])
df
"""
<a id='7'></a>
# Modeling
"""
def kfolds(model, model_name):
    model = cross_val_score(model, X,y, cv=10)
    model_score = np.average(model)
    print(f"{model_name} score on cross validation: {model_score * 100}%")

def train(model, model_name):
    model.fit(X_train, y_train)
    model_train_score = model.score(X_train, y_train)
    model_test_score = model.score(X_test, y_test)
    print(f"{model_name} model score on Training data: {model_train_score * 100}%\n{model_name} model score on Testing data: {model_test_score * 100}%")

def conf_matrix(model):
    y_pred = model.predict(X_test)
    cm = confusion_matrix(y_test, y_pred)
    plt.figure(figsize=(10,8))
    sns.heatmap(cm, annot=True);

def class_report(model):
    y_pred = model.predict(X_test)
    return classification_report(y_test, y_pred)


def roc(model):
    y_binarize = label_binarize(y, classes=[0,1,2,3,4])
    y_binarize.shape
    n_classes = y_binarize.shape[1]
    X_train, X_test, y_train, y_test = train_test_split(X,y_binarize, test_size=0.25, random_state=150)
    classifier = OneVsRestClassifier(
    model)
    y_score = classifier.fit(X_train, y_train).decision_function(X_test)
    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    for i in range(n_classes):
        fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
        roc_auc[i] = auc(fpr[i], tpr[i])
    fpr["micro"], tpr["micro"], _ = roc_curve(y_test.ravel(), y_score.ravel())
    roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
    plt.figure(figsize=(10,7))
    lw = 2
    plt.plot(
        fpr[2],
        tpr[2],
        color="darkorange",
        lw=lw,
        label="ROC curve (area = %0.2f)" % roc_auc[2],
    )
    plt.plot([0, 1], [0, 1], color="navy", lw=lw, linestyle="--")
    plt.xlabel("False Positive Rate")
    plt.ylabel("True Positive Rate")
    plt.title("Receiver operating characteristic")
    plt.legend(loc="lower right")
    plt.show()
"""
<a id='7.1'></a>
## SVM Model
"""
svm_model = SVC(kernel='linear', gamma='auto', C=100)
"""
<a id='7.1.1'></a>
**Cross Validation**
"""
kfolds(svm_model, "SVM")
"""
<a id='7.1.2'></a>
**Score of the model**
"""
train(svm_model, "SVM")
"""
<a id='7.1.3'></a>
**Confusion Matrix**
"""
conf_matrix(svm_model)
"""
<a id='7.1.4'></a>
**Classification Report**
"""
print(class_report(svm_model))
"""
<a id='7.1.5'></a>
**ROC Curve**
"""
roc(svm_model)
"""
<a id='7.3'></a>
## Random Forest
"""
rf_model = RandomForestClassifier(n_estimators=1000, random_state=70, max_features='sqrt')
"""
<a id='7.3.1'></a>
**Cross Validation**
"""
kfolds(rf_model, "Random Forest")
"""
<a id='7.3.2'></a>
**Accuarcy of the model**
"""
train(rf_model, "Random Forest")
"""
<a id='7.3.3'></a>
**Confusion Matrix**
"""
conf_matrix(rf_model)
"""
<a id='7.3.4'></a>
**Classification Report**
"""
print(class_report(rf_model))
"""
<a id='7.4'></a>
## Decision Tree
"""
dt_model = DecisionTreeClassifier()
"""
<a id='7.4.1'></a>
**Cross Validation**
"""
kfolds(dt_model, "Decision Tree")
"""
<a id='7.4.2'></a>
**Accuarcy of the model**
"""
train(dt_model, "Decision Tree")
"""
<a id='7.4.3'></a>
**Confusion Matrix**
"""
conf_matrix(dt_model);
"""
<a id='7.4.4'></a>
**Classification Report**
"""
print(class_report(dt_model))
"""
<a id='7.5'></a>
## Bagging
"""
bagg_model = BaggingClassifier(n_estimators=150)
"""
<a id='7.5.1'></a>
**Cross Validation**
"""
kfolds(bagg_model, "Bagging")
"""
<a id='7.5.2'></a>
**Accuarcy of the model**
"""
train(bagg_model, "Bagging")
"""
<a id='7.5.3'></a>
**Confusion Matrix**
"""
conf_matrix(bagg_model)
"""
<a id='7.5.4'></a>
**Classification Report**
"""
print(class_report(bagg_model))
"""
<a id='7.6'></a>
## Boosting
"""
gb_model = GradientBoostingClassifier()
"""
<a id='7.6.1'></a>
**Cross Validation**
"""
kfolds(gb_model, "Boosting")
"""
<a id='7.6.2'></a>
**Accuarcy of the model**
"""
train(gb_model, "Boosting")
"""
<a id='7.6.3'></a>
**Confusion Matrix**
"""
conf_matrix(gb_model)
"""
<a id='7.6.4'></a>
**Classification Report**
"""
print(class_report(gb_model))
"""
<a id='7.7'></a>
## KNN
"""
knn = KNeighborsClassifier(n_neighbors=6, weights='distance')
"""
<a id='7.7.1'></a>
**Cross Validation**
"""
kfolds(knn, "KNN")
"""
<a id='7.7.2'></a>
**Accuarcy of the model**
"""
train(knn, "KNN")
"""
<a id='7.7.3'></a>
**Confusion Matrix**
"""
conf_matrix(knn)
"""
<a id='7.7.4'></a>
**Classification Report**
"""
print(class_report(knn))
"""
<a id='7.8'></a>
## ANN
"""
FEATURES = [col for col in df.columns if col not in ['Target']]
X_nn = df[FEATURES].to_numpy()
y_nn = df['Target'].to_numpy()
X_nn.shape
y_nn.shape
X_nn[0].shape
y_nn
LE = LabelEncoder()
y_nn_cat = to_categorical(LE.fit_transform(y_nn))
y_nn_cat.shape
X_train_nn, X_test_nn, y_train_nn, y_test_nn = train_test_split(X_nn,y_nn_cat, test_size=0.25, random_state=100)
"""
<a id="7.8.1"></a>
**Build and Train the ANN**
"""
def load_model(): 
    model = Sequential([
        Dense(4096, activation ='relu', input_shape = [X.shape[1]]),
        Dense(2048, activation ='relu'),
        Dense(1024, activation ='relu'),
        Dense(512, activation ='relu'),
        Dense(5, activation='softmax'),
    ])
    model.compile(
        optimizer=  tf.keras.optimizers.Adam(learning_rate = 0.0001),
        loss='categorical_crossentropy',
        metrics=['acc'],
    )
    return model
    
early_stopping = callbacks.EarlyStopping(
        patience=10,
        min_delta=0,
        monitor='val_loss',
        restore_best_weights=True,
        verbose=0,
        mode='min', 
        baseline=None,
    )
plateau = callbacks.ReduceLROnPlateau(
            monitor='val_loss', 
            factor=0.2, 
            patience=4, 
            verbose=0,
            mode='min')


nn_model = load_model()
history = nn_model.fit(  X_train_nn , y_train_nn,
                validation_data = (X_test_nn , y_test_nn),
                epochs = 1000,
                batch_size = 10,
                callbacks = [early_stopping , plateau]
              )
"""
<a id="7.8.2"></a>
**Accuracy of ANN**
"""
nn_model.evaluate(X_test_nn, y_test_nn)
"""
<a id="7.8.3"></a>
**Loss Curve**
"""
plt.figure(figsize=(8,6))
loss_train = history.history['loss']
loss_val = history.history['val_loss']
epochs = range(1,18)
plt.plot(epochs, loss_train, 'g', label='Training loss')
plt.plot(epochs, loss_val, 'b', label='validation loss')
plt.title('Training and Validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.xticks(np.arange(1,20))
plt.legend()
plt.show()
y_predicted = nn_model.predict(X_test_nn)
y_predicted
y_predicted_labels = [np.argmax(i) for i in y_predicted]
y_predicted_labels
y_predicted_labels = np.array(y_predicted_labels)
y_predicted_labels.shape
X_train_nn, X_test_nn, y_train_nn, y_test_nn = train_test_split(X_nn,y_nn, test_size=0.25, random_state=100)
y_test_nn.shape
cm = tf.math.confusion_matrix(labels=y_test_nn, predictions=y_predicted_labels)
cm
"""
<a id="7.8.4"></a>
**Confusion Matrix**
"""
plt.figure(figsize=(9,7))
sns.heatmap(cm, annot=True)
"""
<a id="7.8.5"></a>
**Classification Report**
"""
print(classification_report(y_test_nn, y_predicted_labels))