# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Baseline Classification Model: Random Forest w/ Feature Engineering - Phys Only

# This file is composed of a random forest classification model to evaluate a general accuracy level of traditional ML methods in classifying our HAR data based on activity. We also used Leave-One-Out Cross-Validation to validate our models. 

# __INPUT: .csv files containing the rolled sensor data with feature engineering (engineered_features.csv)__

# __OUTPUT: Random Forest Multi-Classification Model (F1 Score = )__

# ## Imports

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import plot_confusion_matrix
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score
import seaborn as sns
import tensorflow as tf

# ## Read in Data

pd.set_option('display.max_columns', None)
df = pd.read_csv('/Users/N1/Data7/Data-2020/10_code/40_usable_data_for_models/41_Duke_Data/engineered_features.csv')

df = df.assign(count=df.groupby(df.Activity.ne(df.Activity.shift()).cumsum()).cumcount().add(1))

df.head(5)

# +
from sklearn.preprocessing import LabelEncoder
le1 = LabelEncoder()
df['Activity'] = le1.fit_transform(df['Activity'])

activity_name_mapping = dict(zip(le1.classes_, le1.transform(le1.classes_)))
print(activity_name_mapping)
# -

le = LabelEncoder()
df['Subject_ID'] = le.fit_transform(df['Subject_ID'])

# ## Initial RF Model

# + tags=[]
np.random.seed(29)
rands = np.random.choice(df.Subject_ID.unique(),11, replace=False)
print(rands)
# -

# ### Split Subjects into Test and Train Sets (n=44,11)

test = df[df['Subject_ID'].isin(rands)] 
train = df[-df['Subject_ID'].isin(rands)]

# ## Feature Selection

# ### Run this for only Physiological Data

train = train[['TEMP_mean', 'EDA_mean', 'BVP_mean', 'HR_mean', 'TEMP_std',
       'EDA_std', 'BVP_std', 'HR_std', 'Subject_ID', 'count', 'Activity']]
test = test[['TEMP_mean', 'EDA_mean', 'BVP_mean', 'HR_mean', 'TEMP_std',
       'EDA_std', 'BVP_std', 'HR_std', 'Subject_ID', 'count', 'Activity']]   

# This train_df is made so we can use the 

train_SID = train['Subject_ID'].values

# +
train['train'] =1
test['train'] = 0

combined = pd.concat([train, test])
combined = pd.concat([combined, pd.get_dummies(combined['Subject_ID'], prefix = 'SID')], axis =1).drop('Subject_ID', axis =1)
combined = pd.concat([combined, pd.get_dummies(combined['count'], prefix = 'count')], axis =1).drop('count', axis = 1)

# +
train = combined[combined['train'] == 1]
test = combined[combined['train'] == 0]

train.drop(["train"], axis = 1, inplace = True)
test.drop(["train"], axis = 1, inplace = True)
print(train.shape, test.shape)
# -

train.columns

train_f = train.drop("Activity", axis =1)
test_f = test.drop("Activity", axis =1)

# ### Test-Train Split

X_train = train_f
y_train = train.Activity
X_test = test_f
y_test = test.Activity

len(X_train)

# ### Standardize Data

X_train

sc = StandardScaler()
X_train.iloc[:,:16] = sc.fit_transform(X_train.iloc[:,:16])
X_test.iloc[:,:16] = sc.transform(X_test.iloc[:,:16])

X_train = X_train.values
X_test = X_test.values

from keras.utils import np_utils
y_train_dummy = np_utils.to_categorical(y_train)
y_test_dummy = np_utils.to_categorical(y_test)

# ### RF Object Initialization

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout

# ## LOOCV

# __Leave One Out CV:__
# Each observation is considered as a validation set and the rest n-1 observations are a training set. Fit the model and predict using 1 observation validation set. Repeat this for n times for each observation as a validation set.
# Test-error rate is average of all n errors.
#
# __Advantages:__ takes care of both drawbacks of validation-set method
# 1. No randomness of using some observations for training vs. validation set like in validation-set method as each observation is considered for both training a›nd validation. So overall less variability than Validation-set method due to no randomness no matter how many times you run it.
# 2. Less bias than validation-set method as training-set is of n-1 size. Because of this reduced bias, reduced over-estimation of test-error, not as much compared to validation-set method.
#
# __Disadvantages:__
# 1. Even though each iterations test-error is un-biased, it has a high variability as only one-observation validation-set was used for prediction.
# 2. Computationally expensive (time and power) especially if dataset is big with large n as it requires fitting the model n times. Also some statistical models have computationally intensive fitting so with large dataset and these models LOOCV might not be a good choice.

# +
from sklearn.model_selection import LeaveOneGroupOut

# Lists to store metrics
acc_per_fold = []
loss_per_fold = []
f1_per_fold = []

# Define the K-fold Cross Validator
groups = train_SID
inputs = X_train
targets = y_train_dummy
logo = LeaveOneGroupOut()

logo.get_n_splits(inputs, targets, groups)

cv = logo.split(inputs, targets, groups)

# LOGO
fold_no = 1
for train, test in cv:
    #Define the model architecture
    model = Sequential()
    model.add(Dense(32, activation='relu'))
    model.add(Dense(32, activation='relu'))
    model.add(Dense(32, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(32, activation='relu'))
    model.add(Dense(32, activation='relu'))
    model.add(Dense(4, activation='softmax')) #4 outputs are possible 
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    
      # Generate a print
    print('------------------------------------------------------------------------')
    print(f'Training for fold {fold_no} ...')

    # Fit data to model
    history = model.fit(inputs[train], targets[train],
              batch_size=32,
              epochs=10,
              verbose=1)

    # Generate generalization metrics
    scores = model.evaluate(inputs[test], targets[test], verbose=0)
    y_pred = np.argmax(model.predict(inputs[test]), axis=-1)
    f1 = (f1_score(np.argmax(targets[test], axis=1), (y_pred), average = 'weighted'))
    print(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1]*100}%, F1 of {f1}')
    f1_per_fold.append(f1)
    acc_per_fold.append(scores[1] * 100)
    loss_per_fold.append(scores[0])
    
    # Increase fold number
    fold_no = fold_no + 1
    

# == Provide average scores ==
print('------------------------------------------------------------------------')
print('Score per fold')

for i in range(0, len(acc_per_fold)):
    print('------------------------------------------------------------------------')
    print(f'> Fold {i+1} - Loss: {loss_per_fold[i]} - Accuracy: {acc_per_fold[i]}% - F1:{f1_per_fold[i]}%')
print('------------------------------------------------------------------------')
print('Average scores for all folds:')
print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')
print(f'> F1: {np.mean(f1_per_fold)} (+- {np.std(f1_per_fold)})')                     
print(f'> Loss: {np.mean(loss_per_fold)}')
print('------------------------------------------------------------------------')
# -

# !mkdir -p saved_model
model.save('saved_model/30_TF_FE_Phys_Only')

model = tf.keras.models.load_model('saved_model/30_TF_FE_Phys_Only')

y_pred = np.argmax(model.predict(X_test), axis=-1)

results = model.evaluate(X_test, y_test_dummy, batch_size=32)
print("Test loss, Test acc:", results)

cm = confusion_matrix(y_test,y_pred)
cm

cm= cm.astype('float')/cm.sum(axis=1)[:,np.newaxis]
cm

ax = plt.subplot()
sns.heatmap(cm, annot = True, fmt = '.2f',cmap = 'Blues', xticklabels = le1.classes_, yticklabels = le1.classes_)
ax.set_xlabel("Predicted labels")
ax.set_ylabel('Actual labels')
plt.title('Feature Engineered STEP ANN Phys only - Confusion Matrix')
plt.savefig('30_figures/30_feat_engin_window_ANN_phys_only.png')

# +
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score

a_s = accuracy_score(y_test, y_pred)
f1_s = f1_score(y_test, y_pred, average = 'weighted')

print(f'Accuracy Score: {a_s:.3f} \nF1 Score: {f1_s:.3f}')

StarcoderdataJupyterScriptsDedupFiltered