"""
## Importing core libraries and custom estimators
"""
# Importing core libraries
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
from sklearn.metrics import recall_score, precision_score, confusion_matrix
from sklearn.neighbors import LocalOutlierFactor
from sklearn.ensemble import IsolationForest
from sklearn.cluster import KMeans
from sklearn.covariance import EllipticEnvelope
"""
In the following code block, I write the implementation of anomaly detectors from Andrew Ng's Machine Learning lectures (Section 15)
"""
# Custom anomaly detectors
from sklearn.base import BaseEstimator
import numpy as np
from math import gamma, pi
from scipy.stats import multivariate_normal

class MultivariateGaussian(BaseEstimator):
    """
    Anomaly detector using the Multivariate Gaussian class
    Based on Andrew Ng's lecture
    """
    def __init__(self, epsilon=None):
        self.epsilon = epsilon
    
    def fit(self, train_data):
        X = np.array(train_data)
        # Calculating n-dimensional mean vector (mu)
        self.mu = np.mean(X, axis=0)
        # Calculating n-by-n covariance matrix (Sigma)
        self.Sigma = np.cov(X, rowvar=False)
        assert self.Sigma.shape == (X.shape[1], X.shape[1]), f"The covariance matrix must be {X.shape[1]} by {X.shape[1]}"
        return self

    def predict(self, test_data):
        X_test = np.array(test_data)
        p_X = multivariate_normal.pdf(X_test, mean=self.mu, cov=self.Sigma, allow_singular=True)
        assert p_X.shape == (X_test.shape[0],), f"The predicted probability densities must be a {X_test.shape[0]}-dimensional vector"
        # Calculating m-dimensional prediction vector (probability density function)
        predictions = np.where(p_X < self.epsilon, 1, 0)
        return predictions

class MultivariateTDistribution(MultivariateGaussian):
    """
    Anomaly detector using the 'fat-tail' Student's t-Distribution.
    """
    def __init__(self, epsilon=None, df=None):
        self.epsilon = epsilon
        # Degrees of freedom parameter
        self.df = df

    def predict(self, test_data):
        X_test = np.array(test_data)
        df = self.df
        n = test_data.shape[1]
        mu = self.mu
        Sigma = self.Sigma
        pdf_list = []
        # The following codes should be vectorised, any suggestions are welcomed
        for x_test in X_test:
            term_1_num = gamma((n+df)/2)
            term_1_denom = gamma((df/2)) * df**(n/2) * pi**(n/2) * np.linalg.det(Sigma)**0.5
            term_1 = term_1_num / term_1_denom
            term_2 = (1 + 1/df * (x_test - mu).T @ np.linalg.inv(Sigma) @ (x_test - mu)) ** (-1 * (n+df)/2)
            pdf_x = term_1 * term_2
            pdf_list.append(pdf_x)
        pdf_X = np.array(pdf_list)
        assert pdf_X.shape == (X_test.shape[0],), f"The predicted probability densities must be a {X_test.shape[0]}-dimensional vector"
        predictions = np.where(pdf_X < self.epsilon, 1, 0)
        return predictions

class SimpleAnomalyDetector(BaseEstimator):
    """
    Simple anomaly detector, a predicted probability density is the product of all features' probabilities.
    Based on Andrew Ng's lecture.
    """
    def __init__(self, epsilon=None):
        self.epsilon = epsilon

    def fit(self, train_data):
        X = np.array(train_data)
        # Calculating n-dimensional mean vector (mu)
        self.mu = np.mean(X, axis=0).reshape(-1, 1)
        # Calculating n-dimensional vector of feature variances (sigma)
        self.sigma = np.var(X, axis=0).reshape(-1, 1)
        assert self.mu.shape == (X.shape[1], 1)
        assert self.sigma.shape == (X.shape[1], 1)
        return self

    def predict(self, test_data):
        X_test = np.array(test_data)
        mu = self.mu
        sigma = self.sigma
        term_1 = -1 * np.square(X_test - np.tile(mu, len(X_test)).T)
        term_2 = 2 * np.tile(sigma, len(X_test)).T
        term_3 = np.exp(term_1 / term_2)
        term_4 = (1 / ((2*np.pi)**0.5 * np.power(sigma, np.array(0.5))))
        term_4 = np.tile(term_4, len(X_test)).T
        term_5 = term_3 * term_4
        pdf_X = np.prod(term_5, axis=1)
        assert pdf_X.shape == (X_test.shape[0],), f"The predicted probability densities must be a {X_test.shape[0]}-dimensional vector"
        predictions = np.where(pdf_X < self.epsilon, 1, 0)
        return predictions
"""
## Loading data - Credit Card Fraud dataset
#### Context on the dataset

V1 to V28 are PCA components of the data. Only features 'Time' and 'Amount' are untransformed features.

For more details about the dataset, visit the [Kaggle site here](https://www.kaggle.com/mlg-ulb/creditcardfraud).

>The datasets contains transactions made by credit cards in September 2013 by european cardholders.
This dataset presents transactions that occurred in two days, where we have 492 frauds out of 284,807 transactions. 
The dataset is highly unbalanced, the positive class (frauds) account for 0.172% of all transactions.
It contains only numerical input variables which are the result of a PCA transformation.
"""
# Importing the Credit Card Fraud dataset
data = pd.read_csv('/kaggle/input/creditcardfraud/creditcard.csv')
y_data = data.copy()['Class'].values
original_data = data.copy()

# Clean data set
normal_only_data = data[data['Class']==0]
print('Normal only data shape: ', normal_only_data.shape)
# Fraud data set
fraud_only_data = data[data['Class']==1]
print('Fraud only data shape: ', fraud_only_data.shape)

# Shuffling the data
normal_only_data = normal_only_data.sample(frac=1, random_state=42)
fraud_only_data = fraud_only_data.sample(frac=1, random_state=42)

# 80/10/10 data split for normal data
train_set, dev_set, test_set = np.split(normal_only_data, [int(0.8*len(normal_only_data)), int(0.9*len(normal_only_data))])
train_set = train_set.drop('Class', axis=1)

# 50/50 data split for fraud data
fraud_set_1, fraud_set_2 = np.split(fraud_only_data, [int(0.5*len(fraud_only_data))])

# Appending fraud data to dev and test set
dev_set = dev_set.append(fraud_set_1)
y_dev_set = dev_set['Class']
dev_set = dev_set.drop('Class', axis=1)
test_set = test_set.append(fraud_set_2)
y_test_set = test_set['Class']
test_set = test_set.drop('Class', axis=1)

# Showing shapes
for name, data in zip(['Train data shape: ', 'Dev data shape: ', 'Test data shape: '],[train_set, dev_set, test_set]):
    print(name, data.shape)
# Showing the first few rows of the data
original_data.head()
"""
## Trying out Custom Estimators - Novelty detection
For simplicity, I will train and test the custom and a few scikit-learn models on a novelty detection bias.

Novelty detection uses clean data (normal) only to train, and predict on contaminated data. Whereas Outlier detection uses contaminated (normal + fraud) data to train.

For more information, refer to scikit-learn's [Guidance](https://scikit-learn.org/stable/modules/outlier_detection.html#outlier-detection) here
"""
# Helper function to evaluate models
labels = ['Normal', 'Fraud']
def evaluate_model(y_true, y_preds, labels=labels):
    cm = confusion_matrix(y_true, y_preds)
    print('Recall score:\n', recall_score(y_true, y_preds))
    print('Precision score:\n', precision_score(y_true, y_preds))
    print('Confusion matrix:\n')
    cm_df = pd.DataFrame({'Normal (predicted)': (cm[0, 0], cm[1, 0]),
                         'Fraud (predicted)': (cm[0, 1], cm[1, 1])},
                        index=['Normal (true)', 'Fraud (true)'])
    print(cm_df)
# Training Multivariate Gaussian Anomaly Detector
mvg = MultivariateGaussian(epsilon=0.05**30)
mvg.fit(train_set)
# Evaluating on the Dev set
mvg_y_dev_preds = mvg.predict(dev_set)
evaluate_model(y_dev_set, mvg_y_dev_preds)
# Training Simple Anomaly Detector (SAD)
# Side note: I'm still thinking about a new name for this Estimator ... suggestions are welcomed
simple = SimpleAnomalyDetector(epsilon=0.05**30)
simple.fit(train_set)
# Evaluating on the Dev Set
y_simple_dev_preds = simple.predict(dev_set)
evaluate_model(y_dev_set, y_simple_dev_preds)
# Training Multivariate T Anomaly Detector
mvt = MultivariateTDistribution(epsilon=0.05**30, df=3)
mvt.fit(train_set)
# Evaluating on the Dev Set
mvt_y_dev_preds = mvt.predict(dev_set)
evaluate_model(y_dev_set, mvt_y_dev_preds)
# Training Local Outlier Factor Anomaly Detector
lof = LocalOutlierFactor(novelty=True, metric='euclidean')
lof.fit(train_set)
# Evaluating on the Dev set
lof_y_dev_preds = lof.predict(dev_set)
lof_y_dev_preds[lof_y_dev_preds==1] = 0
lof_y_dev_preds[lof_y_dev_preds==-1] = 1
evaluate_model(y_dev_set, lof_y_dev_preds)
# Training Isolation Forest Anomaly Detector
ifr = IsolationForest(random_state=42, behaviour="new")
ifr.fit(train_set)
# Evaluating on the Dev Set
ifr_y_dev_preds = ifr.predict(dev_set)
ifr_y_dev_preds[ifr_y_dev_preds==1] = 0
ifr_y_dev_preds[ifr_y_dev_preds==-1] = 1
evaluate_model(y_dev_set, ifr_y_dev_preds)
# Training Kmeans Clustering
kmeans = KMeans(n_clusters=2, algorithm='auto')
kmeans.fit(train_set)
# Evaluating KMeans on the Dev Set
kmeans_y_dev_preds = kmeans.predict(dev_set)
# Since KMeans only does clustering, we can decide which cluster would be Normal and which cluster would be Fraud
kmeans_y_dev_preds = np.where(kmeans_y_dev_preds==1, 0, 1)
evaluate_model(y_dev_set, kmeans_y_dev_preds)
"""
#### Contaminated dataset
"""
# Trying out Isolation Forest on contaminated dataset
ifr_2 = IsolationForest(random_state=42, behaviour="new")
ifr_2_y_preds = ifr_2.fit_predict(original_data.drop('Class', axis=1))
ifr_2_y_preds[ifr_2_y_preds==1] = 0
ifr_2_y_preds[ifr_2_y_preds==-1] = 1
evaluate_model(y_data, ifr_2_y_preds)
"""
## Trying out Neural Network Autoencoders

#### Intuition
The basic idea behind NN Autoencoders is to learn the very low-level representations of the data. After this process hopefully the 'noise' have been minimised from the data, and the result representations (outputs of NN Autoencoders) can be used as inputs to simplier classifiers such as Logistic Regression.
"""
from keras.layers import Input, Dense
from keras.models import Model, Sequential
from keras import regularizers
from sklearn.preprocessing import MinMaxScaler
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
# Considered using XGBoost, my it would take a while on my EY laptop
# I use smaller sample to save computational time
train_data = normal_only_data[:100000].append(fraud_only_data[:390])
train_data = train_data.sample(frac=1, random_state=42)
print('NN Train data shape:\n', train_data.shape)
dev_data = normal_only_data[100000:120000].append(fraud_only_data[390:])
dev_data = dev_data.sample(frac=1, random_state=42)
print('NN Dev data shape:\n', dev_data.shape)
# Separating X and y
X_train_nn, y_train_nn = train_data.drop('Class', axis=1, inplace=False), train_data['Class'].values
scaler = MinMaxScaler()
X_train_nn = scaler.fit_transform(X_train_nn)
print(X_train_nn.shape, y_train_nn.shape)
X_dev_nn, y_dev_nn = dev_data.drop('Class', axis=1, inplace=False), dev_data['Class'].values
X_dev_nn = scaler.transform(X_dev_nn)
print(X_dev_nn.shape, y_dev_nn.shape)
# Building computational graph for NN Autocencoder
# Input layer
input_layer = Input(shape=(X_train_nn.shape[1],))

# Encoding part
encoded_1 = Dense(200, activation='tanh', activity_regularizer=regularizers.l1(10e-5))(input_layer)
encoded_2 = Dense(100, activation='relu')(encoded_1)

# Decoding part
decoded_1 = Dense(100, activation='tanh')(encoded_2)
decoded_2 = Dense(200, activation='tanh')(decoded_1)

# Output layer
output_layer = Dense(X_train_nn.shape[1], activation='relu')(decoded_2)

# Compiling model
autoencoder = Model(input_layer, output_layer)
autoencoder.compile(optimizer='adam', loss='mse')
# Training the model
autoencoder.fit(X_train_nn, X_train_nn,
                batch_size=256, epochs=20,
                shuffle=True, validation_split=0.2
               )
autoencoder.summary()
# Building a computation graph to get the hidden representations of X
hidden_representation = Sequential()
hidden_representation.add(autoencoder.layers[0])
hidden_representation.add(autoencoder.layers[1])
hidden_representation.add(autoencoder.layers[2])
# Obtaining the hidden representations of X_train
rep_X_train = hidden_representation.predict(X_train_nn)
# Logistic Regression - mapping NN Training output to y
logreg = LogisticRegression(solver='lbfgs')
logreg.fit(rep_X_train, y_train_nn)
# Now moving on to the Dev Set
rep_X_dev = hidden_representation.predict(X_dev_nn)
nn_y_dev_preds = logreg.predict(rep_X_dev)
np.unique(nn_y_dev_preds)
# Evaluating the Dev Set
evaluate_model(y_dev_nn, nn_y_dev_preds)
# Random Forest Classifier - training on hidden representations of X
rf = RandomForestClassifier(random_state=42)
rf.fit(rep_X_train, y_train_nn)
# Evaluating on the Dev Set
nn_y_dev_preds_rf = rf.predict(rep_X_dev)
evaluate_model(y_dev_nn, nn_y_dev_preds_rf)