"""
## Table Content
* 1. Importing Modules

* 2. Loading Data

* 3. Data PreProcessing And Visualizations

* 4. Utility Functions

* 5. Data Balancing

* 6. Modelling And Optimizing The Models

* 7. Results And Conclusion
"""
"""
# Importing Modules
"""
from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_auc_score
import numpy as np
import pandas as pd 
import os
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import plot_tree
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.model_selection import RandomizedSearchCV
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
import seaborn as sns
import matplotlib.pyplot as plt 
from sklearn import metrics
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix,r2_score
from sklearn import model_selection
from sklearn.ensemble import BaggingClassifier
import warnings
from imblearn.over_sampling import SMOTE
warnings.filterwarnings('ignore')
"""
# Loading Data
"""
df = pd.read_csv('/kaggle/input/bank-marketing/bank-additional-full.csv', sep = ';')
df.head()
df.info()
"""
# Checking Categorical Columns
"""
for col in df.columns:
    print()
    if df[col].dtype == 'object':
        print(f'Name of Column is: {col} and unique values are: {df[col].unique()}')
"""
## Utility Function
"""
#this function returns categorical variables
def return_categorical(df):

  categorical_columns = [column_name for column_name in df if df[column_name].dtype == 'O']
  return categorical_columns

#this function returns numerical variables
def return_numerical(df):

  return list(set(df.columns) - set(return_categorical(df)))


def check_normal(df):
  fig, axes = plt.subplots(1,len(return_numerical(df)), figsize =(70, 10))

  for i,numeric_column_name in enumerate(list(set(df.columns) -set(return_categorical(df)))):

    sns.distplot(df[numeric_column_name], ax=axes[i]);
    plt.title(f'Distribution of {numeric_column_name}');
    
def classifier(clf, x_train,x_test,y_train,y_test):
    y_test_pred = clf.predict(x_test)
    y_train_pred = clf.predict(x_train)

    accuracy_test = accuracy_score(y_test,y_test_pred)
    accuracy_train =  accuracy_score(y_train,y_train_pred)
    
    roc_test = roc_auc_score(y_test, y_test_pred, multi_class='ovr')
    roc_train = roc_auc_score(y_train, y_train_pred, multi_class='ovr')
    
    print('Train accuracy is:',accuracy_train )
    print('Test accuracy is:',accuracy_test )
    print()
    print('Train ROC is:', roc_train)
    print('Test ROC is:',roc_test )
    
    # Fscore, precision and recall on test data
    f1 = f1_score(y_test, y_test_pred)
    precision = precision_score(y_test, y_test_pred)
    recall = recall_score(y_test, y_test_pred) 
    print()
    print("F score is:",f1 )
    print("Precision is:",precision)
    print("Recall is:", recall)
  

def random_search(clf,params, x_train,x_test,y_train,y_test):
    
    random_search = RandomizedSearchCV(estimator= clf, param_distributions=params, scoring='roc_auc', cv=5)
    random_search.fit(x_train, y_train)
    optimal_model = random_search.best_estimator_

    print("Best parameters are: ", random_search.best_params_)
    print()
    print("Best estimator is: ", random_search.best_estimator_)
    print()
    print('Scores and accuracies are:')
    print()
    classifier(optimal_model, x_train,x_test,y_train,y_test)
"""
# Data Preprocessing
* Data Visualisation
"""
check_normal(df)
"""
* None of the feature are following a Normal Distribution
"""
"""
# Visualising Categorical Variabels
"""

# plotting graphs for all categorical columns
for col in return_categorical(df):
    counts = df[col].value_counts().sort_index()
    if len(counts) > 10:
      fig = plt.figure(figsize=(30, 10))
    else:
      fig = plt.figure(figsize=(9, 6))
    ax = fig.gca()
    counts.plot.bar(ax = ax, color='steelblue')
    ax.set_title(col + ' counts')
    ax.set_xlabel(col) 
    ax.set_ylabel("Frequency")
plt.show()
"""
* imbalanced categorical data observed
"""
"""
# Checing highly co-related column
"""
corr = df.corr()
corr_greater_than_75 = corr[corr>=.75]
corr_greater_than_75
#Visualising by heatmap
plt.figure(figsize=(12,8))
sns.heatmap(corr_greater_than_75, cmap="Reds", annot = True);
"""
* Some correlation can be seen here between emp. var rate and nr.employed.
* Also, euribor3m and emp.var rate.
* However, I would not be removing any. As I am gonna mostly be training on tree models which doesn't require much preprocessing.
"""
"""
# Checking Unique values
"""
df['pdays'].unique()
"""
* 999 represents no contact has been done, thus i'll be assigning it a very less weight.
"""
"""
## Modefing the column on the basis of importance weighs
"""
df['job'] = df['job'].apply(lambda x: -1 if x=='unknown' or x=='unemployed' else (15 if x=='entrepreneur' else (8 if x == 'blue-collar' else ( 6 if x=='technician' or x=='services' or  x=='admin.' or x=='management' else (4 if x== 'self-employed' or x=='student' else (2 if x=='housemaid' or x=='retired' else None) )))))
df['housing'] = df['housing'].apply(lambda x: 0 if x=='no' else (1 if x=='yes' else -1))
df['loan'] = df['loan'].apply(lambda x: 0 if x=='no' else (1 if x=='yes' else -1))
df['y'] = df['y'].apply(lambda x: 0 if x=='no' else (1 if x=='yes' else -1))
df['default'] = df['default'].apply(lambda x: 0 if x=='no' else (1 if x=='yes' else -1))
df['poutcome'] = df['poutcome'].apply(lambda x: 0 if x=='failure' else (2 if x=='failure' else -1))
df['pdays'] = df['pdays'].apply(lambda x: 0 if x==999 else(20 if x<=10 else(6 if x<=20 else 3)))
"""
# Droping columns
"""
df.drop(['day_of_week', 'contact', 'month'], axis=1, inplace = True)
"""
# One hot encoding for categorical features 
"""
df  = pd.get_dummies(df, drop_first = True)
"""
# Train_Test_Split
"""
x = df.drop("y", axis=1)
y = df['y']
x.sample()

x_train,x_test,y_train,y_test = train_test_split(x,y, random_state=42)
"""
# Balancing data by using SMOTE
"""
smote = SMOTE()

# fit predictor and target variable
x_smote, y_smote = smote.fit_resample(x_train, y_train)

print('Original dataset shape', len(x_train))
print('Resampled dataset shape', len(x_smote))
"""
# Scaling and Optimising
"""
s = StandardScaler()
"""
# Modeling with KNN
"""
knn = KNeighborsClassifier(n_neighbors = 20)
knn.fit( s.fit_transform(x_train), y_train)

classifier(knn, s.fit_transform(x_smote),s.transform(x_test), y_smote,y_test)
"""
# Tuning hyper perameters by using Knn 
"""
error_rate = []
for i in range(1,40):
 knn = KNeighborsClassifier(n_neighbors=i)
 knn.fit( s.fit_transform(x_train), y_train)
 pred_i = knn.predict(s.transform(x_test))
 error_rate.append(np.mean(pred_i != y_test))

acc = []
for i in range(1,40):
    neigh = KNeighborsClassifier(n_neighbors = i).fit(s.fit_transform(x_train), y_train)
    yhat = neigh.predict(s.transform(x_test))
    acc.append(metrics.accuracy_score(y_test, yhat))
    
# Visualising the tuning parameters
plt.figure(figsize=(10,6))
plt.plot(range(1,40),error_rate,color='blue', linestyle='dashed', 
         marker='o',markerfacecolor='red', markersize=10)
plt.title('Error Rate vs. K Value')
plt.xlabel('K')
plt.ylabel('Error Rate')
print("Minimum error:-",min(error_rate),"at K =",error_rate.index(min(error_rate)))
plt.figure(figsize=(10,6))
plt.plot(range(1,40),acc,color = 'blue',linestyle='dashed', 
         marker='o',markerfacecolor='red', markersize=10)
plt.title('accuracy vs. K Value')
plt.xlabel('K')
plt.ylabel('Accuracy')
print("Maximum accuracy:-",max(acc),"at K =",acc.index(max(acc)))
"""
# Modeling value of K gives least error
"""
knn = KNeighborsClassifier(n_neighbors = 10)
knn.fit( s.fit_transform(x_smote), y_smote)

classifier(knn, s.fit_transform(x_smote),s.transform(x_test),y_smote,y_test)
"""
# Bagging Knn as base Model
"""
# bagging classifier
model = BaggingClassifier(base_estimator = KNeighborsClassifier(n_neighbors = 10),
                          n_estimators = 15)
classifier(model.fit( s.fit_transform(x_smote), y_smote), s.fit_transform(x_smote),s.transform(x_test),y_smote,y_test)
"""
# Modeling with Decession Tree
"""
dtree = DecisionTreeClassifier(random_state=0)
dtree.fit(x_train, y_train)
classifier(dtree, x_train,x_test,y_train,y_test)
"""
* Not using balanced data with tree based models because they can handle imbalace pretty well so whynot.
"""
"""
# Tuning hyper parameters for Decission tree
"""
param_grid = {'max_depth':np.arange(3,20),
              'min_samples_split': np.arange(3,20,1),
             'min_samples_leaf':np.arange(3,30),
              'min_samples_split' : np.arange(3,30),
              'criterion': ('gini', 'entropy')}



random_search(DecisionTreeClassifier(random_state=0),param_grid, x_train,x_test,y_train,y_test)
"""
# Decission Tree Base Bagging
"""
# I have chosen tuned hyperparameters here


kfold = model_selection.KFold(n_splits = 3)

# bagging classifier
model = BaggingClassifier(base_estimator = DecisionTreeClassifier(min_samples_split=5, min_samples_leaf=25, max_depth=8, criterion='gini'),n_estimators = 500,)

classifier(model.fit(x_train, y_train), x_train,x_test,y_train,y_test)
"""
# Modeling with Random Forest
"""
rforest = RandomForestClassifier(random_state=0)
classifier(rforest.fit(x_train, y_train), x_train,x_test,y_train,y_test)
"""
# Tuning hyper parameters for Random Forest
"""
params = {'n_estimators' : np.arange(100,1000, 100),
              'max_depth' : np.arange(3,20,1),
              'min_samples_split' : np.arange(3,20,1),
              'min_samples_leaf' : np.arange(3,20,1),
         'max_features': ('sqrt', 'log2'), 'criterion': ('gini', 'entropy')}

random_search(RandomForestClassifier(random_state=0),params, x_train,x_test,y_train,y_test)
"""
* Bagging with Decision tree is performing the best according to recall and roc.
* For a credit insurance problem, I would want to go with recall here.
"""