"""
## Importing Required Libraries
"""
import numpy as np                   # For creating matrices and for number operations
import pandas as pd                  # For manipulating and reading data
import matplotlib.pyplot as plt      # for ploting graphs
import seaborn as sns                # To plot heatmaps
import warnings                      # Hide warnings
warnings.filterwarnings('ignore')
"""
## Loading Dataset
"""
df = pd.read_csv("/kaggle/input/titanic/train.csv")
df.head()
df.info()
plt.figure(figsize = (10 , 5))
sns.heatmap(df.corr() , annot = True , cmap = "coolwarm" , )   # Heatmap of correlations accross columns
plt.figure(figsize = (15 , 8))
sns.heatmap(df.isnull() , cmap = "coolwarm" , yticklabels = False)
sns.set_style("darkgrid")
sns.countplot(data = df , x = "Survived")  # Value_counts of target column
df["First"] = df["Name"].apply(lambda x : x.split(",")[1].split(".")[0])
df.head()
df["First"].dtype
df.info()
df["First"].value_counts()
rest = [" Dr" , " Rev" , " Major" , " Mlle" , " Col" , " Don" , " Lady" , " Sir" , " Capt" , " Ms" , " Jonkheer" , " Mme" , " the Countess"]
df["First"].isna().sum()
df["First"].loc[df["First"].isin(rest)] = "rest"
df["First"][3]
df["First"].value_counts()
df.isna().sum() , len(df)  # Checking for null values
test = pd.read_csv("/kaggle/input/titanic/test.csv")    # Loading test dataset
test.head()
test.info()
test.isna().sum() , len(test)  # Checking for null values
test["First"] = test["Name"].apply(lambda x : x.split(",")[1].split(".")[0])
test["First"].loc[test["First"].isin(rest)] = "rest"
test["First"].value_counts()
test["First"].loc[test["First"] == " Dona"] = "rest"
test["First"].value_counts()
# Droping Cabin columns as it has more percentage of missing values

df.drop(["Cabin"] , axis = 1 , inplace = True)        
test.drop(["Cabin"] , axis = 1 , inplace = True)
df.info()
# Droping columns with unique values as they are not useful

df.drop(["PassengerId" , "Name"] , axis = 1 , inplace = True)
ids = test["PassengerId"]
test.drop(["PassengerId" , "Name"] , axis = 1 , inplace = True)
# Getting all the categorical columns

cats = df.select_dtypes(include="object").columns
cats
# Getting all Integer columns

ints = df.select_dtypes(exclude="object").columns
ints
"""
## Filling Missing Values
"""
for i in df.columns:
    print(f"The number of unique values in {i} column is/are {len(df[i].unique())}")
    print("\n")
    print(f"The unique values in {i} column is/are {df[i].unique()}")
    print("\n")
    print(f"The value counts for each value in {i} column is/are :  \n{df[i].value_counts()}")
    print("\n\n")
    print("*"*100)
    print("\n\n")
# Drpoing ticket column as it has more unique values

df.drop(["Ticket"] , axis = 1 , inplace = True)
test.drop(["Ticket"] , axis = 1 , inplace = True)
df.isna().sum()
test.isna().sum()
sns.countplot(data = df , x =  "Embarked")
# Filling Embarked missing values with the mode.

df["Embarked"].fillna("S" , inplace = True)
sns.countplot(data = df , x =  "Embarked")
# Filling missing intergeral values with the median as mean encounters with outliners

df["Age"].fillna(df["Age"].median() , inplace = True)
test["Age"].fillna(df["Age"].median() , inplace = True)
test["Fare"].fillna(df["Fare"].median() , inplace = True)
df.isna().sum()
test.isna().sum()
df.head()
test.head()
"""
## Data Visualization
"""
sns.set_style("darkgrid")
plt.figure(figsize = (10 , 10))
sns.pairplot(data = df)
# Defining Group by survival function for better visualization

def grouping(x , hue = "Age"):
    res = df.groupby(df[x]).mean()
    res = res.reset_index()
    print(f"Grouping by {x} with DataFrame : \n ")
    print(res)
    print("\n")
    if x in ["Pclass" , "SibSp" , "Parch"]:
        plt.figure(figsize = (12 , 5))
        sns.barplot(data = res , x = x , y = "Survived" , hue = hue)
        plt.show()
    else:
        plt.figure(figsize = (12 , 5))
        sns.histplot(data = res , x = "Survived" , y = x , cbar = True , cmap = "coolwarm")
        plt.show()
    print("\n\n")
grouping("Pclass" , "Age")
grouping("Age")
grouping("SibSp" , "Parch")
grouping("Parch" , "Pclass")
grouping("Fare")
sns.histplot(data = df , x = "Age" , kde = True , hue = "Sex" , bins = 20)
sns.kdeplot(data = df , x = "Age" , hue = "Survived")
sns.kdeplot(data = df , x = "Age" , hue = "Pclass")
sns.histplot(data = df , x = "Fare" , kde = True , bins = 20 , hue = "Pclass")
sns.kdeplot(data = df , x = "Fare" , hue = "Sex")
sns.kdeplot(data = df , x = "Fare" , hue = "Survived")
"""
### Countplots
"""
sns.countplot(data = df , x = "First" , hue = "Survived")
sns.countplot(data = df , x = "Pclass" , hue = "Sex")  # Pclass 3 has more males
sns.countplot(data = df , x = "Sex" , hue = "Survived")  # Males has more rate of survival
plt.figure()
sns.countplot(data = df , x = "SibSp" , hue = "Sex")
plt.figure()
sns.countplot(data = df , x = "Parch" , hue = "Sex")
plt.figure()
sns.countplot(data = df , x = "SibSp" , hue = "Survived")
plt.figure()
sns.countplot(data = df , x = "Parch" , hue = "Survived")
plt.figure()
sns.countplot(data = df , x = "SibSp" , hue = "Pclass")
plt.figure()
sns.countplot(data = df , x = "Parch" , hue = "Pclass")
"""
### Boxplots
"""
sns.boxplot(data = df , x = "First" , y = "Age" , hue = "Survived")
sns.boxplot(data = df , x = "Pclass" , y = "Age" , hue = "Survived")
sns.boxplot(data = df , x = "Sex" , y = "Age" , hue = "Pclass")
sns.boxplot(data = df , x = "Pclass" , y = "Fare" , hue = "Survived")
sns.boxplot(data = df , x = "Sex" , y = "Fare" , hue = "Pclass")
"""
## Data Preprocessing
"""
# Getting rid of skewness in train set

df["Age"] = np.log(df["Age"] + 1)
df["Fare"] = np.log(df["Fare"] + 1)
# Getting rid of skewness in test set

test["Age"] = np.log(test["Age"] + 1)
test["Fare"] = np.log(test["Fare"] + 1)
# Creating dummie variables for categorical columns , droping the first_column

df = pd.get_dummies(df , columns = ["Sex" , "Pclass" , "Embarked" , "First"] , prefix = ["Sex" , "Pclass" , "Embarked" , "First"] , drop_first = True)
test = pd.get_dummies(test , columns = ["Sex" , "Pclass" , "Embarked" , "First"] , prefix = ["Sex" , "Pclass" , "Embarked" , "First"] , drop_first = True)
df.head()
test.head()
"""
## Train Test Split
"""
from sklearn.model_selection import train_test_split
X = df.drop(["Survived"] , axis = 1)
y = df["Survived"]
X_train , X_test , y_train , y_test = train_test_split(X , y , test_size = 0.2 , random_state = 42)
len(X_train) , len(X_test) , len(y_train) , len(y_test)
"""
## Feature Scaling
"""
from sklearn.preprocessing import StandardScaler
scaler1 = StandardScaler()
X_train[["Age" , "Fare"]] = scaler1.fit_transform(X_train[["Age" , "Fare"]])
X_test[["Age" , "Fare"]] = scaler1.transform(X_test[["Age" , "Fare"]])
test[["Age" , "Fare"]] = scaler1.transform(test[["Age" , "Fare"]])
X_train.head()
test.head()
"""
## Importing libraries for model fitting
"""
from xgboost import XGBClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier


from sklearn.metrics import confusion_matrix , roc_auc_score , f1_score , accuracy_score , classification_report , roc_curve , auc , plot_roc_curve
from sklearn.model_selection import cross_val_score
models = []
models.append(['XGBClassifier', XGBClassifier(learning_rate = 0.1 , objective = 'binary:logistic' , random_state = 42 , eval_metric='mlogloss')])
models.append(['AdaBoostClassifier', AdaBoostClassifier(random_state = 42)])
models.append(['RandomForest', RandomForestClassifier(random_state = 42)])
models.append(['Logistic Regression', LogisticRegression(random_state = 42)])
models.append(['KNeigbors', KNeighborsClassifier()])
"""
## Model Evaluation
"""
def metrics(model , X_train , y_train , X_test , y_test , params = False):
    
    model[1].fit(X_train , y_train)
    preds = model[1].predict(X_test)
    accuracies = cross_val_score(estimator = model[1], X = X_train , y = y_train, cv = 10)
    cm = confusion_matrix(y_test , preds)
    cf = classification_report(y_test , preds)
    roc = roc_auc_score(y_test , model[1].predict_proba(X_test)[: , 1])
    fpr, tpr, thresholds = roc_curve(y_test, preds)
    ac = auc(fpr, tpr)
    f1 = f1_score(y_test , preds)
    
    
    print("\n")
    print(model[0])
    
    print("\n")
    if params:
        print(f"Best Parameters are : \n" , model[1].best_params_)
        print("\n")
        
    print(f"Confusion matrix : \n")
    plt.figure(figsize = (8, 5))
    sns.heatmap(cm, cmap = 'coolwarm', annot = True, annot_kws = {'fontsize': 20})
    plt.show()
    print("\n")
    
    print(f"Training score : {model[1].score(X_train , y_train):.4f}")
    print("\n") 
    
    print(f"Test Score : {model[1].score(X_test , y_test):.4f}")
    print("\n")
    
    print(f"K-fold accuracy : {np.mean(accuracies):.4f}")
    print("\n")
    
    print(f"Standard Deviation of Accuracies in k-fold : {np.std(accuracies):.4f}")
    print("\n")
    
    print(f"ROC AUC Score: {roc:.4f}")
    print('\n')
    
    print(f"F1 Score: {f1:.4f}")
    print("\n")
    
    print(f"AUC : {ac:.4f}")
    print("\n")
    
    print(f"Classification report : \n\n{cf}")
    print("\n")

    plt.figure(figsize = (8, 5))
    plot_roc_curve(model[1], X_test, y_test , color = '#FF4500')
    plt.plot([0, 1], [0, 1], linestyle = '--', color = '#7CFC00')
    plt.show()
    print("\n")
    print("*"*100)
    
    print("\n\n")
    
    sam = []
    sam.append(model[0])
    sam.append(model[1].score(X_train , y_train))
    sam.append(model[1].score(X_test , y_test))
    sam.append(np.mean(accuracies))
    sam.append(np.std(accuracies))
    sam.append(roc)
    sam.append(f1)
    sam.append(ac)
    
    return sam
    
    
pre_final = []
for i in models:
    sam = metrics(i , X_train , y_train , X_test , y_test)
    pre_final.append(sam)
pre_final
"""
## Model Evaluation Visualization
"""
me = pd.DataFrame(pre_final , columns = ["Model" , "Train Score" , "Test Score" , "K-fold Accuracy" , "K-fold Std" , "ROC_AUC" , "F1 Score" , "AUC Score"])

me.sort_values(by = ["K-fold Accuracy" , "F1 Score" , "ROC_AUC" , "AUC Score" , "Train Score" , "Test Score"] , inplace = True , ascending = False)
me = me.reset_index(drop = True)
me
plt.figure(figsize = (10 , 5))
sns.barplot(y = "Model" , x = "ROC_AUC" , data = me)
plt.title("Model Comparision based on ROC_AUC");
plt.figure(figsize = (10 , 5))
sns.barplot(y = "Model" , x = "F1 Score" , data = me)
plt.title("Model Comparision based on F1 Score");
plt.figure(figsize = (10 , 5))
sns.barplot(y = "Model" , x = "AUC Score" , data = me)
plt.title("Model Comparision based on AUC Score");
"""
## Model Evaluation with Grid SearchCV
"""
"""
Not fitting XGB , Ada Boost as it already comes with best parameters
"""
from sklearn.model_selection import GridSearchCV


grid_xgb = {"n_estimators" : [100 , 200 , 300]}


grid_ada = {"n_estimators" : [50 , 100 , 200]}


grid_random = {"n_estimators" : [150 , 200 , 250],
              "bootstrap" : [True , False] , 
              "max_features" : ['auto', 'sqrt'] , 
              "min_samples_leaf" : [2, 4] , 
              "class_weight" : ["balanced", "balanced_subsample"]}


grid_linear = {"max_iter" : [100 , 150] , 
              "solver" : ["liblinear"] , 
              "multi_class" : ["ovr"]}


grid_neighbor = {"n_neighbors" : [5 , 7 , 10] , 
                "algorithm" : ["auto", "ball_tree", "kd_tree", "brute"]} 

xgb = metrics(['XGBClassifier', GridSearchCV(XGBClassifier(learning_rate = 0.1, objective = 'binary:logistic' , random_state = 42 , eval_metric='mlogloss') , param_grid = grid_xgb, cv = 5, verbose = 0)] ,  X_train , y_train , X_test , y_test , params = True )
ada = metrics(['AdaBoostClassifier', GridSearchCV(AdaBoostClassifier(random_state = 42) , param_grid = grid_ada , cv = 5 , verbose = 0 )] , X_train , y_train , X_test , y_test , params = True)
random = metrics(['RandomForest', GridSearchCV(RandomForestClassifier(random_state = 42) , param_grid = grid_random, cv = 5, verbose = 0 , scoring = "f1")] ,  X_train , y_train , X_test , y_test , params = True )
linear = metrics(['Logistic Regression', GridSearchCV(LogisticRegression(random_state = 42) , param_grid = grid_linear, cv = 5, verbose = 0 )] ,  X_train , y_train , X_test , y_test , params = True )
knn = metrics(['KNeigbors', GridSearchCV(KNeighborsClassifier() , param_grid = grid_neighbor, cv = 10, verbose = 0)] , X_train , y_train , X_test , y_test , params = True)
final = [xgb , ada , random , linear , knn]
final
"""
## GridsearchCV Model Visualization
"""
me # Without Gridsearchcv
# With gridsearch CV

mef = pd.DataFrame(final , columns = ["Model" , "Train Score" , "Test Score" , "K-fold Accuracy" , "K-fold Std" , "ROC_AUC" , "F1 Score" , "AUC Score"])

mef.sort_values(by = ["K-fold Accuracy" , "F1 Score" , "ROC_AUC" , "AUC Score" , "Train Score" , "Test Score"] , inplace = True , ascending = False)
mef = mef.reset_index(drop = True)
mef
def feature_importance(model , X_train , y_train):
    model[1].fit(X_train , y_train)
    features = model[1].feature_importances_
    print(model[0])
    print("\n")
    print(f"Feature importance list : \n" , features)
    print("\n")
    plt.figure(figsize = (15 , 8))
    sns.barplot(X_train.columns.tolist() , features)
    plt.show()
    print("\n")
    print("*"*100)
    print("\n")
def linear_coeffs(model , X_train , y_train):
    model[1].fit(X_train , y_train)
    features = model[1].coef_
    print(model[0])
    print("\n")
    print(f"Feature importance list : \n" , features)
    print("\n")
    plt.figure(figsize = (15 , 8))
    sns.barplot(X_train.columns.tolist() , features.ravel())
    plt.show()
    print("\n")
    print("*"*100)
    print("\n")
feature_importance(["XGBClassifier" , XGBClassifier(learning_rate = 0.1, objective = 'binary:logistic' , random_state = 42 , eval_metric='mlogloss')] , X_train , y_train)
feature_importance(["Randomforest Classifier" , RandomForestClassifier(random_state = 42 , bootstrap = False, class_weight = 'balanced', max_features = 'auto', min_samples_leaf = 4 , n_estimators = 200 )] , X_train , y_train)
linear_coeffs(["Logestic Regressor" , LogisticRegression(max_iter = 100,  multi_class = 'ovr',  random_state = 42,  solver = 'liblinear')] , X_train , y_train)
"""
## Evaluation on the Test Dataset
"""
test
test.info()
test.isna().sum()
clf_linear = LogisticRegression(random_state = 42)
clf_linear.fit(X_train , y_train)
linear_preds = clf_linear.predict(test)
linear_preds
data_linear = {"PassengerId" : ids , 
       "Survived" : linear_preds}
final_linear = pd.DataFrame(data_linear, columns = ["PassengerId" , "Survived"])
final_linear.head(10)
final_linear["Survived"].value_counts()
clf_xgb = XGBClassifier(learning_rate = 0.1, objective = 'binary:logistic' , random_state = 42 , eval_metric='mlogloss')
clf_xgb.fit(X_train , y_train)
xgb_preds = clf_xgb.predict(test)
xgb_preds
data_xgb = {"PassengerId" : ids , 
       "Survived" : xgb_preds}

final_xgb = pd.DataFrame(data_xgb, columns = ["PassengerId" , "Survived"])
final_xgb.head(10)
final_xgb["Survived"].value_counts()
final_xgb.to_csv("Titanic_linear_1.csv" , index = False)