"""
In my [previous notebook](https://www.kaggle.com/arunima24/exploratory-data-analysis-on-titanic-dataset), I've done an EDA and Feature Engineering on the titanic dataset to understand the data pattern and relationship. Here, I would be continuing on the same pattern to use the different ensemble techniques to predict the accuracy.
"""
"""
# Importing Libraries
"""
import numpy as np 
from numpy import hstack
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import re

from sklearn.model_selection import train_test_split,RandomizedSearchCV
from sklearn.metrics import classification_report, accuracy_score

from sklearn.impute import KNNImputer
from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier,VotingClassifier,AdaBoostClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier

plt.style.use('fivethirtyeight')
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
"""
# Reading Dataset
"""
train_data = pd.read_csv("../input/titanic/train.csv")
test_data = pd.read_csv("../input/titanic/test.csv")
train_data.head(20)
"""
> **Get information about the data like count, mean, standard deviation etc.**
"""
train_data.describe()
train_data.info()
"""
> **Creating an array using the columns which would be used for the various graphs for EDA**
"""
features = []
for i in train_data.columns:
    if i !='Survived':
        features.append(i)

features
"""
> **Getting the information of the number of records per columns with null/NaN value**
"""
train_data.isnull().sum()
"""
> **Checking the impact of each feature on target variable**
"""
for i in features:
    print("\n\n"+i)
    print (train_data[[i, "Survived"]].groupby([i], as_index=False).mean())
"""
# FEATURE ENGINEERING
We see from the above table that some columns have a large amount of similar data or contain NaN values which would be difficult to be used in a model if no changes are made to them. For example, 

**PassengerID**: The ID has no impact on survival, so we might ignore the column altogether.

**Cabin**: The Cabin can be grouped according to the classes, and we can try to derive a pattern from it.

**Fare**: There are 200+ different fares that the passengers have paid. Instead of analysing them individually for a prediction, we can use them as a range.

**Ticket**: The Ticket has no impact on survival, so we might ignore the column altogether.

**Age**: Again, like Fare, we need to treat them as age group instead of considering individual ages 

**Name**: The names are not important, but the salutation is. We can use that in addition to the Sex and Age feature for prediction.
"""
train_data.drop(['PassengerId','Ticket'],axis=1,inplace=True)
"""
**NAME**

Let's look at Name feature. My idea is to get the salutation from each name and store it as a new column 'Title' in the dataframe.
"""

def get_title(name):
    title_search = re.search(' ([A-Za-z]+)\.', name)

    if title_search:
        return title_search.group(1)
    return ""


train_data['Title'] = train_data['Name'].apply(get_title)
train_data.drop('Name',axis=1,inplace=True)
train_data['Title'].unique()
"""
Let's check the number of records for each of these titles.
"""
train_data.groupby(['Title']).size()
"""
We can replace some of the titles (such as Ms -> Miss) and the group the ones which appear lesser compared to others in a new group. We'll then check the mean number of records of each title's relation to the output.
"""
train_data['Title'] = train_data['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Others')

train_data['Title'] = train_data['Title'].replace('Mlle', 'Miss')
train_data['Title'] = train_data['Title'].replace('Ms', 'Miss')
train_data['Title'] = train_data['Title'].replace('Mme', 'Mrs')

train_data[['Title', 'Survived']].groupby(['Title'], as_index=False).mean()
"""
**CABIN**

Next, I'm starting with the Cabin feature. For this, I'll assign -1 to NaN/null vales (and deal with them later) and group the others using the first letter of the values and create a new column 'Cabin_Group'.
"""
def cabin_group(cabin):
    if type(cabin) == str:
        letter =  cabin[0]
    else:
        letter = -1
    
    if type(letter) == str and ord(letter) <= 70:
        return letter
    elif letter == -1:
        return '-1'
    else:
        return 'OtherCabin'
    
    
train_data['Cabin_Group'] = train_data['Cabin'].apply(cabin_group)
train_data.drop('Cabin',axis=1,inplace=True)
train_data.groupby(['Cabin_Group']).size()
"""
Checking the relation between this new feature with target.
"""
train_data[['Cabin_Group', 'Survived']].groupby(['Cabin_Group'], as_index=False).mean()
"""
**FARE**

I'm dividing the fare to 5 buckets or ranges, which I'd be using later for my model.
"""
bucket = pd.qcut(train_data['Fare'], 5)
bucket.unique()
def FareGroup(fare):
    if float(fare):
        if fare > 0.0 and fare <= 7.854:
            return 1
        elif fare > 7.854 and fare <= 10.5:
            return 2
        elif fare > 10.5 and fare <= 21.679:
            return 3
        elif fare > 21.679 and fare <= 39.688:
            return 4
        elif fare > 39.688 and fare <= 512.329:
            return 5
        else:
            return -1
    else:
        return -1
    
    
train_data['FareRange'] = train_data['Fare'].apply(FareGroup)
train_data.drop('Fare',axis=1,inplace=True)
print(train_data.groupby(['FareRange']).size())
print (train_data[['FareRange', 'Survived']].groupby(['FareRange'], as_index=False).mean())
"""
**Age**

Creating a similar range for age as well, like Fare and assigning -1 to Null values to be handled later.
"""
bucket = pd.qcut(train_data['Age'], 5).unique()
bucket
def AgeGroup(age):
    if float(age):
        if age > 19.0 and age <= 25.0:
            return 19
        elif age > 25.0 and age <= 31.8:
            return 25
        elif age > 31.8 and age <= 41.0:
            return 31
        elif age > 41.0 and age <= 80.0:
            return 41
        elif age < 19.0:
            return 0
        else:
            return -1
    else:
        return -1
    
    
train_data['AgeGroup'] = train_data['Age'].apply(AgeGroup)
train_data.drop('Age',axis=1,inplace=True)
print (train_data[['AgeGroup', 'Survived']].groupby(['AgeGroup'], as_index=False).mean())
"""
I'll update my features array with these new features and dropping the old ones.
"""
features = []
for i in train_data.columns:
    if i !='Survived':
        features.append(i)

features
"""
# Handling Null Values

We have null values in Age, Cabin and Embarked features. I'll try different methods o handling null values for these features.
"""
"""
**Embarked**

Since we have only 2 null records for Embarked feature, I'm going to save some time and just drop them.
"""
train_data.dropna(subset=['Embarked'],inplace=True)
train_data.groupby(['Embarked']).size()
"""
**Age and Fare**

Here, I'd be using KNN to predict te missing values (which I've classified as -1 in previous function). For this task, I'd be using KNNImputer.
"""
train_data['AgeGroup'] = train_data['AgeGroup'].apply(lambda x: np.nan if x == -1 else x)
train_data['FareRange'] = train_data['FareRange'].apply(lambda x: np.nan if x == -1 else x)


imputer = KNNImputer(n_neighbors= 5)
AgeGroupImputed = imputer.fit_transform([train_data['AgeGroup']])
FareRangeImputed = imputer.fit_transform([train_data['FareRange']])
FareRangeImputed = pd.Series(FareRangeImputed[0])
AgeGroupImputed = pd.Series(AgeGroupImputed[0])
train_data['FareRange'] = FareRangeImputed
train_data['AgeGroup'] = AgeGroupImputed

train_data.head(20)
"""
I'll replace the new columns with the one hot encoded values and append them to my dataframe.
"""
sex = pd.get_dummies(train_data['Sex'],drop_first=True)
embark = pd.get_dummies(train_data['Embarked'],drop_first=True)
title = pd.get_dummies(train_data['Title'],drop_first=True)
fare = pd.get_dummies(train_data['FareRange'],drop_first=True)
cabin = pd.get_dummies(train_data['Cabin_Group'],drop_first=True)
age = pd.get_dummies(train_data['AgeGroup'],drop_first=True)

train_data.drop(['AgeGroup','Sex',"Embarked",'Title','Cabin_Group','FareRange'],axis=1,inplace=True)

train_data = pd.concat([train_data,sex,embark,title,cabin,fare,age],axis=1)
train_data.head(20)
"""
# Splitting data to train and test dataset
"""
X_train, X_test, y_train, y_test = train_test_split(train_data.drop('Survived',axis=1), 
                                                    train_data['Survived'], test_size=0.30, 
                                                    random_state=101)
"""
# Model Building
"""
RandomForestModel = RandomForestClassifier(random_state=3)
RandomForestModel.fit(X_train,y_train)
accuracy = accuracy_score(y_test,RandomForestModel.predict(X_test))
classificationRep=classification_report(y_test,RandomForestModel.predict(X_test))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
LGBMModel = LGBMClassifier()
LGBMModel.fit(X_train,y_train)
accuracy = accuracy_score(y_test,LGBMModel.predict(X_test))
classificationRep=classification_report(y_test,LGBMModel.predict(X_test))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
XGBModel = XGBClassifier()
XGBModel.fit(X_train,y_train)
accuracy = accuracy_score(y_test,XGBModel.predict(X_test))
classificationRep=classification_report(y_test,XGBModel.predict(X_test))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
AdaBoostModel = AdaBoostClassifier()
AdaBoostModel.fit(X_train,y_train)
accuracy = accuracy_score(y_test,AdaBoostModel.predict(X_test))
classificationRep=classification_report(y_test,AdaBoostModel.predict(X_test))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
"""
# Fine Tuning

Once I've tested with the common ensemble techniques, I'd now fine tune the hyperparamters and test what impact the fine tuning will have on the final prediction.
"""
"""
**Fine tuning Random Forest**
"""

# Number of trees in random forest
n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]
# Number of features to consider at every split
max_features = ['auto', 'sqrt']
# Maximum number of levels in tree
max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
max_depth.append(None)
# Minimum number of samples required to split a node
min_samples_split = [2, 5, 10]
# Minimum number of samples required at each leaf node
min_samples_leaf = [1, 2, 4]
# Method of selecting samples for training each tree
bootstrap = [True, False]

criterion = ['gini', 'entropy']
# Create the random grid
random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}
print(random_grid)
RandomForestModel2 = RandomForestClassifier()
# Random search of parameters, using 3 fold cross validation, 
# search across 100 different combinations, and use all available cores
rf_random = RandomizedSearchCV(estimator = RandomForestModel2, param_distributions = random_grid, n_iter = 50, cv = 3, verbose=2, random_state=42, n_jobs = -1)
# Fit the random search model
rf_random.fit(X_train, y_train)
params = rf_random.best_params_
params
RandomForestModel2 = RandomForestClassifier(n_estimators=params['n_estimators'],min_samples_split=params['min_samples_split'],min_samples_leaf = params['min_samples_leaf'],max_features = params['max_features'],max_depth = params['max_depth'],bootstrap=params['bootstrap'])
RandomForestModel2.fit(X_train,y_train)
accuracy = accuracy_score(y_test,RandomForestModel2.predict(X_test))
classificationRep=classification_report(y_test,RandomForestModel2.predict(X_test))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
"""
**Fine tuning LGBM Classifier**
"""
random_grid_lgbm = { 
                   'num_leaves':[int(x) for x in np.linspace(start = 1, stop = 200, num = 10)],
                   'max_depth':[int(x) for x in np.linspace(start = 10, stop = 100, num = 10)], 
                   'learning_rate':[0.1,0.001], 
                   'n_estimators':[int(x) for x in np.linspace(start = 1, stop = 100, num = 10)]}

LGBMModel2 = LGBMClassifier()
rf_random = RandomizedSearchCV(estimator = LGBMModel2, param_distributions = random_grid_lgbm, n_iter = 50, cv = 3, verbose=2, random_state=42, n_jobs = -1)

rf_random.fit(X_train, y_train)
random_grid_lgbm = rf_random.best_params_
random_grid_lgbm
LGBMModel2 = LGBMClassifier(n_estimators=random_grid_lgbm['n_estimators'],num_leaves=random_grid_lgbm['num_leaves'],max_depth = random_grid_lgbm['max_depth'],learning_rate=random_grid_lgbm['learning_rate'])
LGBMModel2.fit(X_train,y_train)
accuracy = accuracy_score(y_test,LGBMModel2.predict(X_test))
classificationRep=classification_report(y_test,LGBMModel2.predict(X_test))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
"""
**Fine tuning XGBoost**
"""
xgparam = {'eta': [0.01,0.1,0.2],
'min_child_weight': [int(x) for x in np.linspace(start = 1, stop = 10, num = 10)],
'max_depth': [int(x) for x in np.linspace(start = 10, stop = 100, num = 10)],
'gamma':np.linspace(start = 0.0, stop = 0.2, num = 20)}

XGBModel2 = XGBClassifier()
rf_random = RandomizedSearchCV(estimator = XGBModel2, param_distributions = xgparam, n_iter = 50, cv = 3, verbose=2, random_state=42, n_jobs = -1)

rf_random.fit(X_train, y_train)
xgparam = rf_random.best_params_
xgparam
XGBModel2 = XGBClassifier(eta=xgparam['eta'],min_child_weight=xgparam['min_child_weight'],max_depth = xgparam['max_depth'],gamma=xgparam['gamma'])
XGBModel2.fit(X_train,y_train)
accuracy = accuracy_score(y_test,XGBModel2.predict(X_test))
classificationRep=classification_report(y_test,XGBModel2.predict(X_test))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
"""
**Fine Tuning AdaBoost**
"""
adaparam = {
    'n_estimators' : [int(x) for x in np.linspace(start = 10, stop = 100, num = 20)],
    'learning_rate' : np.linspace(start = 0.01, stop = 1, num = 20)
}

AdaBoostModel2 = AdaBoostClassifier()
rf_random = RandomizedSearchCV(estimator = AdaBoostModel2, param_distributions = adaparam, n_iter = 50, cv = 3, verbose=2, random_state=42, n_jobs = -1)

rf_random.fit(X_train, y_train)
adaparam = rf_random.best_params_
adaparam
AdaBoostModel2 = AdaBoostClassifier(n_estimators=adaparam['n_estimators'],learning_rate=adaparam['learning_rate'])
AdaBoostModel2.fit(X_train,y_train)
accuracy = accuracy_score(y_test,AdaBoostModel2.predict(X_test))
classificationRep=classification_report(y_test,AdaBoostModel2.predict(X_test))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
"""
# Stacking (Voting) ensemble
"""

votingModel = VotingClassifier(estimators=[('RF', RandomForestModel2), ('LGBM', LGBMModel2),
('XGB', XGBModel2),('ADA',AdaBoostModel2)], voting='hard')

votingModel = votingModel.fit(X_train, y_train)

accuracy = accuracy_score(y_test,votingModel.predict(X_test))
classificationRep=classification_report(y_test,votingModel.predict(X_test))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
"""
# Blending Ensemble Method
"""
#blender
Blend_X = list()
y1 = RandomForestModel2.predict(X_test)
y1 = y1.reshape(len(y1), 1)
Blend_X.append(y1)

y2 = LGBMModel2.predict(X_test)
y2 = y2.reshape(len(y2), 1)
Blend_X.append(y2)

y3 = XGBModel2.predict(X_test)
y3 = y3.reshape(len(y3), 1)
Blend_X.append(y3)

y4 = AdaBoostModel2.predict(X_test)
y4 = y4.reshape(len(y4), 1)
Blend_X.append(y4)

Blend_X = hstack(Blend_X)
from sklearn.linear_model import LogisticRegression
BlenderModel = LogisticRegression()
BlenderModel.fit(Blend_X, y_test)

accuracy = accuracy_score(y_test,BlenderModel.predict(Blend_X))
classificationRep=classification_report(y_test,BlenderModel.predict(Blend_X))
print("Classification Report: \n"+classificationRep)
print("Accuracy: "+str(accuracy))
"""
# WORKING WITH TEST DATA
"""
test_data.describe()
test_data.info()
test_data_original = test_data.copy()
features = []
for i in test_data.columns:
    if i !='Survived':
        features.append(i)

features
test_data.isnull().sum()
test_data.drop(['PassengerId','Ticket'],axis=1,inplace=True)

test_data['Title'] = test_data['Name'].apply(get_title)
test_data.drop('Name',axis=1,inplace=True)
test_data['Title'] = test_data['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Others')

test_data['Title'] = test_data['Title'].replace('Mlle', 'Miss')
test_data['Title'] = test_data['Title'].replace('Ms', 'Miss')
test_data['Title'] = test_data['Title'].replace('Mme', 'Mrs')


test_data.groupby(['Title']).size()
test_data['Cabin_Group'] = test_data['Cabin'].apply(cabin_group)
print(test_data.groupby(['Cabin_Group']).size())
test_data.drop('Cabin',axis=1,inplace=True)

test_data['FareRange'] = test_data['Fare'].apply(FareGroup)
test_data.drop('Fare',axis=1,inplace=True)
print(test_data.groupby(['FareRange']).size())

test_data['AgeGroup'] = test_data['Age'].apply(AgeGroup)
print(test_data.groupby(['AgeGroup']).size())
test_data.drop('Age',axis=1,inplace=True)
features = []
for i in test_data.columns:
    features.append(i)

features

test_data.dropna(subset=['Embarked'],inplace=True)
test_data.groupby(['Embarked']).size()

test_data['AgeGroup'] = test_data['AgeGroup'].apply(lambda x: np.nan if x == -1 else x)
test_data['FareRange'] = test_data['FareRange'].apply(lambda x: np.nan if x == -1 else x)


imputer = KNNImputer(n_neighbors= 5)
AgeGroupImputed = imputer.fit_transform([test_data['AgeGroup']])
FareRangeImputed = imputer.fit_transform([test_data['FareRange']])
FareRangeImputed = pd.Series(FareRangeImputed[0])
AgeGroupImputed = pd.Series(AgeGroupImputed[0])
test_data['FareRange'] = FareRangeImputed
test_data['AgeGroup'] = AgeGroupImputed

test_data.head(20)

sex = pd.get_dummies(test_data['Sex'],drop_first=True)
embark = pd.get_dummies(test_data['Embarked'],drop_first=True)
title = pd.get_dummies(test_data['Title'],drop_first=True)
fare = pd.get_dummies(test_data['FareRange'],drop_first=True)
cabin = pd.get_dummies(test_data['Cabin_Group'],drop_first=True)
age = pd.get_dummies(test_data['AgeGroup'],drop_first=True)

test_data.drop(['AgeGroup','Sex',"Embarked",'Title','Cabin_Group','FareRange'],axis=1,inplace=True)

test_data = pd.concat([test_data,sex,embark,title,cabin,fare,age],axis=1)
test_data.head(5)



"""
**Predict**

Using the model with best  accuracy for prediction
"""
#blender
Blend_Y = list()
y1 = RandomForestModel2.predict(test_data)
y1 = y1.reshape(len(y1), 1)
Blend_Y.append(y1)

y2 = LGBMModel2.predict(test_data)
y2 = y2.reshape(len(y2), 1)
Blend_Y.append(y2)

y3 = XGBModel2.predict(test_data)
y3 = y3.reshape(len(y3), 1)
Blend_Y.append(y3)

y4 = AdaBoostModel2.predict(test_data)
y4 = y4.reshape(len(y4), 1)
Blend_Y.append(y4)


Blend_Y = hstack(Blend_Y)

y_pred = BlenderModel.predict(Blend_Y)
passengerId = test_data_original.iloc[:,0]
passengerId
submission = pd.DataFrame({'PassengerId':passengerId,'Survived':y_pred})
submission.to_csv('submission.csv',index=False)
"""
If you like it, please upvote. Also let me know in the comments, if my strategy is incorrect somewhere or you have a better solution for any part or any suggestions on how I should improve the accuracy of my models.
"""