"""
# Introduction
One of the most tragic and notorious shipwrecks is the sinking of the Titanic. Most people thought that the passenger liner will not sink ever. But the reality did not match with the thoughts. Titanic collided iceberg and began to sink with many innocent people who cruise with it. Approximately 1500 passengers and crew died in a horrifying accident.

<font color = 'blue'>
Content: 

1. [Load and Check Data](#1)
2. [Variable Description](#2)
    * [Univariate Variable Analysis](#3)
        * [Categorical Variable Analysis](#4)
        * [Numerical Variable Analysis](#5)
3. [Basic Data Analysis](#6)
4. [Outlier Detection](#7)
5. [Missing Values](#8)
    * [Find Missing Value](#9)
    * [Fill Missing Value](#10)
6. [Visualization](#11)
    * [Correlation Between SibSp -- Parch -- Age -- Fare -- Survived](#12)
    * [SibSp -- Survived](#13)
    * [Parch -- Survived](#14)
    * [Pclass -- Survived](#15)
    * [Age -- Survived](#16)
    * [Pclass -- Survived -- Age](#17)
    * [Embarked -- Sex -- Pclass -- Survived](#18)
    * [Embarked -- Sex -- Fare -- Survived](#19)
    * [Fill Missing: Age Feature](#20)
7. [Feature Engineering](#21)
    * [Name -- Title](#22)
    * [Family Size](#23)
    * [Embarked](#24)
    * [Ticket](#25)
    * [Pclass](#26)
    * [Sex](#27)
    * [Drop PassengerId and Cabin](#28)
8. [Modelling](#29)
    * [Train Test Split](#30)
    * [Simple Logistic Regression](#31)
    * [Hyperparameter Tuning -- Grid Search -- Cross Validation](#32)
    * [Ensemble Modeling](#33)
    * [Prediction and Submission](#34)
        
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')
import seaborn as sns
from collections import Counter
import warnings
warnings.filterwarnings('ignore')

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
<a id = "1"></a>
# Load and Check Data
"""
train_df = pd.read_csv('/kaggle/input/titanic/train.csv')
test_df = pd.read_csv('/kaggle/input/titanic/test.csv')
test_passenger_id = test_df["PassengerId"]
train_df.columns
train_df.describe()
"""
<a id = 2></a>
# Variable Description
1. PassengerId: Unique id for each passenger 
1. Survived: Passenger survive(1) or died(0)
1. Pclass: Passenger class
1. Name: Name
1. Sex: Gender of passenger
1. Age: Age of passenger
1. SibSp: Number of siblings/spouses
1. Parch: Number of parents/children
1. Ticket: Ticket number
1. Fare: Amount of money spent on ticket
1. Cabin: Cabin category
1. Embarked: Port where passenger embarked (C = Cherbourg, Q = Queenstown, S = Southampton) 

"""
train_df.info()
"""
* float64(2): Fare and Age
* int64(5): PassengerId, Survived, Pclass, SibSp, Parch
* object(5): Name, Sex, Ticket, Cabin, Embarked
"""
"""
<a id = 3></a>
# Univariate Variable Analysis
* Categorical Variable: Survived, Sex, Pclass, Embarked, Cabin, Name, Ticket, SibSp and Parch
* Numerical Variable: Age, PassengerId, Fare
"""
"""
<a id = 4></a>
## Categorical Variable
"""
def bar_plot(variable):
    """
        input: variable ex: "Sex"
        output: bar plot & value count
    """
    # get features
    var = train_df[variable]
    # count number of categorical varianle(value/sample)
    varValue = var.value_counts()
    
    # visualize
    plt.figure(figsize = (9,3))
    plt.bar(varValue.index, varValue)
    plt.xticks(varValue.index, varValue.index.values)
    plt.ylabel("Frequency")
    plt.title(variable)
    plt.show()
    print('{}: \n {}'.format(variable, varValue))
category1 = ["Survived", "Sex", "Pclass", "Embarked", "SibSp", "Parch"]
for c in category1:
    bar_plot(c)
category2 = ["Cabin", "Name", "Ticket"]
for c in category2:
    print("{} \n".format(train_df[c].value_counts()))
"""
<a id = 5></a>
## Numerical Variable Analysis
"""
def plot_hist(variable):
    plt.figure(figsize = (9,3))
    plt.hist(train_df[variable], bins=50)
    plt.xlabel(variable)
    plt.ylabel("Frequency")
    plt.title("{} Distribution with Hist.".format(variable))
    plt.show()
numericVar = ["Fare", "Age", "PassengerId"]
for n in numericVar:
    plot_hist(n)
"""
<a id = 6></a>
# Basic Data Analysis
* Pclass - Survived
* Sex - Survived
* SibSp - Survived
* Parch - Survived
"""
# Pclass vs Survived
train_df[["Pclass", "Survived"]].groupby(["Pclass"], as_index = False).mean().sort_values(by = "Survived", ascending = False)
# Sex vs Survived
train_df[["Sex", "Survived"]].groupby(["Sex"], as_index = False).mean().sort_values(by = "Survived", ascending = False)
# SibSp vs Survived
b = train_df[["SibSp", "Survived"]].groupby(["SibSp"], as_index = False).mean().sort_values(by = "Survived", ascending = False)
b["count"] = train_df['SibSp'].value_counts()
b
# Parch vs Survived
a = train_df[["Parch", "Survived"]].groupby(["Parch"], as_index = False).mean().sort_values(by = "Survived", ascending = False)
a['count'] = train_df['Parch'].value_counts()
a
train_df['family_sit'] = train_df['Parch'] + train_df['SibSp']
c = train_df[["family_sit", "Survived"]].groupby(["family_sit"], as_index = False).mean().sort_values(by = "Survived", ascending = False)
c['count'] = train_df['family_sit'].value_counts()
c
# Sex vs Survived ????
age_groups = pd.cut(train_df['Age'], bins=[0, 15, 30, 55, np.inf])
b = train_df.groupby(age_groups, as_index = False)["Survived"].mean().sort_values(by = "Survived", ascending = False)
b["count"] = train_df.groupby(age_groups, as_index = False)["Survived"].transform("count")
b
"""
<a id = 7></a>
# Outlier Detection
"""
def detect_outliers(df, features):
    outlier_indices = []
    
    for c in features:
        # 1st quartile
        q1 = np.percentile(df[c], 25)
        
        # 3rd quartile
        q3 = np.percentile(df[c], 75)
        
        # IQR
        iqr = q3 - q1
        
        # Outlier step
        outlier_step = iqr * 1.5
        
        # Detect outlier and their indices
        outlier_list_col = df[(df[c] < q1 - outlier_step) | (df[c] > q3 + outlier_step)].index
        
        # Store indices
        outlier_indices.extend(outlier_list_col)
    
    outlier_indices = Counter(outlier_indices)
    multiple_outliers = list(i for i, v in outlier_indices.items() if v > 2)
    
    return multiple_outliers
train_df.loc[detect_outliers(train_df, ["Age", "SibSp", "Parch", "Fare"])]
# Drop outliers
train_df = train_df.drop(detect_outliers(train_df, ["Age", "SibSp", "Parch", "Fare"]), axis = 0).reset_index(drop=True)
"""
<a id = 8></a>
# Missing Values

* Find Missing Values
* Fill Missing Values
"""
train_df_len = len(train_df)
train_df = pd.concat([train_df, test_df], axis = 0).reset_index(drop = True)
train_df.head()
"""
<a id = 9></a>
## Find Missing Values
"""
train_df.columns[train_df.isnull().any()]
train_df.isnull().sum()
"""
<a id = 10></a>
## Fill Missing Values

* Embarked has 2 missing value
* Fare has only 1
"""
train_df[train_df["Embarked"].isnull()]
train_df.boxplot(column="Fare", by = "Embarked")
plt.show()
train_df["Embarked"] = train_df["Embarked"].fillna("C")
train_df[train_df["Embarked"].isnull()]
train_df[train_df["Fare"].isnull()]
train_df[train_df["Pclass"] == 3]["Fare"]
np.mean(train_df[train_df["Pclass"] == 3]["Fare"])
train_df["Fare"] = train_df["Fare"].fillna(np.mean(train_df[train_df["Pclass"] == 3]["Fare"]))
train_df[train_df["Embarked"].isnull()]
"""
<a id = 11 ></a>
# Visualization
"""
"""
<a id = 12 ></a>
## Correlation Between SibSp -- Parch -- Age -- Fare -- Survived
"""
list1 = ["SibSp", "Parch", "Age", "Fare", "Survived"]
sns.heatmap(train_df[list1].corr(), annot = True, fmt = ".2f")
"""
Fare feature seems to have correlation with survived feature (0.26).
"""
"""
<a id = 13 ></a>
## SibSp -- Survived
"""
g = sns.factorplot(x = "SibSp", y = "Survived", data = train_df, kind = "bar", size = 6)
g.set_ylabels("Survived Probability")
plt.show()

"""
* Having lots of SibSp gives less chance to survive
* if SibSp == 0 or 1 or 2, passenger has more chance to survive but a number of data is important here. if data which more than 2 siblings are fewer, being categorized as having or not having a sibling is a better solution.
* I want to consider this feature as having SibSp or not having.
"""
"""
<a id = 14 ></a>
## Parch -- Survived
"""
g = sns.factorplot(x = "Parch", y = "Survived", data = train_df, kind = "bar", size = 6)
g.set_ylabels("Survived Probability")
plt.show()
"""
* SipSp and Parch can be used for new feature extraction with th = 3
* Small families have more chance to survive
* There is a high std in survival of passenger with parch = 3 
"""
"""
<a id = 15 ></a>
## Pclass -- Survived
"""
g = sns.factorplot(x = "Pclass", y = "Survived", data = train_df, kind = "bar", size = 6)
g.set_ylabels("Survived Probability")
plt.show()
"""
<a id = 16 ></a>
## Age -- Survived
"""
g = sns.FacetGrid(train_df, col = 'Survived')
g.map(sns.distplot, "Age", bins = 25)
plt.show()
"""
* age <= 15 has a high survival rate
* oldest passengers(80)survived,
* large number of 20 years old did not survive,
* most passengers are in 15-35 age range,
* use age feature in training
* use age distribution for missing values of age
"""
"""
<a id = 17 ></a>
## Pclass -- Survived -- Age
"""
g = sns.FacetGrid(train_df, col = 'Survived', row = "Pclass", size = 2)
g.map(plt.hist, "Age", bins = 25)
g.add_legend()
plt.show()
"""
* Pclass is important feature to train data
"""
"""
<a id = 18></a>
## Embarked -- Sex -- Pclass -- Survived
"""
g = sns.FacetGrid(train_df, row = "Embarked", size = 2)
g.map(sns.pointplot, "Pclass", "Survived", "Sex")
g.add_legend()
plt.show()
"""
* Female passengers have much better survival rate than males.
* Male passengers have better survival rate in Pclass 3 in C.
* Embarked and sex will be used in training.
"""
"""
<a id = 19></a>
## Embarked -- Sex -- Fare -- Survived
"""
g = sns.FacetGrid(train_df, row = "Embarked", col = "Survived", size = 3)
g.map(sns.barplot, "Sex", "Fare")
g.add_legend()
plt.show()
"""
* When ticket price is high, survival rate is high, too.
"""
"""
<a id = 20></a>
## Fill Missing: Age Feature
"""
train_df[train_df["Age"].isnull()]
sns.factorplot(x = "Sex", y = "Age", data = train_df, kind = "box")
plt.show()
"""
* Sex is not informative for age prediction, age distribution seems to be same.
"""
sns.factorplot(x = "Sex", y = "Age", hue = "Pclass", data = train_df, kind = "box")
plt.show()
"""
* 1st class passengers are older than 2nd and that are older than 3rd.
"""
sns.factorplot(x = "family_sit", y = "Age", data = train_df, kind = "box")
sns.factorplot(x = "Parch", y = "Age", data = train_df, kind = "box")
sns.factorplot(x = "SibSp", y = "Age", data = train_df, kind = "box")
plt.show()
train_df["Sex"] = [1 if i == "male" else 0 for i in train_df["Sex"]]
sns.heatmap(train_df[["Age", "Sex", "SibSp", "Parch", "Pclass", "family_sit"]].corr(), annot = True)
plt.show()
"""
* Age is not correlated with sex but it is corraleted with parch, sibsp, and pclass
"""
index_nan_age = list(train_df["Age"][train_df["Age"].isnull()].index)
for i in index_nan_age:
    age_pred = train_df["Age"][((train_df["SibSp"] == train_df.iloc[i]["SibSp"]) & (train_df["Parch"] == train_df.iloc[i]["Parch"]) & (train_df["Pclass"] == train_df.iloc[i]["Pclass"]))].median()
    age_med = train_df["Age"].median()
    if not np.isnan(age_pred):
        train_df["Age"].iloc[i] = age_pred
    else:
        train_df["Age"].iloc[i] = age_med
train_df[train_df["Age"].isnull()]
"""
<a id = 21></a>
# Feature Engineering]
"""
"""
<a id = 22></a>
## Name -- Title
"""
train_df["Name"].head(10)
name = train_df["Name"]
train_df["Title"] = [i.split(".")[0].split(",")[-1].strip() for i in name]
train_df["Title"].head(10)
sns.countplot(x = "Title", data = train_df)
plt.xticks(rotation = 60)
plt.show()
# Convert to Categorical
train_df["Title"] = train_df["Title"].replace(["Lady", "the Countess", "Don", "Rev", "Dr", "Major", "Sir", "Col", "Capt", "Jonkheer", "Dona"], "other")
train_df["Title"] = [0 if i == "Master" else 2 if i == "Mr" else 3 if i == "other" else 1 for i in train_df["Title"]]
sns.countplot(x = "Title", data = train_df)
plt.xticks(rotation = 0)
plt.show()
g = sns.factorplot(x = "Title", y = "Survived", data = train_df, kind = "bar")
g.set_xticklabels(["Master", "Mrs", "Mr", "other"])
g.set_ylabels("Survival Probability")
plt.show()
train_df.drop(labels = ["Name"], axis = 1, inplace = True)
train_df.head(10)
train_df = pd.get_dummies(train_df, columns = ["Title"])
train_df.head()
"""
<a id = 23></a>
## Family Size
"""
train_df["family_size"] = [1 if i == 0 else 0 if i < 4 else 2 for i in train_df["family_sit"]]
train_df.head()
sns.countplot(x = "family_size", data = train_df)
plt.show()
g = sns.factorplot(x = "family_size", y = "Survived", data = train_df, kind = "bar")
g.set_ylabels("Survival Probability")
plt.show()
"""
* Individual passengers have more chance to survive than people who have small families and people who have small families have more chance to survive than large families
"""
train_df = pd.get_dummies(train_df, columns = ["family_size"])
train_df.head()
"""
<a id = 24></a>
## Embarked
"""
train_df["Embarked"].head()
sns.countplot(x = "Embarked", data = train_df)
plt.show()
train_df = pd.get_dummies(train_df, columns = ["Embarked"])
train_df.head()
"""
<a id = 25></a>
## Ticket
"""
train_df["Ticket"].head(20)
ticket_list = []
for i in list(train_df.Ticket):
    if not i.isdigit():
        ticket_list.append(i.replace(".","").replace("/","").strip().split(" ")[0])
    else:
        ticket_list.append("x")

train_df["Ticket"] = ticket_list
train_df["Ticket"].head(20)
train_df = pd.get_dummies(train_df, columns = ["Ticket"], prefix = "T")
train_df.head()
"""
<a id = 26></a>
## Pclass
"""
sns.countplot(x = "Pclass", data = train_df)
plt.show()
train_df["Pclass"] = train_df["Pclass"].astype("category")
train_df = pd.get_dummies(train_df, columns = ["Pclass"])
train_df.head()
"""
<a id = 27></a>
## Sex
"""
train_df["Sex"] = train_df["Sex"].astype("category")
train_df = pd.get_dummies(train_df, columns = ["Sex"])
train_df.head()
"""
<a id = 28></a>
## Drop PassengerId and Cabin
"""
try:
    train_df.drop(labels = ["PassengerId", "Cabin", "family_sit"], axis = 1, inplace = True)
except KeyError:
    pass
train_df.columns
"""
<a id = 29></a>
# Modelling
"""
from sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score

"""
<a id = 30></a>
## Train Test Split
"""
test = train_df[train_df_len:]
test.drop(labels = ["Survived"], axis = 1, inplace = True)
test.head()
train = train_df[:train_df_len]
x_train = train.drop(labels = ["Survived"], axis = 1)
y_train = train["Survived"]
x_train, x_test, y_train, y_test = train_test_split(x_train, y_train, test_size = 0.33, random_state = 42)
print("x_train", len(x_train))
print("x_test", len(x_test))
print("y_train", len(y_train))
print("y_test", len(y_test))
print("train", len(train))
"""
<a id = 31></a>
## Simple Logistic Regression
"""
logreg = LogisticRegression(solver = 'liblinear')
logreg.fit(x_train, y_train)
acc_log_train = round(logreg.score(x_train, y_train) * 100, 2)
acc_log_test = round(logreg.score(x_test, y_test) * 100, 2)
print("Training Accuracy: % {}".format(acc_log_train))
print("Test Accuracy: % {}".format(acc_log_test))
"""
<a id = 32></a>
## Hyperparameter Tuning -- Grid Search -- Cross Validation
We will compare 5 ML classifier and evaluate mean accuracy of each of them by stratified cross validation.

* Decision Tree
* SVM
* Random Forest
* KNN
* Logistic Regression
"""
random_state = 42
classifier = [DecisionTreeClassifier(random_state = random_state),
            SVC(random_state = random_state),
            RandomForestClassifier(random_state = random_state),
            LogisticRegression(random_state = random_state, solver = 'liblinear'),
            KNeighborsClassifier()]

dt_parameter_grid = {"min_samples_split": range(10,500,20),
                    "max_depth": range(1,20,2)}

svc_param_grid = {"kernel": ["rbf"],
                 "gamma": [0.001, 0.01, 0.1, 1],
                 "C": [1, 10, 50, 100, 200, 300, 1000]}

rf_param_grid = {"max_features": [1, 3, 10],
                "min_samples_split": [2, 3, 10],
                "min_samples_leaf": [1, 3, 10],
                "bootstrap": [False],
                "n_estimators": [100, 300],
                "criterion": ["gini"]}

logreg_param_grid = {"C": np.logspace(-3, 3, 7),
                    "penalty": ["l1","l2"]}

knn_param_grid = {"n_neighbors": np.linspace(1,19,10, dtype = int).tolist(),
                 "weights": ["uniform", "distance"],
                 "metric": ["euclidean", "manhattan"]}

classifier_param = [dt_parameter_grid,
                   svc_param_grid,
                   rf_param_grid,
                   logreg_param_grid,
                   knn_param_grid]
cv_results = []
best_estimator = []
for i in range(len(classifier)):
    clf = GridSearchCV(classifier[i], param_grid = classifier_param[i], cv = StratifiedKFold(n_splits = 10), scoring = "accuracy", n_jobs = -1, verbose = 1)
    clf.fit(x_train, y_train)
    cv_results.append(clf.best_score_)
    best_estimator.append(clf.best_estimator_)
    print(cv_results[i])
cv_result = pd.DataFrame({"Cross Validation Means": cv_results, "ML Models": ["DecisionTreeClassifier", "SVM", "RandomForest", "LogisticRegression", "KNN"]})
g = sns.barplot("Cross Validation Means", "ML Models", data = cv_result)
g.set_xlabel("Mean Accuracy")
g.set_title("Cross Validation Scores")
plt.show()
"""
<a id = 33></a>
## Ensemble Modeling
"""
votingC = VotingClassifier(estimators = [("dt", best_estimator[0]),
                                        ("rfc", best_estimator[2]),
                                        ("lr", best_estimator[3])],
                          voting = "soft", n_jobs = -1)

votingC = votingC.fit(x_train, y_train)
print(accuracy_score(votingC.predict(x_test), y_test))
"""
<a id = 34></a>
## Prediction and Submission
"""
test_survived = pd.Series(votingC.predict(test), name = "Survived").astype(int)
results = pd.concat([test_passenger_id, test_survived], axis = 1)
results.to_csv("submission.csv", index = False)
test_survived