"""
# Inroduction

RMS Titanic was a British passenger liner, operated by the White Star Line, which sank in the North Atlantic Ocean on 15 April 1912 after striking an iceberg during her maiden voyage from Southampton, UK, to New York City. Of the estimated 2,224 passengers and crew aboard, more than 1,500 died, which made the sinking possibly one of the deadliest for a single ship up to that time.

<font color = "blue">

Content:
   
    
1. [Load and Check Data](#1)
1. [Variable Description](#2)
    1. [Univariate Variable Analysis](#3)
    1. [Categorical Variable Analysis](#4)
    1. [Numerical Variable Analysis](#5)
1. [Basic Data Analysis](#6)
1. [Outlier Detection](#7)
1. [Missing Value](#8)
    1. [Find Missing Value](#9)
    1. [Fill Missing Value](#10)
1. [Visualization](#11)
    1. [Correlation Between Sibsp -- Parch -- Age -- Fare -- Survived](#12)
    1. [SibSp -- Survived](#13)
    1. [Parch -- Survived](#14)
    1. [Pclass -- Survived](#15)
    1. [Age -- Survived](#16)
    1. [Pclass -- Survived -- Age](#17)
    1. [Embarked -- Sex -- Pclass -- Survived](#18)
    1. [Embarked -- Sex -- Fare -- Survived](#19)
    1. [Fill Missing: Age Feature](#20)
1. [Feature Engineering](#21)
    1. [Name -- Title](#22)
    1. [Family Size](#23)
    1. [Embarked](#24)
    1. [Ticket](#25)
    1. [Pclass](#26)
    1. [Sex](#27)
    1. [Drop Passenger ID and Cabin](#28)
1. [Modeling](#29)
    1. [Train - Test Split](#30)
    1. [Simple Logistic Regression](#31)
    1. [Hyperparameter Tuning -- Grid Search -- Cross Validation](#32)
    1. [Ensemble Modeling](#33)
    1. [Prediction and Submission](#34)
        


"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sns
import matplotlib.pyplot as plt
from collections import Counter
import warnings
warnings.filterwarnings("ignore")


# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
<a id="1"></a>
## Load and Check Data
"""
train_df = pd.read_csv("/kaggle/input/titanic-machine-learning-from-disaster/train.csv")
test_df = pd.read_csv("/kaggle/input/titanic-machine-learning-from-disaster/test.csv")
test_PassengerId  = test_df["PassengerId"]
train_df.head()
"""
<a id="2"></a>
## Variable Description

![image.png](attachment:e18ad50c-94b7-413b-b21e-4799a0f00881.png)

![image.png](attachment:637460f4-2334-4986-9d3f-ff6dedfdb5b3.png)
"""
train_df.info()
"""
<a id="1"></a>

"""
"""
<a id="3"></a>

## Univariate Variable Analysis

Categorical Variable Analysis :  Survived, Sex, Pclass, Embarked, Cabin, Name, Ticket, Sibsp and Parch

Numerical Variable Analysis :   Fare, Age and PassengerId
"""
"""
<a id="4"></a>
### Categorical Variable Analysis

"""
def bar_plot(variable): 

    ### input: variable ex: "Sex"
    ### output: barplot & value count 

    
    # get feature
    var = train_df[variable]
    
    # count number of categorical variable
    varValue = var.value_counts()
    
    # visualize
    plt.figure(figsize = (9,3))
    plt.bar(varValue.index, varValue)
    plt.xticks(varValue.index, varValue.index.values)
    plt.ylabel("Frequency")
    plt.title(variable)
    plt.show()
    print("{}: \n  {}".format(variable,varValue))
train_df.columns
category1 = ['Survived', 'Pclass', 'Sex', 'SibSp','Parch','Embarked']
for c in category1:
    bar_plot(c)
    
    ## **Conclusion**
    # 'Survived';  inbalance variable
    # 'Pclass';    related Embarked
    # 'Sex';       inbalance variable
    # 'SibSp';
    # 'Parch';
    # 'Embarked';  related Pclass
category2 = ["Cabin","Name","Ticket"]
for c in category2:
    print("{} \n".format(train_df[c].value_counts()))
"""
<a id="5"></a>
### Numerical Variable Analysis
"""
def plot_hist(variable):
    plt.figure(figsize = (9,3))
    plt.hist(train_df[variable], bins = 50)
    plt.xlabel(variable)
    plt.ylabel("Frequency")
    plt.title("{} distribution with hist".format(variable))
    plt.show()
numericVar = ["Fare","Age","PassengerId"]
for n in numericVar:
    plot_hist(n)
    
    ## **Conclusion**
    # "Fare" ;         
    # "Age" ;         
    # "PassengerId" ;  rubbish

"""
<a id="6"></a>
## Basic Data Analysis

* Pclass - Survived
* Sex - Survived
* SibSp - Survived
* Parch - Survived
"""
train_df[["Pclass" , "Survived"]]
# Pclass - Survived
train_df[["Pclass" , "Survived"]].groupby(["Pclass"], as_index = False).mean().sort_values(by="Survived", ascending = False)
# Sex - Survived
train_df[["Sex" , "Survived"]].groupby(["Sex"], as_index = False).mean().sort_values(by="Survived", ascending = False)
# SibSp - Survived
train_df[["SibSp" , "Survived"]].groupby(["SibSp"], as_index = False).mean().sort_values(by="Survived", ascending = False)
# SibSp - Survived
train_df[["Parch" , "Survived"]].groupby(["Parch"], as_index = False).mean().sort_values(by="Survived", ascending = False)
"""
<a id="7"></a>
## Outlier Detection
"""
# Exp 1
# Check list
a = ["a","a","b","a","c","a","b"]
Counter(a)

def detect_outliers(df,features):
    outlier_indices= []
    
    for c in features:
        # 1st quartile
        Q1 = np.percentile(df[c],25)
        
        # 3rd quartile
        Q3 = np.percentile(df[c],75)

        # IQR
        IQR = Q3 - Q1
        
        # Outlier step
        outlier_step = IQR * 1.5
        
        # Detect outlier and their indeces
        outlier_list_col = df[(df[c] < Q1- outlier_step) | (df[c] >  Q3 + outlier_step)].index
                               
        # Store indeces
        outlier_indices.extend(outlier_list_col)
                               
    outlier_indices = Counter(outlier_indices)
    multiple_outliers = list(i for i, v in outlier_indices.items() if v > 2) # look for Exp1; i = a,b... v = count
                                                            
    return multiple_outliers    
train_df.iloc[detect_outliers(train_df,["Age","SibSp","Parch","Fare"])]
# Drop ouyliers
train_df = train_df.drop(detect_outliers(train_df,["Age","SibSp","Parch","Fare"]), axis = 0).reset_index(drop = True)

"""
<a id="8"></a>
## Missing Value
* Find Missing Value
* Fill Missing Value
"""
train_df_len = len(train_df)
train_df = pd.concat([train_df,test_df],axis = 0).reset_index(drop = True)
train_df.head()
"""
<a id="9"></a>
### Find Missing Value
"""
train_df.columns[train_df.isnull().any()]
train_df.isnull().sum()
"""
<a id="10"></a>
### Fill Missing Value
* Embarked has 2,
* Fare has only 1,
* Age has 256,
* Cabin has 1007  missing value
"""
train_df[train_df["Embarked"].isnull()]
train_df.boxplot(column = "Fare", by="Embarked")
plt.show()
# 
train_df["Embarked"] = train_df["Embarked"].fillna("C")
train_df[train_df["Embarked"].isnull()]
train_df[train_df["Fare"].isnull()]
train_df[train_df["Pclass"] == 3]
np.mean(train_df[train_df["Pclass"] == 3]["Fare"])
train_df["Fare"] = train_df["Fare"].fillna(np.mean(train_df[train_df["Pclass"] == 3]["Fare"]))
train_df[train_df["Fare"].isnull()]
"""
<a id="11"></a>
## Visualization
"""
"""
<a id="12"></a>
### Correlation Between Sibsp -- Parch -- Age -- Fare -- Survived
"""
list1 = ["SibSp" , "Parch" , "Age" , "Fare" , "Survived"]
sns.heatmap(train_df[list1].corr(),annot = True, fmt = ".2f")
"""
<a id="13"></a>
### SibSp -- Survived
"""
g = sns.factorplot(x = "SibSp" , y = "Survived", data = train_df, kind = "bar", size = 6 )
g.set_ylabels("Survived Probability")
plt.show()

"""
* Having a lot of SibSp have less chance to survive.
* if sibsp == 0 or 1 or 2, passenger has more chance to survive
* we can consider a new feature describing these categories.
"""
"""
<a id="14"></a>
### Parch -- Survived
"""
g = sns.factorplot(x = "Parch", y = "Survived", data = train_df, kind = "bar", size = 6)
g.set_ylabels("Survived Probability")
plt.show()

"""
* Sibsp and parch can be used for new feature extraction with th = 3
* small familes have more chance to survive.
* there is a std in survival of passenger with parch = 3
"""
"""
<a id="15"></a>
### Pclass -- Survived
"""
g = sns.factorplot(x = "Pclass", y = "Survived", data = train_df, kind = "bar", size = 6)
g.set_ylabels("Survived Probability")
plt.show()
"""
* It's a clear feature for train our model.
"""
"""
<a id="16"></a>
### Age -- Survived
"""
g = sns.FacetGrid(train_df, col = "Survived")
g.map(sns.distplot, "Age", bins = 25)
plt.show()
"""
* age <= 10 has a high survival rate,
* oldest passengers (80) survived,
* large number of 20 years old did not survive,
* most passengers are in 15-35 age range,
* use age feature in training
* use age distribution for missing value of age
"""
"""
<a id="17"></a>
### Pclass -- Survived -- Age
"""
g = sns.FacetGrid(train_df, col = "Survived", row = "Pclass", size = 3)
g.map(plt.hist, "Age", bins = 25)
g.add_legend()
plt.show()
"""
* pclass is important feature for model training.

"""
"""
<a id="18"></a>
### Embarked -- Sex -- Pclass -- Survived
"""
g = sns.FacetGrid(train_df, row= "Embarked", size = 3)
g.map(sns.pointplot,  "Pclass" , "Survived" , "Sex")
g.add_legend()
plt.show()
"""
* Female passengers have much better survival rate than males.
* males have better survşval rate in pclass 3 in C.
* embarked and sex will be used in training.
"""
"""
<a id="19"></a>
### Embarked -- Sex -- Fare -- Survived
"""
g = sns.FacetGrid(train_df, row = "Embarked", col = "Survived", size = 3)
g.map(sns.barplot, "Sex", "Fare")
g.add_legend()
plt.show()
"""
* Passsengers who pay higher fare have better survival. Fare can be used as categorical for training.

"""
"""
<a id="20"></a>
### Fill Missing: Age Feature
"""
train_df[train_df["Age"].isnull()]
sns.factorplot(x = "Sex", y = "Age", data = train_df, kind="box")
plt.show()
"""
* Sex is not informative for age prediction, age distribution seems to be same.


"""
sns.factorplot(x = "Sex", y = "Age", data = train_df, kind = "box", hue = "Pclass")
plt.show()
"""
* 1st class passengers are older than 2nd, and 2nd is older than 3rd class.


"""
sns.factorplot(x = "Parch", y = "Age", data = train_df, kind = "box")
sns.factorplot(x = "SibSp", y = "Age", data = train_df, kind = "box")

plt.show()
train_df["Sex_N"] = [1 if i == "male" else 0  for i in train_df["Sex"]]
sns.heatmap(train_df[["Age","Sex_N","SibSp","Parch","Pclass"]].corr(), annot = True)
plt.show()
"""
* Age is not correlated with sex but it is correlated with parch, sibsp and pclass.


"""
train_df[train_df["Age"].isnull()]
train_df["Age"][train_df["Age"].isnull()]
train_df.head()
index_nan_age = list(train_df["Age"][train_df["Age"].isnull()].index)
for i in index_nan_age:
    age_pred = train_df["Age"][((train_df["SibSp"] == train_df.iloc[i]["SibSp"])
                               &
                               (train_df["Parch"] == train_df.iloc[i]["Parch"])
                               &
                               (train_df["Pclass"] == train_df.iloc[i]["Pclass"])                    
                               )].median()
    age_med = train_df["Age"].median()
    
    if not np.isnan(age_pred):
        train_df["Age"].iloc[i] = age_pred
    else:
        train_df["Age"].iloc[i] = age_med
train_df[train_df["Age"].isnull()]
"""
<a id="21"></a>
# Feature Engineering

* Name -- Title
* Family Size
* Embarked
* Ticket
* Pclass
* Sex
* Drop Passenger ID and Cabin
"""
"""
<a id="22"></a>
## Name -- Title
"""
train_df["Name"].head()
s = "Futrelle, Mrs. Jacques Heath (Lily May Peel)"
s.split(".")[0].split(",")[1].strip()
name = train_df["Name"]
train_df["Title"] = [i.split(".")[0].split(",")[-1].strip() for i in name]
train_df["Title"].head(10)
sns.countplot(x = "Title", data = train_df)
plt.xticks(rotation = 60)
plt.show()
train_df["Title"].unique()
# convert to categuniquel
train_df["Title_N"] = train_df["Title"].replace(['Don', 'Rev', 'Dr',
       'Major', 'Lady', 'Sir', 'Col', 'Capt', 'the Countess',
       'Jonkheer', 'Dona'],"Other")

train_df["Title_N"] = [0 if i == "Master" else 1 if i == "Miss" or i == "Ms" or i == "Mlle" or i == "Mrs" else 2 if i == "Mr" else 3 for i in train_df["Title"]]
train_df["Title_N"].head(20)
sns.countplot(x = "Title_N", data = train_df)
plt.xticks(rotation = 60)
plt.show()
g = sns.factorplot(x = "Title_N", y = "Survived", data = train_df, kind = "bar")
g.set_xticklabels(["Master","Mrs","Mr","Other"])
g.set_ylabels("Survival Probability")
plt.show()
train_df.drop(labels = ["Name"], axis = 1, inplace = True)
train_df.head()
train_df = pd.get_dummies(train_df,columns = ["Title_N"])
train_df.head()
"""
<a id="23"></a>
## Family Size
"""
train_df["Fsize"] = train_df["SibSp"] + train_df["Parch"] + 1
train_df.head()
g = sns.factorplot(x = "Fsize", y = "Survived", data = train_df, kind = "bar")
g.set_ylabels("Survival")
plt.show()
train_df["Family_Scategory"] = [    0 if i > 4.5
                               else 2 if i < 2
                               else 1 
                           
                               for i in train_df["Fsize"]]
train_df.head(10)
## train_df["Family_size"] = [1 if i < 5 else 0 for i in train_df["Fsize"]] (model upgraded )
sns.countplot(x = "Family_Scategory", data = train_df )
plt.show()
g = sns.factorplot(x = "Family_Scategory", y = "Survived", data = train_df, kind = "bar")
g.set_xticklabels(["LargeFamily","SmallFamily","AloneCowBoy"])
g.set_ylabels("Survival Probability")
plt.show()
"""
* Small familes have more chance to survive than large families.
* Also AloneCowBoy have more chance to survive than large families.
"""
train_df = pd.get_dummies(train_df, columns = ["Family_Scategory"] )
train_df.head()
"""
<a id="24"></a>
## Embarked
"""
train_df["Embarked"].head()
sns.countplot(x = "Embarked",data = train_df)
plt.show()
train_df = pd.get_dummies(train_df, columns=["Embarked"])
train_df.head()
"""
<a id="25"></a>
## Ticket
"""
train_df["Ticket"].head(20)
a = " A/5. 2151  "
a.replace(".","")
a = " A/5. 2151  "
a.replace(".","").replace("/","")
a = " A/5. 2151  "
a.replace(".","").replace("/","").strip()
a = " A/5. 2151  "
a.replace(".","").replace("/","").strip().split(" ")
a = " A/5. 2151  "
a.replace(".","").replace("/","").strip().split(" ")[0]
tickets = []
for i in list(train_df["Ticket"]):
    if not i.isdigit():
            tickets.append(i.replace(".","").replace("/","").strip().split(" ")[0])
    else:
            tickets.append("x")
train_df["Ticket_N"] = tickets
train_df["Ticket_N"].head(20)
train_df.head()
train_df = pd.get_dummies(train_df, columns = ["Ticket_N"], prefix = "T")
train_df.head()
"""
<a id="26"></a>
## Pclass
"""
train_df["Pclass"] = train_df["Pclass"].astype("category")
train_df = pd.get_dummies(train_df, columns = ["Pclass"])
train_df.head()
"""
<a id="27"></a>
## Sex
"""
train_df["Sex"] = train_df["Sex"].astype("category")
train_df = pd.get_dummies(train_df, columns = ["Sex"])
train_df.head()
"""
<a id="28"></a>
## Drop Passenger ID and Cabin
"""
train_df.columns
train_df.head()
train_df.drop(labels = ['PassengerId', 'Cabin', 'Title', 'Fsize', 'Parch', 'SibSp', "Ticket", "Sex_N" ] , axis = 1, inplace = True)
"""
<a id="29"></a>
# Modeling
"""
from sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier,VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
"""
<a id="30"></a>
## Train - Test Split
"""
train_df_len
test = train_df[train_df_len:]
test.drop(labels = ["Survived"], axis = 1, inplace = True)
test.head()
train = train_df[:train_df_len]
X_train = train.drop(labels = "Survived", axis = 1)
y_train = train["Survived"]
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size = 0.33, random_state = 42)

print("X_train",len(X_train))
print("X_test",len(X_test))
print("y_train",len(y_train))
print("y_test",len(y_test))
print("test",len(test))
test.info()
"""
<a id="31"></a>
## Simple Logistic Regression
"""
logreg = LogisticRegression(solver='liblinear')
logreg.fit(X_train,y_train)
acc_log_train = round(logreg.score(X_train,y_train)*100,2)
acc_log_test = round(logreg.score(X_test,y_test)*100,2)
print("Training Accuracy: % {}".format(acc_log_train))
print("Testing Accuracy: % {}".format(acc_log_test))
## https://www.udemy.com/course/machine-learning-ve-python-adan-zye-makine-ogrenmesi-4/learn/lecture/17902832#questions
"""
<a id="32"></a>
## Hyperparameter Tuning -- Grid Search -- Cross Validation

We will compare 5 ml classifier and evaluate mean accuracy of each of them by stratified cross validation.

* Decision Tree
* SVM
* Random Forest
* KNN
* Logistic Regression
"""
random_state = 42
classifier = [DecisionTreeClassifier(random_state = random_state),
             SVC(random_state = random_state),
             RandomForestClassifier(random_state = random_state),
             LogisticRegression(random_state = random_state),
             KNeighborsClassifier()]

dt_param_grid = {"min_samples_split" : range(10,500,20),
                "max_depth": range(1,20,2)}

svc_param_grid = {"kernel" : ["rbf"],
                 "gamma": [0.001, 0.01, 0.1, 1],
                 "C": [1,10,50,100,200,300,1000]}

rf_param_grid = {"max_features": [1,3,10],
                "min_samples_split":[2,3,10],
                "min_samples_leaf":[1,3,10],
                "bootstrap":[False],
                "n_estimators":[100,300],
                "criterion":["gini"]}

logreg_param_grid = {"C":np.logspace(-3,3,7),
                    "penalty": ["l1","l2"]}

knn_param_grid = {"n_neighbors": np.linspace(1,19,10, dtype = int).tolist(),
                 "weights": ["uniform","distance"],
                 "metric":["euclidean","manhattan"]}
classifier_param = [dt_param_grid,
                   svc_param_grid,
                   rf_param_grid,
                   logreg_param_grid,
                   knn_param_grid]
cv_result = []
best_estimators = []
for i in range(len(classifier)):
    clf = GridSearchCV(classifier[i], param_grid=classifier_param[i], cv = StratifiedKFold(n_splits = 10), scoring = "accuracy", n_jobs = -1,verbose = 1)
    clf.fit(X_train,y_train)
    cv_result.append(clf.best_score_)
    best_estimators.append(clf.best_estimator_)
    print(cv_result[i])
cv_results = pd.DataFrame({"Cross Validation Means":cv_result, "ML Models":["DecisionTreeClassifier", "SVM","RandomForestClassifier",
             "LogisticRegression",
             "KNeighborsClassifier"]})

g = sns.barplot("Cross Validation Means", "ML Models", data = cv_results)
g.set_xlabel("Mean Accuracy")
g.set_title("Cross Validation Scores")
plt.show()
best_estimators
"""
<a id="33"></a>
## Ensemble Modeling
"""
votingC = VotingClassifier(estimators = [("DecisionTreeClassifier",best_estimators[0]),
                                        ("SVC",best_estimators[1]),
                                        ("RandomForestClassifier",best_estimators[2])],
                                        voting = "hard", n_jobs = -1)
votingC = votingC.fit(X_train, y_train)
print(accuracy_score(votingC.predict(X_test),y_test))
"""
<a id="34"></a>
## Prediction and Submission
"""
test_survived = pd.Series(votingC.predict(test), name = "Survived").astype(int)
results = pd.concat([test_PassengerId, test_survived],axis = 1)
results.to_csv("titanic.csv", index = False)