# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
import warnings
warnings.filterwarnings('ignore')
titanic = pd.read_csv('/kaggle/input/titanic/train.csv')
titanic.head()
## checking the head of our data set
titanic.info()
## checking info of all columns
titanic.shape
## checking shape of data set
titanic.describe()
## statistical information about numerical variable
"""
# **Data Quality Check**

handling missing values as well

"""
round(100*(titanic.isnull().sum()/len(titanic)),2)
## checking missing value percentage in all columns
titanic.drop('Cabin',axis=1,inplace=True)
## cabin almost have 77% of missing values hence remove this column from data set
age_median = titanic['Age'].median(skipna=True)
titanic['Age'].fillna(age_median,inplace=True)
## as there is 19% of missing values in age column hence it is not a good idea to remove this row wise or column wise hence impute those missing values with the median of age 

titanic = titanic[titanic['Embarked'].isnull()!=True]
## as embarked has a very small amount of missing values hence remove those rows which have missing values in embarked column 

titanic.shape
## checking shape after removing null values
"""
# duplicate check
"""
titanic_dub = titanic.copy()
## creating copy of the data frame to check duplicate values
titanic_dub.shape
## comparing shapes of two data frames
titanic.shape
## shape of original data frame
"""
# EDA
"""
import seaborn as sns
import matplotlib.pyplot as plt
## importing libraries for data visualitation
plt.figure(figsize=(15,5), dpi=80)
plt.subplot(1,4,1)
sns.boxplot(y=titanic['Age'])
plt.title("Outliers in 'Age'")

plt.subplot(1,4,2)
ax = sns.boxplot(y=titanic['Fare'])
ax.set_yscale('log')
plt.title("Outliers in 'Fare'")

plt.subplot(1,4,3)
sns.boxplot(y=titanic['SibSp'])
plt.title("Outliers in 'SibSp'")


plt.subplot(1,4,4)
sns.boxplot(y=titanic['Parch'])
plt.title("Outliers in 'Parch'")
#ax.set_yscale('log')
plt.tight_layout()
plt.show()

## plotting all four variables to check for outliers
## it clearly shows that all four variables has some outliers


sns.catplot(x="SibSp", col = 'Survived', data=titanic, kind = 'count', palette='pastel')
sns.catplot(x="Parch", col = 'Survived', data=titanic, kind = 'count', palette='pastel')
plt.tight_layout()
plt.show()

## plotting of sibsp and parch in basis of survived and not survived
"""
sibsp and parch basically tells us that whether a person is accompanied by someone else or not 
so we can make two category by merging them to find whether a single person is acompanied by some one else or not 
"""
def alone(x):
    if (x['SibSp']+x['Parch']>0):
        return (1)
    else:
        return (0)
titanic['Alone'] = titanic.apply(alone,axis=1)
## creating a function to make one variable which tells us whether a person is single or accompanied by some on the ship
sns.catplot(x="Alone", col = 'Survived', data=titanic, kind = 'count', palette='pastel')
plt.show()
"""
it clearly shows that those person who are not alone survived more
"""
## drop parch and sibsp
titanic = titanic.drop(['Parch','SibSp'],axis=1)
titanic.head()

sns.distplot(titanic['Fare'])
plt.show()
"""
there is some skewness in the fare column 
hence removing the skewness using log function
"""
sns.catplot(x="Sex", y="Survived", col="Pclass", data=titanic, saturation=.5, kind="bar", ci=None, aspect=0.8, palette='deep')
sns.catplot(x="Sex", y="Survived", col="Embarked", data=titanic, saturation=.5, kind="bar", ci=None, aspect=0.8, palette='deep')
plt.show()

## plotting of survive on basis of pclass
"""
females are more likely to be survived
"""
survived_0 = titanic[titanic['Survived']==0]
survived_1 = titanic[titanic['Survived']==1]
## divided our dataset into survived or not survived to check the distribution of age in both the cases 
survived_0.shape
## checking shape of the data set that contains the data of passengers who not survived
survived_1.shape
## checking shape of the data set that contains the data of passengers who survived
sns.distplot(survived_0['Age'])
plt.show()
## checking distribution of age in not survived data set
sns.distplot(survived_1['Age'])
plt.show()
## checking distribution of age in survived dataset
"""
young persons are survived more (age group between 20-40)
"""
sns.boxplot(x='Survived',y='Fare',data=titanic)
plt.show()
## checking survival rate on basis of fare
"""
those who are survived paid more fares
"""
"""
# creating dummy variables
"""
Pclass_dummy = pd.get_dummies(titanic['Pclass'],prefix='Pclass',drop_first=True)
Pclass_dummy.head()
## creating dummy variables for pclass


## joing dummy variables
titanic = pd.concat([titanic,Pclass_dummy],axis=1)
titanic.head()
titanic.drop('Pclass',axis=1,inplace=True)
## as there is no use of pclass after joining the columns that contains dummy variables  for pclass
Embarked_dummy = pd.get_dummies(titanic['Embarked'],drop_first=True)
Embarked_dummy.head()
## creating dummy variables for embarked and dropping first column
titanic = pd.concat([titanic,Embarked_dummy],axis=1)
titanic.drop('Embarked',axis=1,inplace=True)
## joining dummy variables
titanic.head()
## checking head of the data set after joining dummy variables
def sex_map(x):
    if x == 'male':
        return (1)
    elif x == 'female':
        return (0)
titanic['Sex'] = titanic['Sex'].apply(lambda x:sex_map(x))

## creating function for convert sex into binary values
"""
Select variables needed for creating model.
"""
titanic = titanic[['Survived','Sex','Age','Fare','Alone','Pclass_2','Pclass_3','Q','S']]
## let's plot one heatmap to check the corelations 
plt.figure(figsize=(10,10))
sns.heatmap(titanic.corr(),annot=True)
plt.show()
"""
There is not much higher co-relations between two variables.
"""
"""
# Model Build
"""
"""
**XgBoost**

XGBoost is an algorithm that has recently been dominating applied machine learning and Kaggle competitions for structured or tabular data.

XGBoost is an implementation of gradient boosted decision trees designed for speed and performance.

A Gentle Introduction to XGBoost for Applied Machine Learning

**What is XGBoost?**
XGBoost stands for eXtreme Gradient Boosting.

The name xgboost, though, actually refers to the engineering goal to push the limit of computations resources for boosted tree algorithms. Which is the reason why many people use xgboost.

— Tianqi Chen, in answer to the question “What is the difference between the R gbm (gradient boosting machine) and xgboost (extreme gradient boosting)?” on Quora

It is an implementation of gradient boosting machines created by Tianqi Chen, now with contributions from many developers. It belongs to a broader collection of tools under the umbrella of the Distributed Machine Learning Community or DMLC who are also the creators of the popular mxnet deep learning library.

Tianqi Chen provides a brief and interesting back story on the creation of XGBoost in the post Story and Lessons Behind the Evolution of XGBoost.

XGBoost is a software library that you can download and install on your machine, then access from a variety of interfaces. Specifically, XGBoost supports the following main interfaces:

Command Line Interface (CLI).
C++ (the language in which the library is written).
Python interface as well as a model in scikit-learn.
R interface as well as a model in the caret package.
Julia.
Java and JVM languages like Scala and platforms like Hadoop.
XGBoost Features
The library is laser focused on computational speed and model performance, as such there are few frills. Nevertheless, it does offer a number of advanced features.

**Model Features**
The implementation of the model supports the features of the scikit-learn and R implementations, with new additions like regularization. Three main forms of gradient boosting are supported:

Gradient Boosting algorithm also called gradient boosting machine including the learning rate.
Stochastic Gradient Boosting with sub-sampling at the row, column and column per split levels.
Regularized Gradient Boosting with both L1 and L2 regularization.

**System Features**
The library provides a system for use in a range of computing environments, not least:

Parallelization of tree construction using all of your CPU cores during training.
Distributed Computing for training very large models using a cluster of machines.
Out-of-Core Computing for very large datasets that don’t fit into memory.
Cache Optimization of data structures and algorithm to make best use of hardware.

**Algorithm Features**
The implementation of the algorithm was engineered for efficiency of compute time and memory resources. A design goal was to make the best use of available resources to train the model. Some key algorithm implementation features include:

Sparse Aware implementation with automatic handling of missing data values.
Block Structure to support the parallelization of tree construction.
Continued Training so that you can further boost an already fitted model on new data.
XGBoost is free open source software available for use under the permissive Apache-2 license.

**Why Use XGBoost?**
The two reasons to use XGBoost are also the two goals of the project:

Execution Speed.
Model Performance.

Reference:[https://machinelearningmastery.com/gentle-introduction-xgboost-applied-machine-learning/](http://)
"""
## read test data 

titanic_test = pd.read_csv('/kaggle/input/titanic/test.csv')
## check shape and info of titanic_test

titanic_test.shape
titanic_test.info()
## imputing missing values of age 
titanic_test['Age'].fillna(age_median,inplace=True)
## create alone column
titanic_test['Alone'] = titanic_test.apply(alone,axis=1)
## create sex column with binary value 0 for 'female' and 1 for 'male'
titanic_test['Sex'] = titanic_test['Sex'].apply(lambda x:sex_map(x))
## compute dummy encoding in test data 

Pclass_dummy_test = pd.get_dummies(titanic_test['Pclass'],prefix='Pclass',drop_first=True)

titanic_test = pd.concat([titanic_test,Pclass_dummy_test],axis=1)
## compute dummy encoding in test data 


Embarked_dummy_test = pd.get_dummies(titanic_test['Embarked'],drop_first=True)

titanic_test = pd.concat([titanic_test,Embarked_dummy_test],axis=1)
## let's seperate the target variable and divide train data into train and test (validation data) our actual test data is already seperated

target = titanic.pop('Survived')

## define train and test to from train to evaluate our result

## import libraries 
import sklearn
from sklearn.model_selection import train_test_split

X_train,X_test,y_train,y_test = train_test_split(titanic,target,random_state=42,stratify=target)

## 'stratify' makes sure that both train and test contains same percentage of target '0' and '1'
"""
**Default Xgboost Model**

**some info about the parameters used**

Fortunately XGBoost provides a nice way to find the best number of rounds whilst training by **early_stopping_rounds**. Since trees are built sequentially, instead of fixing the number of rounds at the beginning, we can test our model at each step and see if adding a new tree/round improves performance.
To do so, we define a test dataset and a metric that is used to assess performance at each round. If performance haven’t improved for N rounds (N is defined by the variable early_stopping_round), we stop the training and keep the best number of boosting rounds.

**seed**: random seed. It's important to set a seed here, to ensure we are using the same folds for each step so we can properly compare the scores with different parameters.

**metrics**: the metrics to use to evaluate our model, here we use aucpr: Area under the PR curve.

**eval_set** : a list of pairs to evaluate.
"""
import xgboost as xgb

xgb_clf = xgb.XGBClassifier(objective='binary:logistic',seed=42)
xgb_clf.fit(X_train,y_train,verbose=True,early_stopping_rounds=10,eval_metric='aucpr',eval_set=[(X_test,y_test)])
from sklearn.metrics import plot_confusion_matrix
plot_confusion_matrix(xgb_clf,X_test,y_test,values_format='d',display_labels=["Not Survived","Survived"])
plt.show()

## plot confusion matrix on seperated test set of the train data 
"""
# Hyper Parameter Tuning
"""
"""
**Optimize Parameter using cross validation and grid search**

some informations about the parameters used :

**Maximum depth** of a tree. Increasing this value will make the model more complex and more likely to overfit.
max_depth is the maximum number of nodes allowed from the root to the farthest leaf of a tree. Deeper trees can model more complex relationships by adding more nodes, but as we go deeper, splits become less relevant and are sometimes only due to noise, causing the model to overfit.

**learning_rate/ETA** Step size shrinkage used in update to prevents overfitting.
The ETA parameter controls the learning rate. It corresponds to the shrinkage of the weights associated to features after each round, in other words it defines the amount of "correction" we make at each step (remember how each boosting round is correcting the errors of the previous? if not, check our first tutorial here).
In practice, having a lower eta makes our model more robust to overfitting thus, usually, the lower the learning rate, the best. But with a lower eta, we need more boosting rounds, which takes more time to train, sometimes for only marginal improvements.

**gamma** Minimum loss reduction required to make a further partition on a leaf node of the tree (used for pruning)

**lambda** Increasing this value will make model more conservative.

**subsample** corresponds to the fraction of observations (the rows) to subsample at each step. By default it is set to 1 meaning that we use all rows.
**colsample_bytree** corresponds to the fraction of features (the columns) to use. By default it is set to 1 meaning that we will use all features.

Reference [https://xgboost.readthedocs.io/en/latest/parameter.html](http://)
[https://blog.cambridgespark.com/hyperparameter-tuning-in-xgboost-4ff9100a3b2f](http://)
"""
## define my parameters

from sklearn.model_selection import GridSearchCV

##ROUND1

param_grid = {
               'max_depth':[2,3,4],
               'learning_rate':[0.1,0.01,0.05],
               'gamma':[0,0.25,1.0],
               'reg_lambda':[0,1.0,10.0]
    
}

optimal_model1 = GridSearchCV(estimator=xgb.XGBClassifier(objctive='binary:logistic',seed=42,subsample=0.9,colsample_bytree=0.6),
                              param_grid=param_grid,
                              scoring='roc_auc',
                              verbose=0,
                              n_jobs=10,
                              cv=5
                             ).fit(
                                   X_train,y_train,early_stopping_rounds=10,eval_metric='auc',eval_set=[(X_test,y_test)]
)
optimal_model1.best_params_
"""
Those params are end of their range we will explore them more . Hence increase max_depth ,learning rate and gamma.we need to focous on our score so that model will not became overfit. 
"""
##ROUND2
param_grid = {
               'max_depth':[5,6,7],
               'learning_rate':[0.001,0.003,0.005],
               'gamma':[2.0,5.0,10.0],
               'reg_lambda':[0,1.0,10.0]
    
}

optimal_model2 = GridSearchCV(estimator=xgb.XGBClassifier(objctive='binary:logistic',seed=42,subsample=0.9,colsample_bytree=0.6),
                              param_grid=param_grid,
                              scoring='roc_auc',
                              verbose=0,
                              n_jobs=10,
                              cv=5
                             ).fit(
                                   X_train,y_train,early_stopping_rounds=10,eval_metric='auc',eval_set=[(X_test,y_test)]
)
optimal_model2.best_params_
"""
Again scale values.
"""
##ROUND3
param_grid = {
               'max_depth':[8,9,10],
               'learning_rate':[0.0001,0.0003,0.0005],
               'gamma':[1.25,1.50,1.75],
               'reg_lambda':[0,1.0,10.0]
    
}

optimal_model3 = GridSearchCV(estimator=xgb.XGBClassifier(objctive='binary:logistic',seed=42,subsample=0.9,colsample_bytree=0.6),
                              param_grid=param_grid,
                              scoring='roc_auc',
                              verbose=0,
                              n_jobs=10,
                              cv=5
                             ).fit(
                                   X_train,y_train,early_stopping_rounds=10,eval_metric='auc',eval_set=[(X_test,y_test)]
)
optimal_model3.best_params_
"""
As score is getting decreased hence select the previous models params .
"""
optimal_model2.best_estimator_
## hence pick our final params {'gamma': 2.0, 'learning_rate': 0.03, 'max_depth': 7, 'reg_lambda': 0}

xgb_final = optimal_model2.best_estimator_


"""
check on test.
"""
titanic_test['Survived'] = xgb_final.predict(titanic_test[['Sex', 'Age', 'Fare', 'Alone', 'Pclass_2', 'Pclass_3', 'Q', 'S']])
"""
we can use others way to tune our hyper parameters.
"""