"""
# Import necessary libraries 

"""
import pandas as pd # Import Pandas for data manipulation using dataframes
import numpy as np # Import Numpy for data statistical analysis 
import matplotlib.pyplot as plt # Import matplotlib for data visualisation
import seaborn as sns # Statistical data visualization
import missingno as msno# # Import missingno for visualizing missing values in the dataset

"""
# Import Datasets
"""
#import titanic dataset
df_train = pd.read_csv('../input/titanic/train.csv')
df_test  = pd.read_csv('../input/titanic/test.csv')
df_train.shape
"""
Train data has above dimensions which has 891 rows or observations and 12 variables
"""
df_test.shape
"""
Test data has above dimensions which has 418 rows or observations and 11 variables with survived missing column which have to predict
"""
"""
merging the Train and Test data
"""
df_combined= pd.concat([df_train,df_test],ignore_index = True) # code to merge 2 dataframes
"""
We combined the dataset using pandas concat function and assigned it to 'df_combined' Let's check the shape of it to verify that the dataset is combined.
"""
df_combined.shape
"""
Yes, the dataset is combined. Let's proceed
"""
"""
# Let's check the dataset variables and perform EDA
"""
df_combined.head()
"""
we have 11 variables or predictors and 1 Target variable. Below is the small description about each variables
1. PassengerId ( Unique ID for every Passenger )
2. Survived    ( Our Target varibale which takes 2 values 1 = survived, 0 = not survived )  
3. Pclass      ( Represent the class of the passenger which takes 1,2,3 values )
4. Name        ( Name of each Passenger )
5. Sex         ( Represents gender of passenger Male/Female )
6. Age         ( Age of passenger )
7. Sibsp       ( No of Sib = Sibillings and Sp = Spouse shorlty termed as SibSp )
8. Parch       ( No of Par = Parents and ch = Children shorlty termed as Parch )
9. Ticket      ( Ticket Number )
10. Fare       ( Fare of the ticket )
11. Cabin      ( Cabin which passenger assigned to )
12. Embarked   ( Place where the passenger boarded. It takes 3 categories C = Cherbourg,Q = Queenstown,S = Southampton )

"""
df_combined.describe()
"""
From the above basic statistics about the data we can draw some insights such as 
75% of the passengers are below Age of '39'. 
Maximum no of 'SibSp' & 'Parch' are '8' and '9'. 
By the count of Age we can say that there are some missing values in the Age and Fare has one missing value.
describe method can only give the statistics of numerical variables let's explore more to see our categorical variables like 'Sex' & 'Embarked'
"""
df_combined.info()
"""
We can see that there are more missing values in 'Cabin' and 'Embarked' also. Let's count the no of missing values in the dataset. 
"""
df_combined.isnull().sum()
"""
So, we have 418 Survived(Target) missing data which we are going to Predict. More, we have 1014 missing values from Cabin Column, 263 missing values from 'Age',2 from 'Embarked' and 1 from 'Fare'. Let's plot the missing values and check.
"""
msno.matrix(df_combined.drop('Survived', axis='columns'));
"""
We dropped the Survived feature as we have lot of unpredicted values in the test set. We will handle the missing values going further
"""
"""
# Visualisations
"""
# checking Male to female Ratio
sns.factorplot('Sex',data = df_train,kind ='count');
# checking no of passengers survived and not survived
sns.factorplot('Survived',data = df_train,kind ='count');
# Checking the no of male and female passengers survived
sns.catplot(x ="Sex", hue ="Survived", kind ="count", data = df_train);
# plotting the Pclass column to know no of passengers in each class
sns.factorplot('Pclass',data= df_train,kind='count');
"""
We can see from the above plot that more no of female survived and less no of female not Survived but when coming to Male it's quite opposite. 
"""
# checking the passengers survived by class
sns.factorplot('Pclass',data = df_train,hue = 'Sex',kind = 'count');
# Group the dataset by Pclass, Sex ,Survived and then unstack them 
group = df_train.groupby(['Pclass','Sex', 'Survived']) 
pclass_survived_bySex = group.size().unstack() 
  
# Heatmap showing the surived and unsurvived by Pclass
sns.heatmap(pclass_survived_bySex, annot = True, fmt ="d");

"""
From the above heat map we can see that large group of 1st class people survived where as large no of 3rd class people died
"""
# checking the no of passengers boarded Titanic at each port
sns.factorplot('Embarked', data = df_train, kind='count');
# Surivival data grouped by Embarked
sns.factorplot('Embarked',data = df_train,hue='Survived',kind='count');
# Survival data grouped by Embarked and class
sns.catplot(x ='Embarked', hue ='Survived',  
kind ='count', col ='Pclass', data = df_train);
"""
# Data Preprocessing
"""
"""
** Feature Engineering**
"""
"""
Let's go by each column and check whether we can extract any new features from the dataset. If we see name it has Title, Surname and Name we will extract titles from the 'Name' using split and use them for imputing missing 'Age'.
"""
#Extracting Title from name
df_combined['Title'] = df_combined['Name'].str.split(', ',expand = True)[1].str.split('.',expand = True)[0]
df_combined.head()
"""
We got a new column called Title. Let's check what we have got in it.
"""
df_combined['Title'].unique()
"""
Ok.. we have a lot of title in it. Let's get a count using value.counts() 
"""
df_combined['Title'].value_counts()
"""
By the count of the titles I see that we have maximum number titles of 'Mr', 'Miss', 'Mrs', 'Master'. We will combine the other title into this 4 titles by checking the other features like 'Sex' and 'Age'. We proceed by below assumptions

1.  if 'Sex' = Female and Age Greater than 25 we assume them as married and combine them with Title 'Mrs'. If 'Age' Less than 25 we assume them as 'unmarried' and combine them with title 'Miss'.

2. if 'Sex' = Male and 'Age' Greater than 16 we assume as 'Mr' Title .If Age less than 16 then we assume them as children and combine with 'Master' Title 
 
3. Passengers having title as 'Ms' can be combined with 'Miss'

Let's check one by one using titles and use replace the old titles with new
"""
 Rear_titles = ['Don', 'Rev', 'Dr', 'Mme', 'Ms','Major', 'Lady', 'Sir', 
                'Mlle', 'Col', 'Capt', 'the Countess', 'Jonkheer', 'Dona']
for title in Rear_titles:
    print('\n',title)
    print(df_combined[df_combined.Title == title][['Name','Sex','Age','Title','Survived','Pclass']])
"""
By observing the above details of the passengers with Rare Titles we can decide which title to assign according to our assumptions. we replace the title as following                                                                     
old Titles | new Titles                                                          
  Don      :   'Mr'                                                 
  Rev      :   'Mr'                                            
  Dr       :   'Mr'                                       
  Mme      :   'Miss'                                     
  Ms       :   'Miss'                                          
  Major    :   'Mr'                                      
  Mlle     :   'Miss'                                    
  Lady     :   'Mrs'                                           
  Sir      :   'Mr'                                         
  Col      :   'Mr'                                           
  Capt     :   'Mr'                                           
  the Countess : 'Mrs'                                               
  Jonkheer :   'Mr'                                           
  Dona     :   'Mrs'                                                     
  We can see in the above list of Passengers in 'Dr' title that we have 1 female doctor we will deal with it manually later. Further 'Mlle' is French word used for 'Miss'.    
"""
"""
By Observing the Rare title passenger details we can see there is a Passenger Name which has Title 'Mrs' in braces and has 'Mlle' as Title passengerid 710. I will hilight that observation below.
"""
df_combined.iloc[710,12]
#compressing the titles into 4 'Mr', 'Mrs', 'Miss', 'Master' using replace function
df_combined['Title'].replace(['Don','Rev','Dr','Mme','Ms','Major','Mlle','Lady','Sir','Col','Capt','the Countess','Jonkheer','Dona'],
                             ['Mr','Mr','Mr','Miss','Miss','Mr','Miss','Mrs','Mr','Mr','Mr','Mrs','Mr','Mrs'],inplace=True)
"""
We have successfully replaced the Rear Titles with the usual ones Let's check how we did we perform.
"""
df_combined['Title'].unique()
# Lets handle the title of a 'Dr' in which we have a female Dr where we assigned title as 'Mr'.
# we use Sex and title to find out the observation
#df_combined[(df_combined.Sex.str.contains('Female')) & (df_combined.Title.str.contains('Mr'))].Title
df_combined.iloc[796,12] = 'Mrs' # Observation who is Female Doctor
df_combined.iloc[710,12] = 'Mrs' # Observation who has Title Mrs in her Name
df_combined['Title'].value_counts()
"""
Finally, we are done combing the rear titles with usual Titles.
Now, let's see other columns for more feature extraction
"""
# Creating Family size as a new feature using Parch and SibSp
df_combined['Fam_Size'] = df_combined['Parch'] + df_combined['SibSp'] + 1
"""
I can see the letters and numbers in the Ticket Feature but not sure what to extract from it. But let's examine it once
"""
df_combined.Ticket.unique()
"""
Ok.. there are so many unique 'Tickets'. 
"""
len(df_combined.Ticket.unique())
"""
Length of unique Tickets is 929 but we have 1309 Passengers. So, samiliar ticket is shared by other passengers also maybe be all the family members can have same ticket numbers.
"""
# As same Ticket is shared by family members as well we can calculate the Fare per person using 'Fare' and 'Family_Size'
# Calculating Fare_per_person
#feature Engineering Fare per person
df_combined['Fare_per_person'] = df_combined['Fare']/df_combined['Fam_Size']
#Now we came to Cabin let's see what we have for Cabin
df_combined.Cabin.unique()
"""
Above we saw in the Visulisation of missing values that we have '1014' missing values in Cabin. We will extract the letter which means the 'Deck' and the number is 'Room no'. It is possible that Deck can be correlated to the Survival as for Upper Deck passengers it will take less time to reach the top and escape through the life boats. 

Let's extract Deck from Cabin
"""
#Extracting Deck feature from Cabin
df_combined['Deck'] = df_combined["Cabin"].str.slice(0,1)
df_combined['Deck'].unique()
df_combined.Deck.value_counts()
"""
As we have only 1 'T' Deck and 5 'G' Deck we will combine these two into the one Deck which is 'G'. As the less cardinality can make out Model get confused. 
"""
df_combined[df_combined.Deck == 'T']
#we will put all of them in Cabin 'G'
df_combined.loc[df_combined.Deck == 'T','Deck'] = 'G' 
"""
As we have reached the end of the columns in feature engineering now let's enter into the Data Cleansing phase as we have large Nan to clear.
"""
"""
# Data Cleansing
"""
df_combined.isnull().sum()
"""
we claer the age using 'Median Impuation' method. It will be unfair if we assign the median of the whole age column to the missing values. So, we use grouping by 'Title' & 'Pclass' column which we extracted from the name and take the median of age grouping by title and assign the value to respective Nan's. We are using not using mean as it will be effceted by outliers.
"""
#Let's clear the 'Nan' in Age 
#cleaning null values of age
import itertools
for Title,Pclass in itertools.product(df_combined['Title'].unique(),df_combined['Pclass'].unique()):
    df_combined.loc[(df_combined.Age.isnull()) & (df_combined.Title == Title) & (df_combined.Pclass == Pclass),'Age'] = df_combined.Age[(df_combined.Title == Title) & (df_combined.Pclass == Pclass)].median()

#Let's check did we cleared the Nan's in Age
df_combined['Age'].isnull().sum()
# Let's clear the Embarked column we have 2 embareked missing values in the data
# we will replace the Nan with 'mode'(most repeated) of the category
# Let's check the counts of embareked
df_combined['Embarked'].value_counts()
"""
We can see that most of the passengers embarked in S = Southampton. 
"""
#cleaning Embarked Feature
df_combined['Embarked'] = df_combined['Embarked'].fillna(df_combined['Embarked'].mode().iloc[0])
"""
Let's check the Nan in Fare
"""
df_combined[df_combined['Fare'].isnull()]
"""
we can see the observation which has missing Fare it is of Pclass 3. Fare is mostly related to the Pclass. So, we take the mean 'fare' of Pclass 3 and assign it to Nan.
"""
# Let's clear the 'Fare' feature which has 1 missing value  
df_combined.loc[(df_combined.Fare.isnull()),'Fare'] = df_combined.Fare[df_combined.Pclass==3].mean()
df_combined['Fare'].isnull().sum()
"""
As we created the 'Fare_per_person' before clearing the 'Fare' using 'Fare' and 'Family_size'. We have i 1 missing value in 'Fare_per_person' we will clear this now.
"""
df_combined[df_combined['Fare_per_person'].isnull()]
#Clearing the Fare_per_person using index from above
df_combined.loc[1043,'Fare_per_person'] = df_combined.loc[1043,'Fare'] / df_combined.loc[1043,'Fam_Size']  
df_combined['Fare_per_person'].isnull().sum()
# we will assign the 'Nans' in Deck to 'N'
df_combined['Deck'] = df_combined['Deck'].fillna('N')
# Let's check what more missing we have
df_combined.isnull().sum()
"""
We didn't clear the Cabin Values as we don't have data about which cabin is assigned to passengers. We will drop the Cabin coulmn going further.
"""
"""
We finished the data Cleaning part. Now we go to next step
"""
"""
# Dummy Variable for the Categorical values 
"""
# we use get_dummies from pandas to create Dummy Variables for each category and drop the 1st column to avoid dummy variable Trap
df_combined = pd.get_dummies(df_combined, columns = ['Embarked','Title','Deck','Pclass'],drop_first = True)
df_combined.head()
df_combined.columns
# Label encoding 'Sex' variable using LabelEncoder from Sklearn preprocessing module
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
df_combined.Sex = le.fit_transform(df_combined.Sex)
"""
# Drop Unwanted columns
"""
# string the unwanted columns in a list as Name 
drop_columns = ['PassengerId','Name','Ticket','Cabin']
df_combined.drop( drop_columns, axis = 1 , inplace = True)
df_combined.head()
"""
Now, we will separate the datasets into back again into Train_set and Test_set.
"""
# As test data does not contain the Survived column I am using that as my condition to separate the dataset you can use indexing also
df_cleaned_train = df_combined[~df_combined.Survived.isnull()]
df_cleaned_test = df_combined[df_combined.Survived.isnull()]
#we will drop the Survived column from the test data set as we have to predict the values 
df_cleaned_test = df_cleaned_test.drop(['Survived'],axis =1)
# Let's confirm that we don't have Survived in test set
df_cleaned_test.columns
#Correlation Plot to check the relation of the newly created features with survival
plt.figure(figsize=(20,10)) 
sns.heatmap(df_cleaned_train.corr(), annot=True);
"""
We can see from the above correlation plot that 'Survived' is positively correlated with 'Fare_per_person', 'Title_Miss', 'Title_Mrs' and negativily correlated with 'Pclass_3', 'Deck_N', 'Title_Mr', 'Embarked_S'.
"""
#splitting target variable and features
# Let's drop the target label coloumn and we will be left with variables storing them in X
X = df_cleaned_train.drop(['Survived'],axis=1)
X.head()
# Target Variable storing in y
y = df_cleaned_train['Survived']
y.head()
"""
# Splitting the dataset into Train and Test  
"""
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.20, random_state=5)
# Standardizing the varibles in the dataset 
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train.loc[:,['Age','Fare','Fare_per_person','Fam_Size']] = sc.fit_transform(X_train.loc[:,['Age','Fare','Fare_per_person','Fam_Size']])
X_test.loc[:,['Age','Fare','Fare_per_person','Fam_Size']] = sc.transform(X_test.loc[:,['Age','Fare','Fare_per_person','Fam_Size']])
X_train.head()
y_train.head()
"""
# Training Model 
"""
#logistic Regression Model
from sklearn.linear_model import LogisticRegression
lr = LogisticRegression(random_state = 0)
lr.fit(X_train, y_train)
from sklearn.svm import SVC
svc = SVC(kernel = 'rbf', random_state = 0)
svc.fit(X_train, y_train)
# Random Forest Classifier
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_estimators = 10, criterion = 'entropy', random_state = 0)
rf.fit(X_train, y_train)
# Models PREDICTIONS
from sklearn.metrics import confusion_matrix, accuracy_score,recall_score,precision_score,f1_score
y_pred_lr = lr.predict(X_test)
y_pred_svc = svc.predict(X_test)
y_pred_rf = rf.predict(X_test)
print("Logistic Regression",accuracy_score(y_test, y_pred_lr))
print(recall_score(y_test, y_pred_lr))
print(precision_score(y_test, y_pred_lr))
print(f1_score(y_test, y_pred_lr))
print("Support Vector Classifier",accuracy_score(y_test, y_pred_svc))
print(recall_score(y_test, y_pred_svc))
print(precision_score(y_test, y_pred_svc))
print(f1_score(y_test, y_pred_svc))
print("Random Forest",accuracy_score(y_test, y_pred_rf))
print(recall_score(y_test, y_pred_rf))
print(precision_score(y_test, y_pred_rf))
print(f1_score(y_test, y_pred_rf))

"""
We got accuracy more than 84% for both the logistic Regression and Random Forest model. Let's check the confusion matrix generated by Support Vector Machine Model.
"""
from sklearn.metrics import roc_auc_score,roc_curve, auc
#roc_curve(y_test, y_pred_svc) 
#auc(y_test, y_pred_svc)
print(roc_auc_score(y_test, y_pred_svc))
# Confusion matrix 
cm = confusion_matrix(y_test, y_pred_svc)
sns.heatmap(cm, annot = True, fmt = "d");
"""
we can see that we predicted '154' Correct predictions out of '179' test observations and 25 incorrect Predictions
"""
"""
# Hyperparamter tuning using GridSearch
"""
# we finilize Support Vector Machine as our Classifier and Tune its hyperparamters
from sklearn.model_selection import GridSearchCV
parameters = [{'C': [0.25, 0.5, 0.75, 1], 'kernel': ['linear']},
              {'C': [0.25, 0.5, 0.75, 1], 'kernel': ['rbf'], 'gamma': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]}]
grid_search = GridSearchCV(estimator = svc,
                           param_grid = parameters,
                           scoring = 'accuracy',
                           cv = 10,
                           n_jobs = -1)
grid_search.fit(X_train, y_train)
best_accuracy = grid_search.best_score_
best_parameters = grid_search.best_params_
print("Best Accuracy: {:.2f} %".format(best_accuracy*100))
print("Best Parameters:", best_parameters)
"""
we got the best paramters of gamma = 0.1, 'C' = 1, kernal = 'rdf' which is radial basis function
"""
"""
# Applying k-Fold cross validation
"""
from sklearn.model_selection import cross_val_score
accuracies = cross_val_score(estimator = svc, X = X_train, y = y_train, cv = 10)
print("Accuracy: {:.2f} %".format(accuracies.mean()*100))
print("Standard Deviation: {:.2f} %".format(accuracies.std()*100))
"""
Predicting the test set results
"""
# Standardising df_cleaned_test varibales
df_cleaned_test.loc[:,['Age','Fare','Fare_per_person','Fam_Size']] = sc.transform(df_cleaned_test.loc[:,['Age','Fare','Fare_per_person','Fam_Size']])
df_cleaned_test.head()
# standardising the df_cleaned_test set results
y_pred = svc.predict(df_cleaned_test)
submission = pd.DataFrame(df_test['PassengerId'])
submission['Survived'] = y_pred
submission
#exporting submission file into csv 
submission.to_csv('Titanic_survival_prediction.csv',index = False)