# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
[Holistic Overview of data preprocessing and simple model specifications for the Titanic Dataset](https://www.kaggle.com/startupsci/titanic-data-science-solutions)
"""
train_df = pd.read_csv('/kaggle/input/titanic/train.csv')
test_df = pd.read_csv('/kaggle/input/titanic/test.csv')
example_df = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
full_data_df = pd.read_csv('https://www.openml.org/data/get_csv/16826755/phpMYEkMl')
test_df = pd.merge(test_df,full_data_df[['survived']],left_on='PassengerId',right_index=True,how='left').drop_duplicates()
test_df = test_df.dropna(subset=['survived'])
y_test = test_df['survived']
import seaborn as sns
sns.pairplot(train_df[['Sex','Age','Fare','Pclass','Survived']], hue="Survived")
train_df['Survived'].unique()
test_df.info()
train_df['Age'].unique()
# Understand object-type summary statistics
train_df.describe(include=['O'])
# First 5 rows
train_df.head()
# Do we have a categorical variable or do we have a floating variable with integer representation
train_df['Parch'].unique()
"""
# Preprocessing
"""
"""
## Categorical Variables Preprocessing
"""
# One-Hot Encoding of categorical variable
# Each category gets its own column (think vectorization)

# Pclass

classes_df = pd.get_dummies(train_df['Pclass'], prefix='class')
merge_train_df = pd.merge(train_df,classes_df,left_index=True,right_index=True,how='left')

classes_df = pd.get_dummies(test_df['Pclass'], prefix='class')
merge_test_df = pd.merge(test_df,classes_df,left_index=True,right_index=True,how='left')

# Embarked

merge_train_df['Embarked'].fillna('No info', inplace=True)
classes_df = pd.get_dummies(merge_train_df['Embarked'], prefix='embarked')
merge_train_df = pd.merge(merge_train_df,classes_df,left_index=True,right_index=True,how='left')

merge_test_df['Embarked'].fillna('No info', inplace=True)
classes_df = pd.get_dummies(merge_test_df['Embarked'], prefix='embarked')
merge_test_df = pd.merge(merge_test_df,classes_df,left_index=True,right_index=True,how='left')

# Sex

classes_df = pd.get_dummies(merge_train_df['Sex'], prefix='gender')
merge_train_df = pd.merge(merge_train_df,classes_df,left_index=True,right_index=True,how='left')

classes_df = pd.get_dummies(merge_test_df['Sex'], prefix='gender')
merge_test_df = pd.merge(merge_test_df,classes_df,left_index=True,right_index=True,how='left')


# Fill NaN with the median value of the observation
## Why can we do it?? Because only 891-714 observations are missing, and if we want to work with this features, we have to fill it up with a value that is not biasing the outcome, e.g. the median
merge_train_df[merge_train_df['Age'].isnull()]
merge_train_df['Age'].hist()
# we use the median because the distribution is not normal
# Fill up missing values for age using the median
merge_train_df['Age'].fillna(merge_train_df['Age'].dropna().median(), inplace=True)
merge_test_df['Age'].fillna(merge_test_df['Age'].dropna().median(), inplace=True)

# Fill up the missing value for fare using the mean of Pclass == 3
merge_test_df['Fare'].fillna(merge_train_df['Fare'][merge_train_df['Pclass']==3].mean(), inplace=True)
merge_test_df.info()
merge_train_df['Fare'].hist()
merge_train_df['Fare'].hist()
merge_train_df['Fare'][merge_train_df['Survived']==1].hist()
merge_train_df['Fare'][merge_train_df['Survived']==0].hist()
ax = sns.displot(data=merge_train_df, x="Fare", hue="Survived",multiple='stack')
merge_train_df['Fare'][merge_train_df['Survived']==0].max()
# Winsorization helps you to reduce the impact of outliers on your model
# Another way would be to use a quantitative threshold, such as the quantile of the distribution, such as merge_train_df['Fare'].quantile(.99), where your decision is to set the threshold at .99
# In our example we set the threshold to the largest value of the category of the target with the relative smaller maximum value
merge_train_df['Fare_preprocessed'] = merge_train_df['Fare'].clip(upper=merge_train_df['Fare'][merge_train_df['Survived']==0].max())
merge_test_df['Fare_preprocessed'] = merge_test_df['Fare'].clip(upper=merge_train_df['Fare'][merge_train_df['Survived']==0].max())
merge_train_df['Fare_preprocessed'][merge_train_df['Survived']==1].hist()
# Normalize continuous variables
merge_train_df['fare_normalized'] = (merge_train_df['Fare_preprocessed'] - merge_train_df['Fare_preprocessed'].mean())/merge_train_df['Fare_preprocessed'].std()
merge_test_df['fare_normalized'] = (merge_test_df['Fare_preprocessed'] - merge_train_df['Fare_preprocessed'].mean())/merge_train_df['Fare_preprocessed'].std()
merge_train_df['age_normalized'] = (merge_train_df['Age'] - merge_train_df['Age'].mean())/merge_train_df['Age'].std()
merge_test_df['age_normalized'] = (merge_test_df['Age'] - merge_train_df['Age'].mean())/merge_train_df['Age'].std()
merge_train_df.columns
X_train_df = merge_train_df[['class_1', 'class_2','class_3', 'embarked_C', 'embarked_Q', 'embarked_S', 'gender_female','gender_male','fare_normalized','age_normalized']].copy()
X_test_df = merge_test_df[['class_1', 'class_2','class_3', 'embarked_C', 'embarked_Q', 'embarked_S', 'gender_female','gender_male','fare_normalized','age_normalized']].copy()

y_train = merge_train_df['Survived']
"""
## Baseline Model
"""
from sklearn.linear_model import LogisticRegression

# Logistic Regression

logreg = LogisticRegression()
logreg.fit(X_train_df, y_train)
#Y_pred = logreg.predict(X_test_df)
# Check the accuracy based solely on the train dataset. There are NO test targets / label (for the moment)
acc_log = round(logreg.score(X_train_df, y_train) * 100, 2)
acc_log
import statsmodels.api as sm
logreg_sm = sm.Logit(y_train, X_train_df)
logit_res = logreg_sm.fit()
print(logit_res.summary())
y_pred = logreg.predict(X_test_df)
from sklearn.metrics import confusion_matrix, accuracy_score

confusion_matrix(y_test,y_pred)

accuracy_score(y_test,y_pred)
"""
## Feature Engineering to improve the base model
"""
# Multicollinearity problems in classification models
X_train_df
X_train_df
# we create an interaction variable 
X_train_df['class_1_fare'] = X_train_df['class_1']*X_train_df['fare_normalized'][X_train_df['class_1']==True].mean()
X_train_df['class_2_fare'] = X_train_df['class_2']*X_train_df['fare_normalized'][X_train_df['class_2']==True].mean()
X_train_df['class_3_fare'] = X_train_df['class_3']*X_train_df['fare_normalized'][X_train_df['class_3']==True].mean()

X_test_df['class_1_fare'] = X_test_df['class_1']*X_train_df['fare_normalized'][X_train_df['class_1']==True].mean()
X_test_df['class_2_fare'] = X_test_df['class_2']*X_train_df['fare_normalized'][X_train_df['class_1']==True].mean()
X_test_df['class_3_fare'] = X_test_df['class_3']*X_train_df['fare_normalized'][X_train_df['class_1']==True].mean()

# Look-ahead Bias is to take aggregate information NOT from the test data because we assume that we have not seen it, yet
# check the new accuracy having introduced the interaction term 
logreg = LogisticRegression()
logreg.fit(X_train_df, y_train)
#Y_pred = logreg.predict(X_test_df)
# Check the accuracy based solely on the train dataset. There are NO test targets / label (for the moment)
acc_log = round(logreg.score(X_train_df, y_train) * 100, 2)
acc_log
# we merge class and fare normalized 
X_train_df['class_1_fare2'] = X_train_df['class_1']*X_train_df['fare_normalized']
X_train_df['class_2_fare2'] = X_train_df['class_2']*X_train_df['fare_normalized']
X_train_df['class_3_fare2'] = X_train_df['class_3']*X_train_df['fare_normalized']

X_test_df['class_1_fare2'] = X_test_df['class_1']*X_test_df['fare_normalized']
X_test_df['class_2_fare2'] = X_test_df['class_2']*X_test_df['fare_normalized']
X_test_df['class_3_fare2'] = X_test_df['class_3']*X_test_df['fare_normalized']
X_train_df
X_train2_df = X_train_df
X_test2_df = X_test_df
logreg = LogisticRegression()
logreg.fit(X_train2_df, y_train)
#Y_pred = logreg.predict(X_test_df)
# Check the accuracy based solely on the train dataset. There are NO test targets / label (for the moment)
y_pred = logreg.predict(X_test2_df)

acc_log = round(logreg.score(X_train2_df, y_train) * 100, 2)
acc_log

accuracy_score(y_test,y_pred)
"""
## Why is the prediction result getting worse? - model complexity
"""
"""
# Deep Learning Model specification
"""
from sklearn.naive_bayes import GaussianNB
gnb = GaussianNB()
y_pred = gnb.fit(X_train2_df,y_train).predict(X_test2_df)
accuracy_score(y_test,y_pred)
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.layers import Dense, Dropout, Activation, Input
"""
## Nota bene
When you work with classification models, make sure your layer architecture takes into account the characteristics of your target variable. Is it a binary or a multi-class variable?

"""
x
print('Building model sequentially 1...')
model = Sequential()
model.add(Input(shape=(X_train2_df.shape[1],))) # Input
model.add(Dense(512, activation='relu')) # Encoder, we translate data into y = wx+b
#model.add(Dropout(0.5, seed=42)) # Dropout Layer, to make sure you are not overfitting on x
model.add(Dense(1))  # Representation Layer
model.add(Activation('sigmoid')) # Outout Layer of normalized data
model.layers
opt = Adam(learning_rate=0.01)

model.compile(loss='binary_crossentropy',
              optimizer=opt,
              metrics=['accuracy'])
# Translate labels into numpy representation
y_train2 = np.asarray(y_train).astype('float32').reshape((-1,1))
y_test2 = np.asarray(y_test).astype('float32').reshape((-1,1))
"""
# Why do we look at the accuracy instead of the loss itself?

1. "The means to the end" - means: we maniulate "loss" directly, so it is a model "parameter"; end: the purpose of the loss minimization is to improve predictability, which is indirectly represented by accuracy - loss is used internally in the model, accuracy is used for inter-model comparison
2. It enables comparability - I can use "accuracy" to compare different models

"""
%%time
batch_size = 32
epochs = 5
history = model.fit([X_train2_df.values], y_train2,
                    batch_size=batch_size,
                    epochs=epochs,
                    verbose=0,
                    validation_split=0.2)
y_pred = model.predict([X_test2_df.values])
from sklearn.metrics import classification_report, confusion_matrix
y_pred = (y_pred>0.5)
print(classification_report(y_test, y_pred, target_names=['True','False']))
import matplotlib.pyplot as plt

plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()