import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style='whitegrid', palette='muted')
import warnings
warnings.simplefilter("ignore")
np.random.seed(99)
"""
# Load Data
"""
train = pd.read_csv('../input/titanic/train.csv')
test = pd.read_csv('../input/titanic/test.csv')
"""
# EDA
"""
# useful functions

def get_NaN(df):
    N = df.shape[0]
    ndf = pd.DataFrame({'any NaN': df.apply(lambda x: x.isna().any()),  
                        '# NaN': df.apply(lambda x: x.isna().sum()), 
                        '% NaN': df.apply(lambda x: x.isna().sum() / N * 100),
                        '# unique': df.apply(lambda x: x.nunique())
                        })
    ndf['% NaN'] = np.round(ndf['% NaN'], 1)
    return ndf
        
        
def plot_dist(df, feat):
    fig, ax = plt.subplots(1, 2, figsize=(18,4))
    for i, n in enumerate(['no', 'yes']):
        y = df[df['Survived']==i][feat]
        sns.distplot(y, ax=ax[0], label=n)
        sns.distplot(np.log1p(y), ax=ax[1], label=n)
    ax[0].legend()
    ax[1].legend()
    ax[1].set_xlabel(f'log1p {feat}')
"""
## <p style='color:blue'> General Information </p>
"""
train.info()
train.head()
"""
## <p style='color:blue'> Missing Values </p>
"""
# train set
get_NaN(train)
# test set
get_NaN(test)
"""
## <p style='color:blue'> Target </p>
"""
sns.countplot(train['Survived']);

train['Survived'].value_counts() / len(train)
"""
## <p style='color:blue'> Categorical Features </p>
"""
# Pclass
sns.countplot(x='Pclass', hue='Survived', data=train);
# Sex
sns.countplot(x='Sex', hue='Survived', data=train);
# Embarked
sns.countplot(x='Embarked', hue='Survived', data=train);
"""
## <p style='color:blue'> Numerical Features </p>
"""
# Age
plot_dist(train, 'Age')
# SibSp
sns.countplot(x='SibSp', hue='Survived', data=train);
# Parch
# fig, ax = plt.subplots(figsize=(18,4))
sns.countplot(x='Parch', hue='Survived', data=train);
# Fare
plot_dist(train, 'Fare')
"""
# Features Engineering
"""
# Ticket
# Get the number of components separated by "space" in the ticket name
train['Ticket'] = train['Ticket'].apply(lambda x: len(x.split(' ')))
test['Ticket'] = test['Ticket'].apply(lambda x: len(x.split(' ')))
sns.countplot(x='Ticket', hue='Survived', data=train);
# Cabin

# Define NaN as a new category (n)
train['Cabin'].fillna('N', inplace=True)
test['Cabin'].fillna('N', inplace=True)

# Get the first letter of the cabin name
train['Cabin'] = train['Cabin'].apply(lambda x: x[0])
test['Cabin'] = test['Cabin'].apply(lambda x: x[0])
sns.countplot(x='Cabin', hue='Survived', data=train);
# Data Imputation

# Age
med = train['Age'].median()
train['Age'].fillna(med, inplace=True)
test['Age'].fillna(med, inplace=True)

# Fare (test data)
med = test['Fare'].median()
test['Fare'].fillna(med, inplace=True)
"""
# Label Encoding
"""
cols = ['Sex', 'Ticket', 'Cabin', 'Embarked']
for col in cols:
    encoding = {j:i for i,j in enumerate(train[col].unique())}
    train[col] = train[col].map(encoding).values
    test[col] = test[col].map(encoding).values
train.head()
"""
# Correlation
"""
corr = train.corr()
corr.style.background_gradient(cmap='coolwarm').set_precision(2)
"""
# One Hot Encoding
"""
cols = ['Ticket', 'Cabin', 'Embarked']

# copy features
for col in cols:
    train[f'{col}_LE'] = train[col]
    test[f'{col}_LE'] = test[col]

train = pd.get_dummies(train, columns=cols)
test = pd.get_dummies(test, columns=cols)
train.columns
# correlation
corr = train.corr()
corr.style.background_gradient(cmap='coolwarm').set_precision(2)
"""
# Models
"""
# target
y_train = train['Survived']

# features
X_train = train.drop(['Survived'], axis=1)
X_test = test
len(X_train.columns), len(X_test.columns)
[i for i in X_train.columns if i not in X_test.columns]
"""
## <p style='color:blue'> Logistic Regression </p>
"""
# drop some features
X_train.drop(['PassengerId', 'Name', 'Cabin_8', 'Embarked_3'], axis=1, inplace=True)
X_test.drop(['PassengerId', 'Name'], axis=1, inplace=True)

# scale features
for col in X_train.columns:
    min_ = X_train[col].min()
    max_ = X_test[col].max()
    intv = max_ - min_
    X_train[col] = (X_train[col] - min_) / intv
    X_test[col] = (X_test[col] - min_) / intv
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score
def CV_Logistic_Regression(X_train, y_train, X_test, n_folds=5):
    kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 99)
    
    # Create oof sets for prediction storage.
    oof_train = np.zeros(X_train.shape[0])
    oof_test = np.zeros((X_test.shape[0], n_folds))
    
    # Emptly list to store accuracy for each fold
    fold_acc = []

    for ifold, (tr_index, val_index) in enumerate(kf.split(X = X_train, y = y_train)):

        # Create train and validation sets based on KFold indices.
        X_tr = X_train.iloc[tr_index,:]
        X_val = X_train.iloc[val_index,:]
        y_tr = y_train[tr_index]
        y_val = y_train[val_index]

        # Train model
        model = LogisticRegression(random_state=0).fit(X_tr, y_tr)

        # Predict validation and test data and store them in oof sets.
        proba_val = model.predict_proba(X_val)[:,1]
        oof_train[val_index] = np.where(proba_val>0.5, 1, 0)
        proba_test = model.predict_proba(X_test)[:,1]
        oof_test[:, ifold] = np.where(proba_test>0.5, 1, 0)
        
        # Accuracy
        acc = accuracy_score(y_val, oof_train[val_index])
        
        print('FOLD: {} \t Acc: {:.3f}'.format(ifold, acc))
            
        fold_acc.append(acc)

    print(f'\nMEAN Acc\t: {round(np.mean(fold_acc), 3)}')
    acc = round(accuracy_score(y_train, oof_train), 3)
    print(f'ACTUAL Acc\t: {acc}')
    print(f'STD Acc\t\t: {round(np.std(fold_acc), 3)}')
    
    return oof_train, oof_test, acc
oof_train_LogReg, oof_test_LogReg, acc_LogReg = CV_Logistic_Regression(X_train, y_train, X_test)
"""
## <p style='color:blue'> Support Vector Machine (SVM) </p>
"""
from sklearn.svm import SVC
def CV_SVM(X_train, y_train, X_test, n_folds=5):
    kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 99)
    
    # Create oof sets for prediction storage.
    oof_train = np.zeros(X_train.shape[0])
    oof_test = np.zeros((X_test.shape[0], n_folds))
    
    # Emptly list to store accuracy for each fold
    fold_acc = []

    for ifold, (tr_index, val_index) in enumerate(kf.split(X = X_train, y = y_train)):

        # Create train and validation sets based on KFold indices.
        X_tr = X_train.iloc[tr_index,:]
        X_val = X_train.iloc[val_index,:]
        y_tr = y_train[tr_index]
        y_val = y_train[val_index]

        # Train model
        model = SVC()
        model.fit(X_tr, y_tr)

        # Predict validation and test data and store them in oof sets.
        oof_train[val_index] = model.predict(X_val)
        oof_test[:, ifold] = model.predict(X_test)
        
        # Accuracy
        acc = accuracy_score(y_val, oof_train[val_index])
        
        print('FOLD: {} \t Acc: {:.3f}'.format(ifold, acc))
            
        fold_acc.append(acc)

    print(f'\nMEAN Acc\t: {round(np.mean(fold_acc), 3)}')
    acc =round(accuracy_score(y_train, oof_train), 3)
    print(f'ACTUAL Acc\t: {acc}')
    print(f'STD Acc\t\t: {round(np.std(fold_acc), 3)}')
    
    return oof_train, oof_test, acc
oof_train_SVM, oof_test_SVM, acc_SVM = CV_SVM(X_train, y_train, X_test)
"""
## <p style='color:blue'> Decision Tree </p>
"""
from sklearn.tree import DecisionTreeClassifier
def CV_Decision_Tree(X_train, y_train, X_test, n_folds=5):
    kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 99)
    
    # Create oof sets for prediction storage.
    oof_train = np.zeros(X_train.shape[0])
    oof_test = np.zeros((X_test.shape[0], n_folds))
    
    # Emptly list to store accuracy for each fold
    fold_acc = []

    for ifold, (tr_index, val_index) in enumerate(kf.split(X = X_train, y = y_train)):

        # Create train and validation sets based on KFold indices.
        X_tr = X_train.iloc[tr_index,:]
        X_val = X_train.iloc[val_index,:]
        y_tr = y_train[tr_index]
        y_val = y_train[val_index]

        # Train model
        model = DecisionTreeClassifier(random_state=0)
        model.fit(X_tr, y_tr)

        # Predict validation and test data and store them in oof sets.
        oof_train[val_index] = model.predict(X_val)
        oof_test[:, ifold] = model.predict(X_test)
        
        # Accuracy
        acc = accuracy_score(y_val, oof_train[val_index])
        
        print('FOLD: {} \t Acc: {:.3f}'.format(ifold, acc))
            
        fold_acc.append(acc)

    print(f'\nMEAN Acc\t: {round(np.mean(fold_acc), 3)}')
    acc = round(accuracy_score(y_train, oof_train), 3)
    print(f'ACTUAL Acc\t: {acc}')
    print(f'STD Acc\t\t: {round(np.std(fold_acc), 3)}')
    
    return oof_train, oof_test, acc
oof_train_DecisionTree, oof_test_DecisionTree, acc_DecisionTree = CV_Decision_Tree(X_train, y_train, X_test)
"""
### <p style='color:blue'> Random Forest </p>
"""
from sklearn.ensemble import RandomForestClassifier
def CV_Random_Forest(X_train, y_train, X_test, n_folds=5):
    kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 99)
    
    # Create oof sets for prediction storage.
    oof_train = np.zeros(X_train.shape[0])
    oof_test = np.zeros((X_test.shape[0], n_folds))
    
    # Emptly list to store accuracy for each fold
    fold_acc = []

    for ifold, (tr_index, val_index) in enumerate(kf.split(X = X_train, y = y_train)):

        # Create train and validation sets based on KFold indices.
        X_tr = X_train.iloc[tr_index,:]
        X_val = X_train.iloc[val_index,:]
        y_tr = y_train[tr_index]
        y_val = y_train[val_index]

        # Train model
        model = RandomForestClassifier(max_depth=5, random_state=0)
        model.fit(X_tr, y_tr)

        # Predict validation and test data and store them in oof sets.
        oof_train[val_index] = model.predict(X_val)
        oof_test[:, ifold] = model.predict(X_test)
        
        # Accuracy
        acc = accuracy_score(y_val, oof_train[val_index])
        
        print('FOLD: {} \t Acc: {:.3f}'.format(ifold, acc))
            
        fold_acc.append(acc)

    print(f'\nMEAN Acc\t: {round(np.mean(fold_acc), 3)}')
    acc = round(accuracy_score(y_train, oof_train), 3)
    print(f'ACTUAL Acc\t: {acc}')
    print(f'STD Acc\t\t: {round(np.std(fold_acc), 3)}')
    
    return oof_train, oof_test, acc
oof_train_RandomForest, oof_test_RandomForest, acc_RandomForest = CV_Random_Forest(X_train, y_train, X_test)
"""
## <p style='color:blue'> Artificial Neural Networks </p>
"""
import tensorflow as tf
def build_model(Nfeatures):
    tf.keras.backend.clear_session()
    # set random seed for reproducibility
    tf.random.set_seed(99)

    # define a model
    model = tf.keras.Sequential()

    # add the first hidden layer
    model.add(tf.keras.layers.Dense(units = 8, activation = 'linear', input_dim = Nfeatures))
    
#     # add regularization
#     model.add(tf.keras.layers.Dropout(0.2))

#     # add the second hidden layer
#     model.add(tf.keras.layers.Dense(units = 16, activation = 'linear'))
    
#     # add regularization
#     model.add(tf.keras.layers.Dropout(0.2))
    
    # add the output layer
    model.add(tf.keras.layers.Dense(units = 1, activation = 'sigmoid'))

    # compile the model
    model.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy']) 
    
    return model
model = build_model(X_train.shape[1])
model.summary()
def CV_ANN(X_train, y_train, X_test, n_folds=5, seed=99):
    
    # Kfold split
    kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = seed)

    # Create oof sets for prediction storage.
    oof_train = np.zeros((X_train.shape[0]))
    oof_test = np.zeros((X_test.shape[0], n_folds))
    
    # Save all models validation accuracy
    fold_acc = []
    
    # Define early stopping
    earlystop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=2)
        
    for ifold, (train_index, valid_index) in enumerate(kf.split(X=X_train, y=y_train)):
        
        # Create train and validation sets based on KFold indices.
        X_tr = X_train.iloc[train_index,:]
        X_val = X_train.iloc[valid_index,:]
        y_tr = y_train[train_index]
        y_val = y_train[valid_index]
        
        # Restart model
        model = build_model(X_tr.shape[1])
        
        # Train model
        model.fit(
            X_tr,
            y_tr,
            batch_size = 8,
            epochs = 100,
            validation_data = (X_val, y_val),
            verbose = 0,
            callbacks = [earlystop])
        
        # Prediction on validation data and store them in oof (out of folds) sets
        oof_train[valid_index] = np.where(model.predict(X_val)[:,0]>0.5, 1, 0)
        oof_test[:, ifold] = np.where(model.predict(X_test)[:,0]>0.5, 1, 0)
        
        # Accuracy
        acc = accuracy_score(y_val, oof_train[valid_index])
        
        print('FOLD: {} \t Acc: {:.3f}'.format(ifold, acc))
            
        fold_acc.append(acc)
             
    print(f'\nMEAN Acc\t: {round(np.mean(fold_acc), 3)}')
    acc = round(accuracy_score(y_train, oof_train), 3)
    print(f'ACTUAL Acc\t: {acc}')
    print(f'STD Acc\t\t: {round(np.std(fold_acc), 3)}')

    return oof_train, oof_test, acc
oof_train_ANN, oof_test_ANN, acc_ANN = CV_ANN(X_train, y_train, X_test)
"""
## <p style='color:blue'> Gradient Boosted Trees with LGBM </p>
"""
import lightgbm as lgb
def CV_LGBM(X_train, y_train, X_test, n_folds=5, early_stop=None):
    kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 99)
    
    # Create oof sets for prediction storage.
    oof_train = np.zeros(X_train.shape[0])
    oof_test = np.zeros((X_test.shape[0], n_folds))

    gbm_history, fi, fold_acc = {}, [], []
    
    for ifold, (tr_index, val_index) in enumerate(kf.split(X = X_train, y = y_train)):

        # Create train and validation sets based on KFold indices.
        X_tr = X_train.iloc[tr_index,:]
        X_val = X_train.iloc[val_index,:]
        y_tr = y_train[tr_index]
        y_val = y_train[val_index]

        dtrain = lgb.Dataset(X_tr, y_tr)
        dvalid = lgb.Dataset(X_val, y_val)

        # Train LightGBM model
        params ={
            'task': 'train',
            'boosting': 'gbdt',
            'nthread': 2,
            'objective': 'binary',
            'metrics': 'binary',
            'learning_rate': 0.01,
            'num_leaves': 7,
            'max_depth': 20,
            'min_data_in_leaf': 100,
            'seed': 0,
            'feature_fraction': 0.8,
            'bagging_fraction': 0.6,
            'bagging_freq': 1,
            'verbose': -1}
        model = lgb.train(params = params, train_set = dtrain, evals_result = gbm_history, 
                          num_boost_round = 100000, valid_sets = [dtrain, dvalid], 
                          early_stopping_rounds = early_stop, verbose_eval = 0)

        # Predict validation and test data and store them in oof sets.
        proba_train = model.predict(X_val, num_iteration = model.best_iteration)
        oof_train[val_index] = np.where(proba_train > 0.5, 1, 0)
        proba_test = model.predict(X_test, num_iteration = model.best_iteration)
        oof_test[:, ifold] = np.where(proba_test > 0.5, 1, 0)

        # feature importance
        fi.append(model.feature_importance())
    
        # Accuracy
        acc = accuracy_score(y_val, oof_train[val_index])
        
        print('FOLD: {} \t Acc: {:.3f}'.format(ifold, acc))
            
        fold_acc.append(acc)
             
    print(f'\nMEAN Acc\t: {round(np.mean(fold_acc), 3)}')
    acc = round(accuracy_score(y_train, oof_train), 3)
    print(f'ACTUAL Acc\t: {acc}')
    print(f'STD Acc\t\t: {round(np.std(fold_acc), 3)}')
    
    return oof_train, oof_test, acc, fi
# drop one-hot-encoding
dropf = ['Ticket_0', 'Ticket_1', 'Ticket_2', 'Cabin_0', 'Cabin_1', 'Cabin_2', 'Cabin_3', 'Cabin_4', 'Cabin_5',
         'Cabin_6', 'Cabin_7', 'Embarked_0', 'Embarked_1', 'Embarked_2']
oof_train_LGBM, oof_test_LGBM, acc_LGBM, fi = CV_LGBM(X_train.drop(dropf, axis=1), 
                                            y_train, X_test.drop(dropf, axis=1), 
                                            early_stop=200)
usecols = [f for f in X_train.columns if f not in dropf]
Imp_feats = {}
mean_vals = np.mean(fi,axis=0)
for col, val in zip(usecols, mean_vals):
    Imp_feats.update({col: val})
data = pd.Series(Imp_feats).sort_values(ascending=False)
fig, ax = plt.subplots(figsize=(18,12))
sns.barplot(data.values, data.index, ax=ax)
ax.set_title('Features Importance');
"""
# Summary
"""
results = {
    'Logistic_Regression': acc_LogReg,
    'SVM': acc_SVM,
    'Decision Tree': acc_DecisionTree,
    'Random Forest': acc_RandomForest,
    'Neural Networks': acc_ANN,
    'LGBM': acc_LGBM
}
pd.Series(results)
"""
# Submission
"""
submit = pd.read_csv('../input/titanic/gender_submission.csv')
submit.head()
pred = np.mean(oof_test_LGBM, axis=1)
submit['Survived'] = np.where(pred>0.5, 1, 0)
submit.head()
# save
submit.to_csv("pred.csv", index=False)