"""
### This notebook is applying fast.ai Tabular data lesson concepts on titanic training data competition
"""
import fastbook
fastbook.setup_book()
from fastbook import *
from pandas.api.types import is_string_dtype, is_numeric_dtype, is_categorical_dtype
from fastai.tabular.all import *
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from dtreeviz.trees import *
from IPython.display import Image, display_svg, SVG
from sklearn.model_selection import train_test_split
titanic_train_input_data=pd.read_csv('/kaggle/input/titanic/train.csv')
procs = [Categorify, FillMissing]
input_columns=['PassengerId','Pclass', 'Name', 'Sex', 'Age', 'SibSp',
       'Parch', 'Ticket', 'Fare', 'Cabin', 'Embarked']
X_train, X_cv, y_train, y_cv = train_test_split(titanic_train_input_data[input_columns], titanic_train_input_data["Survived"], test_size=0.2, random_state=0)
titanic_train_input_data=titanic_train_input_data.append(titanic_train_input_data.tail(1))
last_added_row=titanic_train_input_data.tail(1)
titanic_train_input_data.loc[last_added_row.index.values,'Fare']=float("NaN")
titanic_train_input_data.tail(1)
splits = (list(X_train.index.values),list(X_cv.index.values))
#Declare dependent variable and automatically deduce the continuous vs categorical variables
dep_var=['Survived']
cont,cat = cont_cat_split(titanic_train_input_data, 1, dep_var=dep_var)
cont,cat
#Tabular Pandas
to = TabularPandas(titanic_train_input_data, procs, cat, cont, y_names=dep_var, splits=splits)
#Display train and validation lengths
len(to.train),len(to.valid)

to.show(3)
to.items.head(3)
titanic_train_input_data.columns
#Build simple decision tree classifier
xs,y = to.train.xs,to.train.y
valid_xs,valid_y = to.valid.xs,to.valid.y
from sklearn.tree import DecisionTreeClassifier
decision_model=DecisionTreeClassifier(max_leaf_nodes=4)
decision_model.fit(xs,y)
#Lets see the tree that was built
draw_tree(decision_model, xs, size=10, leaves_parallel=True, precision=2)
#Dtree visualization
samp_idx = np.random.permutation(len(y))[:500]
dtreeviz(decision_model, xs.iloc[samp_idx], y.iloc[samp_idx], xs.columns, dep_var,
        fontname='DejaVu Sans', scale=1.6, label_fontsize=10,
        orientation='LR')
from sklearn.metrics import confusion_matrix 
from sklearn.metrics import accuracy_score
def calc_accuracy(pred,y):
    tn,fp,fn,tp=confusion_matrix(y,pred).ravel()
    print(tn,fp,fn,tp)
    return round(accuracy_score(y,pred), 6)
def model_accuracy(m, x, y): return calc_accuracy(m.predict(x), y)
model_accuracy(decision_model, valid_xs, valid_y)
decision_model.get_n_leaves(), len(xs)
xs
#Decision modelwith no max leaf nodes
decision_model_nomax=DecisionTreeClassifier()
decision_model_nomax.fit(xs,y)
acc=model_accuracy(decision_model_nomax, valid_xs, valid_y)
print(acc)
decision_model_nomax.get_n_leaves(), len(xs)
decision_model_minsamplesleaf=DecisionTreeClassifier(min_samples_leaf=15)
decision_model_minsamplesleaf.fit(xs,y)
acc=model_accuracy(decision_model_minsamplesleaf, valid_xs, valid_y)
print(acc)
decision_model_nomax.get_n_leaves(), len(xs)
titanic_test_data=pd.read_csv('/kaggle/input/titanic/test.csv')
to_tst = to.new(titanic_test_data)
to_tst.process()
to_tst.items.head(3)
print("Writing submission file:")
y_test=decision_model_minsamplesleaf.predict(to_tst.items)
titanic_test_data["Survived"]=y_test
titanic_test_data[["PassengerId","Survived"]].to_csv("titanic_decision_tree.csv",index=False)
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session

import numpy as np 
import pandas as pd 
from sklearn.ensemble import RandomForestClassifier 
from sklearn.datasets import make_classification 
from sklearn.metrics import confusion_matrix 
from sklearn.model_selection import train_test_split 
from sklearn.metrics import accuracy_score
import scipy.stats
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
import math
from scipy.stats import pearsonr
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from keras.models import Sequential
from keras.layers import Dense
imp = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
from sklearn.preprocessing import LabelEncoder
label_encoder_sex = LabelEncoder()

def map_risk_categorical(data,cv_data,test_data,columns,target_column):
    for column in columns:
        print(column+" unique values:")
        print(data[column].unique())
        mean_encodings=data.groupby(column)[target_column].mean()
        print(mean_encodings)
        mean_encodings['']=data[column].mode()
        data[column]=data[column].map(mean_encodings)
        data[column]=round(data[column].astype(float),2)
        cv_data[column]=cv_data[column].map(mean_encodings)
        cv_data[column]=round(cv_data[column].astype(float),2)
        test_data[column]=test_data[column].map(mean_encodings)
        test_data[column]=round(test_data[column].astype(float),2)
        imp.fit(data[[column]])
        cv_data[[column]]=imp.transform(cv_data[[column]])
        test_data[[column]]=imp.transform(test_data[[column]])
    return (data.copy(),cv_data.copy(),test_data.copy())

def print_correlation(data,columns,class_column):
    dist={}
    for column in columns:
        print("Correlation of "+column+" with "+class_column)
        print(pearsonr(data[column],data[class_column]))
        
def print_summary(data,columns,filter_query):
    normal_stats={}
    for column in columns:
        normal_stats[column]={}
        if filter_query != "":
            total_count=data.query(filter_query)[column].count()
            print("Total count"+str(total_count))
            print(data.query(filter_query)[[column]].describe())
            normal_stats[column]=(data.query(filter_query)[column].mean(),data.query(filter_query)[column].std())
        else:
            print(data[column].describe())
    return normal_stats.copy()

def map_data(data): 
    gender_mapping={"male":1,"female":0} 
    embarked_mapping={"S":1,"C":2,"Q":3} 
    cabin_mapping={"A":1,"B":2,"C":3,"D":4,"E":5,"F":6,"G":7} 
    #data["Sex"]=data["Sex"].map(gender_mapping,na_action='ignore')
    #data["Sex"]=label_encoder_sex.fit_transform(data["Sex"])
    data["Embarked"]=data["Embarked"].map(embarked_mapping,na_action='ignore') 
    data["Cabin"]=data["Cabin"].map(lambda x:x[0],na_action='ignore') 
    data["Cabin"]=data["Cabin"].map(cabin_mapping,na_action='ignore')
    data[["Pclass","Sex","Fare","Embarked","Age"]]=imp.fit_transform(data[["Pclass","Sex","Fare","Embarked","Age"]])
    data["Cabin"]=data["Cabin"].fillna(0)
    return data.copy()

def map_risk_numeric(data,columns,risk_map,complement_flag):
    for column in columns:
        norm_stats_mean,norm_stats_stdev=risk_map[column]
        print("mean:"+str(norm_stats_mean)+" std:"+str(norm_stats_stdev)+" for "+column)
        norm_stats=scipy.stats.norm(norm_stats_mean,norm_stats_stdev)
        data[column]=norm_stats.cdf(data[column])
        data[column]=round(data[column],2)
        #if complement_flag==1:
        #    data[new_column]=round(1-data[new_column],2)
    return data.copy()

titanic_train_input_data=pd.read_csv('/kaggle/input/titanic/train.csv')
titanic_test_data=pd.read_csv('/kaggle/input/titanic/test.csv')
#train_columns=["Pclass","Age","Sex","Fare","Embarked","Cabin","SibSp","Parch"]
train_columns=["Sex","Pclass","Pclass_Sex","Cabin","Age","Pclass_Age","Sex_Fare","Family_Size"]
categorical_columns=["Pclass","Sex","Cabin","Pclass_Sex","Age","Pclass_Age","Family_Size","SibSp","Parch","Fare_Per_Person","Sex_Fare"]
numeric_columns=[]
input_columns=categorical_columns+numeric_columns


titanic_train_input_data=map_data(titanic_train_input_data.copy())
titanic_test_data=map_data(titanic_test_data.copy())

titanic_train_input_data['Family_Size']=titanic_train_input_data['SibSp']+titanic_train_input_data['Parch']+1
titanic_train_input_data['Fare_Per_Person']=titanic_train_input_data['Fare']/(titanic_train_input_data['Family_Size'])
titanic_train_input_data['Fare_Per_Person']=round(titanic_train_input_data['Fare_Per_Person'],2)
titanic_train_input_data['Pclass_Sex']=titanic_train_input_data['Pclass'].map(str)+titanic_train_input_data['Sex'].map(str)
titanic_train_input_data['Age']=round(titanic_train_input_data['Age']/5)
titanic_train_input_data['Fare_Per_Person']=round(titanic_train_input_data['Fare_Per_Person']/20)
titanic_train_input_data['Pclass_Age']=titanic_train_input_data['Pclass'].map(str)+titanic_train_input_data['Age'].map(str)
titanic_train_input_data['Sex_Fare']=titanic_train_input_data['Sex'].map(str)+titanic_train_input_data['Fare_Per_Person'].map(str)
titanic_train_input_data['Sex_Age']=titanic_train_input_data['Sex'].map(str)+titanic_train_input_data['Age'].map(str)
titanic_train_input_data['Age_Cabin']=titanic_train_input_data['Age'].map(str)+titanic_train_input_data['Cabin'].map(str)

X_train, X_cv, y_train, y_cv = train_test_split(titanic_train_input_data[input_columns], titanic_train_input_data["Survived"], test_size=0.2, random_state=0)

titanic_train_data=titanic_train_input_data.iloc[X_train.index.values].copy()

titanic_test_data['Family_Size']=titanic_test_data['SibSp']+titanic_test_data['Parch']
titanic_test_data['Fare_Per_Person']=titanic_test_data['Fare']/(titanic_test_data['Family_Size']+1)
titanic_test_data['Fare_Per_Person']=round(titanic_test_data['Fare_Per_Person']/20)
titanic_test_data['Pclass_Sex']=titanic_test_data['Pclass'].map(str)+titanic_test_data['Sex'].map(str)
titanic_test_data['Age']=round(titanic_test_data['Age']/5)
titanic_test_data['Pclass_Age']=titanic_test_data['Pclass'].map(str)+titanic_test_data['Age'].map(str)
titanic_test_data['Sex_Fare']=titanic_test_data['Sex'].map(str)+titanic_test_data['Fare_Per_Person'].map(str)
titanic_test_data['Sex_Age']=titanic_test_data['Sex'].map(str)+titanic_test_data['Age'].map(str)
titanic_test_data['Age_Cabin']=titanic_test_data['Age'].map(str)+titanic_test_data['Cabin'].map(str)

#Over sampling
#count_survived_0, count_survived_1 = titanic_train_data.Survived.value_counts()

#titanic_survived_0 = titanic_train_data[titanic_train_data["Survived"] == 0]
#titanic_survived_1 = titanic_train_data[titanic_train_data["Survived"] == 1]
#titanic_train_data_survived_1_over = titanic_survived_1.sample(count_survived_0, replace=True)
#titanic_train_data_over = pd.concat([titanic_survived_0, titanic_train_data_survived_1_over], axis=0)


print("Survived class numeric vals:")
#print_unique_vals(titanic_train_data,numeric_columns,"Survived==1")
survived_normal_stats=print_summary(titanic_train_data,numeric_columns,"Survived==1")
print("-------------------------") 
print("Not Survived class numeric vals:")
#print_unique_vals(titanic_train_data,numeric_columns,"Survived==0")
not_survived_normal_stats=print_summary(titanic_train_data,numeric_columns,"Survived==0")
print("--------------------------")
print("Correlation:")
#print_correlation(titanic_train_data,categorical_columns,"Survived")
#print_correlation(titanic_train_data,numeric_columns,"Survived")

titanic_train_data,X_cv,titanic_test_data=map_risk_categorical(titanic_train_data,X_cv,titanic_test_data,categorical_columns,"Survived")
#titanic_train_data=map_risk_numeric(titanic_train_data,numeric_columns,survived_normal_stats,0)
#X_cv=map_risk_numeric(X_cv,numeric_columns,survived_normal_stats,0)
#titanic_test_data=map_risk_numeric(titanic_test_data,numeric_columns,survived_normal_stats,0)
#sc = StandardScaler()
#titanic_train_data[numeric_columns] = sc.fit_transform(titanic_train_data[numeric_columns])
#X_cv[numeric_columns] = sc.fit_transform(X_cv[numeric_columns])
titanic_test_original=titanic_test_data.copy()
#titanic_test_data[numeric_columns] = sc.fit_transform(titanic_test_data[numeric_columns])

titanic_train_data[train_columns+["PassengerId","Name","Survived"]].to_csv("titanic_mapped_risk.csv",index=False)

#from sklearn.naive_bayes import ComplementNB
#clf = ComplementNB()
#clf = MLPClassifier(solver='lbfgs', alpha=1e-5,hidden_layer_sizes=(100,100), random_state=7,max_iter=5000,activation='relu')
#clf = RandomForestClassifier(max_depth=5, random_state=0,max_features=2)
clf = LogisticRegression(random_state=7,C=0.1)

#clf=Sequential()
#clf.add(Dense(units=10, activation="relu",kernel_initializer='glorot_uniform',input_dim=6))
#clf.add(Dense(units=10, activation="relu",kernel_initializer='glorot_uniform'))
#clf.add(Dense(units=1, activation = 'sigmoid',kernel_initializer='glorot_uniform'))
#clf.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
#clf.fit(tf.constant(titanic_train_data[train_columns].values,'float32'), tf.constant(y_train.values,'float32'), epochs = 100)
#y_pred=clf.predict(tf.constant(X_cv[train_columns].values,'float32'))
clf.fit(titanic_train_data[train_columns], y_train)
y_pred=clf.predict_proba(X_cv[train_columns])
y_pred_df=pd.DataFrame(y_cv)
#y_pred_df["Survived_Prob"]=y_pred[:,0]
y_pred_df["Not_Survived_Prob"]=y_pred[:,0]
y_pred_df["Survived_Prob"]=y_pred[:,1]
y_pred_df["Not_Survived_Prob"]=round(y_pred_df["Not_Survived_Prob"],2)
y_pred_df["Survived_Prob"]=round(y_pred_df["Survived_Prob"],2)
y_pred_df.columns=["Survived","Not_Survived_Prob","Survived_Prob"]
#y_pred_df.columns=["Survived","Survived_Prob"]
predictions=X_cv.merge(y_pred_df,left_index=True,right_index=True)
predictions=predictions.merge(titanic_train_input_data,left_index=True,right_index=True)
predictions["Survived_Prediction"]=predictions["Survived_Prob"].map(lambda x:(1 if x>=0.5 else 0))
predictions.query("Survived_x==1 & Survived_Prediction!=1").to_csv("predictions_fn.csv",index=False)
predictions.query("Survived_x==0 & Survived_Prediction==1").to_csv("predictions_fp.csv",index=False)
#print(predictions.query("Survived_Prediction!=Survived"))
#predictions.query("Survived_x==1 & Survived_Prediction!=1").Family_Size.hist()

tn,fp,fn,tp=confusion_matrix(y_cv,predictions["Survived_Prediction"]).ravel()

print(tn,fp,fn,tp)
print(accuracy_score(y_cv,predictions["Survived_Prediction"]))
#print(clf.coef_[0])

print("Writing submission file:")
y_test=clf.predict(titanic_test_data[train_columns])
#y_test=clf.predict(tf.constant(titanic_test_data[train_columns].values,'float32'))
titanic_test_original["Survived"]=y_test
titanic_test_original["Survived"]=titanic_test_original["Survived"].map(lambda x:(1 if x>=0.5 else 0))
titanic_test_original[["PassengerId","Survived"]].to_csv("titanic_random_2.csv",index=False)