# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: p37
#     language: python
#     name: p37
# ---

# # 05.04 - PARTICIPATE IN KAGGLE

# !wget --no-cache -O init.py -q https://raw.githubusercontent.com/rramosp/ai4eng.v1.20211.udea/main/content/init.py
import init; init.init(force_download=False); init.get_weblink()

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
import local.lib.mlutils
import pandas as pd
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, Dropout, Flatten
# %matplotlib inline

# ## We use Titanic data in [Kaggle](http://www.kaggle.com)

# - Register to [Kaggle](http://www.kaggle.com)
# - Enter the competition [Titanic Data at Kaggle](https://www.kaggle.com/c/titanic)
# - Download the `train.csv` and `test.csv` files
# - **UPLOAD THE FILES** to your notebook environment (in Colab, open the Files tab and upload)

d = pd.read_csv("train.csv")
print (d.shape)

d.head()

# **Understand `NaN` values are present**

for i in d.columns:
    print ("%20s"%i, np.sum(d[i].isna()))

d.Embarked.value_counts()

plt.hist(d.Age.dropna().values, bins=30);

# **Remove uninformative columns**

del(d["PassengerId"])
del(d["Name"])
del(d["Ticket"])
del(d["Cabin"])



# **Fix `NaN` values**
#
# - observe the different filling policies we decide to have

d["Embarked"] = d.Embarked.fillna("N")
d["Age"]      = d.Age.fillna(d.Age.mean())
d.head()

plt.hist(d.Age.dropna().values, bins=30);


# **Turn categorical columns to a `one_hot` encoding**

# +
def to_onehot(x):
    values = np.unique(x)
    r = np.r_[[np.argwhere(i==values)[0][0] for i in x]]
    return np.eye(len(values))[r].astype(int)
    
k = to_onehot(d.Embarked.values)
k[:5]


# -

def replace_columns_with_onehot(d, col):
    k = to_onehot(d[col].values)
    r = pd.DataFrame(k, columns=["%s_%d"%(col, i) for i in range(k.shape[1])], index=d.index).join(d)
    del(r[col])
    return r 


d.head()

d = replace_columns_with_onehot(d, "Embarked")
d.head()

d = replace_columns_with_onehot(d, "Sex")
d.head()

d.shape, d.values.sum()


# ### Put all transformations together

def clean_titanic(d):
    del(d["PassengerId"])
    del(d["Name"])
    del(d["Ticket"])
    del(d["Cabin"])
    d["Embarked"] = d.Embarked.fillna("N")
    d["Fare"]     = d.Fare.fillna(d.Fare.mean())
    d["Age"]      = d.Age.fillna(d.Age.mean())
    d = replace_columns_with_onehot(d, "Embarked")
    d = replace_columns_with_onehot(d, "Sex")
    return d


# **transform train and test data together**
#
# - observe that test data **does not have** a `Survival` column. This is the result to submit to Kaggle

dtr = pd.read_csv("train.csv")
dts = pd.read_csv("test.csv")
lentr = len(dtr)
dtr.shape, dts.shape

dts.head()

# **get data ready for training**

# +
source_cols = [i for i in dtr.columns if i!="Survived"]
all_data = pd.concat((dtr[source_cols], dts[source_cols]))
all_data.index = range(len(all_data))
all_data = clean_titanic(all_data)

Xtr, ytr = all_data.iloc[:lentr].values, dtr["Survived"].values
Xts      = all_data.iloc[lentr:].values

print (Xtr.shape, ytr.shape)
print (Xts.shape)
# -

# **cross validate for model selection**

# +
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.model_selection import cross_val_score
rf = RandomForestClassifier()
print (cross_val_score(rf, Xtr, ytr))

svc = SVC()
print (cross_val_score(svc, Xtr, ytr))
# -

# **now train with full dataset and generate submission for Kaggle**

rf.fit(Xtr, ytr)
preds_ts = rf.predict(Xts)
preds_ts

# **get predictions ready to submit to Kaggle**
#
# - see https://www.kaggle.com/c/titanic#evaluation for file format

submission = pd.DataFrame([dts.PassengerId, pd.Series(preds_ts, name="Survived")]).T
submission.head()

submission.to_csv("titanic_kaggle.csv", index=False)

# !head titanic_kaggle.csv





StarcoderdataJupyterScriptsDedupFiltered