# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Exploring and Processing Data - I

# import packages
import pandas as pd
import numpy as np
import os

# ## Import Data

raw_data_path = os.path.join(os.path.abspath('..'),'data','raw')
train_file_path = os.path.join(raw_data_path,'train.csv')
test_file_path = os.path.join(raw_data_path,'test.csv')

# import csv in pandas dataframe, index_col indicates primary key of dataset
train_df = pd.read_csv(train_file_path,index_col="PassengerId")
test_df = pd.read_csv(test_file_path, index_col='PassengerId')

# +
# type(train_df)
# train_df.info()
# test_df.info()

# Add new column 'Survived' to test df with default value of -888
test_df['Survived'] = -888
# -

# dataframes need to be passed as a tuple to concat function
# axis = 0 indicates merge horizontally , axis=1 indicates vertical
df = pd.concat((train_df, test_df), axis=0, sort='False')

# ## Data Munging

df.info()
# Missing values for Age, Cabin, Embarked, Fare

# ### Fix Mssing values for Embarked Column

df[df.Embarked.isnull()]

# How may people embarked at different points
df.Embarked.value_counts()

# Which embarked point has highest survival rate
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Embarked)

df.groupby(['Pclass', 'Embarked']).Fare.median()

# Replace missing embarked values with 'C'
df.Embarked.fillna('C', inplace=True)

df.info()

# ### Fix Missing values for Fare column

df[df.Fare.isnull()]

median_fare = df[(df.Embarked == 'S') & (df.Pclass == 3)].Fare.median()
print(median_fare)

df.Fare.fillna(median_fare, inplace=True)

df[df.Fare.isnull()]

# ### Fix Missing values for Age column

# %matplotlib inline
df.Age.plot(kind="hist", bins=20);


# #### Replace with median of title

# function to get title from name
def GetTitle(name):
        title_group = {'mr' : 'Mr', 
                   'mrs' : 'Mrs', 
                   'miss' : 'Miss', 
                   'master' : 'Master',
                   'don' : 'Sir',
                   'rev' : 'Sir',
                   'dr' : 'Officer',
                   'mme' : 'Mrs',
                   'ms' : 'Mrs',
                   'major' : 'Officer',
                   'lady' : 'Lady',
                   'sir' : 'Sir',
                   'mlle' : 'Miss',
                   'col' : 'Officer',
                   'capt' : 'Officer',
                   'the countess' : 'Lady',
                   'jonkheer' : 'Sir',
                   'dona' : 'Lady'
                     }
        first_name_with_title = name.split(',')[1]
        title = first_name_with_title.split('.')[0]
        title = title.strip().lower()
        return title_group[title]


df['Title'] =  df.Name.map(lambda x : GetTitle(x))

df.head()

title_age_median = df.groupby('Title').Age.transform('median')
df.Age.fillna(title_age_median , inplace=True)

df.info()

# ### Working with Outliers

# Fixing with Binning
df['Fare_Bin'] = pd.qcut(df.Fare, 4, labels=['very_low','low','high','very-high'])

df.Fare_Bin.value_counts().plot(kind='bar', rot=0)

# ## Feature Engineering

# ### Feature : Age State (Adult or Child)

df['AgeState'] = np.where(df['Age'] >= 18, 'Adult', 'Child')

df['AgeState'].value_counts()

# ### Feature : FamilySize

df['FamilySize'] = df.Parch + df.SibSp + 1 # 1 for self

pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].FamilySize)

# ### Feature : IsMother

# a lady aged more thana 18 who has Parch >0 and is married (not Miss)
df['IsMother'] = np.where(((df.Sex == 'female') & (df.Parch > 0) & (df.Age > 18) & (df.Title != 'Miss')), 1, 0)

# Crosstab with IsMother
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].IsMother)

# ### Feature : Deck

# set the value to NaN
df.loc[df.Cabin == 'T', 'Cabin'] = np.NaN


# extract first character of Cabin string to the deck
def get_deck(cabin):
    return np.where(pd.notnull(cabin),str(cabin)[0].upper(),'Z')
df['Deck'] = df['Cabin'].map(lambda x : get_deck(x))

# use crosstab to look into survived feature cabin wise
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Deck)

# ### Categorical Feature Encoding

# sex
df['IsMale'] = np.where(df.Sex == 'male', 1, 0)

# One-hot Encoding
df = pd.get_dummies(df,columns=['Deck', 'Pclass','Title', 'Fare_Bin', 'Embarked','AgeState'])

df.info()

# drop columns
df.drop(['Cabin','Name','Ticket','Parch','SibSp','Sex'], axis=1, inplace=True)

# reorder columns
columns = [column for column in df.columns if column != 'Survived']
columns = ['Survived'] + columns
df = df[columns]

df.info()

# ### Save Processed Data Set

processed_data_path = os.path.join(os.path.pardir,'data','processed')
write_train_path = os.path.join(processed_data_path, 'train.csv')
write_test_path = os.path.join(processed_data_path, 'test.csv')

# train data
df.loc[df.Survived != -888].to_csv(write_train_path) 
# test data, Survived column not in test data
columns = [column for column in df.columns if column != 'Survived']
df.loc[df.Survived == -888, columns].to_csv(write_test_path) 

StarcoderdataJupyterScriptsDedupFiltered