# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/salilathalye/chats-with-austin/blob/main/CWA_EDA_Sharpen_the_Saw.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="CSG27b8yiCD0"
# #Chats with Austin
# ###EDA - Sharpen the Saw
# <p>
# Time to put Pandas back into the rotation for a short EDA rust-removal, cleaning and sharpening. Basic stuff: DataFrame examination, groupby, crosstabs.
# One of the things I still miss about the tidyverse is dplyr chaining using the maggritr pipe operator %>%. After several years (2013 - 2018) of building analytics and data science pipelines in R, I had developed a muscle-memory and state of "flow" in the tidyverse. I could envision what I needed to accomplish and the code would (almost) write itself. I still find method chaining a bit awkward in Pandas and everything seems a tad bit verbose...
#
# Data is a modified version of the Titanic dataset from [DPhi](https://dphi.tech//). The questions I am researching come from a quiz in one of DPhi's learning modules.
# <p>
# <NAME>
#
# www.bokehboy.com

# + id="4mpUj-QNyfmT"
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# + id="XhYNWfv6y7Zq"
data_url = 'https://raw.githubusercontent.com/dphi-official/First_ML_Model/master/titanic.csv'

# + id="lLia2ceqzK0Q"
dataset = pd.read_csv(data_url)

# + colab={"base_uri": "https://localhost:8080/", "height": 195} id="MxVfaCWgzPIv" outputId="ae7b6cdc-41a9-49cc-b6aa-d0701fe10185"
dataset.head()

# + colab={"base_uri": "https://localhost:8080/"} id="pSVum7XmzlQ_" outputId="ea44b429-09d3-42fc-bbf3-b6735c7535cd"
dataset.info()

# + colab={"base_uri": "https://localhost:8080/", "height": 284} id="uoh1aYPBzp1_" outputId="89d03aaa-46d7-493b-8824-e8c7d118585b"
dataset.describe()

# + [markdown] id="FJhTEZDC1xIN"
# The Fare feature has 0 missing values
#

# + id="jPl-m5Hz158r" colab={"base_uri": "https://localhost:8080/"} outputId="78720383-d43e-46ea-e1d2-a2260cb92031"
dataset[['Fare']].isna().sum()

# + [markdown] id="IvcOLiCO1m-9"
# Are there more male passengers than female?
#

# + colab={"base_uri": "https://localhost:8080/"} id="eTwoPcph0ZiH" outputId="37e153e3-0d4a-42c7-a455-e56f110f58c3"
dataset[['Sex','PassengerId']].groupby('Sex')['PassengerId'].count()

# + id="4GcGfHke2Et1"
# Visual method

# + [markdown] id="LnWDeGHt2JRT"
# 2. What is the proportion of passengers who survived?

# + colab={"base_uri": "https://localhost:8080/"} id="sYrF_F7m2ISd" outputId="148a33dc-7d95-4c8c-f14d-7046ecb51c22"
total_passengers = dataset['PassengerId'].nunique()
total_survivors = dataset[dataset['Survived'] == 1]['PassengerId'].nunique()
proportion_survived = total_survivors / total_passengers
proportion_survived

# + [markdown] id="sGqhITQj4qUD"
# 3. What is the median Fare of the passengers? 

# + colab={"base_uri": "https://localhost:8080/"} id="IlB7WP1n2WR9" outputId="e95f70cb-8db0-4e57-f04e-648e65102d4a"
dataset['Fare'].median()

# + [markdown] id="ASXmabKF5Cxa"
# Proportion of survivors, men vs women
#

# + colab={"base_uri": "https://localhost:8080/", "height": 136} id="tXWqxnNX5SyS" outputId="9301d3b8-ce9e-49f8-913a-e20b981f0b72"
subset_by_sex = dataset[['PassengerId','Sex','Survived']].copy()
# axis=1 will run the proportions by row, so we get a proportion of survivors by sex
contingency_table = pd.crosstab(subset_by_sex['Sex'], subset_by_sex['Survived']).apply(lambda r: r/r.sum(), axis=1)
contingency_table

# + [markdown] id="dbpXCtNT7lMR"
# 74% of women survived, compared to 19% of men

# + colab={"base_uri": "https://localhost:8080/", "height": 166} id="SaDATTha5r7K" outputId="b04a7fa1-2bfc-408b-cd11-6b64d6fca6eb"
subset_by_class = dataset[['PassengerId','Pclass','Survived']].copy()
contingency_table = pd.crosstab(subset_by_class['Pclass'], subset_by_class['Survived']).apply(lambda r: r/r.sum(), axis=1)
contingency_table

# + [markdown] id="WyRfl2N7811w"
# 63% of First Class passengers survived, compared to 47% of Second Class and only 24% of Third Class passengers.
#

# + [markdown] id="oWUJq39A9MAw"
# Proportion of Survivors by Age

# + colab={"base_uri": "https://localhost:8080/", "height": 136} id="YEEyXiRD9EIv" outputId="b8552e54-b19d-47d3-b5f4-1522a204b501"
subset_by_age = dataset[['PassengerId','Age','Survived']].copy()
bins= [0,20,120]
labels = ['Child','Adult']
subset_by_age['AgeGroup'] = pd.cut(subset_by_age['Age'], bins=bins, labels=labels, right=False)
contingency_table = pd.crosstab(subset_by_age['AgeGroup'], subset_by_class['Survived']).apply(lambda r: r/r.sum(), axis=1)
contingency_table

# + [markdown] id="ee0UQJ3kAwgt"
# 48% of Children survived compared to 38% of Adults.

# + [markdown] id="3brNoTYxBAAM"
# How many of the survivors embarked from Southamption 'S'

# + colab={"base_uri": "https://localhost:8080/"} id="ACQuBYR8BaIK" outputId="420938cb-09c4-45c2-cd05-78d530187a05"
survivors = dataset[dataset['Survived'] == 1]
survivors[survivors['Embarked'] == 'S'].shape[0]

# + [markdown] id="ODpkp7KKCoOa"
# 6. Five highest fares of passengers (not unique)

# + colab={"base_uri": "https://localhost:8080/", "height": 195} id="vAoHzCSFBkoa" outputId="530c1108-06fe-45a6-9ecc-4f3c975eb204"
subset = dataset[['Fare']].copy()
subset.sort_values(by='Fare', ascending=False).head(5)

# + [markdown] id="u9Ow1iiTFSEh"
# Median Age of Passengers

# + colab={"base_uri": "https://localhost:8080/"} id="E7dEyMJOC9Dn" outputId="f5346e7f-3b11-4eb1-88f0-e5b52b10c4f0"
dataset[['Age']].median()

# + [markdown] id="vuBta50yFsg4"
# Unique Names

# + colab={"base_uri": "https://localhost:8080/"} id="Yl1HdLvQFvdl" outputId="851fa5a5-80a2-44ca-a46c-14ee5f3b6897"
dataset[['Name']].nunique()

# + [markdown] id="5eTcBqgVGFgx"
# Most of the passengers have x siblings/spouses

# + colab={"base_uri": "https://localhost:8080/", "height": 77} id="XyCJm7KlGNkp" outputId="364b99f3-50e1-422a-8510-d62fbaf6bc4e"
dataset[['SibSp']].mode()

StarcoderdataJupyterScriptsDedupFiltered