# Import the packages we will use
import pandas as pd
import numpy as np
from IPython.display import display, HTML

import matplotlib
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns

"""
Explanation: Rough outline brainstorm
Reading in data

briefly address encoding issues. I believe pandas default to ASCII? (-- it's acutally UTF-8)

basic manipulations

Subsetting
Accessing rows/columns/individual items
changing column headers
creating calculated values
using lambda functions (or functions in general) to manipulate the data

more advanced manipulations

group by
melting
merging

miscellanous thoughts

TBD
End of explanation
"""


# Read some data into a frame
# A frame is like an table in a spreadsheet. 
# It contains columns (which usually have names) and rows (which can be indexed by number, 
# but may also have names)
df = pd.read_csv('https://raw.githubusercontent.com/rmcelreath/rethinking/master/data/Howell1.csv', sep=";")
df.head()

"""
Explanation: Lesson 1 : Demonstration
This first example follows Chapter 4, section 3 of Richard McElreath's book Statistical Rethinking.
The task is understand height in a population, in this case using data about the !Kung San people. Anthropologist Nancy Howell conducted interviews with the !Kung San and collected the data used here.
The data are available in the github repository for the book: https://github.com/rmcelreath/rethinking/blob/master/data/Howell1.csv
Throughout this lesson, I'll provide the R equivalent for these actions in markdown as such:
R equivalent:
```r

df <- read.csv("https://raw.githubusercontent.com/rmcelreath/rethinking/master/data/Howell1.csv", sep = ";", header = TRUE)
head(df)
```
End of explanation
"""


# Graph the data -- let's look at height vs. age
df.plot.scatter(x='age', y='height')

"""
Explanation: R equivalent:
Installing ggplot2 is optional if you already have it.
```R

install.packages("ggplot2")
library(ggplot2)
ggplot(data=df, mapping=aes(x=age, y=height)) + geom_point()
```
End of explanation
"""


# Filter to adults, since height and age are correlated in children
adults_df = df[df['age'] >= 18]
# Look at height vs. age again
adults_df.plot.scatter(x='age', y='height')

"""
Explanation: R equivalent
```R

install.packages('dplyr')
library(dplyr)
adults_df <- filter(df, age>=18)
ggplot(data=adults_df, mapping=aes(x=age, y=height)) + geom_point()
```
End of explanation
"""


# Print out how many rows are in each frame
len(df), len(adults_df)

"""
Explanation: R equivalent:
``` R

nrow(adults_df); nrow(df)
```
End of explanation
"""


# Let's look at how the data are distributed
adults_df['height'].plot.hist()

"""
Explanation: R equivalent:
``` R

ggplot(data=adults_df, mapping=aes(x=age)) + geom_histogram()
```
End of explanation
"""


# Split data in to male and female
# -- first add in a sex column to make it less confusing
df['sex'] = df.apply(lambda row: 'Male' if row['male'] == 1 else 'Female', axis=1)
# -- re-apply the filter, since we modified the data
adults_df = df[df['age'] >= 18]
adults_df.head()

# Let's summarize the data
adults_df[['age', 'height', 'weight']].describe()

# Let's look at the data broken down by sex
adults_df[['age', 'height', 'weight', 'sex']].groupby('sex').describe()

# Let's focus on the means and std
summary_df = adults_df[['age', 'height', 'weight', 'sex']].groupby('sex').describe()
summary_df.loc[(slice(None),['mean', 'std']), :]

# Let's look at this visually -- plot height broken down by sex
g = sns.FacetGrid(adults_df, hue='sex', size=6)
g.map(sns.distplot, "height")
g.add_legend()

# Actually, let's look at everything
# -- first, get rid of the male column, it's redundant and confusing
del adults_df['male']
adults_df.head()

# -- now flatten the data -- very confusing, it will be explained later
flat_df = adults_df.set_index('sex', append=True)
flat_df = flat_df.stack().reset_index([1, 2])
flat_df.columns = ['sex', 'measurement', 'value']
flat_df.head()

# Plot!
g = sns.FacetGrid(flat_df, col='measurement', hue='sex', size=6, sharex=False)
g.map(sns.distplot, "value")
g.add_legend()

"""
Explanation: R equivalent:
```R

```
End of explanation
"""


df = pd.read_csv('https://raw.githubusercontent.com/rmcelreath/rethinking/master/data/Howell1.csv', sep=";")
df.head()

"""
Explanation: Lesson 2: Details
What just happened!? Let's take a deeper look at what was done above.
Reading in data
End of explanation
"""


df['height'].head()

"""
Explanation: One central abstraction in pandas is the DataFrame, which is similar to a data frame in R &mdash; that is, basically a spreadsheet. It is made up of columns, which are usually names, and rows, which may be named or just accessed by index.
Pandas is designed to be fast and efficient, so the table isn't necessarily stored the way you think it is internally. In particular, data is stored in columns, and each column is a pandas Series, which itself builds on numpy arrays, not native Python arrays.
Pandas can read data in many formats. CSV and JSON are common ones to use. You can control many aspects about how the data is read. Above, you see that the structure of the file is csv-like, but instead the ';' is used as the column separator. This is not a problem. Pandas can also handle different file encodings (UTF-8 is the default), etc. 
Basic frame manipulations — Accessing columns, rows, elements
End of explanation
"""


df.loc[0]

"""
Explanation: In many cases, columns of a frame can be accessed like an array. The result is a pandas Series object, which, as you see, has a name, index, and type.
Aside — why all the calls to 'head()'?
Series and frames can be very large. The methods head() and tail() can be used get a few of the first and last rows, respectively. By default, first/last 5 rows are returned. It's used here to limit output to a small number of rows, since there is no need to see the whole table.
End of explanation
"""


summary_df = df.describe()
summary_df.loc['mean']

"""
Explanation: Rows are accessed using the method loc or iloc. The method 'loc' takes the index, 'iloc' takes the row index. In the above case, these are the same, but that is not always true. For example...
End of explanation
"""


summary_df.loc['mean', 'age']

"""
Explanation: To access an individual cell, specify a row and column using loc.
End of explanation
"""


# select row index 0, and all the columns in that row
df.iloc[0,:]

# select all the rows in column 0 by index
df.iloc[:,0]

"""
Explanation: Another aside -- loc vs. iloc
As you saw above, the method loc takes the "label" of the index. The method iloc takes the index as arguments, with the parameters [row-index, col-index]
End of explanation
"""


df[['age', 'height', 'weight']].head()

"""
Explanation: Basic frame manipulations — data subsetting
End of explanation
"""


df.iloc[0:5]

"""
Explanation: Specifiying an array of column names returns a frame containing just those columns.
End of explanation
"""


df[df['age'] >= 18].head()

"""
Explanation: It's also possible to access a subset of the rows by index. More commonly, though, you will want to subset the data by some property.
End of explanation
"""


(df['age'] >= 18).head()

(df['male'] == 0).head()

"""
Explanation: This is intiutive to understand, but may seem a little magical at first. It is worth understanding what is going on underneath the covers.
The expression
df['age'] &gt;= 18

returns an series of bool indicating whether the expression is true or false for that row (identified by index).
End of explanation
"""


((df['age'] >= 18) & (df['male'] == 0)).head()

df[(df['age'] >= 18) & (df['male'] == 0)].head()

"""
Explanation: When such a series is the argument to the indexing operator, [], pandas returns a frame containing the rows where the value is True. These kinds of expressions can be combined as well, using the bitwise operators (not and/or).
End of explanation
"""


df.loc[(df['age'] >= 18) & (df['male'] == 0), ['height', 'weight', 'age']].head()

"""
Explanation: This way, code for subsetting is intuitive to understand. It is also possible to subset rows and columns simultaneously.
End of explanation
"""


df.columns = ['new_height', 'new_weight', 'new_age', 'coded_gender']

"""
Explanation: Basic frame manipulations — renaming columns
Renaming columns: just feed a list of new columns and pass it to df.columns
End of explanation
"""


df['new_id'] = df['new_weight'] + df['new_age']
df.head(2)

"""
Explanation: Creating columns based on other columns
If I wanted to create a new column based on adding up the weight and age, I could do this:
End of explanation
"""


gender_text = {1: 'Male', 0: 'Female'}
df['text_gender'] = df['coded_gender'].map(gender_text)
df.head(2)

"""
Explanation: If I wanted to create a calculated column using a dictionary replacement, I could use the map function
End of explanation
"""


df['double_age'] = df['new_age'].apply(lambda x: x*2)

df.head(2)

"""
Explanation: What about using a lambda function to create a new column?
End of explanation
"""


