# Read some data into a frame
# A frame is like an table in a spreadsheet. 
# It contains columns (which usually have names) and rows (which can be indexed by number, 
# but may also have names)
df = pd.read_csv('https://raw.githubusercontent.com/rmcelreath/rethinking/master/data/Howell1.csv', sep=";")
df.head()

# Graph the data -- let's look at height vs. age
df.plot.scatter(x='age', y='height')

# Filter to adults, since height and age are correlated in children
adults_df = df[df['age'] >= 18]
# Look at height vs. age again
adults_df.plot.scatter(x='age', y='height')

# Print out how many rows are in each frame
len(df), len(adults_df)

# Let's look at how the data are distributed
adults_df['height'].plot.hist()

# Split data in to male and female
# -- first add in a sex column to make it less confusing
df['sex'] = df.apply(lambda row: 'Male' if row['male'] == 1 else 'Female', axis=1)
# -- re-apply the filter, since we modified the data
adults_df = df[df['age'] >= 18]
adults_df.head()

# Let's summarize the data
adults_df[['age', 'height', 'weight']].describe()

# Let's look at the data broken down by sex
adults_df[['age', 'height', 'weight', 'sex']].groupby('sex').describe()

# Let's focus on the means and std
summary_df = adults_df[['age', 'height', 'weight', 'sex']].groupby('sex').describe()
summary_df.loc[(slice(None),['mean', 'std']), :]

# Let's look at this visually -- plot height broken down by sex
g = sns.FacetGrid(adults_df, hue='sex', size=6)
g.map(sns.distplot, "height")
g.add_legend()

# Actually, let's look at everything
# -- first, get rid of the male column, it's redundant and confusing
del adults_df['male']
adults_df.head()

# -- now flatten the data -- very confusing, it will be explained later
flat_df = adults_df.set_index('sex', append=True)
flat_df = flat_df.stack().reset_index([1, 2])
flat_df.columns = ['sex', 'measurement', 'value']
flat_df.head()

# Plot!
g = sns.FacetGrid(flat_df, col='measurement', hue='sex', size=6, sharex=False)
g.map(sns.distplot, "value")
g.add_legend()

"""
Explanation: Lesson 1 : Demonstration
This first example follows Chapter 4, section 3 of Richard McElreath's book Statistical Rethinking.
The task is understand height in a population, in this case using data about the !Kung San people. Anthropologist Nancy Howell conducted interviews with the !Kung San and collected the data used here.
The data are available in the github repository for the book: https://github.com/rmcelreath/rethinking/blob/master/data/Howell1.csv
End of explanation
"""


dummy_df = pd.DataFrame([['a', 'b'], ['c','d']])
dummy_df

"""
Explanation: Lesson 2: Details
What just happened!? Let's take a deeper look at what was done above.
Key abstractions in Pandas
Series
Series are one-dimensional arrays, which can also be "labelled" - have labeled indices. You've probably encountered this already. Let's create a dummy dataframe to illustrate this point.
End of explanation
"""


type(dummy_df[0])

"""
Explanation: A column of a dataframe is a series.
End of explanation
"""


type(dummy_df.loc[0])

"""
Explanation: As is the row of a datafrmae.
End of explanation
"""


pd.Series([1,2,3,4])

"""
Explanation: One can also construct a series from a list.
End of explanation
"""


pd.Series([1,2,3,4], index=['one', 'two', 'three', 'four'])

"""
Explanation: Unlike normal Python list, you can label the axis.
End of explanation
"""


df = pd.read_csv('https://raw.githubusercontent.com/rmcelreath/rethinking/master/data/Howell1.csv', sep=";")
df.head()

"""
Explanation: Dataframes
One central abstraction in pandas is the DataFrame. It's similar to a dataframe in R &mdash basically a spreadsheet.
which is similar to a data frame in R &mdash; that is, basically a spreadsheet, with columns and rows.
Pandas is designed to be fast and efficient, so the table isn't necessarily stored the way you think it is internally. In particular, data is stored in columns, and each column is a pandas Series, which itself builds on numpy arrays, not native Python arrays.
Pandas can read data in many formats. CSV and JSON are common ones to use. You can control many aspects about how the data is read. Below, you see that the structure of the file is csv-like, but instead the ';' is used as the column separator. This is not a problem. Pandas can also handle different file encodings (UTF-8 is the default), etc. 
Reading in data from a file
End of explanation
"""


col1 = pd.Series([0,1,2,3])
col2 = pd.Series([4,5,6,7])

pd.DataFrame({"col1": col1, "col2": col2})

"""
Explanation: Constructing a dataframe
If you were already manipulating data in python and had them as series - the pandas object type we discussed above, you could construct a dataframe from pandas series objects.
End of explanation
"""


df['height'].head()

"""
Explanation: Basic frame manipulations — Accessing columns, rows, elements
End of explanation
"""


df.loc[0]

"""
Explanation: In many cases, columns of a frame can be accessed like an array. The result is a pandas Series object, which, as you see, has a name, index, and type.
Aside — why all the calls to 'head()'?
Series and frames can be very large. The methods head() and tail() can be used get a few of the first and last rows, respectively. By default, first/last 5 rows are returned. It's used here to limit output to a small number of rows, since there is no need to see the whole table.
Accessing a dataframe using loc
The method loc allows you to access the dataframe using the "label" of the index. 
If you look at the dataframe we read in at the beginning of the lesson, the row labels are actually numbers, so you can access a single row using what's functionally an index.
End of explanation
"""


summary_df = df.describe()
summary_df

# summary_df.loc[0]

summary_df.loc['mean']

"""
Explanation: However, if you look at the summary_df below, you'll see that we can't actually access rows by their index using the method loc.
End of explanation
"""


summary_df.loc['mean', 'age']

"""
Explanation: To access an individual cell, specify a row and column using loc.
End of explanation
"""


# select row index 0, and all the columns in that row
df.iloc[0,:]

# select all the rows in column 0 by index
df.iloc[:,0].head(3)

"""
Explanation: Accessing elements of a dataframe using iloc
The method iloc takes the index as arguments, with the parameters [row-index, col-index].
End of explanation
"""


df[['age', 'height', 'weight']].head()

"""
Explanation: Basic frame manipulations — data subsetting
What if you had more columns and rows than you cared for in a dataframe? How do you subset the data?
End of explanation
"""


df.iloc[0:5]

"""
Explanation: Specifiying an array of column names returns a frame containing just those columns.
End of explanation
"""


df[df['age'] >= 18].head()

"""
Explanation: It's also possible to access a subset of the rows by index. More commonly, though, you will want to subset the data by some property.
End of explanation
"""


(df['age'] >= 18).head()

(df['male'] == 0).head()

"""
Explanation: This is intiutive to understand, but may seem a little magical at first. It is worth understanding what is going on underneath the covers.
The expression
df['age'] &gt;= 18

returns an series of bool indicating whether the expression is true or false for that row (identified by index).
End of explanation
"""


((df['age'] >= 18) & (df['male'] == 0)).head()

df[(df['age'] >= 18) & (df['male'] == 0)].head()

"""
Explanation: When such a series is the argument to the indexing operator, [], pandas returns a frame containing the rows where the value is True. These kinds of expressions can be combined as well, using the bitwise operators (not and/or).
End of explanation
"""


df.loc[(df['age'] >= 18) & (df['male'] == 0), ['height', 'weight', 'age']].head()

"""
Explanation: This way, code for subsetting is intuitive to understand. It is also possible to subset rows and columns simultaneously.
End of explanation
"""


df.columns = ['new_height', 'new_weight', 'new_age', 'coded_gender']

"""
Explanation: Basic frame manipulations — renaming columns
Renaming columns: just feed a list of new columns and pass it to df.columns
End of explanation
"""


df['new_id'] = df['new_weight'] + df['new_age']
df.head(2)

"""
Explanation: Creating columns based on other columns
If I wanted to create a new column based on adding up the weight and age, I could do this:
End of explanation
"""


gender_text = {1: 'Male', 0: 'Female'}
df['text_gender'] = df['coded_gender'].map(gender_text)
df.head(2)

"""
Explanation: If I wanted to create a calculated column using a dictionary replacement, I could use the map function
End of explanation
"""


df['double_age'] = df['new_age'].apply(lambda x: x*2)

df.head(2)

"""
Explanation: What about using a lambda function to create a new column?
End of explanation
"""


df.groupby('male')

"""
Explanation: More advanced stuff
In lesson 1, you saw a groupby: notice that it multi-indexed things for you. 
What happens when you group by? Well, for one, you get a Pandas groupby object.
End of explanation
"""


df.groupby('male').get_group(0).head(5)

"""
Explanation: If you choose to "get_group", you can return all of the values which have a particular value of the column you "grouped by."
End of explanation
"""


df.groupby('male').sum()

"""
Explanation: What I wanted to "sum by" gender?
End of explanation
"""


df.groupby('male').sum().reset_index()

"""
Explanation: That's great, but I actually want that returned as a normally-indexed DataFrame, so I'll "reset-index".
End of explanation
"""


# insert nlargest/nsmallest stuff here
pd.DataFrame(df.groupby('male')['height'].nlargest(1))

"""
Explanation: How about a similar (but different) question. I want to know this: who is the tallest male, and who is the tallest female?
End of explanation
"""


pd.DataFrame(df.groupby('male')['height'].nsmallest(2))

"""
Explanation: If I wanted to know the shortest 2 males/females?
End of explanation
"""


df1 = pd.DataFrame({'A': ['A0', 'A1', 'A2', 'A3'],\
                    'B': ['B0', 'B1', 'B2', 'B3'],\
                    'C': ['C0', 'C1', 'C2', 'C3'],\
                    'D': ['D0', 'D1', 'D2', 'D3']})

df2 = pd.DataFrame({'A': ['E0', 'E1', 'E2', 'E3'],\
                    'B': ['F0', 'F1', 'F2', 'F3'],\
                    'C': ['G0', 'G1', 'G2', 'G3'],\
                    'D': ['H0', 'H1', 'H2', 'H3']})
df1.head(2)

pd.concat([df1, df2])

"""
Explanation: What about multiple dataframes that I want to work with together? In pandas, "concatenation" of dataframes adds them together. If you don't specify a default axis along which it should concatenate, it'll assume that you want to add rows together, and figure out how the columns match.
End of explanation
"""


df1.append(df2)

"""
Explanation: Now if all you needed to do was append one data frame to another along axis "0" (add rows to each other), all you would do is this:
End of explanation
"""


df1 = pd.DataFrame({'A': ['A0', 'A1', 'A2', 'A3', 'A4'],\
                    'B': ['B0', 'B1', 'B2', 'B3', 'B4'],\
                    'C': ['C0', 'C1', 'C2', 'C3', 'C4'],\
                    'D': ['D0', 'D1', 'D2', 'D3', 'D4']})

df2 = pd.DataFrame({'A': ['A0', 'A1', 'A2', 'A3'],\
                    'F': ['F0', 'F1', 'F2', 'F3'],\
                    'G': ['G0', 'G1', 'G2', 'G3'],\
                    'H': ['H0', 'H1', 'H2', 'H3']})
pd.merge(df1, df2, on='A', how='left')

"""
Explanation: If I wanted to maintain the same number of rows and add columns to each other, I would specify that was use axis "1".
Merging in pandas is similar to the JOIN keyword in SQL. You can specify which column(s) match so that you can "add" relevant columns together, and also specify what kind of join: left, right, outer, or inner.
End of explanation
"""


df1 = pd.DataFrame({'A': ['A0', None , 'A2', 'A3', 'A4'], 'B': ['B0', 'B1', 'B2', None , None]})
df1

df1.dropna()

df1.fillna('No longer missing!')

"""
Explanation: PS - how do I deal with missing data?
Pandas provides two really useful functions: dropna, and fillna, to get rid of datapoints where there exist null values, or replace them. You can specify things like the axis along which to drop, as well as the columns to consier.
End of explanation
"""


