import pandas as pd
import numpy as np

"""
Explanation: Pandas
http://pandas.pydata.org/
Data analysis library that introduces the concepts of data-frames and series to Python. Powerful tool for time-series analysis and fast visualizations of data.
Excellent introduction by the author: https://vimeo.com/59324550
Software development rigor applied to spread-sheets.
http://www.eusprig.org/horror-stories.htm
See pages 131-132 of the JP Morgan Task Force Report "...further errors were discovered in the Basel II.5 model, including, most significantly, an operational error in the calculation of the relative changes in hazard rates and correlation estimates. Specifically, after subtracting the old rate from the new rate, the spreadsheet divided by their sum instead of their average, as the modeler had intended. This error likely had the effect of muting volatility by a factor of two and of lowering the VaR" 
As reported in "A tempest in a spreadsheet" http://ftalphaville.ft.com/2013/01/17/1342082/a-tempest-in-a-spreadsheet/? Lisa Pollack comments "On a number of occasions, he asked the trader to whom he reported for additional resources to support his work on the VaR model, but he did not receive any. Also it appears that he (had to?) cut a number of corners, which resulted increased operational risk and artificially low volatility numbers ... pressure was put on the reviewers to get on with approving the model"
End of explanation
"""


data = np.random.rand(5, 5)
data

"""
Explanation: Dataframes
A flexible data structure for tabular data manipulation.
End of explanation
"""


pd.DataFrame(data)

table = pd.DataFrame(data, columns=["a","b","c","d","e"])
table

"""
Explanation: Matrix representation of data, provided by numpy, often isn't enough and this is where Pandas can help, through the introduction of a "data-frame", which is a tabular representation.
End of explanation
"""


table.ix[0]

"""
Explanation: Note the automatic selection of an index (left hand column). This can be used to pull out rows of interest.
End of explanation
"""


table["a"]

table[["a","e"]]

"""
Explanation: Similarly we can select data from columns.
End of explanation
"""


missing_data = table[table > 0.6]
missing_data

"""
Explanation: A table behave similarly to a numpy array (in fact underneath it is a nd-array), meaning we can use fancy indexing:
End of explanation
"""


missing_data.describe()

missing_data.count()

missing_data.min()

"""
Explanation: Q. What are the summary statistics of the "missing_data" table?
End of explanation
"""


%matplotlib inline

missing_data.describe().plot(kind='bar', figsize=(10,5))

"""
Explanation: Visualization of data is made simple with pandas (for simple things).
End of explanation
"""


dates = pd.date_range(start="1/1/14", end="31/12/14")
dates

values = np.sin(np.linspace(0,2*np.pi,365)) * np.random.rand(365)
values[:10]

time_series = pd.Series(index=dates, data=values)
time_series.head()

time_series.plot(figsize=(10,5))

"""
Explanation: Time series
Pandas has an extensive set of functions for working with time series.
http://pandas.pydata.org/pandas-docs/dev/timeseries.html#time-series-date-functionality
Creating a time series of data requires a time based index.
End of explanation
"""


time_series.resample("M").plot(figsize=(10,5))

"""
Explanation: Q. Whats the monthly variation of the time-series. Smooth out the noise by down-sampling - note any change in the signal.
http://pandas.pydata.org/pandas-docs/dev/timeseries.html#up-and-downsampling
End of explanation
"""


dates = pd.date_range(start="1/1/14", end="31/12/14")

categories = ["a", "b", "c", "d"]
noisey_signal = np.c_[np.sin(np.linspace(0, 2*np.pi, 365)) * np.random.rand(365)]
data = np.hstack([0.5 * noisey_signal, 3.5 * noisey_signal, 0.01 * noisey_signal, noisey_signal])
table = pd.DataFrame(data=data, columns=categories)

table["date"] = dates
table["month"] = [x.strftime("%B") for x in dates]

table.head()

table.set_index("date")[["a","b","c","d"]].plot(figsize=(10,5))

"""
Explanation: Split - Apply - Combine
A powerful concept that will help you to perform complex operations on datasets.
http://pandas.pydata.org/pandas-docs/stable/groupby.html
End of explanation
"""


table[table["month"] == "December"].describe()

groups = table.groupby("month").describe().loc["December"]

"""
Explanation: Q. What are the summary statistics for December?
End of explanation
"""


groups = table.groupby( [x[0] for x in table["month"]] ).describe().loc["J"]
groups

"""
Explanation: Q. What are the summary statistics for all months starting with the letter "J".
End of explanation
"""


table.groupby("month").quantile(.95)

"""
Explanation: Q. What is the 95 quantile for each month.
End of explanation
"""


data = table[["date", "a"]].set_index("date")
data.head()

ax = data.plot(figsize=(10,5), color='g')
data.resample('w').plot(color='m', ax=ax)
data.resample('w', how="min").plot(color='b', ax=ax)
data.resample('w', how="max").plot(color='r', ax=ax)
ax.legend(['a', 'mean(a)', 'min(a)', 'max(a)']);

ax = data.plot(figsize=(10,5), color='g')
data.resample("w", how=["mean", "min", "max"]).plot(ax=ax, color=["m","b","r"])
ax.legend(['a', 'mean(a)', 'min(a)', 'max(a)']);

"""
Explanation: Q. Plot the monthly statistics for the "a" category.
End of explanation
"""


