import pandas as pd

"""
Explanation: <p style="text-align: center;"> Y Py?</p>

<img src="https://dl.dropboxusercontent.com/u/5880397/zendesk.jpg" width=500>
Chris Hausler
Data Engineer @ Zendesk
<br><br>
 <center><img src="https://dl.dropboxusercontent.com/u/5880397/anaconda_logo_web.png"/><br>
 <b><a href="http://continuum.io/downloads">http://continuum.io/downloads</a></b><br/><br/><br>
Completely free enterprise-ready Python distribution for large-scale<br><br> data processing, predictive analytics, and scientific computing
</center>
What I want from my data stack

Munging
Plotting
Learning
Productionising
Collaborating

Munging
And introduction to Pandas
Pandas
End of explanation
"""


ages = pd.DataFrame([['John', 25],
                     ['Mary', 9],
                     ['Radek', 16],
                     ['Mia', 64],
                     ['Geroge', 4],
                     ['Katrin', 21]])
ages

"""
Explanation: Data Frames
End of explanation
"""


ages.columns = ['Name', 'Age']
ages.set_index('Name', inplace=True)
ages

"""
Explanation: Column Names and Index
End of explanation
"""


ages.ix[['John', 'Mary']]

ages.ix[['John', 'Mary', 'Thomas']]

"""
Explanation: Using the Index
End of explanation
"""


ages * 2

"""
Explanation: Basic Arithmetic
End of explanation
"""


genders = pd.DataFrame(['male', 'female', 'male', 'female'],
                       index=['John', 'Mary', 'Alberto', 'Karyn'],
                       columns=['Gender'])
genders

"""
Explanation: Another DataFrame
End of explanation
"""


people = genders.join(ages, how='left')
people

"""
Explanation: Joins
End of explanation
"""


people

"""
Explanation: Dealing with Missing Values
End of explanation
"""


people.dropna()

"""
Explanation: 1. Get rid of them
End of explanation
"""


people.fillna(people.Age.mean())

"""
Explanation: 2. Fill them with something
End of explanation
"""


people.fillna(method='ffill')

"""
Explanation: 3. Use a function like forward fill
End of explanation
"""


# get the data from here
# https://data.melbourne.vic.gov.au/api/views/b2ak-trbp/rows.csv?accessType=DOWNLOAD
data = pd.read_csv('pedestrian_count.csv', index_col=0, parse_dates=[0])
data.head()

"""
Explanation: Loading Data

read_csv
read_excel
read_hdf
read_sql
read_json
read_clipboard
....

Get some real data
End of explanation
"""


data.info()

"""
Explanation: Basic Info
End of explanation
"""


data.describe()

"""
Explanation: Descriptive stats
End of explanation
"""


import pylab as plt
import seaborn as sns
sns.set_context('poster')

"""
Explanation: Plotting
And advanced Pandas operations
End of explanation
"""


bridges = data[['Webb Bridge', 'Princes Bridge', 'Sandridge Bridge']]
ax = bridges.plot(figsize=(10, 6))
_ = ax.set_ylabel('# Pedestrians')

"""
Explanation: Bridge pedestrian counts
End of explanation
"""


ax = bridges.resample('D', how='sum').plot(figsize=(10, 6))
_ = ax.set_ylabel('# Pedestrians')

"""
Explanation: Daily totals for pedestrians
End of explanation
"""


axs = bridges.resample('M', how='sum').plot(subplots=True, figsize=(12, 7.5))
_ = axs[1].set_ylabel('# Pedestrians over the Month')

"""
Explanation: Monthly Pedestrian Volume
End of explanation
"""


dt = data.ix['2014-07']['Flagstaff Station']
ax = dt.groupby(dt.index.hour).median().plot(kind='bar', figsize=(10, 6))
ax.set_ylabel('# Pedestrians')
_ = ax.set_xlabel('Hour of Day')

"""
Explanation: Median Pedestrians per hour, Flagstaff July 2014
End of explanation
"""


fig, ax = plt.subplots(1, figsize=(8, 4))
data.resample('D', how='sum').boxplot(ax=ax)
ax.set_xticklabels(data.columns, rotation=90, size=16)
ax.set_ylim(0, 40000)
_ = ax.set_ylabel('# Pedestrians')

"""
Explanation: Daily Pedestrian Variation
End of explanation
"""


with sns.axes_style("white"):
    sns.jointplot('Princes Bridge', 'Flinders St Underpass', data, size=7);

"""
Explanation: Scatter Plots
End of explanation
"""


# get data here https://www.kaggle.com/c/titanic/download/train.csv
titanic = pd.read_csv('train.csv').drop(['Name', 'Ticket', 'PassengerId'],
                                        axis=1)
titanic.head()

"""
Explanation: (Machine) Learning
And Introduction to Scikit-Learn
<img src="https://dl.dropboxusercontent.com/u/5880397/ml_map.png" width=1000/>
Load some Titanic Data
End of explanation
"""


titanic["Alone"] = ((titanic.Parch + titanic.SibSp) == 0) * 1
titanic['Cabin'] = titanic.Cabin.fillna('NA')
titanic['Embarked'] = titanic.Embarked.fillna('NA')
titanic['Age'] = titanic.Age.fillna(titanic.Age.median())
titanic.head()

"""
Explanation: Make some new features / cleanup
End of explanation
"""


g = sns.factorplot("Pclass", "Survived", "Sex",
                    data=titanic, kind="bar",
                    size=6, palette="muted")
g.despine(left=True)
_ = g.set_ylabels("Survival Probability")

"""
Explanation: Survival Probability
End of explanation
"""


from sklearn.preprocessing import LabelBinarizer
lbl = LabelBinarizer().fit(titanic.Embarked)
print lbl.classes_
print lbl.transform(titanic.Embarked)

"""
Explanation: Building Categorical Features
End of explanation
"""


import numpy as np

lbl = LabelBinarizer()
X_categorical = np.hstack([lbl.fit_transform(titanic[c])
                           for c in ['Cabin', 'Embarked', 'Sex']])

print 'Array shape:', X_categorical.shape
X_categorical

"""
Explanation: We can do them all at once
End of explanation
"""


y = titanic.pop('Survived').values

X_numeric = titanic._get_numeric_data().values
X = np.hstack([X_numeric, X_categorical])

"""
Explanation: The Dataset
End of explanation
"""


from sklearn.cross_validation import cross_val_score
from sklearn.linear_model import SGDClassifier

clf = SGDClassifier(loss='log')
scores = cross_val_score(clf, X, y, cv=3, scoring='accuracy')
print "Accuracy: {:.2f}".format(scores.mean())

"""
Explanation: First Pass Cross-Validation
End of explanation
"""


for a in [0.0001, 0.001, 0.01, 0.1, 1., 10, 100]:
    clf = SGDClassifier(loss='log', alpha=a)
    scores = cross_val_score(clf, X, y, cv=3, scoring='accuracy')    
    print "Alpha: {:.4f}\tAccuracy: {:.2f}".format(a, scores.mean())

"""
Explanation: Can we do better?
End of explanation
"""


from sklearn.grid_search import RandomizedSearchCV

params = {'alpha': np.logspace(-4, 4, 50),
          'loss': ['log', 'modified_huber', 'perceptron'],
          'penalty': ['l1', 'l2'],
          'n_iter': [50, 100, 200]}
clf = SGDClassifier()
random_search  = RandomizedSearchCV(clf, params, n_iter=100,
                                    scoring='accuracy', n_jobs=4,
                                    verbose=1)
random_search.fit(X, y)
print "Best Accuracy: {:.2f}".format(random_search.best_score_)
print random_search.best_params_

"""
Explanation: Can we do even better??
End of explanation
"""


from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB

classifiers = [
    SGDClassifier(),
    KNeighborsClassifier(3),
    SVC(kernel="linear"),
    SVC(gamma=2),
    DecisionTreeClassifier(),
    RandomForestClassifier(),
    AdaBoostClassifier(),
    GaussianNB(),
    LogisticRegression(class_weight='auto')]


"""
Explanation: What about different classifiers?
End of explanation
"""


res = []
names = []
for clf in classifiers:
    scores = cross_val_score(clf, X, y, cv=5, scoring='accuracy')
    names.append(clf.__class__.__name__)
    res.append(scores.mean())

"""
Explanation: Train them...
End of explanation
"""


fig, ax = plt.subplots(1, figsize=(14, 6))
sns.barplot(np.array(names), np.array(res), ci=None,  palette="muted", ax=ax)
ax.set_ylabel("Accuracy")
_ = plt.xticks(rotation=50, ha='right', size=12)


"""
Explanation: Compare them
End of explanation
"""


