from __future__ import print_function, unicode_literals, absolute_import, division
from six.moves import range, zip, map, reduce, filter

import numpy as np
import matplotlib.pyplot as plt
from IPython import display
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

import seaborn as sns
sns.set_style('whitegrid')
plt.rc('figure', figsize=(7.0, 5.0))

import keras
from keras import backend as K
from keras.models import Sequential, Model
from keras.layers import Input, Dense, Activation
from keras.optimizers import Adam
from keras.callbacks import LambdaCallback
from keras.utils import np_utils

def plot_callback(func,p=20):
    def plot_epoch_end(epoch,logs):
        if epoch == 0 or (epoch+1) % p == 0:
            plt.clf(); func(); plt.title('epoch %d' % (epoch+1))
            display.clear_output(wait=True); display.display(plt.gcf())
    def clear(*args):
        plt.clf()
    return LambdaCallback(on_epoch_end=plot_epoch_end,on_train_end=clear)

def plot_loss_acc(hist):
    plt.figure(figsize=(15,4));
    if len(hist.params['metrics']) == 2:
        plt.subplot(121); plt.semilogy(hist.epoch,hist.history['loss'])
        plt.xlabel('epoch'); plt.ylabel('loss'); plt.legend(['train'],loc='upper right')
        plt.subplot(122); plt.plot(hist.epoch,hist.history['acc']);
        plt.xlabel('epoch'); plt.ylabel('accuracy'); plt.legend(['train'],loc='lower right');
    else:
        plt.subplot(121); plt.semilogy(hist.epoch,hist.history['loss'], hist.epoch,hist.history['val_loss']);
        plt.xlabel('epoch'); plt.ylabel('loss'); plt.legend(['train','test'],loc='upper right')
        plt.subplot(122); plt.plot(hist.epoch,hist.history['acc'], hist.epoch,hist.history['val_acc'])
        plt.xlabel('epoch'); plt.ylabel('accuracy'); plt.legend(['train','test'],loc='lower right');    

"""
Explanation: Setup
End of explanation
"""


iris = sns.load_dataset("iris")
iris.sample(10)

"""
Explanation: Adapted from http://blog.fastforwardlabs.com/2016/02/24/hello-world-in-keras-or-scikit-learn-versus.html

Dataset
Iris flower data set
End of explanation
"""


sns.pairplot(iris, hue='species');

"""
Explanation: Image from https://commons.wikimedia.org/wiki/File:Petal-sepal.jpg
End of explanation
"""


def label_encode(arr):
    uniques, ids = np.unique(arr, return_inverse=True)
    return ids

classes = ('setosa', 'versicolor', 'virginica')
labels  = label_encode(classes)
for i,c in enumerate(classes):
    print('%10s → %d' % (c, labels[i]))

def onehot_encode(arr):
    uniques, ids = np.unique(arr, return_inverse=True)
    return np_utils.to_categorical(ids, len(uniques))

classes = ('setosa', 'versicolor', 'virginica')
onehot = onehot_encode(classes)
for i,c in enumerate(classes):
    print('%10s → [%d,%d,%d]' % (c, onehot[i,0], onehot[i,1], onehot[i,2]))

"""
Explanation: Label Encoding
End of explanation
"""


data = iris
feature_name = 'petal_length'
data = data[[feature_name,'species']]

X    = data.values[:,0]
y    = label_encode(data.values[:,1])
y_oh = onehot_encode(data.values[:,1])
N    = len(y)

"""
Explanation: Multiclass Classification
Data subset (1 feature only)
End of explanation
"""


R = np.linspace(X.min()-1,X.max()+1,100)
Xp = np.zeros(X.shape[0])-.1
Rp = np.zeros(R.shape[0])-.2
def plot_all(model=None):
    plt.scatter(X, Xp, c=y, cmap='jet');
    plt.xlabel(feature_name)
    if model is not None:
        prob = model.predict(R)
        yhat = np.argmax(prob,axis=1)
        plt.scatter(R, Rp, c=yhat);
        plt.plot(R,prob)
        leg = plt.legend(map(lambda s:'p("%s")'%s,classes),loc='upper center',frameon=False,ncol=3)
    plt.xlim(X.min()-1.5,X.max()+1.5)
    plt.ylim(-.4,1.2)
plot_all()

"""
Explanation: Plotting
End of explanation
"""


model = Sequential()
model.add(Dense(16, input_shape=(1,)))
model.add(Activation('tanh'))
model.add(Dense(3))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()

"""
Explanation: Model (Logistic Regression)

https://en.wikipedia.org/wiki/Multinomial_logistic_regression
https://en.wikipedia.org/wiki/Softmax_function
https://en.wikipedia.org/wiki/Cross_entropy
End of explanation
"""


hist = model.fit(X,y_oh,batch_size=5,epochs=300,verbose=0,
                 callbacks=[plot_callback(lambda:plot_all(model))]);

plot_loss_acc(hist)

"""
Explanation: Softmax
Converts arbitrary "scores" to normalized probabilities:
$
\large
\sigma(\mathbf{z})_i = \frac{\exp(z_i)}{\sum_j \exp(z_j)}
$
Example: for $\mathbf{z} = [0.451, -0.599, 0.006]$, we get
$\sigma(\mathbf{z}) = [ 0.50232021,  0.1757808 ,  0.32189899]$.
Cross entropy
$H(p, q) = \mathrm{E}p[-\log q] = H(p) + D{\mathrm{KL}}(p \| q)$
defines the cross entropy for distributions $p$ and $q$, where
- $H(p)$ is the entropy of $p$, and
- $D_{\mathrm{KL}}(p \| q)$ is the Kullback–Leibler divergence of $q$ from $p$.
End of explanation
"""


N = iris.shape[0] # number of data points / table rows
data = iris.sample(N,replace=False) # shuffle data
X    = data.values[:,0:4]
y_oh = onehot_encode(data.values[:,4])

N_train = N//2 # random 50/50 train/test split
X_train, y_train = X[:N_train], y_oh[:N_train]
X_test,  y_test  = X[N_train:], y_oh[N_train:]

model = Sequential()
model.add(Dense(16, input_shape=(4,)))
model.add(Activation('tanh'))
model.add(Dense(3))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()

hist = model.fit(X_train, y_train, validation_data=(X_test,y_test), epochs=200, verbose=0, batch_size=5)

plot_loss_acc(hist)

loss, accuracy = model.evaluate(X_train, y_train, verbose=0)
print('train set: loss = %.5f, accuracy = %.5f' % (loss,accuracy))
loss, accuracy = model.evaluate(X_test, y_test, verbose=0)
print('test  set: loss = %.5f, accuracy = %.5f' % (loss,accuracy))

"""
Explanation: Full dataset (all 4 features)
Use all features and split dataset in train and test subsets:
End of explanation
"""


N_train = 20 # only 20 of 150 samples for training, rest for testing
X_train, y_train = X[:N_train], y_oh[:N_train]
X_test,  y_test  = X[N_train:], y_oh[N_train:]

model = Sequential()
model.add(Dense(16, input_shape=(4,)))
model.add(Activation('tanh'))
model.add(Dense(16))
model.add(Activation('tanh'))
model.add(Dense(16))
model.add(Activation('tanh'))
model.add(Dense(3))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()

hist = model.fit(X_train, y_train, validation_data=(X_test,y_test), epochs=1000, verbose=0, batch_size=5)

plot_loss_acc(hist)

loss, accuracy = model.evaluate(X_train, y_train, verbose=0)
print('train set: loss = %.5f, accuracy = %.5f' % (loss,accuracy))
loss, accuracy = model.evaluate(X_test, y_test, verbose=0)
print('test  set: loss = %.5f, accuracy = %.5f' % (loss,accuracy))

"""
Explanation: Overfitting
End of explanation
"""


