%%bash
cd ~/Downloads
wget http://cybertron.cg.tu-berlin.de/eitz/projects/classifysketch/sketches_png.zip
unzip sketches_png.zip

files = !find ~/Downloads/png -name "*.png"
len(files)

"""
Explanation: Eitz et al asked non-expert humans to sketch objects of a given category and gather 20,000 unique sketches evenly distributed over 250 object categories.
@article{eitz2012hdhso,
    author={Eitz, Mathias and Hays, James and Alexa, Marc},
    title={How Do Humans Sketch Objects?},
    journal={ACM Trans. Graph. (Proc. SIGGRAPH)},
    year={2012},
    volume={31},
    number={4},
    pages = {44:1--44:10}
}

Download
Download png images (525MB):
End of explanation
"""


#Import libraries for doing image analysis
from skimage.io import imread
from skimage.transform import resize
from sklearn.ensemble import RandomForestClassifier as RF
import glob
import os
from sklearn import cross_validation
from sklearn.cross_validation import StratifiedKFold as KFold
from sklearn.metrics import classification_report
from matplotlib import pyplot as plt
from matplotlib import colors
from pylab import cm
from skimage import segmentation
from skimage.morphology import watershed
from skimage import measure
from skimage import morphology
import numpy as np
import pandas as pd
from scipy import ndimage
from skimage.feature import peak_local_max
# make graphics inline
%matplotlib inline

plt.figure(figsize=(12,3))
image = imread(files[0], as_grey=True)
imagethr = np.where(image > np.mean(image),0.,1.0)

plt.subplot(1,3,1)
plt.imshow(imagethr, cmap=cm.gray);
imdilated = morphology.dilation(imagethr, np.ones((16,16)))
plt.subplot(1,3,2)
plt.imshow(imdilated, cmap=cm.gray);

im1 = resize(imdilated,[56,56])
plt.subplot(1,3,3)
plt.imshow(im1, cmap=cm.gray);
# plt.show()

"""
Explanation: show sample sketch
End of explanation
"""


def process(fname):
    image = imread(fname, as_grey=True)
    imagethr = np.where(image > np.mean(image),0.,1.)
    imdilated = morphology.dilation(imagethr, np.ones((16,16)))
    im1 = resize(imdilated,[56,56])
    return im1.ravel().astype(np.int8)

"""
Explanation: Read
End of explanation
"""


import multiprocessing as mp
NUM_PROCESSES = 8
pool = mp.Pool(NUM_PROCESSES)
results = pool.map(process, files, chunksize=100)
pool.close()
pool.join()

y = np.array(map(lambda f: f.split('/')[-2], files))

X = np.array(results)
N, image_size = X.shape
D = int(np.sqrt(image_size))
N, image_size, D

"""
Explanation: forever divided by eight is also forever :(
End of explanation
"""


import random
s = random.sample(xrange(N),N)
X=X[s]
y=y[s]

y.dtype

N_train = int(0.9 * N)
N_test = N - N_train
N_train, N_test

"""
Explanation: shuffle the order of the images
End of explanation
"""


import fuel
datasource = 'sketch'
datasource_dir = os.path.join(fuel.config.data_path, datasource)
!mkdir -p {datasource_dir}
datasource_fname = os.path.join(datasource_dir , datasource+'.hdf5')
datasource_fname

import h5py
fp = h5py.File(datasource_fname, mode='w')
image_features = fp.create_dataset('features', (N, image_size), dtype='uint8')

image_features[...] = X

targets = fp.create_dataset('targets', (N, 1), dtype='S18')

targets[...] = y.reshape(-1,1)

from fuel.datasets.hdf5 import H5PYDataset
split_dict = {
    'train': {'features': (0, N_train), 'targets': (0, N_train)},
    'test': {'features': (N_train, N), 'targets': (N_train, N)}
}
fp.attrs['split'] = H5PYDataset.create_split_array(split_dict)

fp.flush()
fp.close()

!ls -l {datasource_fname}

!aws s3 cp {datasource_fname} s3://udidraw/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers

"""
Explanation: HDF5
Save all images in a format that can be used by fuel
End of explanation
"""


train_set = H5PYDataset(datasource_fname, which_set='train')

train_set.num_examples

train_set.provides_sources

handle = train_set.open()
data = train_set.get_data(handle, slice(0, 16))
data[0].shape,data[1].shape

plt.figure(figsize=(12,12))
for i in range(16):
    plt.subplot(4,4,i+1)
    plt.imshow(data[0][i].reshape(D,D), cmap=cm.gray)
    plt.title(data[1][i][0]);

train_set.close(handle)

"""
Explanation: Test
End of explanation
"""


