# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# %matplotlib inline
from __future__ import division, print_function
from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets
import matplotlib.pyplot as plt
from sklearn.cross_validation import StratifiedShuffleSplit
from sklearn.neighbors import KNeighborsClassifier
from sklearn.decomposition import PCA
import numpy as np
import time

mnist_dir = '../dat'
dat = read_data_sets(mnist_dir)
train_size = 100
sss = StratifiedShuffleSplit(dat.train.labels,train_size=train_size, 
                             test_size=dat.train.labels.shape[0]-train_size, n_iter=1, random_state = 1234)
i_tr, i_unlab = [i for i in sss][0]
Xtr = dat.train.images[i_tr]
ytr = dat.train.labels[i_tr]
Xunl = dat.train.images[i_unlab]
yunl = dat.train.labels[i_unlab]
Xval = dat.validation.images
yval = dat.validation.labels

# # Ordinary discriminative CNN using propagated labels

y_train = np.hstack((ytr, yprop))
y_train = np.eye(10)[y_train]
ytr_oh = np.eye(10)[ytr]
yprop_oh = np.eye(10)[yprop]

# +
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import Adam
from keras.optimizers import SGD
from keras.callbacks import EarlyStopping
from keras.utils import np_utils
from sklearn.metrics import log_loss
from keras.regularizers import WeightRegularizer


def create_model(img_rows, img_cols,learning_rate, reg, decay, momentum=.9, dropout = .5):
    model = Sequential()
    model.add(Convolution2D(128, 5, 5, border_mode='same', init='he_normal', W_regularizer=WeightRegularizer(l1=reg),
                            input_shape=(1, img_rows, img_cols)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(312, 3, 3, border_mode='same', init='he_normal', W_regularizer=WeightRegularizer(l1=reg)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(172, 3, 3, border_mode='same', init='he_normal', W_regularizer=WeightRegularizer(l1=reg)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(172, 3, 3, border_mode='same', init='he_normal', W_regularizer=WeightRegularizer(l1=reg)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Flatten())
    model.add(Dense(48))
    
    model.add(Dense(10))
    model.add(Activation('softmax'))

    sgd = SGD(lr=learning_rate, decay=decay, momentum=0.9, nesterov=True)
    model.compile(optimizer=sgd, loss='categorical_crossentropy')
    return model


# +
batch_size = 32
nb_epoch = 50
dropout = .60
reg = 1e-5
learning_rate = 1e-4
decay = 1e-5
momentum = .95
prop_weight = .1

sample_weight = np.ones((x_train.shape[0],))
sample_weight[train_size:] /= prop_weight

model = create_model(28, 28, learning_rate=learning_rate, reg=reg, decay = decay, dropout = dropout)

model.fit(x_train, np.vstack((ytr_oh, yprop_oh)), batch_size=batch_size, nb_epoch=nb_epoch,
      shuffle=True, verbose=2, validation_data=(x_test, y_test),
      callbacks=[EarlyStopping(monitor='val_loss', patience=4, verbose=0)],
            sample_weight=sample_weight)

predictions_valid = model.predict(x_test, verbose=1)

print('Accuracy: {}'.format((y_test.argmax(axis=1)==predictions_valid.argmax(axis=1)).mean()))
# -

#doublecheck on holdout set
x_ho = dat.test.images.reshape(dat.test.images.shape[0],1,28,28)
predictions_test = model.predict(x_ho)
print('Accuracy: {}'.format((dat.test.labels==predictions_test.argmax(axis=1)).mean()))

StarcoderdataJupyterScriptsDedupFiltered