stackedAutoencoder.py
def executeStackedAutoencoder():
    
    """ Define the parameters of the first Autoencoder """
    
    visible_size   = 784    # size of input vector
    hidden_size1   = 200    # size of hidden layer vector of first autoencoder
    hidden_size2   = 200    # size of hidden layer vector of second autoencoder
    rho            = 0.1    # desired average activation of hidden units
    lamda          = 0.003  # weight decay parameter
    beta           = 3      # weight of sparsity penalty term
    max_iterations = 200    # number of optimization iterations
    num_classes    = 10     # number of classes
    
    """ Load MNIST images for training and testing """
    
    train_data    = loadMNISTImages('train-images.idx3-ubyte')
    train_labels  = loadMNISTLabels('train-labels.idx1-ubyte')
    
    """ Initialize the first Autoencoder with the above parameters """
    
    encoder1 = SparseAutoencoder(visible_size, hidden_size1, rho, lamda, beta)
    
    """ Run the L-BFGS algorithm to get the optimal parameter values """
    
    opt_solution   = scipy.optimize.minimize(encoder1.sparseAutoencoderCost, encoder1.theta, 
                                             args = (train_data,), method = 'L-BFGS-B', 
                                             jac = True, options = {'maxiter': max_iterations})
    sae1_opt_theta = opt_solution.x
    
    """ Get the features corresponding to first Autoencoder """
    
    sae1_features = feedForwardAutoencoder(sae1_opt_theta, hidden_size1, visible_size, train_data)
    
    """ Initialize the second Autoencoder with the above parameters """
    
    encoder2 = SparseAutoencoder(hidden_size1, hidden_size2, rho, lamda, beta)
    
    """ Run the L-BFGS algorithm to get the optimal parameter values """
    
    opt_solution   = scipy.optimize.minimize(encoder2.sparseAutoencoderCost, encoder2.theta, 
                                             args = (sae1_features,), method = 'L-BFGS-B', 
                                             jac = True, options = {'maxiter': max_iterations})
    sae2_opt_theta = opt_solution.x
    
    """ Get the features corresponding to second Autoencoder """
    
    sae2_features = feedForwardAutoencoder(sae2_opt_theta, hidden_size2, hidden_size1, sae1_features)
    
    """ Initialize Softmax Regressor with the above parameters """
    
    regressor = SoftmaxRegression(hidden_size2, num_classes, lamda)
    
    """ Run the L-BFGS algorithm to get the optimal parameter values """
    
    opt_solution      = scipy.optimize.minimize(regressor.softmaxCost, regressor.theta, 
                                                args = (sae2_features, train_labels,), method = 'L-BFGS-B', 
                                                jac = True, options = {'maxiter': max_iterations})
    softmax_opt_theta = opt_solution.x
    
    """ Create a stack of the Stacked Autoencoder parameters """
    
    stack = {}
    stack[0, "W"] = sae1_opt_theta[encoder1.limit0 : encoder1.limit1].reshape(hidden_size1, visible_size)
    stack[1, "W"] = sae2_opt_theta[encoder2.limit0 : encoder2.limit1].reshape(hidden_size2, hidden_size1)
    stack[0, "b"] = sae1_opt_theta[encoder1.limit2 : encoder1.limit3].reshape(hidden_size1, 1)
    stack[1, "b"] = sae2_opt_theta[encoder2.limit2 : encoder2.limit3].reshape(hidden_size2, 1)
    
    """ Create a vector of the Stacked Autoencoder parameters for optimization """
    
    stack_params     = stack2Params(stack)
    stacked_ae_theta = numpy.concatenate((softmax_opt_theta.flatten(), stack_params.flatten()))
    
    """ Create a neural network configuration, with number of units in each layer """
    
    net_config = [visible_size, hidden_size1, hidden_size2, num_classes]
    
    """ Load MNIST test images and labels """
    
    test_data   = loadMNISTImages('t10k-images.idx3-ubyte') 
    test_labels = loadMNISTLabels('t10k-labels.idx1-ubyte')
    
    """ Get predictions after greedy training """
    
    predictions = stackedAutoencoderPredict(stacked_ae_theta, net_config, test_data)
    
    """ Print accuracy of the trained model """
    
    correct = test_labels[:, 0] == predictions[:, 0]
    print """Accuracy after greedy training :""", numpy.mean(correct)
    
    """ Finetune the greedily trained model """
    
    opt_solution = scipy.optimize.minimize(stackedAutoencoderCost, stacked_ae_theta, 
                                           args = (net_config, lamda, train_data, train_labels,),
                                           method = 'L-BFGS-B', jac = True, options = {'maxiter': max_iterations})
    stacked_ae_opt_theta = opt_solution.x
    
    """ Get predictions after finetuning """
    
    predictions = stackedAutoencoderPredict(stacked_ae_opt_theta, net_config, test_data)
    
    """ Print accuracy of the trained model """
    
    correct = test_labels[:, 0] == predictions[:, 0]
    print """Accuracy after finetuning :""", numpy.mean(correct)
 Define the parameters of the first Autoencoder 
Define the parameters of the first Autoencoder
Define the parameters of the first Autoencoder
def executeStackedAutoencoder():
    visible_size   = 784    
    hidden_size1   = 200    
    hidden_size2   = 200    
    rho            = 0.1    
    lamda          = 0.003  
    beta           = 3      
    max_iterations = 200    
    num_classes    = 10     
    train_data    = loadMNISTImages('train-images.idx3-ubyte')
    train_labels  = loadMNISTLabels('train-labels.idx1-ubyte')
    encoder1 = SparseAutoencoder(visible_size, hidden_size1, rho, lamda, beta)
    opt_solution   = scipy.optimize.minimize(encoder1.sparseAutoencoderCost, encoder1.theta, 
                                             args = (train_data,), method = 'L-BFGS-B', 
                                             jac = True, options = {'maxiter': max_iterations})
    sae1_opt_theta = opt_solution.x
    sae1_features = feedForwardAutoencoder(sae1_opt_theta, hidden_size1, visible_size, train_data)
    encoder2 = SparseAutoencoder(hidden_size1, hidden_size2, rho, lamda, beta)
    opt_solution   = scipy.optimize.minimize(encoder2.sparseAutoencoderCost, encoder2.theta, 
                                             args = (sae1_features,), method = 'L-BFGS-B', 
                                             jac = True, options = {'maxiter': max_iterations})
    sae2_opt_theta = opt_solution.x
    sae2_features = feedForwardAutoencoder(sae2_opt_theta, hidden_size2, hidden_size1, sae1_features)
    regressor = SoftmaxRegression(hidden_size2, num_classes, lamda)
    opt_solution      = scipy.optimize.minimize(regressor.softmaxCost, regressor.theta, 
                                                args = (sae2_features, train_labels,), method = 'L-BFGS-B', 
                                                jac = True, options = {'maxiter': max_iterations})
    softmax_opt_theta = opt_solution.x
    stack = {}
    stack[0, "W"] = sae1_opt_theta[encoder1.limit0 : encoder1.limit1].reshape(hidden_size1, visible_size)
    stack[1, "W"] = sae2_opt_theta[encoder2.limit0 : encoder2.limit1].reshape(hidden_size2, hidden_size1)
    stack[0, "b"] = sae1_opt_theta[encoder1.limit2 : encoder1.limit3].reshape(hidden_size1, 1)
    stack[1, "b"] = sae2_opt_theta[encoder2.limit2 : encoder2.limit3].reshape(hidden_size2, 1)
    stack_params     = stack2Params(stack)
    stacked_ae_theta = numpy.concatenate((softmax_opt_theta.flatten(), stack_params.flatten()))
    net_config = [visible_size, hidden_size1, hidden_size2, num_classes]
    test_data   = loadMNISTImages('t10k-images.idx3-ubyte') 
    test_labels = loadMNISTLabels('t10k-labels.idx1-ubyte')
    predictions = stackedAutoencoderPredict(stacked_ae_theta, net_config, test_data)
    correct = test_labels[:, 0] == predictions[:, 0]
    print """Accuracy after greedy training :""", numpy.mean(correct)
    opt_solution = scipy.optimize.minimize(stackedAutoencoderCost, stacked_ae_theta, 
                                           args = (net_config, lamda, train_data, train_labels,),
                                           method = 'L-BFGS-B', jac = True, options = {'maxiter': max_iterations})
    stacked_ae_opt_theta = opt_solution.x
    predictions = stackedAutoencoderPredict(stacked_ae_opt_theta, net_config, test_data)
    correct = test_labels[:, 0] == predictions[:, 0]
    print """Accuracy after finetuning :""", numpy.mean(correct)
def executeStackedAutoencoder ( ) : visible_size = 784 hidden_size1 = 200 hidden_size2 = 200 rho = 0.1 lamda = 0.003 beta = 3 max_iterations = 200 num_classes = 10 """ Load MNIST images for training and testing """ train_data = loadMNISTImages ( 'train-images.idx3-ubyte' ) train_labels = loadMNISTLabels ( 'train-labels.idx1-ubyte' ) """ Initialize the first Autoencoder with the above parameters """ encoder1 = SparseAutoencoder ( visible_size , hidden_size1 , rho , lamda , beta ) """ Run the L-BFGS algorithm to get the optimal parameter values """ opt_solution = scipy . optimize . minimize ( encoder1 . sparseAutoencoderCost , encoder1 . theta , args = ( train_data , ) , method = 'L-BFGS-B' , jac = True , options = { 'maxiter' : max_iterations } ) sae1_opt_theta = opt_solution . x """ Get the features corresponding to first Autoencoder """ sae1_features = feedForwardAutoencoder ( sae1_opt_theta , hidden_size1 , visible_size , train_data ) """ Initialize the second Autoencoder with the above parameters """ encoder2 = SparseAutoencoder ( hidden_size1 , hidden_size2 , rho , lamda , beta ) """ Run the L-BFGS algorithm to get the optimal parameter values """ opt_solution = scipy . optimize . minimize ( encoder2 . sparseAutoencoderCost , encoder2 . theta , args = ( sae1_features , ) , method = 'L-BFGS-B' , jac = True , options = { 'maxiter' : max_iterations } ) sae2_opt_theta = opt_solution . x """ Get the features corresponding to second Autoencoder """ sae2_features = feedForwardAutoencoder ( sae2_opt_theta , hidden_size2 , hidden_size1 , sae1_features ) """ Initialize Softmax Regressor with the above parameters """ regressor = SoftmaxRegression ( hidden_size2 , num_classes , lamda ) """ Run the L-BFGS algorithm to get the optimal parameter values """ opt_solution = scipy . optimize . minimize ( regressor . softmaxCost , regressor . theta , args = ( sae2_features , train_labels , ) , method = 'L-BFGS-B' , jac = True , options = { 'maxiter' : max_iterations } ) softmax_opt_theta = opt_solution . x """ Create a stack of the Stacked Autoencoder parameters """ stack = { } stack [ 0 , "W" ] = sae1_opt_theta [ encoder1 . limit0 : encoder1 . limit1 ] . reshape ( hidden_size1 , visible_size ) stack [ 1 , "W" ] = sae2_opt_theta [ encoder2 . limit0 : encoder2 . limit1 ] . reshape ( hidden_size2 , hidden_size1 ) stack [ 0 , "b" ] = sae1_opt_theta [ encoder1 . limit2 : encoder1 . limit3 ] . reshape ( hidden_size1 , 1 ) stack [ 1 , "b" ] = sae2_opt_theta [ encoder2 . limit2 : encoder2 . limit3 ] . reshape ( hidden_size2 , 1 ) """ Create a vector of the Stacked Autoencoder parameters for optimization """ stack_params = stack2Params ( stack ) stacked_ae_theta = numpy . concatenate ( ( softmax_opt_theta . flatten ( ) , stack_params . flatten ( ) ) ) """ Create a neural network configuration, with number of units in each layer """ net_config = [ visible_size , hidden_size1 , hidden_size2 , num_classes ] """ Load MNIST test images and labels """ test_data = loadMNISTImages ( 't10k-images.idx3-ubyte' ) test_labels = loadMNISTLabels ( 't10k-labels.idx1-ubyte' ) """ Get predictions after greedy training """ predictions = stackedAutoencoderPredict ( stacked_ae_theta , net_config , test_data ) """ Print accuracy of the trained model """ correct = test_labels [ : , 0 ] == predictions [ : , 0 ] print """Accuracy after greedy training :""" , numpy . mean ( correct ) """ Finetune the greedily trained model """ opt_solution = scipy . optimize . minimize ( stackedAutoencoderCost , stacked_ae_theta , args = ( net_config , lamda , train_data , train_labels , ) , method = 'L-BFGS-B' , jac = True , options = { 'maxiter' : max_iterations } ) stacked_ae_opt_theta = opt_solution . x """ Get predictions after finetuning """ predictions = stackedAutoencoderPredict ( stacked_ae_opt_theta , net_config , test_data ) """ Print accuracy of the trained model """ correct = test_labels [ : , 0 ] == predictions [ : , 0 ] print """Accuracy after finetuning :""" , numpy . mean ( correct )
Define the parameters of the first Autoencoder
Define the parameters of the first Autoencoder
""" Define the parameters of the first Autoencoder """ # size of input vector # size of hidden layer vector of first autoencoder # size of hidden layer vector of second autoencoder # desired average activation of hidden units # weight decay parameter # weight of sparsity penalty term # number of optimization iterations # number of classes """ Load MNIST images for training and testing """ """ Initialize the first Autoencoder with the above parameters """ """ Run the L-BFGS algorithm to get the optimal parameter values """ """ Get the features corresponding to first Autoencoder """ """ Initialize the second Autoencoder with the above parameters """ """ Run the L-BFGS algorithm to get the optimal parameter values """ """ Get the features corresponding to second Autoencoder """ """ Initialize Softmax Regressor with the above parameters """ """ Run the L-BFGS algorithm to get the optimal parameter values """ """ Create a stack of the Stacked Autoencoder parameters """ """ Create a vector of the Stacked Autoencoder parameters for optimization """ """ Create a neural network configuration, with number of units in each layer """ """ Load MNIST test images and labels """ """ Get predictions after greedy training """ """ Print accuracy of the trained model """ """ Finetune the greedily trained model """ """ Get predictions after finetuning """ """ Print accuracy of the trained model """