stackedAutoencoder.py
def sparseAutoencoderCost(self, theta, input):
        
        """ Extract weights and biases from 'theta' input """
        
        W1 = theta[self.limit0 : self.limit1].reshape(self.hidden_size, self.visible_size)
        W2 = theta[self.limit1 : self.limit2].reshape(self.visible_size, self.hidden_size)
        b1 = theta[self.limit2 : self.limit3].reshape(self.hidden_size, 1)
        b2 = theta[self.limit3 : self.limit4].reshape(self.visible_size, 1)
        
        """ Compute output layers by performing a feedforward pass
            Computation is done for all the training inputs simultaneously """
        
        hidden_layer = sigmoid(numpy.dot(W1, input) + b1)
        output_layer = sigmoid(numpy.dot(W2, hidden_layer) + b2)
        
        """ Estimate the average activation value of the hidden layers """
        
        rho_cap = numpy.sum(hidden_layer, axis = 1) / input.shape[1]
        
        """ Compute intermediate difference values using Backpropagation algorithm """
        
        diff = output_layer - input
        
        sum_of_squares_error = 0.5 * numpy.sum(numpy.multiply(diff, diff)) / input.shape[1]
        weight_decay         = 0.5 * self.lamda * (numpy.sum(numpy.multiply(W1, W1)) +
                                                   numpy.sum(numpy.multiply(W2, W2)))
        KL_divergence        = self.beta * numpy.sum(self.rho * numpy.log(self.rho / rho_cap) +
                                                    (1 - self.rho) * numpy.log((1 - self.rho) / (1 - rho_cap)))
        cost                 = sum_of_squares_error + weight_decay + KL_divergence
        
        KL_div_grad = self.beta * (-(self.rho / rho_cap) + ((1 - self.rho) / (1 - rho_cap)))
        
        del_out = numpy.multiply(diff, numpy.multiply(output_layer, 1 - output_layer))
        del_hid = numpy.multiply(numpy.dot(numpy.transpose(W2), del_out) + numpy.transpose(numpy.matrix(KL_div_grad)), 
                                 numpy.multiply(hidden_layer, 1 - hidden_layer))
        
        """ Compute the gradient values by averaging partial derivatives
            Partial derivatives are averaged over all training examples """
            
        W1_grad = numpy.dot(del_hid, numpy.transpose(input))
        W2_grad = numpy.dot(del_out, numpy.transpose(hidden_layer))
        b1_grad = numpy.sum(del_hid, axis = 1)
        b2_grad = numpy.sum(del_out, axis = 1)
            
        W1_grad = W1_grad / input.shape[1] + self.lamda * W1
        W2_grad = W2_grad / input.shape[1] + self.lamda * W2
        b1_grad = b1_grad / input.shape[1]
        b2_grad = b2_grad / input.shape[1]
        
        """ Transform numpy matrices into arrays """
        
        W1_grad = numpy.array(W1_grad)
        W2_grad = numpy.array(W2_grad)
        b1_grad = numpy.array(b1_grad)
        b2_grad = numpy.array(b2_grad)
        
        """ Unroll the gradient values and return as 'theta' gradient """
        
        theta_grad = numpy.concatenate((W1_grad.flatten(), W2_grad.flatten(),
                                        b1_grad.flatten(), b2_grad.flatten()))
                                        
        return [cost, theta_grad]
 Extract weights and biases from 'theta' input 
Extract weights and biases from 'theta' input
Extract weights and biases from ' theta ' input
def sparseAutoencoderCost(self, theta, input):
        W1 = theta[self.limit0 : self.limit1].reshape(self.hidden_size, self.visible_size)
        W2 = theta[self.limit1 : self.limit2].reshape(self.visible_size, self.hidden_size)
        b1 = theta[self.limit2 : self.limit3].reshape(self.hidden_size, 1)
        b2 = theta[self.limit3 : self.limit4].reshape(self.visible_size, 1)
        hidden_layer = sigmoid(numpy.dot(W1, input) + b1)
        output_layer = sigmoid(numpy.dot(W2, hidden_layer) + b2)
        rho_cap = numpy.sum(hidden_layer, axis = 1) / input.shape[1]
        diff = output_layer - input
        sum_of_squares_error = 0.5 * numpy.sum(numpy.multiply(diff, diff)) / input.shape[1]
        weight_decay         = 0.5 * self.lamda * (numpy.sum(numpy.multiply(W1, W1)) +
                                                   numpy.sum(numpy.multiply(W2, W2)))
        KL_divergence        = self.beta * numpy.sum(self.rho * numpy.log(self.rho / rho_cap) +
                                                    (1 - self.rho) * numpy.log((1 - self.rho) / (1 - rho_cap)))
        cost                 = sum_of_squares_error + weight_decay + KL_divergence
        KL_div_grad = self.beta * (-(self.rho / rho_cap) + ((1 - self.rho) / (1 - rho_cap)))
        del_out = numpy.multiply(diff, numpy.multiply(output_layer, 1 - output_layer))
        del_hid = numpy.multiply(numpy.dot(numpy.transpose(W2), del_out) + numpy.transpose(numpy.matrix(KL_div_grad)), 
                                 numpy.multiply(hidden_layer, 1 - hidden_layer))
        W1_grad = numpy.dot(del_hid, numpy.transpose(input))
        W2_grad = numpy.dot(del_out, numpy.transpose(hidden_layer))
        b1_grad = numpy.sum(del_hid, axis = 1)
        b2_grad = numpy.sum(del_out, axis = 1)
        W1_grad = W1_grad / input.shape[1] + self.lamda * W1
        W2_grad = W2_grad / input.shape[1] + self.lamda * W2
        b1_grad = b1_grad / input.shape[1]
        b2_grad = b2_grad / input.shape[1]
        W1_grad = numpy.array(W1_grad)
        W2_grad = numpy.array(W2_grad)
        b1_grad = numpy.array(b1_grad)
        b2_grad = numpy.array(b2_grad)
        theta_grad = numpy.concatenate((W1_grad.flatten(), W2_grad.flatten(),
                                        b1_grad.flatten(), b2_grad.flatten()))
        return [cost, theta_grad]
def sparseAutoencoderCost ( self , theta , input ) : W1 = theta [ self . limit0 : self . limit1 ] . reshape ( self . hidden_size , self . visible_size ) W2 = theta [ self . limit1 : self . limit2 ] . reshape ( self . visible_size , self . hidden_size ) b1 = theta [ self . limit2 : self . limit3 ] . reshape ( self . hidden_size , 1 ) b2 = theta [ self . limit3 : self . limit4 ] . reshape ( self . visible_size , 1 ) """ Compute output layers by performing a feedforward pass
            Computation is done for all the training inputs simultaneously """ hidden_layer = sigmoid ( numpy . dot ( W1 , input ) + b1 ) output_layer = sigmoid ( numpy . dot ( W2 , hidden_layer ) + b2 ) """ Estimate the average activation value of the hidden layers """ rho_cap = numpy . sum ( hidden_layer , axis = 1 ) / input . shape [ 1 ] """ Compute intermediate difference values using Backpropagation algorithm """ diff = output_layer - input sum_of_squares_error = 0.5 * numpy . sum ( numpy . multiply ( diff , diff ) ) / input . shape [ 1 ] weight_decay = 0.5 * self . lamda * ( numpy . sum ( numpy . multiply ( W1 , W1 ) ) + numpy . sum ( numpy . multiply ( W2 , W2 ) ) ) KL_divergence = self . beta * numpy . sum ( self . rho * numpy . log ( self . rho / rho_cap ) + ( 1 - self . rho ) * numpy . log ( ( 1 - self . rho ) / ( 1 - rho_cap ) ) ) cost = sum_of_squares_error + weight_decay + KL_divergence KL_div_grad = self . beta * ( - ( self . rho / rho_cap ) + ( ( 1 - self . rho ) / ( 1 - rho_cap ) ) ) del_out = numpy . multiply ( diff , numpy . multiply ( output_layer , 1 - output_layer ) ) del_hid = numpy . multiply ( numpy . dot ( numpy . transpose ( W2 ) , del_out ) + numpy . transpose ( numpy . matrix ( KL_div_grad ) ) , numpy . multiply ( hidden_layer , 1 - hidden_layer ) ) """ Compute the gradient values by averaging partial derivatives
            Partial derivatives are averaged over all training examples """ W1_grad = numpy . dot ( del_hid , numpy . transpose ( input ) ) W2_grad = numpy . dot ( del_out , numpy . transpose ( hidden_layer ) ) b1_grad = numpy . sum ( del_hid , axis = 1 ) b2_grad = numpy . sum ( del_out , axis = 1 ) W1_grad = W1_grad / input . shape [ 1 ] + self . lamda * W1 W2_grad = W2_grad / input . shape [ 1 ] + self . lamda * W2 b1_grad = b1_grad / input . shape [ 1 ] b2_grad = b2_grad / input . shape [ 1 ] """ Transform numpy matrices into arrays """ W1_grad = numpy . array ( W1_grad ) W2_grad = numpy . array ( W2_grad ) b1_grad = numpy . array ( b1_grad ) b2_grad = numpy . array ( b2_grad ) """ Unroll the gradient values and return as 'theta' gradient """ theta_grad = numpy . concatenate ( ( W1_grad . flatten ( ) , W2_grad . flatten ( ) , b1_grad . flatten ( ) , b2_grad . flatten ( ) ) ) return [ cost , theta_grad ]
Extract weights and biases from 'theta' input
Extract weights and biases from ' theta ' input
""" Extract weights and biases from 'theta' input """ """ Compute output layers by performing a feedforward pass
            Computation is done for all the training inputs simultaneously """ """ Estimate the average activation value of the hidden layers """ """ Compute intermediate difference values using Backpropagation algorithm """ """ Compute the gradient values by averaging partial derivatives
            Partial derivatives are averaged over all training examples """ """ Transform numpy matrices into arrays """ """ Unroll the gradient values and return as 'theta' gradient """
self
theta
input