neural_network.py
def train_model(self, num_iterations, reg_lambda = 1e-2,
                    print_loss=False):
        """This function calculates the cost function and backpropagates the error"""
        dW = [np.zeros(self.weights[i].shape) for i in range(len(self.layers)-1)]
        db = [np.zeros((1, self.layers[i + 1])) for i in range(len(self.layers) - 1)]

        for i in range(0, num_iterations):
            random_indices = np.arange(self.num_examples)
            np.random.shuffle(random_indices)
            selection_array = random_indices[0:self.batch_size]
            batch_input = self.X[selection_array] # make sure this gets the whole set
            batch_output = self.Y[selection_array]

            probs = self.__forward_prop__(batch_input)
            derivative = self.output_layer.loss_function(probs, self.batch_size, batch_output)

            for j in range(len(self.layers) - 2,0,-1):
                dW[j] = (self.a[j-1].T).dot(derivative)
                dW[j] += reg_lambda *self.weights[j]
                db[j] = np.sum(derivative, axis=0, keepdims=True)
                derivative = derivative.dot(self.weights[j].T) * self.activation_function.activation_derivative(self.a[j-1])

            dW[0] = np.dot(batch_input.T, derivative)
            dW[0] += reg_lambda * self.weights[0]
            db[0] = np.sum(derivative, axis=0)

            # Gradient descent parameter update
            learning_rate = self.anneal.learning_function(i)
            for j in range(0, len(self.layers) - 1):
                self.weights[j] -= learning_rate * dW[j]
                self.biases[j] -= learning_rate * db[j]

            # TODO ADD GRADIENT CHECK
            if print_loss and i % 1000 == 0:
                print("Loss after iteration %i: %f" % (i, self.calculate_loss()))
This function calculates the cost function and backpropagates the error
This function calculates the cost function and backpropagates the error
This function calculates the cost function and backpropagates the error
def train_model(self, num_iterations, reg_lambda = 1e-2,
                    print_loss=False):
        dW = [np.zeros(self.weights[i].shape) for i in range(len(self.layers)-1)]
        db = [np.zeros((1, self.layers[i + 1])) for i in range(len(self.layers) - 1)]
        for i in range(0, num_iterations):
            random_indices = np.arange(self.num_examples)
            np.random.shuffle(random_indices)
            selection_array = random_indices[0:self.batch_size]
            batch_input = self.X[selection_array] 
            batch_output = self.Y[selection_array]
            probs = self.__forward_prop__(batch_input)
            derivative = self.output_layer.loss_function(probs, self.batch_size, batch_output)
            for j in range(len(self.layers) - 2,0,-1):
                dW[j] = (self.a[j-1].T).dot(derivative)
                dW[j] += reg_lambda *self.weights[j]
                db[j] = np.sum(derivative, axis=0, keepdims=True)
                derivative = derivative.dot(self.weights[j].T) * self.activation_function.activation_derivative(self.a[j-1])
            dW[0] = np.dot(batch_input.T, derivative)
            dW[0] += reg_lambda * self.weights[0]
            db[0] = np.sum(derivative, axis=0)
            learning_rate = self.anneal.learning_function(i)
            for j in range(0, len(self.layers) - 1):
                self.weights[j] -= learning_rate * dW[j]
                self.biases[j] -= learning_rate * db[j]
            if print_loss and i % 1000 == 0:
                print("Loss after iteration %i: %f" % (i, self.calculate_loss()))
def train_model ( self , num_iterations , reg_lambda = 1e-2 , print_loss = False ) : dW = [ np . zeros ( self . weights [ i ] . shape ) for i in range ( len ( self . layers ) - 1 ) ] db = [ np . zeros ( ( 1 , self . layers [ i + 1 ] ) ) for i in range ( len ( self . layers ) - 1 ) ] for i in range ( 0 , num_iterations ) : random_indices = np . arange ( self . num_examples ) np . random . shuffle ( random_indices ) selection_array = random_indices [ 0 : self . batch_size ] batch_input = self . X [ selection_array ] batch_output = self . Y [ selection_array ] probs = self . __forward_prop__ ( batch_input ) derivative = self . output_layer . loss_function ( probs , self . batch_size , batch_output ) for j in range ( len ( self . layers ) - 2 , 0 , - 1 ) : dW [ j ] = ( self . a [ j - 1 ] . T ) . dot ( derivative ) dW [ j ] += reg_lambda * self . weights [ j ] db [ j ] = np . sum ( derivative , axis = 0 , keepdims = True ) derivative = derivative . dot ( self . weights [ j ] . T ) * self . activation_function . activation_derivative ( self . a [ j - 1 ] ) dW [ 0 ] = np . dot ( batch_input . T , derivative ) dW [ 0 ] += reg_lambda * self . weights [ 0 ] db [ 0 ] = np . sum ( derivative , axis = 0 ) learning_rate = self . anneal . learning_function ( i ) for j in range ( 0 , len ( self . layers ) - 1 ) : self . weights [ j ] -= learning_rate * dW [ j ] self . biases [ j ] -= learning_rate * db [ j ] if print_loss and i % 1000 == 0 : print ( "Loss after iteration %i: %f" % ( i , self . calculate_loss ( ) ) )
This function calculates the cost function and backpropagates the error
This function calculates the cost function and backpropagates the error
"""This function calculates the cost function and backpropagates the error""" # make sure this gets the whole set # Gradient descent parameter update # TODO ADD GRADIENT CHECK
self
num_iterations
reg_lambda
print_loss