models/dense_net.py
def train_all_epochs(self, train_params):
        """
        Trains the model for a certain number of epochs, using parameters
        specified in the train_params argument.

        Args (in train_params):
            batch_size: `int`, number of examples in a training batch;
            max_n_ep: `int`, maximum number of training epochs to run;
            initial_learning_rate: `int`, initial learning rate for optimizer;
            reduce_lr_1: `float`, if not self-constructing the network,
                first fraction of max_n_ep after which the current
                learning rate is divided by 10 (initial_learning_rate/10);
            reduce_lr_2: `float`, if not self-constructing the network,
                second fraction of max_n_ep after which the current
                learning rate is divided by 10 (initial_learning_rate/100);
            validation_set: `bool`, should a validation set be used or not;
            validation_split: `float` or None;
                `float`: chunk of the training set used as the validation set;
                None: use the testing set as the validation set;
            shuffle: `str` or None, or `bool`;
                `str` or None: used with CIFAR datasets, should we shuffle the
                    data only before training ('once_prior_train'), on every
                    epoch ('every_epoch') or not at all (None);
                `bool`: used with SVHN, should we shuffle the data or not;
            normalisation: `str` or None;
                None: don't use any normalisation for pixels;
                'divide_255': divide all pixels by 255;
                'divide_256': divide all pixels by 256;
                'by_chanels': substract the mean of the pixel's chanel and
                    divide the result by the channel's standard deviation.
        """
        self.max_n_ep = train_params['max_n_ep']
        initial_lr = train_params['initial_learning_rate']
        learning_rate = train_params['initial_learning_rate']
        batch_size = train_params['batch_size']
        rlr_1 = train_params['reduce_lr_1']
        rlr_2 = train_params['reduce_lr_2']
        validation_set = train_params.get('validation_set', False)
        total_start_time = time.time()

        epoch = 1         # current training epoch
        epoch_last_b = 0  # epoch at which the last block was added
        while True:
            # only print epoch name on certain epochs
            if (epoch-1) % self.ft_period == 0:
                print('\n', '-'*30, "Train epoch: %d" % epoch, '-'*30, '\n')
            start_time = time.time()

            # if not self-constructing, may reduce learning rate at some epochs
            if not self.should_self_construct and self.should_change_lr:
                if (epoch == int(self.max_n_ep * rlr_1)) or (
                        epoch == int(self.max_n_ep * rlr_2)):
                    learning_rate = learning_rate / 10
                    print("Learning rate has been divided by 10, new lr = %f" %
                          learning_rate)

            # training step for one epoch
            print("Training...", end=' ')
            loss, acc = self.train_one_epoch(
                self.data_provider.train, batch_size, learning_rate)
            # save logs
            if self.should_save_logs:
                self.log_loss_accuracy(loss, acc, epoch, prefix='train')

            # validation step after the epoch
            if validation_set:
                print("Validation...")
                loss, acc = self.test(
                    self.data_provider.validation, batch_size)
                # save logs
                if self.should_save_logs:
                    self.log_loss_accuracy(loss[-1], acc, epoch,
                                           prefix='valid')

            # save feature logs (on certain epochs)
            if (epoch-1) % self.ft_period == 0:
                self.print_pertinent_features(loss, acc, epoch, validation_set)

            # save model if required
            if self.should_save_model:
                self.save_model()

            # step of the self-constructing algorithm
            if self.should_self_construct:
                if epoch - epoch_last_b != 1:
                    # can break here if self-constructing algorithm is over
                    if not self.self_constructing_step(
                            epoch - epoch_last_b, acc):
                        # add another block if block_count not yet exceeded
                        if self.total_blocks < self.block_count:
                            self._new_block()
                        else:
                            break
                    # optional learning rate reduction for self-constructing
                    if self.should_change_lr:
                        if self.has_micro_algo and self.micro_stage == 3:
                            # micro-recovery uses rlr0 for proper recovery
                            learning_rate = self.self_constr_rlr0(
                                learning_rate, initial_lr, rlr_1, rlr_2)
                        else:
                            learning_rate = self.self_constr_rlr(
                                learning_rate, initial_lr, rlr_1, rlr_2)
                # if this is a new block, reset the algorithm's variables
                else:
                    self.settled_layers_ceil = 0  # highest num of settled lay
                    self.algorithm_stage = 0  # start with ascension stage
                    self.patience_cntdwn = self.patience_param
                    if self.has_micro_algo:
                        self.useful_kernels_ceil = 0  # highest n of settled k
                        self.micro_stage = 0  # kernel-level stages (if needed)
                        self.m_patience_cntdwn = self.m_patience_param
                        self.accuracy_pre_pruning = 0
                        self.accuracy_last_layer = 0

            # measure training time for this epoch
            time_per_epoch = time.time() - start_time
            seconds_left = int((self.max_n_ep - epoch) * time_per_epoch)
            print("Time per epoch: %s, Est. complete (%d epochs) in: %s" % (
                str(timedelta(seconds=time_per_epoch)),
                self.max_n_ep,
                str(timedelta(seconds=seconds_left))))

            # increase epoch, break at max_n_ep if not self-constructing
            epoch += 1
            if not self.should_self_construct and epoch >= self.max_n_ep+1:
                break

        # measure total training time
        total_training_time = time.time() - total_start_time
        print("\nTOTAL TRAINING TIME: %s\n" % str(timedelta(
            seconds=total_training_time)))
        if self.should_save_ft_logs:
            self.feature_writer.write("\nTOTAL TRAINING TIME: %s\n" % str(
                timedelta(seconds=total_training_time)))
        self._count_trainable_params_in_use(write_to_ft_logs=True)

        Trains the model for a certain number of epochs, using parameters
        specified in the train_params argument.

        Args (in train_params):
            batch_size: `int`, number of examples in a training batch;
            max_n_ep: `int`, maximum number of training epochs to run;
            initial_learning_rate: `int`, initial learning rate for optimizer;
            reduce_lr_1: `float`, if not self-constructing the network,
                first fraction of max_n_ep after which the current
                learning rate is divided by 10 (initial_learning_rate/10);
            reduce_lr_2: `float`, if not self-constructing the network,
                second fraction of max_n_ep after which the current
                learning rate is divided by 10 (initial_learning_rate/100);
            validation_set: `bool`, should a validation set be used or not;
            validation_split: `float` or None;
                `float`: chunk of the training set used as the validation set;
                None: use the testing set as the validation set;
            shuffle: `str` or None, or `bool`;
                `str` or None: used with CIFAR datasets, should we shuffle the
                    data only before training ('once_prior_train'), on every
                    epoch ('every_epoch') or not at all (None);
                `bool`: used with SVHN, should we shuffle the data or not;
            normalisation: `str` or None;
                None: don't use any normalisation for pixels;
                'divide_255': divide all pixels by 255;
                'divide_256': divide all pixels by 256;
                'by_chanels': substract the mean of the pixel's chanel and
                    divide the result by the channel's standard deviation.
        
Trains the model for a certain number of epochs, using parameters
specified in the train_params argument.


Trains the model for a certain number of epochs using parameters specified in the train_params argument .
def train_all_epochs(self, train_params):
        self.max_n_ep = train_params['max_n_ep']
        initial_lr = train_params['initial_learning_rate']
        learning_rate = train_params['initial_learning_rate']
        batch_size = train_params['batch_size']
        rlr_1 = train_params['reduce_lr_1']
        rlr_2 = train_params['reduce_lr_2']
        validation_set = train_params.get('validation_set', False)
        total_start_time = time.time()
        epoch = 1         
        epoch_last_b = 0  
        while True:
            if (epoch-1) % self.ft_period == 0:
                print('\n', '-'*30, "Train epoch: %d" % epoch, '-'*30, '\n')
            start_time = time.time()
            if not self.should_self_construct and self.should_change_lr:
                if (epoch == int(self.max_n_ep * rlr_1)) or (
                        epoch == int(self.max_n_ep * rlr_2)):
                    learning_rate = learning_rate / 10
                    print("Learning rate has been divided by 10, new lr = %f" %
                          learning_rate)
            print("Training...", end=' ')
            loss, acc = self.train_one_epoch(
                self.data_provider.train, batch_size, learning_rate)
            if self.should_save_logs:
                self.log_loss_accuracy(loss, acc, epoch, prefix='train')
            if validation_set:
                print("Validation...")
                loss, acc = self.test(
                    self.data_provider.validation, batch_size)
                if self.should_save_logs:
                    self.log_loss_accuracy(loss[-1], acc, epoch,
                                           prefix='valid')
            if (epoch-1) % self.ft_period == 0:
                self.print_pertinent_features(loss, acc, epoch, validation_set)
            if self.should_save_model:
                self.save_model()
            if self.should_self_construct:
                if epoch - epoch_last_b != 1:
                    if not self.self_constructing_step(
                            epoch - epoch_last_b, acc):
                        if self.total_blocks < self.block_count:
                            self._new_block()
                        else:
                            break
                    if self.should_change_lr:
                        if self.has_micro_algo and self.micro_stage == 3:
                            learning_rate = self.self_constr_rlr0(
                                learning_rate, initial_lr, rlr_1, rlr_2)
                        else:
                            learning_rate = self.self_constr_rlr(
                                learning_rate, initial_lr, rlr_1, rlr_2)
                else:
                    self.settled_layers_ceil = 0  
                    self.algorithm_stage = 0  
                    self.patience_cntdwn = self.patience_param
                    if self.has_micro_algo:
                        self.useful_kernels_ceil = 0  
                        self.micro_stage = 0  
                        self.m_patience_cntdwn = self.m_patience_param
                        self.accuracy_pre_pruning = 0
                        self.accuracy_last_layer = 0
            time_per_epoch = time.time() - start_time
            seconds_left = int((self.max_n_ep - epoch) * time_per_epoch)
            print("Time per epoch: %s, Est. complete (%d epochs) in: %s" % (
                str(timedelta(seconds=time_per_epoch)),
                self.max_n_ep,
                str(timedelta(seconds=seconds_left))))
            epoch += 1
            if not self.should_self_construct and epoch >= self.max_n_ep+1:
                break
        total_training_time = time.time() - total_start_time
        print("\nTOTAL TRAINING TIME: %s\n" % str(timedelta(
            seconds=total_training_time)))
        if self.should_save_ft_logs:
            self.feature_writer.write("\nTOTAL TRAINING TIME: %s\n" % str(
                timedelta(seconds=total_training_time)))
        self._count_trainable_params_in_use(write_to_ft_logs=True)
def train_all_epochs ( self , train_params ) : self . max_n_ep = train_params [ 'max_n_ep' ] initial_lr = train_params [ 'initial_learning_rate' ] learning_rate = train_params [ 'initial_learning_rate' ] batch_size = train_params [ 'batch_size' ] rlr_1 = train_params [ 'reduce_lr_1' ] rlr_2 = train_params [ 'reduce_lr_2' ] validation_set = train_params . get ( 'validation_set' , False ) total_start_time = time . time ( ) epoch = 1 epoch_last_b = 0 while True : if ( epoch - 1 ) % self . ft_period == 0 : print ( '\n' , '-' * 30 , "Train epoch: %d" % epoch , '-' * 30 , '\n' ) start_time = time . time ( ) if not self . should_self_construct and self . should_change_lr : if ( epoch == int ( self . max_n_ep * rlr_1 ) ) or ( epoch == int ( self . max_n_ep * rlr_2 ) ) : learning_rate = learning_rate / 10 print ( "Learning rate has been divided by 10, new lr = %f" % learning_rate ) print ( "Training..." , end = ' ' ) loss , acc = self . train_one_epoch ( self . data_provider . train , batch_size , learning_rate ) if self . should_save_logs : self . log_loss_accuracy ( loss , acc , epoch , prefix = 'train' ) if validation_set : print ( "Validation..." ) loss , acc = self . test ( self . data_provider . validation , batch_size ) if self . should_save_logs : self . log_loss_accuracy ( loss [ - 1 ] , acc , epoch , prefix = 'valid' ) if ( epoch - 1 ) % self . ft_period == 0 : self . print_pertinent_features ( loss , acc , epoch , validation_set ) if self . should_save_model : self . save_model ( ) if self . should_self_construct : if epoch - epoch_last_b != 1 : if not self . self_constructing_step ( epoch - epoch_last_b , acc ) : if self . total_blocks < self . block_count : self . _new_block ( ) else : break if self . should_change_lr : if self . has_micro_algo and self . micro_stage == 3 : learning_rate = self . self_constr_rlr0 ( learning_rate , initial_lr , rlr_1 , rlr_2 ) else : learning_rate = self . self_constr_rlr ( learning_rate , initial_lr , rlr_1 , rlr_2 ) else : self . settled_layers_ceil = 0 self . algorithm_stage = 0 self . patience_cntdwn = self . patience_param if self . has_micro_algo : self . useful_kernels_ceil = 0 self . micro_stage = 0 self . m_patience_cntdwn = self . m_patience_param self . accuracy_pre_pruning = 0 self . accuracy_last_layer = 0 time_per_epoch = time . time ( ) - start_time seconds_left = int ( ( self . max_n_ep - epoch ) * time_per_epoch ) print ( "Time per epoch: %s, Est. complete (%d epochs) in: %s" % ( str ( timedelta ( seconds = time_per_epoch ) ) , self . max_n_ep , str ( timedelta ( seconds = seconds_left ) ) ) ) epoch += 1 if not self . should_self_construct and epoch >= self . max_n_ep + 1 : break total_training_time = time . time ( ) - total_start_time print ( "\nTOTAL TRAINING TIME: %s\n" % str ( timedelta ( seconds = total_training_time ) ) ) if self . should_save_ft_logs : self . feature_writer . write ( "\nTOTAL TRAINING TIME: %s\n" % str ( timedelta ( seconds = total_training_time ) ) ) self . _count_trainable_params_in_use ( write_to_ft_logs = True )
Trains the model for a certain number of epochs, using parameters
specified in the train_params argument.
Trains the model for a certain number of epochs using parameters specified in the train_params argument .
"""
        Trains the model for a certain number of epochs, using parameters
        specified in the train_params argument.

        Args (in train_params):
            batch_size: `int`, number of examples in a training batch;
            max_n_ep: `int`, maximum number of training epochs to run;
            initial_learning_rate: `int`, initial learning rate for optimizer;
            reduce_lr_1: `float`, if not self-constructing the network,
                first fraction of max_n_ep after which the current
                learning rate is divided by 10 (initial_learning_rate/10);
            reduce_lr_2: `float`, if not self-constructing the network,
                second fraction of max_n_ep after which the current
                learning rate is divided by 10 (initial_learning_rate/100);
            validation_set: `bool`, should a validation set be used or not;
            validation_split: `float` or None;
                `float`: chunk of the training set used as the validation set;
                None: use the testing set as the validation set;
            shuffle: `str` or None, or `bool`;
                `str` or None: used with CIFAR datasets, should we shuffle the
                    data only before training ('once_prior_train'), on every
                    epoch ('every_epoch') or not at all (None);
                `bool`: used with SVHN, should we shuffle the data or not;
            normalisation: `str` or None;
                None: don't use any normalisation for pixels;
                'divide_255': divide all pixels by 255;
                'divide_256': divide all pixels by 256;
                'by_chanels': substract the mean of the pixel's chanel and
                    divide the result by the channel's standard deviation.
        """ # current training epoch # epoch at which the last block was added # only print epoch name on certain epochs # if not self-constructing, may reduce learning rate at some epochs # training step for one epoch # save logs # validation step after the epoch # save logs # save feature logs (on certain epochs) # save model if required # step of the self-constructing algorithm # can break here if self-constructing algorithm is over # add another block if block_count not yet exceeded # optional learning rate reduction for self-constructing # micro-recovery uses rlr0 for proper recovery # if this is a new block, reset the algorithm's variables # highest num of settled lay # start with ascension stage # highest n of settled k # kernel-level stages (if needed) # measure training time for this epoch # increase epoch, break at max_n_ep if not self-constructing # measure total training time
self
train_params