models/dense_net.py
def self_constructing_var4(self, epoch, accuracy):
        """
        A step of the self-constructing algorithm (variant #4) for one
        training epoch.
        Builds new layers in the last block depending on parameters.
        Returns True if training should continue, False otherwise.

        This algorithm consists of an macro-algorithm, which adds layers to the
        last block, and a micro-algorithm, which builds those layers kernel by
        kernel.
        - The macro-algorithm adds a new layer with one kernel and runs the
          micro-algorithm to build it (i.e. to add/prune kernels in it), then
          checks if the accuracy has improved significantly since the previous
          layer addition. If so it adds a new layer, else the algorithm ends.
        - The micro-algorithm consists of a succession of four stages:
            - Ascension: add one kernel every m_asc_thresh training epochs,
              break the loop (end the stage) when one of the kernels settles
              (its CS has remained stable for a certain number of epochs).
            - Improvement: countdown of m_patience_param epochs until the stage
              ends; if another kernel settles AND if it is useful (CS above
              usefulness_thresh, set by the user), add a kernel and restart the
              countdown.
            - Pruning: first wait until all kernels have settled (useful or
              not), then save the current accuracy and prune all useless
              kernels (CS below uselessness_thresh, set by the user) to end the
              stage.
            - Recovery: wait for one last countdown of m_patience_param epochs
              (optionally resetting the learning rate to its initial value and
              reducing it according to rlr0); after this countdown wait until
              reaching pre-pruning accuracy, then end the stage.

        Args:
            epoch: `int`, current training epoch (since adding the last block);
            accuracy: `float`, accuracy for this epoch.
        """
        continue_training = True

        settled_kernels_count = 0
        useful_kernels_count = 0
        useless_kernels_list = []
        # Update the kernel CS lists, count settled and useful kernels
        cs_kernels = self.process_layer_kernels(
            self.total_blocks-1, self.layer_num_list[-1]-1, epoch)
        for k in range(len(self.kCS_FIFO)):
            self.kCS_FIFO[k].append(cs_kernels[k])
            if len(self.kCS_FIFO[k]) == self.dkCS_softening:
                self.dkCS_FIFO[k].append(
                    (self.kCS_FIFO[k][-1] - self.kCS_FIFO[k][0])/(
                        self.dkCS_softening-1))
                # Settled = kCS remained close to 0 during the last epochs
                if ((len(self.dkCS_FIFO[k]) == self.dkCS_std_window) and (
                        np.abs(np.mean(self.dkCS_FIFO[k])
                               ) <= self.dkCS_stl_thresh) and
                        (np.abs(np.std(self.dkCS_FIFO[k])
                                ) <= self.dkCS_stl_thresh)):
                    settled_kernels_count += 1
                    # Useful = settled, and kCS above the usefulness thresh
                    if np.mean(self.kCS_FIFO[k]) >= self.usefulness_thresh:
                        useful_kernels_count += 1
                    # Useless = settled, and kCS below the uselessness thresh
                    if np.mean(self.kCS_FIFO[k]) <= self.uselessness_thresh:
                        useless_kernels_list.append(k)

        # stage #0 = ascension stage (currently does nothing)
        if self.algorithm_stage == 0:
            self.algorithm_stage += 1

        # stage #1 = improvement stage
        if self.algorithm_stage == 1:
            # micro-stage #0 = micro-ascension stage
            if self.micro_stage == 0:
                if settled_kernels_count >= 1:
                    # end stage when one or various kernels have settled
                    self.useful_kernels_ceil = useful_kernels_count
                    self.micro_stage += 1
                    # max_n_ep is used to estimate completion time
                    self.max_n_ep = epoch + 2*(self.m_patience_param + 1)
                elif (epoch-1) % self.m_asc_thresh == 0:
                    self._new_kernels_to_last_layer(
                        self.expansion_rate,
                        complementarity=self.complementarity)

            # micro-stage #1 = micro-improvement stage
            if self.micro_stage == 1:
                if self.m_patience_cntdwn <= 0:
                    # at the end of the patience countdown, end stage
                    self.micro_stage += 1
                elif useful_kernels_count > self.useful_kernels_ceil:
                    # if a new kernel is useful, add a kernel and restart ctdwn
                    self.useful_kernels_ceil = useful_kernels_count
                    self._new_kernels_to_last_layer(
                        self.expansion_rate,
                        complementarity=self.complementarity)
                    self.m_patience_cntdwn = self.m_patience_param
                    self.max_n_ep = epoch + 2*(self.m_patience_param + 1)
                else:
                    # patience countdown progress
                    self.m_patience_cntdwn -= 1

            # micro-stage #2 = micro-pruning stage
            if self.micro_stage == 2:
                # wait until all kernels have settled (bound to happen?)
                if settled_kernels_count == len(self.kCS_FIFO):
                    # save the accuracy, prune useless kernels and end stage
                    self.accuracy_pre_pruning = accuracy
                    self._prune_kernels_in_last_layer(useless_kernels_list)
                    self.micro_stage += 1
                    # run one last patience countdown for recovery
                    self.m_patience_cntdwn = self.m_patience_param
                    self.max_n_ep = epoch + self.m_patience_param + 1

            # micro-stage #3 = micro-recovery stage (accessed in next epoch)
            elif self.micro_stage == 3:
                # patience countdown to ensure recovery
                if self.m_patience_cntdwn > 0:
                    self.m_patience_cntdwn -= 1
                # wait until reaching pre-pruning accuracy
                elif accuracy >= self.accuracy_pre_pruning:
                    # prune again if there are useless kernels, else end stage
                    # if len(useless_kernels_list) >= 1:
                    #     self.micro_stage = 2
                    # else:
                    self.micro_stage += 1
                    # self.algorithm_stage = 2

            # at the end of the micro-algorithm, try to add a new layer
            if self.micro_stage == 4:
                # reset everything for the micro-algorithm
                self.micro_stage = 0
                self.useful_kernels_ceil = 0
                self.m_patience_cntdwn = self.m_patience_param
                self.accuracy_pre_pruning = 0
                # check if the accuracy has improved since the last layer
                # if so, add a layer, else end the improvement stage
                if abs(accuracy-self.accuracy_last_layer) >= self.impr_thresh:
                    self.accuracy_last_layer = accuracy
                    self._new_layer(self.growth_rate)
                    # different uselessness threshold for new layers
                    # self.uselessness_thresh = self.alt_uselessness_thresh
                    # alt. number of kernels = half the previous
                    # layer's number if during the ascension stage.
                    # self._new_layer(floor(
                    #     len(self.kernels_ref_list[-1][-1])/2))
                # else:
                self.algorithm_stage += 1

        # stage #2 (nothing yet, stop the algorithm and reset everything)
        if self.algorithm_stage == 2:
            continue_training = False
            self.algorithm_stage = 0
            self.patience_cntdwn = self.patience_param
            self.accuracy_last_layer = 0

        return continue_training

        A step of the self-constructing algorithm (variant #4) for one
        training epoch.
        Builds new layers in the last block depending on parameters.
        Returns True if training should continue, False otherwise.

        This algorithm consists of an macro-algorithm, which adds layers to the
        last block, and a micro-algorithm, which builds those layers kernel by
        kernel.
        - The macro-algorithm adds a new layer with one kernel and runs the
          micro-algorithm to build it (i.e. to add/prune kernels in it), then
          checks if the accuracy has improved significantly since the previous
          layer addition. If so it adds a new layer, else the algorithm ends.
        - The micro-algorithm consists of a succession of four stages:
            - Ascension: add one kernel every m_asc_thresh training epochs,
              break the loop (end the stage) when one of the kernels settles
              (its CS has remained stable for a certain number of epochs).
            - Improvement: countdown of m_patience_param epochs until the stage
              ends; if another kernel settles AND if it is useful (CS above
              usefulness_thresh, set by the user), add a kernel and restart the
              countdown.
            - Pruning: first wait until all kernels have settled (useful or
              not), then save the current accuracy and prune all useless
              kernels (CS below uselessness_thresh, set by the user) to end the
              stage.
            - Recovery: wait for one last countdown of m_patience_param epochs
              (optionally resetting the learning rate to its initial value and
              reducing it according to rlr0); after this countdown wait until
              reaching pre-pruning accuracy, then end the stage.

        Args:
            epoch: `int`, current training epoch (since adding the last block);
            accuracy: `float`, accuracy for this epoch.
        
A step of the self-constructing algorithm (variant #4) for one
training epoch.
Builds new layers in the last block depending on parameters.
Returns True if training should continue, False otherwise.

This algorithm consists of an macro-algorithm, which adds layers to the
last block, and a micro-algorithm, which builds those layers kernel by
kernel.
The macro-algorithm adds a new layer with one kernel and runs the
micro-algorithm to build it , then
checks if the accuracy has improved significantly since the previous
layer addition. If so it adds a new layer, else the algorithm ends.
The micro-algorithm consists of a succession of four stages:
Ascension: add one kernel every m_asc_thresh training epochs,
break the loop (end the stage) when one of the kernels settles
(its CS has remained stable for a certain number of epochs).
Improvement: countdown of m_patience_param epochs until the stage
ends; if another kernel settles AND if it is useful (CS above
usefulness_thresh, set by the user), add a kernel and restart the
countdown.
Pruning: first wait until all kernels have settled (useful or
not), then save the current accuracy and prune all useless
kernels (CS below uselessness_thresh, set by the user) to end the
stage.
Recovery: wait for one last countdown of m_patience_param epochs
(optionally resetting the learning rate to its initial value and
reducing it according to rlr0); after this countdown wait until
reaching pre-pruning accuracy, then end the stage.
A step of the self - constructing algorithm ( variant #4 ) for one training epoch . Builds new layers in the last block depending on parameters . Returns True if training should continue False otherwise . This algorithm consists of an macro - algorithm which adds layers to the last block and a micro - algorithm which builds those layers kernel by kernel . The macro - algorithm adds a new layer with one kernel and runs the micro - algorithm to build it then checks if the accuracy has improved significantly since the previous layer addition . If so it adds a new layer else the algorithm ends . The micro - algorithm consists of a succession of four stages : Ascension : add one kernel every m_asc_thresh training epochs break the loop ( end the stage ) when one of the kernels settles ( its CS has remained stable for a certain number of epochs ) . Improvement : countdown of m_patience_param epochs until the stage ends ; if another kernel settles AND if it is useful ( CS above usefulness_thresh set by the user ) add a kernel and restart the countdown . Pruning : first wait until all kernels have settled ( useful or not ) then save the current accuracy and prune all useless kernels ( CS below uselessness_thresh set by the user ) to end the stage . Recovery : wait for one last countdown of m_patience_param epochs ( optionally resetting the learning rate to its initial value and reducing it according to rlr0 ) ; after this countdown wait until reaching pre - pruning accuracy then end the stage .
def self_constructing_var4(self, epoch, accuracy):
        continue_training = True
        settled_kernels_count = 0
        useful_kernels_count = 0
        useless_kernels_list = []
        cs_kernels = self.process_layer_kernels(
            self.total_blocks-1, self.layer_num_list[-1]-1, epoch)
        for k in range(len(self.kCS_FIFO)):
            self.kCS_FIFO[k].append(cs_kernels[k])
            if len(self.kCS_FIFO[k]) == self.dkCS_softening:
                self.dkCS_FIFO[k].append(
                    (self.kCS_FIFO[k][-1] - self.kCS_FIFO[k][0])/(
                        self.dkCS_softening-1))
                if ((len(self.dkCS_FIFO[k]) == self.dkCS_std_window) and (
                        np.abs(np.mean(self.dkCS_FIFO[k])
                               ) <= self.dkCS_stl_thresh) and
                        (np.abs(np.std(self.dkCS_FIFO[k])
                                ) <= self.dkCS_stl_thresh)):
                    settled_kernels_count += 1
                    if np.mean(self.kCS_FIFO[k]) >= self.usefulness_thresh:
                        useful_kernels_count += 1
                    if np.mean(self.kCS_FIFO[k]) <= self.uselessness_thresh:
                        useless_kernels_list.append(k)
        if self.algorithm_stage == 0:
            self.algorithm_stage += 1
        if self.algorithm_stage == 1:
            if self.micro_stage == 0:
                if settled_kernels_count >= 1:
                    self.useful_kernels_ceil = useful_kernels_count
                    self.micro_stage += 1
                    self.max_n_ep = epoch + 2*(self.m_patience_param + 1)
                elif (epoch-1) % self.m_asc_thresh == 0:
                    self._new_kernels_to_last_layer(
                        self.expansion_rate,
                        complementarity=self.complementarity)
            if self.micro_stage == 1:
                if self.m_patience_cntdwn <= 0:
                    self.micro_stage += 1
                elif useful_kernels_count > self.useful_kernels_ceil:
                    self.useful_kernels_ceil = useful_kernels_count
                    self._new_kernels_to_last_layer(
                        self.expansion_rate,
                        complementarity=self.complementarity)
                    self.m_patience_cntdwn = self.m_patience_param
                    self.max_n_ep = epoch + 2*(self.m_patience_param + 1)
                else:
                    self.m_patience_cntdwn -= 1
            if self.micro_stage == 2:
                if settled_kernels_count == len(self.kCS_FIFO):
                    self.accuracy_pre_pruning = accuracy
                    self._prune_kernels_in_last_layer(useless_kernels_list)
                    self.micro_stage += 1
                    self.m_patience_cntdwn = self.m_patience_param
                    self.max_n_ep = epoch + self.m_patience_param + 1
            elif self.micro_stage == 3:
                if self.m_patience_cntdwn > 0:
                    self.m_patience_cntdwn -= 1
                elif accuracy >= self.accuracy_pre_pruning:
                    self.micro_stage += 1
            if self.micro_stage == 4:
                self.micro_stage = 0
                self.useful_kernels_ceil = 0
                self.m_patience_cntdwn = self.m_patience_param
                self.accuracy_pre_pruning = 0
                if abs(accuracy-self.accuracy_last_layer) >= self.impr_thresh:
                    self.accuracy_last_layer = accuracy
                    self._new_layer(self.growth_rate)
                self.algorithm_stage += 1
        if self.algorithm_stage == 2:
            continue_training = False
            self.algorithm_stage = 0
            self.patience_cntdwn = self.patience_param
            self.accuracy_last_layer = 0
        return continue_training
def self_constructing_var4 ( self , epoch , accuracy ) : continue_training = True settled_kernels_count = 0 useful_kernels_count = 0 useless_kernels_list = [ ] cs_kernels = self . process_layer_kernels ( self . total_blocks - 1 , self . layer_num_list [ - 1 ] - 1 , epoch ) for k in range ( len ( self . kCS_FIFO ) ) : self . kCS_FIFO [ k ] . append ( cs_kernels [ k ] ) if len ( self . kCS_FIFO [ k ] ) == self . dkCS_softening : self . dkCS_FIFO [ k ] . append ( ( self . kCS_FIFO [ k ] [ - 1 ] - self . kCS_FIFO [ k ] [ 0 ] ) / ( self . dkCS_softening - 1 ) ) if ( ( len ( self . dkCS_FIFO [ k ] ) == self . dkCS_std_window ) and ( np . abs ( np . mean ( self . dkCS_FIFO [ k ] ) ) <= self . dkCS_stl_thresh ) and ( np . abs ( np . std ( self . dkCS_FIFO [ k ] ) ) <= self . dkCS_stl_thresh ) ) : settled_kernels_count += 1 if np . mean ( self . kCS_FIFO [ k ] ) >= self . usefulness_thresh : useful_kernels_count += 1 if np . mean ( self . kCS_FIFO [ k ] ) <= self . uselessness_thresh : useless_kernels_list . append ( k ) if self . algorithm_stage == 0 : self . algorithm_stage += 1 if self . algorithm_stage == 1 : if self . micro_stage == 0 : if settled_kernels_count >= 1 : self . useful_kernels_ceil = useful_kernels_count self . micro_stage += 1 self . max_n_ep = epoch + 2 * ( self . m_patience_param + 1 ) elif ( epoch - 1 ) % self . m_asc_thresh == 0 : self . _new_kernels_to_last_layer ( self . expansion_rate , complementarity = self . complementarity ) if self . micro_stage == 1 : if self . m_patience_cntdwn <= 0 : self . micro_stage += 1 elif useful_kernels_count > self . useful_kernels_ceil : self . useful_kernels_ceil = useful_kernels_count self . _new_kernels_to_last_layer ( self . expansion_rate , complementarity = self . complementarity ) self . m_patience_cntdwn = self . m_patience_param self . max_n_ep = epoch + 2 * ( self . m_patience_param + 1 ) else : self . m_patience_cntdwn -= 1 if self . micro_stage == 2 : if settled_kernels_count == len ( self . kCS_FIFO ) : self . accuracy_pre_pruning = accuracy self . _prune_kernels_in_last_layer ( useless_kernels_list ) self . micro_stage += 1 self . m_patience_cntdwn = self . m_patience_param self . max_n_ep = epoch + self . m_patience_param + 1 elif self . micro_stage == 3 : if self . m_patience_cntdwn > 0 : self . m_patience_cntdwn -= 1 elif accuracy >= self . accuracy_pre_pruning : self . micro_stage += 1 if self . micro_stage == 4 : self . micro_stage = 0 self . useful_kernels_ceil = 0 self . m_patience_cntdwn = self . m_patience_param self . accuracy_pre_pruning = 0 if abs ( accuracy - self . accuracy_last_layer ) >= self . impr_thresh : self . accuracy_last_layer = accuracy self . _new_layer ( self . growth_rate ) self . algorithm_stage += 1 if self . algorithm_stage == 2 : continue_training = False self . algorithm_stage = 0 self . patience_cntdwn = self . patience_param self . accuracy_last_layer = 0 return continue_training
A step of the self-constructing algorithm (variant #4) for one
training epoch.
A step of the self - constructing algorithm ( variant #4 ) for one training epoch .
"""
        A step of the self-constructing algorithm (variant #4) for one
        training epoch.
        Builds new layers in the last block depending on parameters.
        Returns True if training should continue, False otherwise.

        This algorithm consists of an macro-algorithm, which adds layers to the
        last block, and a micro-algorithm, which builds those layers kernel by
        kernel.
        - The macro-algorithm adds a new layer with one kernel and runs the
          micro-algorithm to build it (i.e. to add/prune kernels in it), then
          checks if the accuracy has improved significantly since the previous
          layer addition. If so it adds a new layer, else the algorithm ends.
        - The micro-algorithm consists of a succession of four stages:
            - Ascension: add one kernel every m_asc_thresh training epochs,
              break the loop (end the stage) when one of the kernels settles
              (its CS has remained stable for a certain number of epochs).
            - Improvement: countdown of m_patience_param epochs until the stage
              ends; if another kernel settles AND if it is useful (CS above
              usefulness_thresh, set by the user), add a kernel and restart the
              countdown.
            - Pruning: first wait until all kernels have settled (useful or
              not), then save the current accuracy and prune all useless
              kernels (CS below uselessness_thresh, set by the user) to end the
              stage.
            - Recovery: wait for one last countdown of m_patience_param epochs
              (optionally resetting the learning rate to its initial value and
              reducing it according to rlr0); after this countdown wait until
              reaching pre-pruning accuracy, then end the stage.

        Args:
            epoch: `int`, current training epoch (since adding the last block);
            accuracy: `float`, accuracy for this epoch.
        """ # Update the kernel CS lists, count settled and useful kernels # Settled = kCS remained close to 0 during the last epochs # Useful = settled, and kCS above the usefulness thresh # Useless = settled, and kCS below the uselessness thresh # stage #0 = ascension stage (currently does nothing) # stage #1 = improvement stage # micro-stage #0 = micro-ascension stage # end stage when one or various kernels have settled # max_n_ep is used to estimate completion time # micro-stage #1 = micro-improvement stage # at the end of the patience countdown, end stage # if a new kernel is useful, add a kernel and restart ctdwn # patience countdown progress # micro-stage #2 = micro-pruning stage # wait until all kernels have settled (bound to happen?) # save the accuracy, prune useless kernels and end stage # run one last patience countdown for recovery # micro-stage #3 = micro-recovery stage (accessed in next epoch) # patience countdown to ensure recovery # wait until reaching pre-pruning accuracy # prune again if there are useless kernels, else end stage # if len(useless_kernels_list) >= 1: #     self.micro_stage = 2 # else: # self.algorithm_stage = 2 # at the end of the micro-algorithm, try to add a new layer # reset everything for the micro-algorithm # check if the accuracy has improved since the last layer # if so, add a layer, else end the improvement stage # different uselessness threshold for new layers # self.uselessness_thresh = self.alt_uselessness_thresh # alt. number of kernels = half the previous # layer's number if during the ascension stage. # self._new_layer(floor( #     len(self.kernels_ref_list[-1][-1])/2)) # else: # stage #2 (nothing yet, stop the algorithm and reset everything)
self
epoch
`int`, current training epoch (since adding the last block).
` int ` current training epoch ( since adding the last block ) .
accuracy
`float`, accuracy for this epoch.
` float ` accuracy for this epoch .