models/dense_net.py
def self_constructing_var5(self, epoch, accuracy):
        """
        A step of the self-constructing algorithm (variant #5) for one
        training epoch.
        Builds new layers in the last block depending on parameters.
        Returns True if training should continue, False otherwise.

        This algorithm consists of an macro-algorithm, which adds layers to the
        last block, and a micro-algorithm, which builds those layers kernel by
        kernel.
        - The macro-algorithm adds a new layer with one kernel and runs the
          micro-algorithm to build it (i.e. to add/prune kernels in it), then
          checks if the accuracy has improved significantly since the previous
          layer addition. If so it adds a new layer, else the algorithm ends.
        - The micro-algorithm consists of a succession of three stages:
            - Improvement: countdown of m_patience_param epochs; if the number
              of useful kernels (CS above usefulness_thresh, automatically set)
              is above the latest max number of useful kernels, add a kernel
              and restart the countdown; if the countdown ends, wait until all
              kernels have settled and end the stage.
            - Pruning: save the current accuracy and prune all useless kernels
              (CS below uselessness_thresh, automatically set) to end the
              stage.
            - Recovery: wait for one last countdown of m_patience_param epochs
              (optionally resetting the learning rate to its initial value and
              reducing it according to rlr0); after this countdown wait until
              reaching pre-pruning accuracy, then if there are any new useless
              kernels wait for all kernels to settle and return to pruning,
              else end the stage.

        Args:
            epoch: `int`, current training epoch (since adding the last block);
            accuracy: `float`, accuracy for this epoch.
        """
        continue_training = True

        settled_kernels_count = 0
        useful_kernels_count = 0
        useless_kernels_list = []
        kCS_settled = []
        # Update the kernel CS lists, count settled kernels
        cs_kernels = self.process_layer_kernels(
            self.total_blocks-1, self.layer_num_list[-1]-1, epoch)
        for k in range(len(self.kCS_FIFO)):
            self.kCS_FIFO[k].append(cs_kernels[k])
            if len(self.kCS_FIFO[k]) == self.dkCS_softening:
                self.dkCS_FIFO[k].append(
                    (self.kCS_FIFO[k][-1] - self.kCS_FIFO[k][0])/(
                        self.dkCS_softening-1))
                # Settled = kCS remained close to 0 during the last epochs
                if ((len(self.dkCS_FIFO[k]) == self.dkCS_std_window) and (
                        np.abs(np.mean(self.dkCS_FIFO[k])
                               ) <= self.dkCS_stl_thresh) and
                        (np.abs(np.std(self.dkCS_FIFO[k])
                                ) <= self.dkCS_stl_thresh)):
                    settled_kernels_count += 1
                    if self.micro_stage == 1:
                        kCS_settled.append(self.kCS_FIFO[k][-1])

        # If half of the original kernels have settled
        if settled_kernels_count >= 0.5*self.growth_rate:
            # During impr. stage, calculate usefulness and uselessness thresh
            if self.micro_stage == 1:
                self.usefulness_thresh = min(kCS_settled) + (
                    max(kCS_settled) - min(kCS_settled)
                    )*self.auto_usefulness_thresh
                self.uselessness_thresh = min(kCS_settled) + (
                    max(kCS_settled) - min(kCS_settled)
                    )*self.auto_uselessness_thresh
            # Detect and count useful and useless kernels
            for k in range(len(self.kCS_FIFO)):
                # Useful = kCS above the usefulness thresh
                if np.mean(self.kCS_FIFO[k]) >= self.usefulness_thresh:
                    useful_kernels_count += 1
                # Useless = kCS below the uselessness thresh
                if np.mean(self.kCS_FIFO[k]) <= self.uselessness_thresh:
                    useless_kernels_list.append(k)

        # stage #0 = ascension stage (currently does nothing)
        if self.algorithm_stage == 0:
            self.algorithm_stage += 1

        # stage #1 = improvement stage
        if self.algorithm_stage == 1:
            # micro-stage #0 = just some settings for the next (actual) stage
            if self.micro_stage == 0:
                self.micro_stage += 1
                # max_n_ep is used to estimate completion time
                self.max_n_ep = epoch + 2*(self.m_patience_param + 1)

            # micro-stage #1 = micro-improvement stage
            if self.micro_stage == 1:
                if self.m_patience_cntdwn <= 0:
                    # at the end of the patience countdown, end stage when all
                    # the kernels have settled
                    if settled_kernels_count == len(self.kCS_FIFO):
                        self.micro_stage += 1
                elif useful_kernels_count > self.useful_kernels_ceil:
                    # if the number of useful kernels is above the latest max,
                    # add a kernel and restart ctdwn
                    self.useful_kernels_ceil = useful_kernels_count
                    self._new_kernels_to_last_layer(
                        self.expansion_rate,
                        complementarity=self.complementarity)
                    self.m_patience_cntdwn = self.m_patience_param
                    self.max_n_ep = epoch + 2*(self.m_patience_param + 1)
                else:
                    # patience countdown progress
                    self.m_patience_cntdwn -= 1

            # micro-stage #2 = micro-pruning stage
            if self.micro_stage == 2:
                # save the accuracy, prune useless kernels and end stage
                self.accuracy_pre_pruning = accuracy
                self._prune_kernels_in_last_layer(useless_kernels_list)
                self.micro_stage += 1
                # run one last patience countdown for recovery
                self.m_patience_cntdwn = self.m_patience_param
                self.max_n_ep = epoch + self.m_patience_param + 1

            # micro-stage #3 = micro-recovery stage (accessed in next epoch)
            elif self.micro_stage == 3:
                # patience countdown to ensure recovery
                if self.m_patience_cntdwn > 0:
                    self.m_patience_cntdwn -= 1
                # wait until reaching pre-pruning accuracy
                elif accuracy >= self.accuracy_pre_pruning:
                    # prune again if there are useless kernels, else end stage
                    if len(useless_kernels_list) >= 1:
                        # but first, wait for all kernels to settle
                        if settled_kernels_count == len(self.kCS_FIFO):
                            self.micro_stage = 2
                    else:
                        self.micro_stage += 1

            # at the end of the micro-algorithm, try to add a new layer
            if self.micro_stage == 4:
                # reset everything for the micro-algorithm
                self.micro_stage = 0
                self.useful_kernels_ceil = 0
                self.m_patience_cntdwn = self.m_patience_param
                self.accuracy_pre_pruning = 0
                # check if the accuracy has improved since the last layer
                # if so, add a layer, else end the improvement stage
                if abs(accuracy-self.accuracy_last_layer) >= self.impr_thresh:
                    self.accuracy_last_layer = accuracy
                    self._new_layer(self.growth_rate)
                    # alt. number of kernels = half the previous
                    # layer's number if during the ascension stage.
                    # self._new_layer(floor(
                    #     len(self.kernels_ref_list[-1][-1])/2))
                else:
                    self.algorithm_stage += 1

        # stage #2 (nothing yet, stop the algorithm and reset everything)
        if self.algorithm_stage == 2:
            continue_training = False
            self.algorithm_stage = 0
            self.patience_cntdwn = self.patience_param
            self.accuracy_last_layer = 0

        return continue_training

        A step of the self-constructing algorithm (variant #5) for one
        training epoch.
        Builds new layers in the last block depending on parameters.
        Returns True if training should continue, False otherwise.

        This algorithm consists of an macro-algorithm, which adds layers to the
        last block, and a micro-algorithm, which builds those layers kernel by
        kernel.
        - The macro-algorithm adds a new layer with one kernel and runs the
          micro-algorithm to build it (i.e. to add/prune kernels in it), then
          checks if the accuracy has improved significantly since the previous
          layer addition. If so it adds a new layer, else the algorithm ends.
        - The micro-algorithm consists of a succession of three stages:
            - Improvement: countdown of m_patience_param epochs; if the number
              of useful kernels (CS above usefulness_thresh, automatically set)
              is above the latest max number of useful kernels, add a kernel
              and restart the countdown; if the countdown ends, wait until all
              kernels have settled and end the stage.
            - Pruning: save the current accuracy and prune all useless kernels
              (CS below uselessness_thresh, automatically set) to end the
              stage.
            - Recovery: wait for one last countdown of m_patience_param epochs
              (optionally resetting the learning rate to its initial value and
              reducing it according to rlr0); after this countdown wait until
              reaching pre-pruning accuracy, then if there are any new useless
              kernels wait for all kernels to settle and return to pruning,
              else end the stage.

        Args:
            epoch: `int`, current training epoch (since adding the last block);
            accuracy: `float`, accuracy for this epoch.
        
A step of the self-constructing algorithm (variant #5) for one
training epoch.
Builds new layers in the last block depending on parameters.
Returns True if training should continue, False otherwise.

This algorithm consists of an macro-algorithm, which adds layers to the
last block, and a micro-algorithm, which builds those layers kernel by
kernel.
The macro-algorithm adds a new layer with one kernel and runs the
micro-algorithm to build it , then
checks if the accuracy has improved significantly since the previous
layer addition. If so it adds a new layer, else the algorithm ends.
The micro-algorithm consists of a succession of three stages:
Improvement: countdown of m_patience_param epochs; if the number
of useful kernels (CS above usefulness_thresh, automatically set)
is above the latest max number of useful kernels, add a kernel
and restart the countdown; if the countdown ends, wait until all
kernels have settled and end the stage.
Pruning: save the current accuracy and prune all useless kernels
(CS below uselessness_thresh, automatically set) to end the
stage.
Recovery: wait for one last countdown of m_patience_param epochs
(optionally resetting the learning rate to its initial value and
reducing it according to rlr0); after this countdown wait until
reaching pre-pruning accuracy, then if there are any new useless
kernels wait for all kernels to settle and return to pruning,
else end the stage.
A step of the self - constructing algorithm ( variant #5 ) for one training epoch . Builds new layers in the last block depending on parameters . Returns True if training should continue False otherwise . This algorithm consists of an macro - algorithm which adds layers to the last block and a micro - algorithm which builds those layers kernel by kernel . The macro - algorithm adds a new layer with one kernel and runs the micro - algorithm to build it then checks if the accuracy has improved significantly since the previous layer addition . If so it adds a new layer else the algorithm ends . The micro - algorithm consists of a succession of three stages : Improvement : countdown of m_patience_param epochs ; if the number of useful kernels ( CS above usefulness_thresh automatically set ) is above the latest max number of useful kernels add a kernel and restart the countdown ; if the countdown ends wait until all kernels have settled and end the stage . Pruning : save the current accuracy and prune all useless kernels ( CS below uselessness_thresh automatically set ) to end the stage . Recovery : wait for one last countdown of m_patience_param epochs ( optionally resetting the learning rate to its initial value and reducing it according to rlr0 ) ; after this countdown wait until reaching pre - pruning accuracy then if there are any new useless kernels wait for all kernels to settle and return to pruning else end the stage .
def self_constructing_var5(self, epoch, accuracy):
        continue_training = True
        settled_kernels_count = 0
        useful_kernels_count = 0
        useless_kernels_list = []
        kCS_settled = []
        cs_kernels = self.process_layer_kernels(
            self.total_blocks-1, self.layer_num_list[-1]-1, epoch)
        for k in range(len(self.kCS_FIFO)):
            self.kCS_FIFO[k].append(cs_kernels[k])
            if len(self.kCS_FIFO[k]) == self.dkCS_softening:
                self.dkCS_FIFO[k].append(
                    (self.kCS_FIFO[k][-1] - self.kCS_FIFO[k][0])/(
                        self.dkCS_softening-1))
                if ((len(self.dkCS_FIFO[k]) == self.dkCS_std_window) and (
                        np.abs(np.mean(self.dkCS_FIFO[k])
                               ) <= self.dkCS_stl_thresh) and
                        (np.abs(np.std(self.dkCS_FIFO[k])
                                ) <= self.dkCS_stl_thresh)):
                    settled_kernels_count += 1
                    if self.micro_stage == 1:
                        kCS_settled.append(self.kCS_FIFO[k][-1])
        if settled_kernels_count >= 0.5*self.growth_rate:
            if self.micro_stage == 1:
                self.usefulness_thresh = min(kCS_settled) + (
                    max(kCS_settled) - min(kCS_settled)
                    )*self.auto_usefulness_thresh
                self.uselessness_thresh = min(kCS_settled) + (
                    max(kCS_settled) - min(kCS_settled)
                    )*self.auto_uselessness_thresh
            for k in range(len(self.kCS_FIFO)):
                if np.mean(self.kCS_FIFO[k]) >= self.usefulness_thresh:
                    useful_kernels_count += 1
                if np.mean(self.kCS_FIFO[k]) <= self.uselessness_thresh:
                    useless_kernels_list.append(k)
        if self.algorithm_stage == 0:
            self.algorithm_stage += 1
        if self.algorithm_stage == 1:
            if self.micro_stage == 0:
                self.micro_stage += 1
                self.max_n_ep = epoch + 2*(self.m_patience_param + 1)
            if self.micro_stage == 1:
                if self.m_patience_cntdwn <= 0:
                    if settled_kernels_count == len(self.kCS_FIFO):
                        self.micro_stage += 1
                elif useful_kernels_count > self.useful_kernels_ceil:
                    self.useful_kernels_ceil = useful_kernels_count
                    self._new_kernels_to_last_layer(
                        self.expansion_rate,
                        complementarity=self.complementarity)
                    self.m_patience_cntdwn = self.m_patience_param
                    self.max_n_ep = epoch + 2*(self.m_patience_param + 1)
                else:
                    self.m_patience_cntdwn -= 1
            if self.micro_stage == 2:
                self.accuracy_pre_pruning = accuracy
                self._prune_kernels_in_last_layer(useless_kernels_list)
                self.micro_stage += 1
                self.m_patience_cntdwn = self.m_patience_param
                self.max_n_ep = epoch + self.m_patience_param + 1
            elif self.micro_stage == 3:
                if self.m_patience_cntdwn > 0:
                    self.m_patience_cntdwn -= 1
                elif accuracy >= self.accuracy_pre_pruning:
                    if len(useless_kernels_list) >= 1:
                        if settled_kernels_count == len(self.kCS_FIFO):
                            self.micro_stage = 2
                    else:
                        self.micro_stage += 1
            if self.micro_stage == 4:
                self.micro_stage = 0
                self.useful_kernels_ceil = 0
                self.m_patience_cntdwn = self.m_patience_param
                self.accuracy_pre_pruning = 0
                if abs(accuracy-self.accuracy_last_layer) >= self.impr_thresh:
                    self.accuracy_last_layer = accuracy
                    self._new_layer(self.growth_rate)
                else:
                    self.algorithm_stage += 1
        if self.algorithm_stage == 2:
            continue_training = False
            self.algorithm_stage = 0
            self.patience_cntdwn = self.patience_param
            self.accuracy_last_layer = 0
        return continue_training
def self_constructing_var5 ( self , epoch , accuracy ) : continue_training = True settled_kernels_count = 0 useful_kernels_count = 0 useless_kernels_list = [ ] kCS_settled = [ ] cs_kernels = self . process_layer_kernels ( self . total_blocks - 1 , self . layer_num_list [ - 1 ] - 1 , epoch ) for k in range ( len ( self . kCS_FIFO ) ) : self . kCS_FIFO [ k ] . append ( cs_kernels [ k ] ) if len ( self . kCS_FIFO [ k ] ) == self . dkCS_softening : self . dkCS_FIFO [ k ] . append ( ( self . kCS_FIFO [ k ] [ - 1 ] - self . kCS_FIFO [ k ] [ 0 ] ) / ( self . dkCS_softening - 1 ) ) if ( ( len ( self . dkCS_FIFO [ k ] ) == self . dkCS_std_window ) and ( np . abs ( np . mean ( self . dkCS_FIFO [ k ] ) ) <= self . dkCS_stl_thresh ) and ( np . abs ( np . std ( self . dkCS_FIFO [ k ] ) ) <= self . dkCS_stl_thresh ) ) : settled_kernels_count += 1 if self . micro_stage == 1 : kCS_settled . append ( self . kCS_FIFO [ k ] [ - 1 ] ) if settled_kernels_count >= 0.5 * self . growth_rate : if self . micro_stage == 1 : self . usefulness_thresh = min ( kCS_settled ) + ( max ( kCS_settled ) - min ( kCS_settled ) ) * self . auto_usefulness_thresh self . uselessness_thresh = min ( kCS_settled ) + ( max ( kCS_settled ) - min ( kCS_settled ) ) * self . auto_uselessness_thresh for k in range ( len ( self . kCS_FIFO ) ) : if np . mean ( self . kCS_FIFO [ k ] ) >= self . usefulness_thresh : useful_kernels_count += 1 if np . mean ( self . kCS_FIFO [ k ] ) <= self . uselessness_thresh : useless_kernels_list . append ( k ) if self . algorithm_stage == 0 : self . algorithm_stage += 1 if self . algorithm_stage == 1 : if self . micro_stage == 0 : self . micro_stage += 1 self . max_n_ep = epoch + 2 * ( self . m_patience_param + 1 ) if self . micro_stage == 1 : if self . m_patience_cntdwn <= 0 : if settled_kernels_count == len ( self . kCS_FIFO ) : self . micro_stage += 1 elif useful_kernels_count > self . useful_kernels_ceil : self . useful_kernels_ceil = useful_kernels_count self . _new_kernels_to_last_layer ( self . expansion_rate , complementarity = self . complementarity ) self . m_patience_cntdwn = self . m_patience_param self . max_n_ep = epoch + 2 * ( self . m_patience_param + 1 ) else : self . m_patience_cntdwn -= 1 if self . micro_stage == 2 : self . accuracy_pre_pruning = accuracy self . _prune_kernels_in_last_layer ( useless_kernels_list ) self . micro_stage += 1 self . m_patience_cntdwn = self . m_patience_param self . max_n_ep = epoch + self . m_patience_param + 1 elif self . micro_stage == 3 : if self . m_patience_cntdwn > 0 : self . m_patience_cntdwn -= 1 elif accuracy >= self . accuracy_pre_pruning : if len ( useless_kernels_list ) >= 1 : if settled_kernels_count == len ( self . kCS_FIFO ) : self . micro_stage = 2 else : self . micro_stage += 1 if self . micro_stage == 4 : self . micro_stage = 0 self . useful_kernels_ceil = 0 self . m_patience_cntdwn = self . m_patience_param self . accuracy_pre_pruning = 0 if abs ( accuracy - self . accuracy_last_layer ) >= self . impr_thresh : self . accuracy_last_layer = accuracy self . _new_layer ( self . growth_rate ) else : self . algorithm_stage += 1 if self . algorithm_stage == 2 : continue_training = False self . algorithm_stage = 0 self . patience_cntdwn = self . patience_param self . accuracy_last_layer = 0 return continue_training
A step of the self-constructing algorithm (variant #5) for one
training epoch.
A step of the self - constructing algorithm ( variant #5 ) for one training epoch .
"""
        A step of the self-constructing algorithm (variant #5) for one
        training epoch.
        Builds new layers in the last block depending on parameters.
        Returns True if training should continue, False otherwise.

        This algorithm consists of an macro-algorithm, which adds layers to the
        last block, and a micro-algorithm, which builds those layers kernel by
        kernel.
        - The macro-algorithm adds a new layer with one kernel and runs the
          micro-algorithm to build it (i.e. to add/prune kernels in it), then
          checks if the accuracy has improved significantly since the previous
          layer addition. If so it adds a new layer, else the algorithm ends.
        - The micro-algorithm consists of a succession of three stages:
            - Improvement: countdown of m_patience_param epochs; if the number
              of useful kernels (CS above usefulness_thresh, automatically set)
              is above the latest max number of useful kernels, add a kernel
              and restart the countdown; if the countdown ends, wait until all
              kernels have settled and end the stage.
            - Pruning: save the current accuracy and prune all useless kernels
              (CS below uselessness_thresh, automatically set) to end the
              stage.
            - Recovery: wait for one last countdown of m_patience_param epochs
              (optionally resetting the learning rate to its initial value and
              reducing it according to rlr0); after this countdown wait until
              reaching pre-pruning accuracy, then if there are any new useless
              kernels wait for all kernels to settle and return to pruning,
              else end the stage.

        Args:
            epoch: `int`, current training epoch (since adding the last block);
            accuracy: `float`, accuracy for this epoch.
        """ # Update the kernel CS lists, count settled kernels # Settled = kCS remained close to 0 during the last epochs # If half of the original kernels have settled # During impr. stage, calculate usefulness and uselessness thresh # Detect and count useful and useless kernels # Useful = kCS above the usefulness thresh # Useless = kCS below the uselessness thresh # stage #0 = ascension stage (currently does nothing) # stage #1 = improvement stage # micro-stage #0 = just some settings for the next (actual) stage # max_n_ep is used to estimate completion time # micro-stage #1 = micro-improvement stage # at the end of the patience countdown, end stage when all # the kernels have settled # if the number of useful kernels is above the latest max, # add a kernel and restart ctdwn # patience countdown progress # micro-stage #2 = micro-pruning stage # save the accuracy, prune useless kernels and end stage # run one last patience countdown for recovery # micro-stage #3 = micro-recovery stage (accessed in next epoch) # patience countdown to ensure recovery # wait until reaching pre-pruning accuracy # prune again if there are useless kernels, else end stage # but first, wait for all kernels to settle # at the end of the micro-algorithm, try to add a new layer # reset everything for the micro-algorithm # check if the accuracy has improved since the last layer # if so, add a layer, else end the improvement stage # alt. number of kernels = half the previous # layer's number if during the ascension stage. # self._new_layer(floor( #     len(self.kernels_ref_list[-1][-1])/2)) # stage #2 (nothing yet, stop the algorithm and reset everything)
self
epoch
`int`, current training epoch (since adding the last block).
` int ` current training epoch ( since adding the last block ) .
accuracy
`float`, accuracy for this epoch.
` float ` accuracy for this epoch .