deeppavlov/core/models/lr_scheduled_model.py
def fit(self, *args):
        """
        Find the best learning rate schedule, and set obtained values of learning rate
        and momentum for further model training. Best learning rate will be divided
        by `fit_learning_rate_div` for further training model.

        Args:
            *args: arguments

        Returns:

        """
        data = list(zip(*args))
        self.save()
        if self._fit_batch_size is None:
            raise ConfigError("in order to use fit() method"
                              " set `fit_batch_size` parameter")
        bs = int(self._fit_batch_size)
        data_len = len(data)
        num_batches = self._fit_max_batches or ((data_len - 1) // bs + 1)

        avg_loss = 0.
        best_loss = float('inf')
        lrs, losses = [], []
        _lr_find_schedule = DecayScheduler(start_val=self._fit_learning_rate[0],
                                           end_val=self._fit_learning_rate[1],
                                           dec_type="exponential",
                                           num_it=num_batches)
        self._lr = _lr_find_schedule.start_val
        self._mom = 0.
        self._update_graph_variables(learning_rate=self._lr, momentum=self._mom)
        best_lr = _lr_find_schedule.start_val
        for i in range(num_batches):
            batch_start = (i * bs) % data_len
            batch_end = batch_start + bs
            report = self.train_on_batch(*zip(*data[batch_start:batch_end]))
            if not isinstance(report, dict):
                report = {'loss': report}
            # Calculating smoothed loss
            avg_loss = self._fit_beta*avg_loss + (1 - self._fit_beta)*report['loss']
            smoothed_loss = avg_loss / (1 - self._fit_beta**(i + 1))
            lrs.append(self._lr)
            losses.append(smoothed_loss)
            log.info(f"Batch {i}/{num_batches}: smooth_loss = {smoothed_loss}"
                     f", lr = {self._lr}, best_lr = {best_lr}")
            if math.isnan(smoothed_loss) or (smoothed_loss > 4 * best_loss):
                break
            if (smoothed_loss < best_loss) and (i >= self._fit_min_batches):
                best_loss = smoothed_loss
                best_lr = self._lr
            self._lr = _lr_find_schedule.next_val()
            self._update_graph_variables(learning_rate=self._lr)

            if i >= num_batches:
                break
        # best_lr /= 10
        end_val = self._get_best(lrs, losses)

        start_val = end_val
        if self._lr_schedule.dec_type in (DecayType.ONECYCLE, DecayType.TRAPEZOID):
            start_val = end_val / self._fit_learning_rate_div
        elif self._lr_schedule.dec_type in (DecayType.POLYNOMIAL, DecayType.EXPONENTIAL,
                                            DecayType.LINEAR, DecayType.COSINE):
            start_val = end_val
            end_val = end_val / self._fit_learning_rate_div
        self._lr_schedule = DecayScheduler(start_val=start_val,
                                           end_val=end_val,
                                           num_it=self._lr_schedule.nb,
                                           dec_type=self._lr_schedule.dec_type,
                                           extra=self._lr_schedule.extra)
        log.info(f"Found best learning rate value = {best_lr}"
                 f", setting new learning rate schedule with {self._lr_schedule}.")

        self.load()
        self._lr = self._lr_schedule.start_val
        self._mom = self._mom_schedule.start_val
        self._update_graph_variables(learning_rate=self._lr, momentum=self._mom)
        return {'smoothed_loss': losses, 'learning_rate': lrs}
def fit(self, *args):
        """
        Find the best learning rate schedule, and set obtained values of learning rate
        and momentum for further model training. Best learning rate will be divided
        by `fit_learning_rate_div` for further training model.

        Args:
            *args: arguments

        Returns:

        """
        data = list(zip(*args))
        self.save()
        if self._fit_batch_size is None:
            raise ConfigError("in order to use fit() method"
                              " set `fit_batch_size` parameter")
        bs = int(self._fit_batch_size)
        data_len = len(data)
        num_batches = self._fit_max_batches or ((data_len - 1) // bs + 1)

        avg_loss = 0.
        best_loss = float('inf')
        lrs, losses = [], []
        _lr_find_schedule = DecayScheduler(start_val=self._fit_learning_rate[0],
                                           end_val=self._fit_learning_rate[1],
                                           dec_type="exponential",
                                           num_it=num_batches)
        self._lr = _lr_find_schedule.start_val
        self._mom = 0.
        self._update_graph_variables(learning_rate=self._lr, momentum=self._mom)
        best_lr = _lr_find_schedule.start_val
        for i in range(num_batches):
            batch_start = (i * bs) % data_len
            batch_end = batch_start + bs
            report = self.train_on_batch(*zip(*data[batch_start:batch_end]))
            if not isinstance(report, dict):
                report = {'loss': report}
            # Calculating smoothed loss
            avg_loss = self._fit_beta*avg_loss + (1 - self._fit_beta)*report['loss']
            smoothed_loss = avg_loss / (1 - self._fit_beta**(i + 1))
            lrs.append(self._lr)
            losses.append(smoothed_loss)
            log.info(f"Batch {i}/{num_batches}: smooth_loss = {smoothed_loss}"
                     f", lr = {self._lr}, best_lr = {best_lr}")
            if math.isnan(smoothed_loss) or (smoothed_loss > 4 * best_loss):
                break
            if (smoothed_loss < best_loss) and (i >= self._fit_min_batches):
                best_loss = smoothed_loss
                best_lr = self._lr
            self._lr = _lr_find_schedule.next_val()
            self._update_graph_variables(learning_rate=self._lr)

            if i >= num_batches:
                break
        # best_lr /= 10
        end_val = self._get_best(lrs, losses)

        start_val = end_val
        if self._lr_schedule.dec_type in (DecayType.ONECYCLE, DecayType.TRAPEZOID):
            start_val = end_val / self._fit_learning_rate_div
        elif self._lr_schedule.dec_type in (DecayType.POLYNOMIAL, DecayType.EXPONENTIAL,
                                            DecayType.LINEAR, DecayType.COSINE):
            start_val = end_val
            end_val = end_val / self._fit_learning_rate_div
        self._lr_schedule = DecayScheduler(start_val=start_val,
                                           end_val=end_val,
                                           num_it=self._lr_schedule.nb,
                                           dec_type=self._lr_schedule.dec_type,
                                           extra=self._lr_schedule.extra)
        log.info(f"Found best learning rate value = {best_lr}"
                 f", setting new learning rate schedule with {self._lr_schedule}.")

        self.load()
        self._lr = self._lr_schedule.start_val
        self._mom = self._mom_schedule.start_val
        self._update_graph_variables(learning_rate=self._lr, momentum=self._mom)
        return {'smoothed_loss': losses, 'learning_rate': lrs}
def fit ( self , * args ) : data = list ( zip ( * args ) ) self . save ( ) if self . _fit_batch_size is None : raise ConfigError ( "in order to use fit() method" " set `fit_batch_size` parameter" ) bs = int ( self . _fit_batch_size ) data_len = len ( data ) num_batches = self . _fit_max_batches or ( ( data_len - 1 ) // bs + 1 ) avg_loss = 0. best_loss = float ( 'inf' ) lrs , losses = [ ] , [ ] _lr_find_schedule = DecayScheduler ( start_val = self . _fit_learning_rate [ 0 ] , end_val = self . _fit_learning_rate [ 1 ] , dec_type = "exponential" , num_it = num_batches ) self . _lr = _lr_find_schedule . start_val self . _mom = 0. self . _update_graph_variables ( learning_rate = self . _lr , momentum = self . _mom ) best_lr = _lr_find_schedule . start_val for i in range ( num_batches ) : batch_start = ( i * bs ) % data_len batch_end = batch_start + bs report = self . train_on_batch ( * zip ( * data [ batch_start : batch_end ] ) ) if not isinstance ( report , dict ) : report = { 'loss' : report } # Calculating smoothed loss avg_loss = self . _fit_beta * avg_loss + ( 1 - self . _fit_beta ) * report [ 'loss' ] smoothed_loss = avg_loss / ( 1 - self . _fit_beta ** ( i + 1 ) ) lrs . append ( self . _lr ) losses . append ( smoothed_loss ) log . info ( f"Batch {i}/{num_batches}: smooth_loss = {smoothed_loss}" f", lr = {self._lr}, best_lr = {best_lr}" ) if math . isnan ( smoothed_loss ) or ( smoothed_loss > 4 * best_loss ) : break if ( smoothed_loss < best_loss ) and ( i >= self . _fit_min_batches ) : best_loss = smoothed_loss best_lr = self . _lr self . _lr = _lr_find_schedule . next_val ( ) self . _update_graph_variables ( learning_rate = self . _lr ) if i >= num_batches : break # best_lr /= 10 end_val = self . _get_best ( lrs , losses ) start_val = end_val if self . _lr_schedule . dec_type in ( DecayType . ONECYCLE , DecayType . TRAPEZOID ) : start_val = end_val / self . _fit_learning_rate_div elif self . _lr_schedule . dec_type in ( DecayType . POLYNOMIAL , DecayType . EXPONENTIAL , DecayType . LINEAR , DecayType . COSINE ) : start_val = end_val end_val = end_val / self . _fit_learning_rate_div self . _lr_schedule = DecayScheduler ( start_val = start_val , end_val = end_val , num_it = self . _lr_schedule . nb , dec_type = self . _lr_schedule . dec_type , extra = self . _lr_schedule . extra ) log . info ( f"Found best learning rate value = {best_lr}" f", setting new learning rate schedule with {self._lr_schedule}." ) self . load ( ) self . _lr = self . _lr_schedule . start_val self . _mom = self . _mom_schedule . start_val self . _update_graph_variables ( learning_rate = self . _lr , momentum = self . _mom ) return { 'smoothed_loss' : losses , 'learning_rate' : lrs }
Find the best learning rate schedule, and set obtained values of learning rate
        and momentum for further model training. Best learning rate will be divided
        by `fit_learning_rate_div` for further training model.

        Args:
            *args: arguments

        Returns:
Find the best learning rate schedule and set obtained values of learning rate and momentum for further model training . Best learning rate will be divided by fit_learning_rate_div for further training model .