carpet/lista_base.py
def _fit_sub_net_batch_gd(self, x, lbda, params, layer_id, max_iter,
                              output_layer=None, eps=1e-20):
        """ Fit the parameters of the sub-network. """
        if output_layer is None:
            output_layer = layer_id
        with torch.no_grad():
            z = self(x, lbda, output_layer=output_layer)
            prev_loss = self._loss_fn(x, lbda, z)
            self.training_loss_ = [float(prev_loss)]
        self.norm_grad_ = []

        verbose_rate = 50
        lr = 1.0
        is_converged = True

        i = 0

        if self.verbose > 1:
            print(f"\r[{self.name} - layer{layer_id}] "  # noqa: E999
                  f"Fitting, step_size={lr:.2e}, "
                  f"iter={i}/{max_iter}, "
                  f"loss={self.training_loss_[-1]:.3e}")

        while i < max_iter:

            i += 1

            # Verbosity
            if self.verbose > 1 and i % verbose_rate == 0:
                print(f"\r[{self.name} - layer{layer_id}] "  # noqa: E999
                      f"Fitting, step_size={lr:.2e}, "
                      f"iter={i}/{max_iter}, "
                      f"loss={self.training_loss_[-1]:.3e}")
            if self.verbose > 0:
                p = (layer_id - 1 + i/max_iter) / self.n_layers
                print(f"\rTraining... {p:7.2%}", end='', flush=True)

            # Gradient computation
            self._compute_gradient(x, lbda, output_layer)

            # Compute a gradient step `-lr * grad`
            self._update_parameters(params, lr=lr)

            # Back-tracking line search descent step with parameter c = 1/2
            while lr >= 1e-20:

                # Compute new possible loss
                with torch.no_grad():
                    z = self(x, lbda, output_layer=output_layer)
                    loss_value = self._loss_fn(x, lbda, z)

                # Accepting the point when the loss decreases
                if prev_loss > loss_value:
                    prev_loss = loss_value
                    self.training_loss_.append(float(loss_value))
                    # Increase the learning rate to make sure we don't have a
                    # learning rate too low for the next step. Here we chose
                    # arbitrarily to increase by (1 / c) ** 3. This results
                    # from a tradeoff between computations at convergence and
                    # finding a large enough step size.
                    lr *= 8
                    break
                # Rejecting the point and taking c * lr as our new
                # learning rate.
                else:
                    # Back track with step `+ (1 - c)lr * grad`
                    # This overall results with a step `- c * lr * grad`
                    self._update_parameters(params, lr=-lr/2)
                    lr /= 2.0
            else:
                # Stopping criterion lr < 1e-20 was reached.
                # Make sure we get back to the original parameters before the
                # line search with a full backtracking step.
                self._update_parameters(params, lr=-lr)
                break

            # Early stopping when the training loss is not moving anymore
            if len(self.training_loss_) > 1:
                if self.training_loss_[-2] - self.training_loss_[-1] < eps:
                    break
        else:
            is_converged = False

        if self.verbose:
            converging_status = '' if is_converged else 'not '
            print(f"\r[{self.name} - layer{layer_id}] Finished "
                  f"({converging_status}converged), step_size={lr:.2e}, "
                  f"iter={i}/{max_iter}, "
                  f"final-loss={self.training_loss_[-1]:.6e}")

        msg = "Loss function has increased during training!"
        assert np.all(np.diff(self.training_loss_) < 0.0), msg
 Fit the parameters of the sub-network. 
Fit the parameters of the sub-network.
Fit the parameters of the sub - network .
def _fit_sub_net_batch_gd(self, x, lbda, params, layer_id, max_iter,
                              output_layer=None, eps=1e-20):
        if output_layer is None:
            output_layer = layer_id
        with torch.no_grad():
            z = self(x, lbda, output_layer=output_layer)
            prev_loss = self._loss_fn(x, lbda, z)
            self.training_loss_ = [float(prev_loss)]
        self.norm_grad_ = []
        verbose_rate = 50
        lr = 1.0
        is_converged = True
        i = 0
        if self.verbose > 1:
            print(f"\r[{self.name} - layer{layer_id}] "  
                  f"Fitting, step_size={lr:.2e}, "
                  f"iter={i}/{max_iter}, "
                  f"loss={self.training_loss_[-1]:.3e}")
        while i < max_iter:
            i += 1
            if self.verbose > 1 and i % verbose_rate == 0:
                print(f"\r[{self.name} - layer{layer_id}] "  
                      f"Fitting, step_size={lr:.2e}, "
                      f"iter={i}/{max_iter}, "
                      f"loss={self.training_loss_[-1]:.3e}")
            if self.verbose > 0:
                p = (layer_id - 1 + i/max_iter) / self.n_layers
                print(f"\rTraining... {p:7.2%}", end='', flush=True)
            self._compute_gradient(x, lbda, output_layer)
            self._update_parameters(params, lr=lr)
            while lr >= 1e-20:
                with torch.no_grad():
                    z = self(x, lbda, output_layer=output_layer)
                    loss_value = self._loss_fn(x, lbda, z)
                if prev_loss > loss_value:
                    prev_loss = loss_value
                    self.training_loss_.append(float(loss_value))
                    lr *= 8
                    break
                else:
                    self._update_parameters(params, lr=-lr/2)
                    lr /= 2.0
            else:
                self._update_parameters(params, lr=-lr)
                break
            if len(self.training_loss_) > 1:
                if self.training_loss_[-2] - self.training_loss_[-1] < eps:
                    break
        else:
            is_converged = False
        if self.verbose:
            converging_status = '' if is_converged else 'not '
            print(f"\r[{self.name} - layer{layer_id}] Finished "
                  f"({converging_status}converged), step_size={lr:.2e}, "
                  f"iter={i}/{max_iter}, "
                  f"final-loss={self.training_loss_[-1]:.6e}")
        msg = "Loss function has increased during training!"
        assert np.all(np.diff(self.training_loss_) < 0.0), msg
def _fit_sub_net_batch_gd ( self , x , lbda , params , layer_id , max_iter , output_layer = None , eps = 1e-20 ) : if output_layer is None : output_layer = layer_id with torch . no_grad ( ) : z = self ( x , lbda , output_layer = output_layer ) prev_loss = self . _loss_fn ( x , lbda , z ) self . training_loss_ = [ float ( prev_loss ) ] self . norm_grad_ = [ ] verbose_rate = 50 lr = 1.0 is_converged = True i = 0 if self . verbose > 1 : print ( f"\r[{self.name} - layer{layer_id}] " f"Fitting, step_size={lr:.2e}, " f"iter={i}/{max_iter}, " f"loss={self.training_loss_[-1]:.3e}" ) while i < max_iter : i += 1 if self . verbose > 1 and i % verbose_rate == 0 : print ( f"\r[{self.name} - layer{layer_id}] " f"Fitting, step_size={lr:.2e}, " f"iter={i}/{max_iter}, " f"loss={self.training_loss_[-1]:.3e}" ) if self . verbose > 0 : p = ( layer_id - 1 + i / max_iter ) / self . n_layers print ( f"\rTraining... {p:7.2%}" , end = '' , flush = True ) self . _compute_gradient ( x , lbda , output_layer ) self . _update_parameters ( params , lr = lr ) while lr >= 1e-20 : with torch . no_grad ( ) : z = self ( x , lbda , output_layer = output_layer ) loss_value = self . _loss_fn ( x , lbda , z ) if prev_loss > loss_value : prev_loss = loss_value self . training_loss_ . append ( float ( loss_value ) ) lr *= 8 break else : self . _update_parameters ( params , lr = - lr / 2 ) lr /= 2.0 else : self . _update_parameters ( params , lr = - lr ) break if len ( self . training_loss_ ) > 1 : if self . training_loss_ [ - 2 ] - self . training_loss_ [ - 1 ] < eps : break else : is_converged = False if self . verbose : converging_status = '' if is_converged else 'not ' print ( f"\r[{self.name} - layer{layer_id}] Finished " f"({converging_status}converged), step_size={lr:.2e}, " f"iter={i}/{max_iter}, " f"final-loss={self.training_loss_[-1]:.6e}" ) msg = "Loss function has increased during training!" assert np . all ( np . diff ( self . training_loss_ ) < 0.0 ) , msg
Fit the parameters of the sub-network.
Fit the parameters of the sub - network .
""" Fit the parameters of the sub-network. """ # noqa: E999 # Verbosity # noqa: E999 # Gradient computation # Compute a gradient step `-lr * grad` # Back-tracking line search descent step with parameter c = 1/2 # Compute new possible loss # Accepting the point when the loss decreases # Increase the learning rate to make sure we don't have a # learning rate too low for the next step. Here we chose # arbitrarily to increase by (1 / c) ** 3. This results # from a tradeoff between computations at convergence and # finding a large enough step size. # Rejecting the point and taking c * lr as our new # learning rate. # Back track with step `+ (1 - c)lr * grad` # This overall results with a step `- c * lr * grad` # Stopping criterion lr < 1e-20 was reached. # Make sure we get back to the original parameters before the # line search with a full backtracking step. # Early stopping when the training loss is not moving anymore
self
x
lbda
params
layer_id
max_iter
output_layer
eps