opt_algorithms/st_optimizers.py
def step(self, closure):
        """Performs a single optimization step.

        Arguments:
            closure (callable): A closure that reevaluates the model
                and returns the loss.
        """
        group = self.param_groups[0]
        p = group['params'][0].t() #use H as N x s
        N, s = p.shape
        g = group['params'][0].grad.t() #use H as N x s
        lr = group['lr']
        beta = group['beta']
        loss = float(closure(p.t()))

        # Backtracking line-search
        p1 = None
        u, s, v = torch.empty((s,s), device=p.device), torch.empty((s,), device=p.device), torch.empty((N,s), device=p.device)
        i = 0
        while i < 1000:
            try:
                p1 = self.st_project(p - lr * g, (u, s, v))
            except:
                break
            loss1 = float(closure(p1.t()))
            condition = loss1 <= loss + torch.trace(torch.mm(g.t(), (p1-p))) + torch.trace(torch.mm(p1-p, (p1-p).t())) / (2*lr)
            if condition:
                break
            else:
                lr *= beta
            i += 1
        group['params'][0][:] = p1.t()[:]

        return loss, lr
Performs a single optimization step.

        Arguments:
            closure (callable): A closure that reevaluates the model
                and returns the loss.
        
Performs a single optimization step.
Performs a single optimization step .
def step(self, closure):
        group = self.param_groups[0]
        p = group['params'][0].t() 
        N, s = p.shape
        g = group['params'][0].grad.t() 
        lr = group['lr']
        beta = group['beta']
        loss = float(closure(p.t()))
        p1 = None
        u, s, v = torch.empty((s,s), device=p.device), torch.empty((s,), device=p.device), torch.empty((N,s), device=p.device)
        i = 0
        while i < 1000:
            try:
                p1 = self.st_project(p - lr * g, (u, s, v))
            except:
                break
            loss1 = float(closure(p1.t()))
            condition = loss1 <= loss + torch.trace(torch.mm(g.t(), (p1-p))) + torch.trace(torch.mm(p1-p, (p1-p).t())) / (2*lr)
            if condition:
                break
            else:
                lr *= beta
            i += 1
        group['params'][0][:] = p1.t()[:]
        return loss, lr
def step ( self , closure ) : group = self . param_groups [ 0 ] p = group [ 'params' ] [ 0 ] . t ( ) N , s = p . shape g = group [ 'params' ] [ 0 ] . grad . t ( ) lr = group [ 'lr' ] beta = group [ 'beta' ] loss = float ( closure ( p . t ( ) ) ) p1 = None u , s , v = torch . empty ( ( s , s ) , device = p . device ) , torch . empty ( ( s , ) , device = p . device ) , torch . empty ( ( N , s ) , device = p . device ) i = 0 while i < 1000 : try : p1 = self . st_project ( p - lr * g , ( u , s , v ) ) except : break loss1 = float ( closure ( p1 . t ( ) ) ) condition = loss1 <= loss + torch . trace ( torch . mm ( g . t ( ) , ( p1 - p ) ) ) + torch . trace ( torch . mm ( p1 - p , ( p1 - p ) . t ( ) ) ) / ( 2 * lr ) if condition : break else : lr *= beta i += 1 group [ 'params' ] [ 0 ] [ : ] = p1 . t ( ) [ : ] return loss , lr
Performs a single optimization step.
Performs a single optimization step .
"""Performs a single optimization step.

        Arguments:
            closure (callable): A closure that reevaluates the model
                and returns the loss.
        """ #use H as N x s
 #use H as N x s
 # Backtracking line-search

self
closure
A closure that reevaluates the model
and returns the loss.
A closure that reevaluates the model and returns the loss .