models/optimizer/neumann.py
def step(self, closure=None):
        """
        Performs a single optimization step.
        
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        self.iter += 1


        loss = None
        if closure is not None: #checkout what's the deal with this. present in multiple pytorch optimizers
            loss = closure()

        for group in self.param_groups:

            sgd_steps = group['sgd_steps']

            alpha = group['alpha']
            beta = group['beta']
            gamma = group['gamma']
            K = group['K']
            momentum = group['momentum']
            mu = momentum*(1 - (1/(1+self.iter)))
            
            if mu >= 0.9:
                mu = 0.9
            elif mu <= 0.5:
                mu = 0.5


            eta = group['lr']/self.iter ## update with time ## changed
            # print("here")
            
            
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data 

                state = self.state[p]

                if len(state) == 0:
                    state['step'] = 0
                    state['m'] = torch.zeros_like(p.data).float()
                    state['d'] = torch.zeros_like(p.data).float()
                    # state['moving_avg'] = p.data


                if self.iter <= sgd_steps:
                
                    p.data.add_(-group['lr'], grad)
                    # self.sgd.step()
                    continue

                state['step'] += 1


                # Reset neumann iterate 
                if self.iter%K == 0:
                    state['m'] = grad.mul(-eta)
                    ## changed                  

                else:   
                    ## Compute update d_t
                    # diff = p.data.sub(state['moving_avg'])
                    # # print(diff)
                    # diff_norm = p.data.sub(state['moving_avg']).norm()
                    # # if np.count_nonzero(diff) and diff_norm > 0:
                    #     state['d'] = grad.add( (( (diff_norm.pow(2)).mul(alpha) ).sub( (diff_norm.pow(-2)).mul(beta) )).mul( diff.div(diff_norm)) )
                    # # else:
                    # state['d'].add_(grad)
                    state['d'] = grad

                    ## Update Neumann iterate
                    (state['m'].mul_(mu)).sub_( state['d'].mul(eta) )

                    ## Update Weights
                    p.data.add_((state['m'].mul(mu)).sub( state['d'].mul(eta)))

                    ## Update Moving Average
                    # state['moving_avg'] = p.data.add( (state['moving_avg'].sub(p.data)).mul(gamma) )

                # print(p.data)

        ## changed
        if self.iter%K == 0:
            group['K'] = group['K']*2
        
        # return loss

        Performs a single optimization step.
        
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        
Performs a single optimization step.
Performs a single optimization step .
def step(self, closure=None):
        self.iter += 1
        loss = None
        if closure is not None: 
            loss = closure()
        for group in self.param_groups:
            sgd_steps = group['sgd_steps']
            alpha = group['alpha']
            beta = group['beta']
            gamma = group['gamma']
            K = group['K']
            momentum = group['momentum']
            mu = momentum*(1 - (1/(1+self.iter)))
            if mu >= 0.9:
                mu = 0.9
            elif mu <= 0.5:
                mu = 0.5
            eta = group['lr']/self.iter 
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data 
                state = self.state[p]
                if len(state) == 0:
                    state['step'] = 0
                    state['m'] = torch.zeros_like(p.data).float()
                    state['d'] = torch.zeros_like(p.data).float()
                if self.iter <= sgd_steps:
                    p.data.add_(-group['lr'], grad)
                    continue
                state['step'] += 1
                if self.iter%K == 0:
                    state['m'] = grad.mul(-eta)
                else:   
                    state['d'] = grad
                    (state['m'].mul_(mu)).sub_( state['d'].mul(eta) )
                    p.data.add_((state['m'].mul(mu)).sub( state['d'].mul(eta)))
                    .add( (state['moving_avg'].sub(p.data)).mul(gamma) )
        if self.iter%K == 0:
            group['K'] = group['K']*2
def step ( self , closure = None ) : self . iter += 1 loss = None if closure is not None : loss = closure ( ) for group in self . param_groups : sgd_steps = group [ 'sgd_steps' ] alpha = group [ 'alpha' ] beta = group [ 'beta' ] gamma = group [ 'gamma' ] K = group [ 'K' ] momentum = group [ 'momentum' ] mu = momentum * ( 1 - ( 1 / ( 1 + self . iter ) ) ) if mu >= 0.9 : mu = 0.9 elif mu <= 0.5 : mu = 0.5 eta = group [ 'lr' ] / self . iter for p in group [ 'params' ] : if p . grad is None : continue grad = p . grad . data state = self . state [ p ] if len ( state ) == 0 : state [ 'step' ] = 0 state [ 'm' ] = torch . zeros_like ( p . data ) . float ( ) state [ 'd' ] = torch . zeros_like ( p . data ) . float ( ) if self . iter <= sgd_steps : p . data . add_ ( - group [ 'lr' ] , grad ) continue state [ 'step' ] += 1 if self . iter % K == 0 : state [ 'm' ] = grad . mul ( - eta ) else : state [ 'd' ] = grad ( state [ 'm' ] . mul_ ( mu ) ) . sub_ ( state [ 'd' ] . mul ( eta ) ) p . data . add_ ( ( state [ 'm' ] . mul ( mu ) ) . sub ( state [ 'd' ] . mul ( eta ) ) ) if self . iter % K == 0 : group [ 'K' ] = group [ 'K' ] * 2
Performs a single optimization step.
Performs a single optimization step .
"""
        Performs a single optimization step.
        
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """ #checkout what's the deal with this. present in multiple pytorch optimizers ## update with time ## changed # print("here") # state['moving_avg'] = p.data # self.sgd.step() # Reset neumann iterate  ## changed                   ## Compute update d_t # diff = p.data.sub(state['moving_avg']) # # print(diff) # diff_norm = p.data.sub(state['moving_avg']).norm() # # if np.count_nonzero(diff) and diff_norm > 0: #     state['d'] = grad.add( (( (diff_norm.pow(2)).mul(alpha) ).sub( (diff_norm.pow(-2)).mul(beta) )).mul( diff.div(diff_norm)) ) # # else: # state['d'].add_(grad) ## Update Neumann iterate ## Update Weights ## Update Moving Average # state['moving_avg'] = p.data.add( (state['moving_avg'].sub(p.data)).mul(gamma) ) # print(p.data) ## changed # return loss
self
closure
A closure that reevaluates the model
and returns the loss.
A closure that reevaluates the model and returns the loss .