opt_algorithms/st_optimizers.py
def step(self, closure):
        """Performs a single optimization step.

        Arguments:
            closure (callable): A closure that reevaluates the model
                and returns the loss.
        """
        assert len(self.param_groups) == 1

        group = self.param_groups[0]
        lr = group['lr']
        max_iter = group['max_iter']
        max_eval = group['max_eval']
        tolerance_grad = group['tolerance_grad']
        tolerance_change = group['tolerance_change']
        line_search_fn = group['line_search_fn']
        history_size = group['history_size']

        # NOTE: LBFGS has only global state, but we register it as state for
        # the first param, because this helps with casting in load_state_dict
        state = self.state[self._params[0]]
        state.setdefault('func_evals', 0)
        state.setdefault('n_iter', 0)

        # evaluate initial f(x) and df/dx
        orig_loss = closure()
        loss = float(orig_loss)
        current_evals = 1
        state['func_evals'] += 1

        flat_grad = self._gather_flat_grad()
        opt_cond = flat_grad.abs().max() <= tolerance_grad

        # optimal condition
        if opt_cond:
            return orig_loss

        # tensors cached in state (for tracing)
        d = state.get('d')
        t = state.get('t')
        old_dirs = state.get('old_dirs')
        old_stps = state.get('old_stps')
        ro = state.get('ro')
        H_diag = state.get('H_diag')
        prev_flat_grad = state.get('prev_flat_grad')
        prev_loss = state.get('prev_loss')

        n_iter = 0
        # optimize for a max of max_iter iterations
        while n_iter < max_iter:
            # keep track of nb of iterations
            n_iter += 1
            state['n_iter'] += 1

            ############################################################
            # compute gradient descent direction
            ############################################################
            if state['n_iter'] == 1:
                d = flat_grad.neg()
                old_dirs = []
                old_stps = []
                ro = []
                H_diag = 1
            else:
                # do lbfgs update (update memory)
                y = flat_grad.sub(prev_flat_grad)
                s = d.mul(t)
                ys = y.dot(s)  # y*s
                if ys > 1e-10:
                    # updating memory
                    if len(old_dirs) == history_size:
                        # shift history by one (limited-memory)
                        old_dirs.pop(0)
                        old_stps.pop(0)
                        ro.pop(0)

                    # store new direction/step
                    old_dirs.append(y)
                    old_stps.append(s)
                    ro.append(1. / ys)

                    # update scale of initial Hessian approximation
                    H_diag = ys / y.dot(y)  # (y*y)

                # compute the approximate (L-BFGS) inverse Hessian
                # multiplied by the gradient
                num_old = len(old_dirs)

                if 'al' not in state:
                    state['al'] = [None] * history_size
                al = state['al']

                # iteration in L-BFGS loop collapsed to use just one buffer
                q = flat_grad.neg()
                for i in range(num_old - 1, -1, -1):
                    al[i] = old_stps[i].dot(q) * ro[i]
                    q.add_(-al[i], old_dirs[i])

                # multiply by initial Hessian
                # r/d is the final direction
                d = r = torch.mul(q, H_diag)
                for i in range(num_old):
                    be_i = old_dirs[i].dot(r) * ro[i]
                    r.add_(al[i] - be_i, old_stps[i])

            if prev_flat_grad is None:
                prev_flat_grad = flat_grad.clone()
            else:
                prev_flat_grad.copy_(flat_grad)
            prev_loss = loss

            ############################################################
            # compute step length
            ############################################################
            # reset initial guess for step size
            if state['n_iter'] == 1:
                t = min(1., 1. / flat_grad.abs().sum()) * lr
            else:
                t = lr

            # directional derivative
            gtd = flat_grad.dot(d)  # g * d

            # directional derivative is below tolerance
            if gtd > -tolerance_change:
                break

            # optional line search: user function
            ls_func_evals = 0
            if line_search_fn is not None:
                # perform line search, using user function
                if line_search_fn != "wolfe":
                    raise RuntimeError("only 'wolfe' is supported")
                else:
                    x_init = self._clone_param()

                    def obj_func(x, t, d):
                        return self._directional_evaluate(closure, x, t, d)

                    t = self._my_wolfe(obj_func, x_init, d, loss, flat_grad)
                self._add_grad(t, d)
                loss = float(closure())
                flat_grad = self._gather_flat_grad()
                ls_func_evals = 1
                opt_cond = flat_grad.abs().max() <= tolerance_grad
            else:
                # no line search, simply move with fixed-step
                self._add_grad(t, d)
                if n_iter != max_iter:
                    # re-evaluate function only if not in last iteration
                    # the reason we do this: in a stochastic setting,
                    # no use to re-evaluate that function here
                    loss = float(closure())
                    flat_grad = self._gather_flat_grad()
                    opt_cond = flat_grad.abs().max() <= tolerance_grad
                    ls_func_evals = 1

            # update func eval
            current_evals += ls_func_evals
            state['func_evals'] += ls_func_evals

            ############################################################
            # check conditions
            ############################################################
            if n_iter == max_iter:
                break

            if current_evals >= max_eval:
                break

            # optimal condition
            if opt_cond:
                break

            # lack of progress
            if d.mul(t).abs().max() <= tolerance_change:
                break

            if abs(loss - prev_loss) < tolerance_change:
                break

        state['d'] = d
        state['t'] = t
        state['old_dirs'] = old_dirs
        state['old_stps'] = old_stps
        state['ro'] = ro
        state['H_diag'] = H_diag
        state['prev_flat_grad'] = prev_flat_grad
        state['prev_loss'] = prev_loss

        return orig_loss, t
Performs a single optimization step.

        Arguments:
            closure (callable): A closure that reevaluates the model
                and returns the loss.
        
Performs a single optimization step.
Performs a single optimization step .
def step(self, closure):
        assert len(self.param_groups) == 1
        group = self.param_groups[0]
        lr = group['lr']
        max_iter = group['max_iter']
        max_eval = group['max_eval']
        tolerance_grad = group['tolerance_grad']
        tolerance_change = group['tolerance_change']
        line_search_fn = group['line_search_fn']
        history_size = group['history_size']
        state = self.state[self._params[0]]
        state.setdefault('func_evals', 0)
        state.setdefault('n_iter', 0)
        orig_loss = closure()
        loss = float(orig_loss)
        current_evals = 1
        state['func_evals'] += 1
        flat_grad = self._gather_flat_grad()
        opt_cond = flat_grad.abs().max() <= tolerance_grad
        if opt_cond:
            return orig_loss
        d = state.get('d')
        t = state.get('t')
        old_dirs = state.get('old_dirs')
        old_stps = state.get('old_stps')
        ro = state.get('ro')
        H_diag = state.get('H_diag')
        prev_flat_grad = state.get('prev_flat_grad')
        prev_loss = state.get('prev_loss')
        n_iter = 0
        while n_iter < max_iter:
            n_iter += 1
            state['n_iter'] += 1
            if state['n_iter'] == 1:
                d = flat_grad.neg()
                old_dirs = []
                old_stps = []
                ro = []
                H_diag = 1
            else:
                y = flat_grad.sub(prev_flat_grad)
                s = d.mul(t)
                ys = y.dot(s)  
                if ys > 1e-10:
                    if len(old_dirs) == history_size:
                        old_dirs.pop(0)
                        old_stps.pop(0)
                        ro.pop(0)
                    old_dirs.append(y)
                    old_stps.append(s)
                    ro.append(1. / ys)
                    H_diag = ys / y.dot(y)  
                num_old = len(old_dirs)
                if 'al' not in state:
                    state['al'] = [None] * history_size
                al = state['al']
                q = flat_grad.neg()
                for i in range(num_old - 1, -1, -1):
                    al[i] = old_stps[i].dot(q) * ro[i]
                    q.add_(-al[i], old_dirs[i])
                d = r = torch.mul(q, H_diag)
                for i in range(num_old):
                    be_i = old_dirs[i].dot(r) * ro[i]
                    r.add_(al[i] - be_i, old_stps[i])
            if prev_flat_grad is None:
                prev_flat_grad = flat_grad.clone()
            else:
                prev_flat_grad.copy_(flat_grad)
            prev_loss = loss
            if state['n_iter'] == 1:
                t = min(1., 1. / flat_grad.abs().sum()) * lr
            else:
                t = lr
            gtd = flat_grad.dot(d)  
            if gtd > -tolerance_change:
                break
            ls_func_evals = 0
            if line_search_fn is not None:
                if line_search_fn != "wolfe":
                    raise RuntimeError("only 'wolfe' is supported")
                else:
                    x_init = self._clone_param()
                    def obj_func(x, t, d):
                        return self._directional_evaluate(closure, x, t, d)
                    t = self._my_wolfe(obj_func, x_init, d, loss, flat_grad)
                self._add_grad(t, d)
                loss = float(closure())
                flat_grad = self._gather_flat_grad()
                ls_func_evals = 1
                opt_cond = flat_grad.abs().max() <= tolerance_grad
            else:
                self._add_grad(t, d)
                if n_iter != max_iter:
                    loss = float(closure())
                    flat_grad = self._gather_flat_grad()
                    opt_cond = flat_grad.abs().max() <= tolerance_grad
                    ls_func_evals = 1
            current_evals += ls_func_evals
            state['func_evals'] += ls_func_evals
            if n_iter == max_iter:
                break
            if current_evals >= max_eval:
                break
            if opt_cond:
                break
            if d.mul(t).abs().max() <= tolerance_change:
                break
            if abs(loss - prev_loss) < tolerance_change:
                break
        state['d'] = d
        state['t'] = t
        state['old_dirs'] = old_dirs
        state['old_stps'] = old_stps
        state['ro'] = ro
        state['H_diag'] = H_diag
        state['prev_flat_grad'] = prev_flat_grad
        state['prev_loss'] = prev_loss
        return orig_loss, t
def step ( self , closure ) : assert len ( self . param_groups ) == 1 group = self . param_groups [ 0 ] lr = group [ 'lr' ] max_iter = group [ 'max_iter' ] max_eval = group [ 'max_eval' ] tolerance_grad = group [ 'tolerance_grad' ] tolerance_change = group [ 'tolerance_change' ] line_search_fn = group [ 'line_search_fn' ] history_size = group [ 'history_size' ] state = self . state [ self . _params [ 0 ] ] state . setdefault ( 'func_evals' , 0 ) state . setdefault ( 'n_iter' , 0 ) orig_loss = closure ( ) loss = float ( orig_loss ) current_evals = 1 state [ 'func_evals' ] += 1 flat_grad = self . _gather_flat_grad ( ) opt_cond = flat_grad . abs ( ) . max ( ) <= tolerance_grad if opt_cond : return orig_loss d = state . get ( 'd' ) t = state . get ( 't' ) old_dirs = state . get ( 'old_dirs' ) old_stps = state . get ( 'old_stps' ) ro = state . get ( 'ro' ) H_diag = state . get ( 'H_diag' ) prev_flat_grad = state . get ( 'prev_flat_grad' ) prev_loss = state . get ( 'prev_loss' ) n_iter = 0 while n_iter < max_iter : n_iter += 1 state [ 'n_iter' ] += 1 if state [ 'n_iter' ] == 1 : d = flat_grad . neg ( ) old_dirs = [ ] old_stps = [ ] ro = [ ] H_diag = 1 else : y = flat_grad . sub ( prev_flat_grad ) s = d . mul ( t ) ys = y . dot ( s ) if ys > 1e-10 : if len ( old_dirs ) == history_size : old_dirs . pop ( 0 ) old_stps . pop ( 0 ) ro . pop ( 0 ) old_dirs . append ( y ) old_stps . append ( s ) ro . append ( 1. / ys ) H_diag = ys / y . dot ( y ) num_old = len ( old_dirs ) if 'al' not in state : state [ 'al' ] = [ None ] * history_size al = state [ 'al' ] q = flat_grad . neg ( ) for i in range ( num_old - 1 , - 1 , - 1 ) : al [ i ] = old_stps [ i ] . dot ( q ) * ro [ i ] q . add_ ( - al [ i ] , old_dirs [ i ] ) d = r = torch . mul ( q , H_diag ) for i in range ( num_old ) : be_i = old_dirs [ i ] . dot ( r ) * ro [ i ] r . add_ ( al [ i ] - be_i , old_stps [ i ] ) if prev_flat_grad is None : prev_flat_grad = flat_grad . clone ( ) else : prev_flat_grad . copy_ ( flat_grad ) prev_loss = loss if state [ 'n_iter' ] == 1 : t = min ( 1. , 1. / flat_grad . abs ( ) . sum ( ) ) * lr else : t = lr gtd = flat_grad . dot ( d ) if gtd > - tolerance_change : break ls_func_evals = 0 if line_search_fn is not None : if line_search_fn != "wolfe" : raise RuntimeError ( "only 'wolfe' is supported" ) else : x_init = self . _clone_param ( ) def obj_func ( x , t , d ) : return self . _directional_evaluate ( closure , x , t , d ) t = self . _my_wolfe ( obj_func , x_init , d , loss , flat_grad ) self . _add_grad ( t , d ) loss = float ( closure ( ) ) flat_grad = self . _gather_flat_grad ( ) ls_func_evals = 1 opt_cond = flat_grad . abs ( ) . max ( ) <= tolerance_grad else : self . _add_grad ( t , d ) if n_iter != max_iter : loss = float ( closure ( ) ) flat_grad = self . _gather_flat_grad ( ) opt_cond = flat_grad . abs ( ) . max ( ) <= tolerance_grad ls_func_evals = 1 current_evals += ls_func_evals state [ 'func_evals' ] += ls_func_evals if n_iter == max_iter : break if current_evals >= max_eval : break if opt_cond : break if d . mul ( t ) . abs ( ) . max ( ) <= tolerance_change : break if abs ( loss - prev_loss ) < tolerance_change : break state [ 'd' ] = d state [ 't' ] = t state [ 'old_dirs' ] = old_dirs state [ 'old_stps' ] = old_stps state [ 'ro' ] = ro state [ 'H_diag' ] = H_diag state [ 'prev_flat_grad' ] = prev_flat_grad state [ 'prev_loss' ] = prev_loss return orig_loss , t
Performs a single optimization step.
Performs a single optimization step .
"""Performs a single optimization step.

        Arguments:
            closure (callable): A closure that reevaluates the model
                and returns the loss.
        """ # NOTE: LBFGS has only global state, but we register it as state for
 # the first param, because this helps with casting in load_state_dict
 # evaluate initial f(x) and df/dx
 # optimal condition
 # tensors cached in state (for tracing)
 # optimize for a max of max_iter iterations
 # keep track of nb of iterations
 ############################################################
 # compute gradient descent direction
 ############################################################
 # do lbfgs update (update memory)
 # y*s
 # updating memory
 # shift history by one (limited-memory)
 # store new direction/step
 # update scale of initial Hessian approximation
 # (y*y)
 # compute the approximate (L-BFGS) inverse Hessian
 # multiplied by the gradient
 # iteration in L-BFGS loop collapsed to use just one buffer
 # multiply by initial Hessian
 # r/d is the final direction
 ############################################################
 # compute step length
 ############################################################
 # reset initial guess for step size
 # directional derivative
 # g * d
 # directional derivative is below tolerance
 # optional line search: user function
 # perform line search, using user function
 # no line search, simply move with fixed-step
 # re-evaluate function only if not in last iteration
 # the reason we do this: in a stochastic setting,
 # no use to re-evaluate that function here
 # update func eval
 ############################################################
 # check conditions
 ############################################################
 # optimal condition
 # lack of progress

self
closure
A closure that reevaluates the model
and returns the loss.
A closure that reevaluates the model and returns the loss .