neuroprob/utils/pytorch.py
def step(self, closure, b=None, M_inv=None):
        """
        Performs a single optimization step.
        Arguments:
            closure (callable): A closure that re-evaluates the model
                and returns a tuple of the loss and the output.
            b (callable, optional): A closure that calculates the vector b in
                the minimization problem x^T . A . x + x^T b.
            M (callable, optional): The INVERSE preconditioner of A
        """
        assert len(self.param_groups) == 1

        group = self.param_groups[0]
        alpha = group['alpha']
        delta_decay = group['delta_decay']
        cg_max_iter = group['cg_max_iter']
        damping = group['damping']
        use_gnm = group['use_gnm']
        verbose = group['verbose']

        state = self.state[self._params[0]]
        state.setdefault('func_evals', 0)
        state.setdefault('n_iter', 0)

        loss_before, output = closure()
        current_evals = 1
        state['func_evals'] += 1

        # Gather current parameters and respective gradients
        flat_params = parameters_to_vector(self._params)
        flat_grad = self._gather_flat_grad()

        # Define linear operator
        if use_gnm:
            # Generalized Gauss-Newton vector product
            def A(x):
                return self._Gv(loss_before, output, x, damping)
        else:
            # Hessian-vector product
            def A(x):
                return self._Hv(flat_grad, x, damping)

        if M_inv is not None:
            m_inv = M_inv()

            # Preconditioner recipe (Section 20.13)
            if m_inv.dim() == 1:
                m = (m_inv + damping) ** (-0.85)

                def M(x):
                    return m * x
            else:
                m = torch.inverse(m_inv + damping * torch.eye(*m_inv.shape))

                def M(x):
                    return m @ x
        else:
            M = None

        b = flat_grad.detach() if b is None else b().detach().flatten()

        # Initializing Conjugate-Gradient (Section 20.10)
        if state.get('init_delta') is not None:
            init_delta = delta_decay * state.get('init_delta')
        else:
            init_delta = torch.zeros_like(flat_params)

        eps = torch.finfo(b.dtype).eps

        # Conjugate-Gradient
        deltas, Ms = self._CG(A=A, b=b.neg(), x0=init_delta,
                              M=M, max_iter=cg_max_iter,
                              tol=1e1 * eps, eps=eps, martens=True)

        # Update parameters
        delta = state['init_delta'] = deltas[-1]
        M = Ms[-1]

        vector_to_parameters(flat_params + delta, self._params)
        loss_now = closure()[0]
        current_evals += 1
        state['func_evals'] += 1

        # Conjugate-Gradient backtracking (Section 20.8.7)
        if verbose:
            print("Loss before CG: {}".format(float(loss_before)))
            print("Loss before BT: {}".format(float(loss_now)))

        for (d, m) in zip(reversed(deltas[:-1][::2]), reversed(Ms[:-1][::2])):
            vector_to_parameters(flat_params + d, self._params)
            loss_prev = closure()[0]
            if float(loss_prev) > float(loss_now):
                break
            delta = d
            M = m
            loss_now = loss_prev

        if verbose:
            print("Loss after BT:  {}".format(float(loss_now)))

        # The Levenberg-Marquardt Heuristic (Section 20.8.5)
        reduction_ratio = (float(loss_now) -
                           float(loss_before)) / M if M != 0 else 1

        if reduction_ratio < 0.25:
            group['damping'] *= 3 / 2
        elif reduction_ratio > 0.75:
            group['damping'] *= 2 / 3
        if reduction_ratio < 0:
            group['init_delta'] = 0

        # Line Searching (Section 20.8.8)
        beta = 0.8
        c = 1e-2
        min_improv = min(c * torch.dot(b, delta), 0)

        for _ in range(60):
            if float(loss_now) <= float(loss_before) + alpha * min_improv:
                break

            alpha *= beta
            vector_to_parameters(flat_params + alpha * delta, self._params)
            loss_now = closure()[0]
        else:  # No good update found
            alpha = 0.0
            loss_now = loss_before

        # Update the parameters (this time fo real)
        vector_to_parameters(flat_params + alpha * delta, self._params)

        if verbose:
            print("Loss after LS:  {0} (lr: {1:.3f})".format(
                float(loss_now), alpha))
            print("Tikhonov damping: {0:.3f} (reduction ratio: {1:.3f})".format(
                group['damping'], reduction_ratio), end='\n\n')

        return loss_now

        Performs a single optimization step.
        Arguments:
            closure (callable): A closure that re-evaluates the model
                and returns a tuple of the loss and the output.
            b (callable, optional): A closure that calculates the vector b in
                the minimization problem x^T . A . x + x^T b.
            M (callable, optional): The INVERSE preconditioner of A
        
Performs a single optimization step.
Performs a single optimization step .
def step(self, closure, b=None, M_inv=None):
        assert len(self.param_groups) == 1
        group = self.param_groups[0]
        alpha = group['alpha']
        delta_decay = group['delta_decay']
        cg_max_iter = group['cg_max_iter']
        damping = group['damping']
        use_gnm = group['use_gnm']
        verbose = group['verbose']
        state = self.state[self._params[0]]
        state.setdefault('func_evals', 0)
        state.setdefault('n_iter', 0)
        loss_before, output = closure()
        current_evals = 1
        state['func_evals'] += 1
        flat_params = parameters_to_vector(self._params)
        flat_grad = self._gather_flat_grad()
        if use_gnm:
            def A(x):
                return self._Gv(loss_before, output, x, damping)
        else:
            def A(x):
                return self._Hv(flat_grad, x, damping)
        if M_inv is not None:
            m_inv = M_inv()
            if m_inv.dim() == 1:
                m = (m_inv + damping) ** (-0.85)
                def M(x):
                    return m * x
            else:
                m = torch.inverse(m_inv + damping * torch.eye(*m_inv.shape))
                def M(x):
                    return m @ x
        else:
            M = None
        b = flat_grad.detach() if b is None else b().detach().flatten()
        if state.get('init_delta') is not None:
            init_delta = delta_decay * state.get('init_delta')
        else:
            init_delta = torch.zeros_like(flat_params)
        eps = torch.finfo(b.dtype).eps
        deltas, Ms = self._CG(A=A, b=b.neg(), x0=init_delta,
                              M=M, max_iter=cg_max_iter,
                              tol=1e1 * eps, eps=eps, martens=True)
        delta = state['init_delta'] = deltas[-1]
        M = Ms[-1]
        vector_to_parameters(flat_params + delta, self._params)
        loss_now = closure()[0]
        current_evals += 1
        state['func_evals'] += 1
         backtracking (Section 20.8.7)
        if verbose:
            print("Loss before CG: {}".format(float(loss_before)))
            print("Loss before BT: {}".format(float(loss_now)))
        for (d, m) in zip(reversed(deltas[:-1][::2]), reversed(Ms[:-1][::2])):
            vector_to_parameters(flat_params + d, self._params)
            loss_prev = closure()[0]
            if float(loss_prev) > float(loss_now):
                break
            delta = d
            M = m
            loss_now = loss_prev
        if verbose:
            print("Loss after BT:  {}".format(float(loss_now)))
        reduction_ratio = (float(loss_now) -
                           float(loss_before)) / M if M != 0 else 1
        if reduction_ratio < 0.25:
            group['damping'] *= 3 / 2
        elif reduction_ratio > 0.75:
            group['damping'] *= 2 / 3
        if reduction_ratio < 0:
            group['init_delta'] = 0
        beta = 0.8
        c = 1e-2
        min_improv = min(c * torch.dot(b, delta), 0)
        for _ in range(60):
            if float(loss_now) <= float(loss_before) + alpha * min_improv:
                break
            alpha *= beta
            vector_to_parameters(flat_params + alpha * delta, self._params)
            loss_now = closure()[0]
        else:  
            alpha = 0.0
            loss_now = loss_before
        vector_to_parameters(flat_params + alpha * delta, self._params)
        if verbose:
            print("Loss after LS:  {0} (lr: {1:.3f})".format(
                float(loss_now), alpha))
            print("Tikhonov damping: {0:.3f} (reduction ratio: {1:.3f})".format(
                group['damping'], reduction_ratio), end='\n\n')
        return loss_now
def step ( self , closure , b = None , M_inv = None ) : assert len ( self . param_groups ) == 1 group = self . param_groups [ 0 ] alpha = group [ 'alpha' ] delta_decay = group [ 'delta_decay' ] cg_max_iter = group [ 'cg_max_iter' ] damping = group [ 'damping' ] use_gnm = group [ 'use_gnm' ] verbose = group [ 'verbose' ] state = self . state [ self . _params [ 0 ] ] state . setdefault ( 'func_evals' , 0 ) state . setdefault ( 'n_iter' , 0 ) loss_before , output = closure ( ) current_evals = 1 state [ 'func_evals' ] += 1 flat_params = parameters_to_vector ( self . _params ) flat_grad = self . _gather_flat_grad ( ) if use_gnm : def A ( x ) : return self . _Gv ( loss_before , output , x , damping ) else : def A ( x ) : return self . _Hv ( flat_grad , x , damping ) if M_inv is not None : m_inv = M_inv ( ) if m_inv . dim ( ) == 1 : m = ( m_inv + damping ) ** ( - 0.85 ) def M ( x ) : return m * x else : m = torch . inverse ( m_inv + damping * torch . eye ( * m_inv . shape ) ) def M ( x ) : return m @ x else : M = None b = flat_grad . detach ( ) if b is None else b ( ) . detach ( ) . flatten ( ) if state . get ( 'init_delta' ) is not None : init_delta = delta_decay * state . get ( 'init_delta' ) else : init_delta = torch . zeros_like ( flat_params ) eps = torch . finfo ( b . dtype ) . eps deltas , Ms = self . _CG ( A = A , b = b . neg ( ) , x0 = init_delta , M = M , max_iter = cg_max_iter , tol = 1e1 * eps , eps = eps , martens = True ) delta = state [ 'init_delta' ] = deltas [ - 1 ] M = Ms [ - 1 ] vector_to_parameters ( flat_params + delta , self . _params ) loss_now = closure ( ) [ 0 ] current_evals += 1 state [ 'func_evals' ] += 1 if verbose : print ( "Loss before CG: {}" . format ( float ( loss_before ) ) ) print ( "Loss before BT: {}" . format ( float ( loss_now ) ) ) for ( d , m ) in zip ( reversed ( deltas [ : - 1 ] [ : : 2 ] ) , reversed ( Ms [ : - 1 ] [ : : 2 ] ) ) : vector_to_parameters ( flat_params + d , self . _params ) loss_prev = closure ( ) [ 0 ] if float ( loss_prev ) > float ( loss_now ) : break delta = d M = m loss_now = loss_prev if verbose : print ( "Loss after BT:  {}" . format ( float ( loss_now ) ) ) reduction_ratio = ( float ( loss_now ) - float ( loss_before ) ) / M if M != 0 else 1 if reduction_ratio < 0.25 : group [ 'damping' ] *= 3 / 2 elif reduction_ratio > 0.75 : group [ 'damping' ] *= 2 / 3 if reduction_ratio < 0 : group [ 'init_delta' ] = 0 beta = 0.8 c = 1e-2 min_improv = min ( c * torch . dot ( b , delta ) , 0 ) for _ in range ( 60 ) : if float ( loss_now ) <= float ( loss_before ) + alpha * min_improv : break alpha *= beta vector_to_parameters ( flat_params + alpha * delta , self . _params ) loss_now = closure ( ) [ 0 ] else : alpha = 0.0 loss_now = loss_before vector_to_parameters ( flat_params + alpha * delta , self . _params ) if verbose : print ( "Loss after LS:  {0} (lr: {1:.3f})" . format ( float ( loss_now ) , alpha ) ) print ( "Tikhonov damping: {0:.3f} (reduction ratio: {1:.3f})" . format ( group [ 'damping' ] , reduction_ratio ) , end = '\n\n' ) return loss_now
Performs a single optimization step.
Performs a single optimization step .
"""
        Performs a single optimization step.
        Arguments:
            closure (callable): A closure that re-evaluates the model
                and returns a tuple of the loss and the output.
            b (callable, optional): A closure that calculates the vector b in
                the minimization problem x^T . A . x + x^T b.
            M (callable, optional): The INVERSE preconditioner of A
        """ # Gather current parameters and respective gradients # Define linear operator # Generalized Gauss-Newton vector product # Hessian-vector product # Preconditioner recipe (Section 20.13) # Initializing Conjugate-Gradient (Section 20.10) # Conjugate-Gradient # Update parameters # Conjugate-Gradient backtracking (Section 20.8.7) # The Levenberg-Marquardt Heuristic (Section 20.8.5) # Line Searching (Section 20.8.8) # No good update found # Update the parameters (this time fo real)
self
closure
A closure that re-evaluates the model
and returns a tuple of the loss and the output.
A closure that re - evaluates the model and returns a tuple of the loss and the output .
b
A closure that calculates the vector b in
the minimization problem x^T . A .
A closure that calculates the vector b in the minimization problem x^T . A .
M_inv
M
The INVERSE preconditioner of A
The INVERSE preconditioner of A