neuroprob/utils/pytorch.py
def compute_newton_grads(params, loss, mode='inverse'):
    """
    Newton's method for optimization.
    Compute the inverse Hessian gradient product (jvp) to replace the gradients for a chosen 
    subset of second-order optimized parameters.
    """
    #grad_p = torch.cat([p.grad.flatten() for p in self.newton_grad])
    grad_p = torch.cat([torch.autograd.grad(loss, p, retain_graph=True, create_graph=True)[0].flatten() 
                        for p in params])

    # compute Hessian vector product
    v = torch.ones(len(grad_p), device=grad_p.device, requires_grad=True)
    a = torch.dot(grad_p, v)
    Hv = torch.cat([torch.autograd.grad(grad_p, p, v, retain_graph=True, create_graph=True)[0].flatten() 
                     for p in params])
    H = torch.stack([torch.autograd.grad(h, v, retain_graph=True, create_graph=True)[0] 
                     for h in Hv])

    if mode == 'inverse': # invert the Hessian matrix
        L = torch.cholesky(H, upper=False)
        b = torch.eye(M, device=L.device)
        L_ = torch.triangular_solve(b, L_inv, upper=False)[0]
        H_inv = L_ @ L_.permute(1, 0)
        #H_inv = torch.inverse(H)
    elif mode == 'Hessian_free': # conjugate gradients (Hessian free algorithm)
        pass
    else:
        raise NotImplementedError

    params_grad = H_inv @ grad_p.data

    accum_l = 0
    for p in params:
        l = len(p.flatten())
        p.grad = params_grad[accum_l:accum_l+l].view(*p.shape).clone() # shape back
        accum_l += l

    Newton's method for optimization.
    Compute the inverse Hessian gradient product (jvp) to replace the gradients for a chosen 
    subset of second-order optimized parameters.
    
Newton's method for optimization.
Compute the inverse Hessian gradient product (jvp) to replace the gradients for a chosen
subset of second-order optimized parameters.
Newton ' s method for optimization . Compute the inverse Hessian gradient product ( jvp ) to replace the gradients for a chosen subset of second - order optimized parameters .
def compute_newton_grads(params, loss, mode='inverse'):
    grad_p = torch.cat([torch.autograd.grad(loss, p, retain_graph=True, create_graph=True)[0].flatten() 
                        for p in params])
    v = torch.ones(len(grad_p), device=grad_p.device, requires_grad=True)
    a = torch.dot(grad_p, v)
    Hv = torch.cat([torch.autograd.grad(grad_p, p, v, retain_graph=True, create_graph=True)[0].flatten() 
                     for p in params])
    H = torch.stack([torch.autograd.grad(h, v, retain_graph=True, create_graph=True)[0] 
                     for h in Hv])
    if mode == 'inverse': 
        L = torch.cholesky(H, upper=False)
        b = torch.eye(M, device=L.device)
        L_ = torch.triangular_solve(b, L_inv, upper=False)[0]
        H_inv = L_ @ L_.permute(1, 0)
    elif mode == 'Hessian_free': 
        pass
    else:
        raise NotImplementedError
    params_grad = H_inv @ grad_p.data
    accum_l = 0
    for p in params:
        l = len(p.flatten())
        p.grad = params_grad[accum_l:accum_l+l].view(*p.shape).clone() 
        accum_l += l
def compute_newton_grads ( params , loss , mode = 'inverse' ) : grad_p = torch . cat ( [ torch . autograd . grad ( loss , p , retain_graph = True , create_graph = True ) [ 0 ] . flatten ( ) for p in params ] ) v = torch . ones ( len ( grad_p ) , device = grad_p . device , requires_grad = True ) a = torch . dot ( grad_p , v ) Hv = torch . cat ( [ torch . autograd . grad ( grad_p , p , v , retain_graph = True , create_graph = True ) [ 0 ] . flatten ( ) for p in params ] ) H = torch . stack ( [ torch . autograd . grad ( h , v , retain_graph = True , create_graph = True ) [ 0 ] for h in Hv ] ) if mode == 'inverse' : L = torch . cholesky ( H , upper = False ) b = torch . eye ( M , device = L . device ) L_ = torch . triangular_solve ( b , L_inv , upper = False ) [ 0 ] H_inv = L_ @ L_ . permute ( 1 , 0 ) elif mode == 'Hessian_free' : pass else : raise NotImplementedError params_grad = H_inv @ grad_p . data accum_l = 0 for p in params : l = len ( p . flatten ( ) ) p . grad = params_grad [ accum_l : accum_l + l ] . view ( * p . shape ) . clone ( ) accum_l += l
Newton's method for optimization.
Newton ' s method for optimization .
"""
    Newton's method for optimization.
    Compute the inverse Hessian gradient product (jvp) to replace the gradients for a chosen 
    subset of second-order optimized parameters.
    """ #grad_p = torch.cat([p.grad.flatten() for p in self.newton_grad]) # compute Hessian vector product # invert the Hessian matrix #H_inv = torch.inverse(H) # conjugate gradients (Hessian free algorithm) # shape back
params
loss
mode