supertensortools/apg.py
def fit_apg(
        model, *, patience, rtol=1e-1, atol=1e-4, max_iter=1000,
        trace_decoding_loss=False, trace_reconstruction_loss=False,
        verbose=True,
    ):
    """
    Fits model using acceleration proximal gradient method.
    """

    # Define proximal operators.
    optimizer = AcceleratedProxGrad(
        list(model.parameters()),
        lr=1.0, init_momentum=1.0,
        momentum_backtrack_factor=0.9,
        lr_backtrack_factor=0.5,
    )

    # Keep history of optimization progress.
    trace = {
        "converged": False,
        "loss": [],
        "learning_rate": [],
        "momentum": [],
    }
    if trace_decoding_loss:
        trace["decoding_loss"] = []
    if trace_reconstruction_loss:
        trace["reconstruction_loss"] = []

    if verbose:
        pbar = tqdm(total=max_iter)

    itercount = 0
    while (not trace["converged"]) and (itercount < max_iter):
        
        # Update parameters.
        trace["learning_rate"].append(optimizer.prxgrad.param_groups[0]["lr"])
        trace["momentum"].append(optimizer.param_groups[0]["beta"])
        loss = optimizer.step(
            model.total_loss, # function that evaluates objective.
            model.apply_prox  # function that implement prox operator.
        )
        trace["loss"].append(loss.item())
        
        # Trace additional losses.
        with torch.no_grad():
            if "reconstruction_loss" in trace:
                trace["reconstruction_loss"].append(model.reconstruction_loss())
            if "decoding_loss" in trace:
                trace["decoding_loss"].append(model.decoding_loss())

        # Count iterations.
        itercount += 1
        if verbose:
            pbar.update(1)

        # Check convergence.
        if itercount > patience:
            abs_imp = trace["loss"][-patience] - trace["loss"][-1]
            rel_imp = abs_imp / trace["loss"][-1]
            if abs_imp < atol:
                print("Converged: reached absolute tolerance. aImp: ", abs_imp)
                trace["converged"] = True
            if rel_imp < rtol:
                print("Converged: reached relative tolerance. rImp: ", rel_imp)
                trace["converged"] = True

    if verbose:
        pbar.close()

    return trace

    Fits model using acceleration proximal gradient method.
    
Fits model using acceleration proximal gradient method.
Fits model using acceleration proximal gradient method .
def fit_apg(
        model, *, patience, rtol=1e-1, atol=1e-4, max_iter=1000,
        trace_decoding_loss=False, trace_reconstruction_loss=False,
        verbose=True,
    ):
    optimizer = AcceleratedProxGrad(
        list(model.parameters()),
        lr=1.0, init_momentum=1.0,
        momentum_backtrack_factor=0.9,
        lr_backtrack_factor=0.5,
    )
    trace = {
        "converged": False,
        "loss": [],
        "learning_rate": [],
        "momentum": [],
    }
    if trace_decoding_loss:
        trace["decoding_loss"] = []
    if trace_reconstruction_loss:
        trace["reconstruction_loss"] = []
    if verbose:
        pbar = tqdm(total=max_iter)
    itercount = 0
    while (not trace["converged"]) and (itercount < max_iter):
        trace["learning_rate"].append(optimizer.prxgrad.param_groups[0]["lr"])
        trace["momentum"].append(optimizer.param_groups[0]["beta"])
        loss = optimizer.step(
            model.total_loss, 
            model.apply_prox  
        )
        trace["loss"].append(loss.item())
        with torch.no_grad():
            if "reconstruction_loss" in trace:
                trace["reconstruction_loss"].append(model.reconstruction_loss())
            if "decoding_loss" in trace:
                trace["decoding_loss"].append(model.decoding_loss())
        itercount += 1
        if verbose:
            pbar.update(1)
        if itercount > patience:
            abs_imp = trace["loss"][-patience] - trace["loss"][-1]
            rel_imp = abs_imp / trace["loss"][-1]
            if abs_imp < atol:
                print("Converged: reached absolute tolerance. aImp: ", abs_imp)
                trace["converged"] = True
            if rel_imp < rtol:
                print("Converged: reached relative tolerance. rImp: ", rel_imp)
                trace["converged"] = True
    if verbose:
        pbar.close()
    return trace
def fit_apg ( model , * , patience , rtol = 1e-1 , atol = 1e-4 , max_iter = 1000 , trace_decoding_loss = False , trace_reconstruction_loss = False , verbose = True , ) : optimizer = AcceleratedProxGrad ( list ( model . parameters ( ) ) , lr = 1.0 , init_momentum = 1.0 , momentum_backtrack_factor = 0.9 , lr_backtrack_factor = 0.5 , ) trace = { "converged" : False , "loss" : [ ] , "learning_rate" : [ ] , "momentum" : [ ] , } if trace_decoding_loss : trace [ "decoding_loss" ] = [ ] if trace_reconstruction_loss : trace [ "reconstruction_loss" ] = [ ] if verbose : pbar = tqdm ( total = max_iter ) itercount = 0 while ( not trace [ "converged" ] ) and ( itercount < max_iter ) : trace [ "learning_rate" ] . append ( optimizer . prxgrad . param_groups [ 0 ] [ "lr" ] ) trace [ "momentum" ] . append ( optimizer . param_groups [ 0 ] [ "beta" ] ) loss = optimizer . step ( model . total_loss , model . apply_prox ) trace [ "loss" ] . append ( loss . item ( ) ) with torch . no_grad ( ) : if "reconstruction_loss" in trace : trace [ "reconstruction_loss" ] . append ( model . reconstruction_loss ( ) ) if "decoding_loss" in trace : trace [ "decoding_loss" ] . append ( model . decoding_loss ( ) ) itercount += 1 if verbose : pbar . update ( 1 ) if itercount > patience : abs_imp = trace [ "loss" ] [ - patience ] - trace [ "loss" ] [ - 1 ] rel_imp = abs_imp / trace [ "loss" ] [ - 1 ] if abs_imp < atol : print ( "Converged: reached absolute tolerance. aImp: " , abs_imp ) trace [ "converged" ] = True if rel_imp < rtol : print ( "Converged: reached relative tolerance. rImp: " , rel_imp ) trace [ "converged" ] = True if verbose : pbar . close ( ) return trace
Fits model using acceleration proximal gradient method.
Fits model using acceleration proximal gradient method .
"""
    Fits model using acceleration proximal gradient method.
    """ # Define proximal operators. # Keep history of optimization progress. # Update parameters. # function that evaluates objective. # function that implement prox operator. # Trace additional losses. # Count iterations. # Check convergence.
model
patience
rtol
atol
max_iter
trace_decoding_loss
trace_reconstruction_loss
verbose