neuroprob/inference.py
def fit(self, max_epochs, margin_epochs=10, loss_margin=0.0, neuron=None, kl_anneal_func=None,
            z_anneal_func=None, retain_graph=False, cov_samples=1, ll_samples=1, bound='ELBO', ll_mode='MC', 
            lv_input=None, callback=None):
        """
        Can fit to all neurons present in the data (default), or fits only a subset of neurons.
        margin_epochs sets the early stop when the loss is still above lowest loss in given iterations.
        
        :param int max_epochs: maximum number of iterations over the dataset
        :param int margin_epochs: number of iterations above margin loss reduction to tolerate
        :param float loss_margin: tolerated loss increase during training
        :param list neuron: neuron indices to fit to
        :param string grad_type: gradient type used in optimization (`vanilla` or `natural` [1])
        :returns: time series of loss during training
        :rtype: list
        """
        self.validate_model()
        batches = self.likelihood.batches
        
        if kl_anneal_func is None:
            kl_anneal_func = lambda x: 1.0
        if z_anneal_func is None:
            z_anneal_func = lambda x: 1.0
            
        tracked_loss = []
        minloss = np.inf
        cnt = 0
        iterator = tqdm(range(max_epochs))
        
        optimizer = [self.optim_base]
        if len(self.nat_grad) > 0:
            optimizer += [self.optim_nat]
        if len(self.newton_grad) > 0:
            optimizer += [self.optim_newton]

        for epoch in iterator:
            
            sloss = 0
            for b in range(batches):
                
                def closure():
                    if torch.is_grad_enabled():
                        for o in optimizer:
                            o.zero_grad()
                            
                    anneal_t = epoch/max_epochs
                    loss = self.objective(b, neuron, beta=kl_anneal_func(anneal_t), beta_z=z_anneal_func(anneal_t), 
                                          cov_samples=cov_samples, ll_samples=ll_samples, bound=bound, 
                                          ll_mode=ll_mode, lv_input=lv_input)
                    
                    if loss.requires_grad:
                        #with profiler.profile(record_shapes=True, use_cuda=True) as profi:
                        #    with profiler.record_function("backwards"):
                        loss.backward(retain_graph=retain_graph)
                        
                        if len(self.nat_grad) > 0: # compute natural gradient
                            pytorch.compute_nat_grads(self.nat_grad)
                        if len(self.newton_grad) > 0: # compute Newton gradient
                            pytorch.compute_newton_grads(self.newton_grad, loss)
                    return loss
                
                if isinstance(optimizer[0], optim.LBFGS):
                    optimizer[0].step(closure)
                    with torch.no_grad():
                        loss = closure()
                else:
                    loss = closure()
                    optimizer[0].step()
                    
                for o in optimizer[1:]: # additional optimizers
                    o.step()

                self.constrain()
                if callback is not None: # additional operations per gradient step
                    callback(self)
                    
                if torch.isnan(loss.data).any() or torch.isinf(loss.data).any():
                    raise ValueError('Loss diverged')
                sloss += loss.item()
                
            if self.sch_st is not None and epoch % self.sch_st == self.sch_st-1:
                self.sch.step()

            sloss /= batches # average over batches, each batch loss is subsampled estimator of full loss
            iterator.set_postfix(loss=sloss)
            tracked_loss.append(sloss)
            
            if sloss <= minloss + loss_margin:
                cnt = 0
            else:
                cnt += 1
                
            if sloss < minloss:
                minloss = sloss

            if cnt > margin_epochs:
                print("\nStopped at epoch {}.".format(epoch+1))
                break
        
        return tracked_loss

        Can fit to all neurons present in the data (default), or fits only a subset of neurons.
        margin_epochs sets the early stop when the loss is still above lowest loss in given iterations.
        
        :param int max_epochs: maximum number of iterations over the dataset
        :param int margin_epochs: number of iterations above margin loss reduction to tolerate
        :param float loss_margin: tolerated loss increase during training
        :param list neuron: neuron indices to fit to
        :param string grad_type: gradient type used in optimization (`vanilla` or `natural` [1])
        :returns: time series of loss during training
        :rtype: list
        
Can fit to all neurons present in the data (default), or fits only a subset of neurons.
margin_epochs sets the early stop when the loss is still above lowest loss in given iterations.
Can fit to all neurons present in the data ( default ) or fits only a subset of neurons . margin_epochs sets the early stop when the loss is still above lowest loss in given iterations .
def fit(self, max_epochs, margin_epochs=10, loss_margin=0.0, neuron=None, kl_anneal_func=None,
            z_anneal_func=None, retain_graph=False, cov_samples=1, ll_samples=1, bound='ELBO', ll_mode='MC', 
            lv_input=None, callback=None):
        self.validate_model()
        batches = self.likelihood.batches
        if kl_anneal_func is None:
            kl_anneal_func = lambda x: 1.0
        if z_anneal_func is None:
            z_anneal_func = lambda x: 1.0
        tracked_loss = []
        minloss = np.inf
        cnt = 0
        iterator = tqdm(range(max_epochs))
        optimizer = [self.optim_base]
        if len(self.nat_grad) > 0:
            optimizer += [self.optim_nat]
        if len(self.newton_grad) > 0:
            optimizer += [self.optim_newton]
        for epoch in iterator:
            sloss = 0
            for b in range(batches):
                def closure():
                    if torch.is_grad_enabled():
                        for o in optimizer:
                            o.zero_grad()
                    anneal_t = epoch/max_epochs
                    loss = self.objective(b, neuron, beta=kl_anneal_func(anneal_t), beta_z=z_anneal_func(anneal_t), 
                                          cov_samples=cov_samples, ll_samples=ll_samples, bound=bound, 
                                          ll_mode=ll_mode, lv_input=lv_input)
                    if loss.requires_grad:
                        loss.backward(retain_graph=retain_graph)
                        if len(self.nat_grad) > 0: 
                            pytorch.compute_nat_grads(self.nat_grad)
                        if len(self.newton_grad) > 0: 
                            pytorch.compute_newton_grads(self.newton_grad, loss)
                    return loss
                if isinstance(optimizer[0], optim.LBFGS):
                    optimizer[0].step(closure)
                    with torch.no_grad():
                        loss = closure()
                else:
                    loss = closure()
                    optimizer[0].step()
                for o in optimizer[1:]: 
                    o.step()
                self.constrain()
                if callback is not None: 
                    callback(self)
                if torch.isnan(loss.data).any() or torch.isinf(loss.data).any():
                    raise ValueError('Loss diverged')
                sloss += loss.item()
            if self.sch_st is not None and epoch % self.sch_st == self.sch_st-1:
                self.sch.step()
            sloss /= batches 
            iterator.set_postfix(loss=sloss)
            tracked_loss.append(sloss)
            if sloss <= minloss + loss_margin:
                cnt = 0
            else:
                cnt += 1
            if sloss < minloss:
                minloss = sloss
            if cnt > margin_epochs:
                print("\nStopped at epoch {}.".format(epoch+1))
                break
        return tracked_loss
def fit ( self , max_epochs , margin_epochs = 10 , loss_margin = 0.0 , neuron = None , kl_anneal_func = None , z_anneal_func = None , retain_graph = False , cov_samples = 1 , ll_samples = 1 , bound = 'ELBO' , ll_mode = 'MC' , lv_input = None , callback = None ) : self . validate_model ( ) batches = self . likelihood . batches if kl_anneal_func is None : kl_anneal_func = lambda x : 1.0 if z_anneal_func is None : z_anneal_func = lambda x : 1.0 tracked_loss = [ ] minloss = np . inf cnt = 0 iterator = tqdm ( range ( max_epochs ) ) optimizer = [ self . optim_base ] if len ( self . nat_grad ) > 0 : optimizer += [ self . optim_nat ] if len ( self . newton_grad ) > 0 : optimizer += [ self . optim_newton ] for epoch in iterator : sloss = 0 for b in range ( batches ) : def closure ( ) : if torch . is_grad_enabled ( ) : for o in optimizer : o . zero_grad ( ) anneal_t = epoch / max_epochs loss = self . objective ( b , neuron , beta = kl_anneal_func ( anneal_t ) , beta_z = z_anneal_func ( anneal_t ) , cov_samples = cov_samples , ll_samples = ll_samples , bound = bound , ll_mode = ll_mode , lv_input = lv_input ) if loss . requires_grad : loss . backward ( retain_graph = retain_graph ) if len ( self . nat_grad ) > 0 : pytorch . compute_nat_grads ( self . nat_grad ) if len ( self . newton_grad ) > 0 : pytorch . compute_newton_grads ( self . newton_grad , loss ) return loss if isinstance ( optimizer [ 0 ] , optim . LBFGS ) : optimizer [ 0 ] . step ( closure ) with torch . no_grad ( ) : loss = closure ( ) else : loss = closure ( ) optimizer [ 0 ] . step ( ) for o in optimizer [ 1 : ] : o . step ( ) self . constrain ( ) if callback is not None : callback ( self ) if torch . isnan ( loss . data ) . any ( ) or torch . isinf ( loss . data ) . any ( ) : raise ValueError ( 'Loss diverged' ) sloss += loss . item ( ) if self . sch_st is not None and epoch % self . sch_st == self . sch_st - 1 : self . sch . step ( ) sloss /= batches iterator . set_postfix ( loss = sloss ) tracked_loss . append ( sloss ) if sloss <= minloss + loss_margin : cnt = 0 else : cnt += 1 if sloss < minloss : minloss = sloss if cnt > margin_epochs : print ( "\nStopped at epoch {}." . format ( epoch + 1 ) ) break return tracked_loss
Can fit to all neurons present in the data (default), or fits only a subset of neurons.
Can fit to all neurons present in the data ( default ) or fits only a subset of neurons .
"""
        Can fit to all neurons present in the data (default), or fits only a subset of neurons.
        margin_epochs sets the early stop when the loss is still above lowest loss in given iterations.
        
        :param int max_epochs: maximum number of iterations over the dataset
        :param int margin_epochs: number of iterations above margin loss reduction to tolerate
        :param float loss_margin: tolerated loss increase during training
        :param list neuron: neuron indices to fit to
        :param string grad_type: gradient type used in optimization (`vanilla` or `natural` [1])
        :returns: time series of loss during training
        :rtype: list
        """ #with profiler.profile(record_shapes=True, use_cuda=True) as profi: #    with profiler.record_function("backwards"): # compute natural gradient # compute Newton gradient # additional optimizers # additional operations per gradient step # average over batches, each batch loss is subsampled estimator of full loss
time series of loss during training
time series of loss during training
list
self
max_epochs
maximum number of iterations over the dataset
maximum number of iterations over the dataset
margin_epochs
number of iterations above margin loss reduction to tolerate
number of iterations above margin loss reduction to tolerate
loss_margin
tolerated loss increase during training
tolerated loss increase during training
neuron
neuron indices to fit to
neuron indices to fit to
kl_anneal_func
z_anneal_func
retain_graph
cov_samples
ll_samples
bound
ll_mode
lv_input
callback
grad_type
gradient type used in optimization (`vanilla` or `natural` [1])
gradient type used in optimization ( ` vanilla ` or ` natural ` [ 1 ] )