neuroprob/inference.py
def objective(self, b, neuron=None, beta=1.0, beta_z=1.0, cov_samples=1, ll_samples=10, bound='ELBO', ll_mode='MC', 
                  lv_input=None):
        """
        Compute the rate and then spike train log likelihood.
        Mapping prior is for SVGP the KL term in the variational ELBO.
        When rate output is deterministic (inner_var=0) the ll_samples parameter is irrelevant in the 
        'MC' in that case, where no sampling will occur. Note that MC is used for the expectation over 
        the latent variable distribution as well, hence ll_mode 'MC' leads to double MC estimation.
        
        :param int b: batch index used for picking the batch to fit
        :param np.array neuron: indices of neurons for which to evaluate the likelihood term (observed neurons)
        :param float beta: prior annealing or KL annealing via beta
        :param int cov_samples: number of samples to draw from covariates distribution
        :param int ll_samples: number of samples to evaluate likelihood per covariate sample
        :param string bound: type of variational objective ('ELBO', 'PLL' [1] or 'IWAE' [2])
        :param string ll_mode: likelihood evaluation mode (Monte Carlo `MC` or Gauss-Hermite quadratures `GH`)
        :param bool enuerate_z: use enumerate for discrete latent (marginalizing out all states)
        :param torch.tensor lv_input: input tensor for the latent variable computation (e.g. in the case 
                                      of amortized inference we want self.likelihood.all_spikes)
        :returns: variational approximation to negative marginal log likelihood
        :rtype: torch.tensor
        
        References:
        
        [1] `Parametric Gaussian Process Regressors`, Martin Jankowiak, Geoff Pleiss, Jacob R. Gardner (2019)
        
        [2] `Importance Weighted Autoencoders`, Yuri Burda, Roger Grosse, Ruslan Salakhutdinov (2015)
            
        """
        if (bound != 'ELBO') and (bound != 'PLL') and (bound != 'IWAE'):
            raise ValueError('Bound parameter not known')
        entropy = False if bound == 'IWAE' else True
        neuron = self.likelihood._validate_neuron(neuron)
            
        ### sample input ###
        XZ, lvm_lprior, lvm_nlq, KL_prior_in = self.inputs.sample_XZ(b, cov_samples, lv_input, entropy) # samples, timesteps, dims
        
        ### mean and covariance of input mapping ###
        F_mu, F_var = self.mapping.compute_F(XZ) # samples, neurons, timesteps
        KL_prior_m = self.mapping.KL_prior() # prior may need quantities computed in compute_F, e.g. Luu
        
        ### evaluating likelihood expectation and prior terms ###
        maps = len(F_mu) if isinstance(F_mu, list) else 1
        batch_size = F_mu[0].shape[-1] if isinstance(F_mu, list) else F_mu.shape[-1]
        fac = self.likelihood.tsteps / batch_size # n_data/batch_size, as we subsample temporally (batching)
        KL_prior_l = self.likelihood.KL_prior()

        if bound == 'IWAE':
            if maps != 1:
                raise ValueError('Enumeration not supported for IWAE')
                
            _nll, ws = self._nll(F_mu, F_var, maps, ll_samples, ll_mode, b, neuron, XZ, sum_time=True) # sum over time or dummy
            ll_samples = _nll.shape[0] // cov_samples # could be less than requested if exact nll
            
            tot_samp = ll_samples*cov_samples*self.inputs.trials
            lvm_lprior = lvm_lprior.view(cov_samples, self.inputs.trials).repeat(ll_samples, 1) / tot_samp
            lvm_nlq = lvm_nlq.view(cov_samples, self.inputs.trials).repeat(ll_samples, 1) / tot_samp
            
            log_pq = (lvm_lprior + lvm_nlq) * beta_z
            model_nlog_pq = -torch.logsumexp((-_nll + log_pq + \
                                              torch.log(ws)), dim=0).mean() * fac # mean over trials

        else: # lvm_nlq is scalar as entropy is True
            if maps > 1:
                nll = []
                for k in range(maps): # enumeration
                    mean = F_mu[k] if maps > 1 else F_mu
                    variance = F_var[k] if maps > 1 else F_var

                    _nll, ws = self._nll(mean, variance, maps, ll_samples, ll_mode, b, neuron, XZ)
                    if bound == 'PLL': # mean is taken over the sample dimension i.e. log E_{q(f)q(z)}[...]
                        nll.append(-torch.logsumexp(-_nll+torch.log(ws), dim=0))
                    else:
                        nll.append((_nll*ws).sum(0)) # sum over MC/GH samples
                nll_term = self.mapping.objective(torch.stack(nll))
                
            else:
                _nll, ws = self._nll(F_mu, F_var, maps, ll_samples, ll_mode, b, neuron, XZ, sum_time=True)
                if bound == 'PLL': # mean is taken over the sample dimension i.e. log E_{q(f)q(z)}[...]
                    nll_term = -torch.logsumexp(-_nll+torch.log(ws), dim=0) # sum over MC/GH samples
                else:
                    nll_term = (_nll*ws).sum(0) # sum over MC/GH samples
                nll_term = nll_term.mean() # mean over trials
            
            log_pq = (lvm_lprior.mean(0) + lvm_nlq) * beta_z
            model_nlog_pq = (nll_term - log_pq) * fac
            
        KL_prior = KL_prior_in + KL_prior_m + KL_prior_l # model priors integrated out (KL terms)
        return model_nlog_pq - beta*KL_prior

        Compute the rate and then spike train log likelihood.
        Mapping prior is for SVGP the KL term in the variational ELBO.
        When rate output is deterministic (inner_var=0) the ll_samples parameter is irrelevant in the 
        'MC' in that case, where no sampling will occur. Note that MC is used for the expectation over 
        the latent variable distribution as well, hence ll_mode 'MC' leads to double MC estimation.
        
        :param int b: batch index used for picking the batch to fit
        :param np.array neuron: indices of neurons for which to evaluate the likelihood term (observed neurons)
        :param float beta: prior annealing or KL annealing via beta
        :param int cov_samples: number of samples to draw from covariates distribution
        :param int ll_samples: number of samples to evaluate likelihood per covariate sample
        :param string bound: type of variational objective ('ELBO', 'PLL' [1] or 'IWAE' [2])
        :param string ll_mode: likelihood evaluation mode (Monte Carlo `MC` or Gauss-Hermite quadratures `GH`)
        :param bool enuerate_z: use enumerate for discrete latent (marginalizing out all states)
        :param torch.tensor lv_input: input tensor for the latent variable computation (e.g. in the case 
                                      of amortized inference we want self.likelihood.all_spikes)
        :returns: variational approximation to negative marginal log likelihood
        :rtype: torch.tensor
        
        References:
        
        [1] `Parametric Gaussian Process Regressors`, Martin Jankowiak, Geoff Pleiss, Jacob R. Gardner (2019)
        
        [2] `Importance Weighted Autoencoders`, Yuri Burda, Roger Grosse, Ruslan Salakhutdinov (2015)
            
        
Compute the rate and then spike train log likelihood.
Mapping prior is for SVGP the KL term in the variational ELBO.
When rate output is deterministic (inner_var=0) the ll_samples parameter is irrelevant in the
'MC' in that case, where no sampling will occur. Note that MC is used for the expectation over
the latent variable distribution as well, hence ll_mode 'MC' leads to double MC estimation.
Compute the rate and then spike train log likelihood . Mapping prior is for SVGP the KL term in the variational ELBO . When rate output is deterministic ( inner_var = 0 ) the ll_samples parameter is irrelevant in the ' MC ' in that case where no sampling will occur . Note that MC is used for the expectation over the latent variable distribution as well hence ll_mode ' MC ' leads to double MC estimation .
def objective(self, b, neuron=None, beta=1.0, beta_z=1.0, cov_samples=1, ll_samples=10, bound='ELBO', ll_mode='MC', 
                  lv_input=None):
        if (bound != 'ELBO') and (bound != 'PLL') and (bound != 'IWAE'):
            raise ValueError('Bound parameter not known')
        entropy = False if bound == 'IWAE' else True
        neuron = self.likelihood._validate_neuron(neuron)
        XZ, lvm_lprior, lvm_nlq, KL_prior_in = self.inputs.sample_XZ(b, cov_samples, lv_input, entropy) 
        F_mu, F_var = self.mapping.compute_F(XZ) 
        KL_prior_m = self.mapping.KL_prior() 
        maps = len(F_mu) if isinstance(F_mu, list) else 1
        batch_size = F_mu[0].shape[-1] if isinstance(F_mu, list) else F_mu.shape[-1]
        fac = self.likelihood.tsteps / batch_size 
        KL_prior_l = self.likelihood.KL_prior()
        if bound == 'IWAE':
            if maps != 1:
                raise ValueError('Enumeration not supported for IWAE')
            _nll, ws = self._nll(F_mu, F_var, maps, ll_samples, ll_mode, b, neuron, XZ, sum_time=True) 
            ll_samples = _nll.shape[0] // cov_samples 
            tot_samp = ll_samples*cov_samples*self.inputs.trials
            lvm_lprior = lvm_lprior.view(cov_samples, self.inputs.trials).repeat(ll_samples, 1) / tot_samp
            lvm_nlq = lvm_nlq.view(cov_samples, self.inputs.trials).repeat(ll_samples, 1) / tot_samp
            log_pq = (lvm_lprior + lvm_nlq) * beta_z
            model_nlog_pq = -torch.logsumexp((-_nll + log_pq + \
                                              torch.log(ws)), dim=0).mean() * fac 
        else: 
            if maps > 1:
                nll = []
                for k in range(maps): 
                    mean = F_mu[k] if maps > 1 else F_mu
                    variance = F_var[k] if maps > 1 else F_var
                    _nll, ws = self._nll(mean, variance, maps, ll_samples, ll_mode, b, neuron, XZ)
                    if bound == 'PLL': 
                        nll.append(-torch.logsumexp(-_nll+torch.log(ws), dim=0))
                    else:
                        nll.append((_nll*ws).sum(0)) 
                nll_term = self.mapping.objective(torch.stack(nll))
            else:
                _nll, ws = self._nll(F_mu, F_var, maps, ll_samples, ll_mode, b, neuron, XZ, sum_time=True)
                if bound == 'PLL': 
                    nll_term = -torch.logsumexp(-_nll+torch.log(ws), dim=0) 
                else:
                    nll_term = (_nll*ws).sum(0) 
                nll_term = nll_term.mean() 
            log_pq = (lvm_lprior.mean(0) + lvm_nlq) * beta_z
            model_nlog_pq = (nll_term - log_pq) * fac
        KL_prior = KL_prior_in + KL_prior_m + KL_prior_l 
        return model_nlog_pq - beta*KL_prior
def objective ( self , b , neuron = None , beta = 1.0 , beta_z = 1.0 , cov_samples = 1 , ll_samples = 10 , bound = 'ELBO' , ll_mode = 'MC' , lv_input = None ) : if ( bound != 'ELBO' ) and ( bound != 'PLL' ) and ( bound != 'IWAE' ) : raise ValueError ( 'Bound parameter not known' ) entropy = False if bound == 'IWAE' else True neuron = self . likelihood . _validate_neuron ( neuron ) XZ , lvm_lprior , lvm_nlq , KL_prior_in = self . inputs . sample_XZ ( b , cov_samples , lv_input , entropy ) F_mu , F_var = self . mapping . compute_F ( XZ ) KL_prior_m = self . mapping . KL_prior ( ) maps = len ( F_mu ) if isinstance ( F_mu , list ) else 1 batch_size = F_mu [ 0 ] . shape [ - 1 ] if isinstance ( F_mu , list ) else F_mu . shape [ - 1 ] fac = self . likelihood . tsteps / batch_size KL_prior_l = self . likelihood . KL_prior ( ) if bound == 'IWAE' : if maps != 1 : raise ValueError ( 'Enumeration not supported for IWAE' ) _nll , ws = self . _nll ( F_mu , F_var , maps , ll_samples , ll_mode , b , neuron , XZ , sum_time = True ) ll_samples = _nll . shape [ 0 ] // cov_samples tot_samp = ll_samples * cov_samples * self . inputs . trials lvm_lprior = lvm_lprior . view ( cov_samples , self . inputs . trials ) . repeat ( ll_samples , 1 ) / tot_samp lvm_nlq = lvm_nlq . view ( cov_samples , self . inputs . trials ) . repeat ( ll_samples , 1 ) / tot_samp log_pq = ( lvm_lprior + lvm_nlq ) * beta_z model_nlog_pq = - torch . logsumexp ( ( - _nll + log_pq + torch . log ( ws ) ) , dim = 0 ) . mean ( ) * fac else : if maps > 1 : nll = [ ] for k in range ( maps ) : mean = F_mu [ k ] if maps > 1 else F_mu variance = F_var [ k ] if maps > 1 else F_var _nll , ws = self . _nll ( mean , variance , maps , ll_samples , ll_mode , b , neuron , XZ ) if bound == 'PLL' : nll . append ( - torch . logsumexp ( - _nll + torch . log ( ws ) , dim = 0 ) ) else : nll . append ( ( _nll * ws ) . sum ( 0 ) ) nll_term = self . mapping . objective ( torch . stack ( nll ) ) else : _nll , ws = self . _nll ( F_mu , F_var , maps , ll_samples , ll_mode , b , neuron , XZ , sum_time = True ) if bound == 'PLL' : nll_term = - torch . logsumexp ( - _nll + torch . log ( ws ) , dim = 0 ) else : nll_term = ( _nll * ws ) . sum ( 0 ) nll_term = nll_term . mean ( ) log_pq = ( lvm_lprior . mean ( 0 ) + lvm_nlq ) * beta_z model_nlog_pq = ( nll_term - log_pq ) * fac KL_prior = KL_prior_in + KL_prior_m + KL_prior_l return model_nlog_pq - beta * KL_prior
Compute the rate and then spike train log likelihood.
Compute the rate and then spike train log likelihood .
"""
        Compute the rate and then spike train log likelihood.
        Mapping prior is for SVGP the KL term in the variational ELBO.
        When rate output is deterministic (inner_var=0) the ll_samples parameter is irrelevant in the 
        'MC' in that case, where no sampling will occur. Note that MC is used for the expectation over 
        the latent variable distribution as well, hence ll_mode 'MC' leads to double MC estimation.
        
        :param int b: batch index used for picking the batch to fit
        :param np.array neuron: indices of neurons for which to evaluate the likelihood term (observed neurons)
        :param float beta: prior annealing or KL annealing via beta
        :param int cov_samples: number of samples to draw from covariates distribution
        :param int ll_samples: number of samples to evaluate likelihood per covariate sample
        :param string bound: type of variational objective ('ELBO', 'PLL' [1] or 'IWAE' [2])
        :param string ll_mode: likelihood evaluation mode (Monte Carlo `MC` or Gauss-Hermite quadratures `GH`)
        :param bool enuerate_z: use enumerate for discrete latent (marginalizing out all states)
        :param torch.tensor lv_input: input tensor for the latent variable computation (e.g. in the case 
                                      of amortized inference we want self.likelihood.all_spikes)
        :returns: variational approximation to negative marginal log likelihood
        :rtype: torch.tensor
        
        References:
        
        [1] `Parametric Gaussian Process Regressors`, Martin Jankowiak, Geoff Pleiss, Jacob R. Gardner (2019)
        
        [2] `Importance Weighted Autoencoders`, Yuri Burda, Roger Grosse, Ruslan Salakhutdinov (2015)
            
        """ ### sample input ### # samples, timesteps, dims ### mean and covariance of input mapping ### # samples, neurons, timesteps # prior may need quantities computed in compute_F, e.g. Luu ### evaluating likelihood expectation and prior terms ### # n_data/batch_size, as we subsample temporally (batching) # sum over time or dummy # could be less than requested if exact nll # mean over trials # lvm_nlq is scalar as entropy is True # enumeration # mean is taken over the sample dimension i.e. log E_{q(f)q(z)}[...] # sum over MC/GH samples # mean is taken over the sample dimension i.e. log E_{q(f)q(z)}[...] # sum over MC/GH samples # sum over MC/GH samples # mean over trials # model priors integrated out (KL terms)
variational approximation to negative marginal log likelihood
variational approximation to negative marginal log likelihood
torch.tensor
References:

[1] `Parametric Gaussian Process Regressors`, Martin Jankowiak, Geoff Pleiss, Jacob R. Gardner (2019)

[2] `Importance Weighted Autoencoders`, Yuri Burda, Roger Grosse, Ruslan Salakhutdinov (2015)
self
b
batch index used for picking the batch to fit
batch index used for picking the batch to fit
neuron
indices of neurons for which to evaluate the likelihood term (observed neurons)
indices of neurons for which to evaluate the likelihood term ( observed neurons )
beta
prior annealing or KL annealing via beta
prior annealing or KL annealing via beta
beta_z
cov_samples
number of samples to draw from covariates distribution
number of samples to draw from covariates distribution
ll_samples
number of samples to evaluate likelihood per covariate sample
number of samples to evaluate likelihood per covariate sample
bound
type of variational objective ('ELBO', 'PLL' [1] or 'IWAE' [2])
type of variational objective ( ' ELBO ' ' PLL ' [ 1 ] or ' IWAE ' [ 2 ] )
ll_mode
likelihood evaluation mode (Monte Carlo `MC` or Gauss-Hermite quadratures `GH`)
likelihood evaluation mode ( Monte Carlo ` MC ` or Gauss - Hermite quadratures ` GH ` )
lv_input
input tensor for the latent variable computation
input tensor for the latent variable computation
enuerate_z
use enumerate for discrete latent (marginalizing out all states)
use enumerate for discrete latent ( marginalizing out all states )