docstr: Compute the loglikelihood of the given set of texts.
docstr_tok: ["Compute","the","loglikelihood","of","the","given","set","of","texts","."]
code: def get_batch_loglikelihood(self, texts):
      """
      Compute the loglikelihood of the given set of texts.
      """
      perplexities = []
      total_likelihoods = []
      for text in texts:
          tokenized = self.tokenizer(
              [text], return_tensors="pt", padding=True
          ).to(self.device)
          labels = tokenized.input_ids
          outputs = self.model(
              input_ids=tokenized["input_ids"],
              attention_mask=tokenized["attention_mask"],
              labels=labels,
          )
          logits = outputs.logits.cpu()
          labels = labels.cpu()
  
          shift_logits = logits[..., :-1, :].contiguous()
          shift_labels = labels[..., 1:].contiguous()
          shift_labels[
              shift_labels == self.tokenizer.pad_token_id
          ] = -100
          loss = F.cross_entropy(
              shift_logits.view(-1, shift_logits.size(-1)),
              shift_labels.view(-1),
              reduction="none",
          ).detach()
          ll_per_sample = -loss.view(
              shift_logits.shape[0], shift_logits.shape[1]
          )
          nonpad_per_row = (shift_labels != -100).sum(dim=1)
          ll_mean = ll_per_sample.sum(dim=1) / nonpad_per_row
  
          ll_per_sample[(shift_labels == -100)] = 0
          ll_total = ll_per_sample.sum(dim=1)
          perplexities.append(ll_mean.cpu().numpy())
          total_likelihoods.append(ll_total.cpu().numpy())
      perplexities = np.concatenate(perplexities, axis=0)
      total_likelihoods = np.concatenate(total_likelihoods, axis=0)
      return perplexities, total_likelihoods
  
file_path: interpret-text/python/interpret_text/generative/model_lib/hf_tooling.py
imports: import torch
  import numpy
  from tqdm import tqdm
  import torch.nn.functional
  from scipy.special import log_softmax, logsumexp
label_desc: Use image node_id 2 for calling the HF_LM obj's underlying member method code with example usage: obj.get_batch_loglikelihood(texts) and returns: perplexities,  total_likelihoods