code: def get_conditional_loglikelihood(
      self,
      texts: list,
      completion_offset: torch.Tensor,
      tokens_to_explain: list = [],
  ):
      tokenized = self.tokenizer(
          texts,
          return_tensors="pt",
          truncation=True,
          return_token_type_ids=False,
          padding=True,
      ).to(self.device)
      labels = tokenized.input_ids
      outputs = self.model(**tokenized)
      logits = (
          outputs["logits"]
          .detach()
          .to(device="cpu", dtype=torch.float32)
      )
      labels = labels.cpu()
  
      shift_logits = logits[:, :-1, :].contiguous()
      shift_labels = labels[:, 1:].contiguous()
      shift_labels[shift_labels == self.tokenizer.pad_token_id] = -100
  
      # Ignore the tokens prior to the completion offset.
      mask = torch.arange(shift_labels.size(1)).unsqueeze(0) < (
          shift_labels.shape[1] - completion_offset.unsqueeze(1)
      )
  
      # Choose which tokens to mask from LL computation.
      shift_labels[
          (mask | (shift_labels == self.tokenizer.pad_token_id))
      ] = -100
  
      loss = F.cross_entropy(
          shift_logits.view(-1, shift_logits.size(-1)),
          shift_labels.view(-1),
          reduction="none",
      ).detach()
      ll_per_sample = -loss.view(
          shift_logits.shape[0], shift_logits.shape[1]
      )
      # nonpad_per_row = (shift_labels != -100).sum(dim=1)
      # ll_mean = ll_per_sample.sum(dim=1)/nonpad_per_row
  
      ll_per_sample[(shift_labels == -100)] = 0
      ll_total = ll_per_sample.sum(dim=1)
      torch.cuda.empty_cache()
      return (
          ll_per_sample.float().cpu().numpy(),
          ll_total.float().cpu().numpy(),
      )
  
file_path: interpret-text/python/interpret_text/generative/model_lib/hf_tooling.py
imports: import torch
  import numpy
  from tqdm import tqdm
  import torch.nn.functional
  from scipy.special import log_softmax, logsumexp
label_desc: Use image node_id 3 for calling the HF_LM obj's underlying member method code with example usage: obj.get_conditional_loglikelihood(texts, completion_offset, tokens_to_explain) and returns: ll_per_sample,  ll_total