docstr: logodds is much like classification, except we need to compare across prompts so we just
  compute the log odds here
docstr_tok: ["logodds","is","much","like","classification",",","except","we","need","to","compare","across","prompts","so","we","just","compute","the","log","odds","here"]
code: def _evaluate_logodds(
      self,
      examples: list[LogoddsExample],
      take_absolute_value: bool = False,
  ) -> dict[str, Union[Sequence[float], Sequence[int]]]:
      """logodds is much like classification, except we need to compare across prompts so we just
      compute the log odds here"""
      prompts = [example.prompt for example in examples]
      other_prompts = [example.other_prompt for example in examples]
      tokenized_inputs = self.tokenizer(
          prompts,
          return_tensors="pt",
          truncation=True,
          return_token_type_ids=False,
      ).to(self.device)
      other_tokenized_inputs = self.tokenizer(
          other_prompts,
          return_tensors="pt",
          truncation=True,
          return_token_type_ids=False,
      ).to(self.device)
      outputs = self.model(**tokenized_inputs)
      other_outputs = self.model(**other_tokenized_inputs)
      # we only need the logits for the final (new) token
      # NOTE: this may need to change if we use batch size > 1 with padding
  
      logits = (
          outputs["logits"][:, -1]
          .detach()
          .to(device="cpu", dtype=torch.float32)
      )
  
      other_logits = (
          other_outputs["logits"][:, -1]
          .detach()
          .to(device="cpu", dtype=torch.float32)
      )
      logodds = self._logodds_from_logits(examples, logits)
      other_logodds = self._logodds_from_logits(examples, other_logits)
  
      logodds_differences = list(
          np.array(logodds)
          - np.array(other_logodds)  # type: ignore (np typing bad)
      )
      answer_indices = [example.answer_index for example in examples]
      # flip the order (and hence the sign) if the answer is "no"
      # (unless we are taking absolute values)
      for i, answer_index in enumerate(answer_indices):
          if answer_index == 1:
              logodds_differences[i] *= -1
          if take_absolute_value:
              logodds_differences[i] = np.abs(logodds_differences[i])
  
      accuracies = self._accuracies_from_logits(examples, other_logits)
      total_logprob = list(
          torch.logsumexp(
              torch.stack(
                  (
                      torch.tensor(
                          self._total_logprobs_from_logits(
                              examples, logits
                          )
                      ),
                      torch.tensor(
                          self._total_logprobs_from_logits(
                              examples, other_logits
                          )
                      ),
                  )
              ),
              dim=0,
          )
      )
      return {
          "logodds_difference": logodds_differences,
          "correct": accuracies,
          # type: ignore (they should be floats)
          "total_logprob": total_logprob,
      }
  
imports: from __future__ import annotations
  from abc import ABC, abstractmethod
  from typing import Union, cast, Sequence
  from typing_extensions import Literal, get_args
  import numpy
  import torch
  import torch.nn.functional
  from transformers import AutoModelForCausalLM, AutoTokenizer
  from accelerate import Accelerator
  from .isp_dataset import ClassificationExample, Example, ExampleWithClasses, LogoddsExample, SequenceProbExample, NumericExample, TaskType
  from .isp_utils import BasicParser
  from .isp_utils import OpenAIModel, OpenaAIModelList
label_desc: Use image node_id 7 for calling the HFModel obj's underlying member method code with example usage: obj._evaluate_logodds(examples, take_absolute_value) and returns: dict