code: def _loglikelihood_tokens(self, requests, disable_tqdm=False):
      res = []
  
      def _collate(x):
          # this doesn't efficiently handle last-token differences yet, but those are kinda annoying because
          # it's not guaranteed that the 100 or so logprobs we get to see actually contain all the continuations
          # we care about and so we need some kind of backup for when it isn't
          toks = x[1] + x[2]
          return -len(toks), tuple(toks)
  
      re_ord = utils.Reorderer(requests, _collate)
  
      for chunk in tqdm(
          list(
              utils.chunks(re_ord.get_reordered(), self.REQ_CHUNK_SIZE)
          ),
          disable=disable_tqdm,
      ):
          inps = []
          ctxlens = []
          for cache_key, context_enc, continuation_enc in chunk:
              # max_length+1 because the API takes up to 2049 tokens, including the first context token
              inp = (context_enc + continuation_enc)[
                  -(self.max_length + 1) :
              ]
              # TODO: the logic is much simpler if we just look at the length of continuation tokens
              ctxlen = len(context_enc) - max(
                  0,
                  len(context_enc)
                  + len(continuation_enc)
                  - (self.max_length + 1),
              )
  
              inps.append(inp)
              ctxlens.append(ctxlen)
  
          response = oa_completion(
              engine=self.engine,
              prompt=inps,
              echo=True,
              max_tokens=0,
              temperature=0.0,
              logprobs=10,
          )
  
          for (
              resp,
              ctxlen,
              (cache_key, context_enc, continuation_enc),
          ) in zip(response.choices, ctxlens, chunk):
              answer = get_result(resp, ctxlen)
  
              res.append(answer)
  
              # partial caching
              if cache_key is not None:
                  self.cache_hook.add_partial(
                      "loglikelihood", cache_key, answer
                  )
  
      return re_ord.get_original(res)
  
imports: import os
  import numpy
  import transformers
  from lm_eval.base import BaseLM
  from lm_eval import utils
  from tqdm import tqdm
  import time
label_desc: Use image node_id 9 for calling the GPT3LM obj's underlying member method code with example usage: obj._loglikelihood_tokens(requests, disable_tqdm) and returns: re_ord, unknown,  tuple