class_docstr: Transformers handler class for sequence, token classification and question answering.
class_docstr_tok: ["Transformers","handler","class","for","sequence",",","token","classification","and","question","answering","."]
code: def _run_prefill(self, req_id):
      self._vacate_kv_cache_before_prefill()
      self.batch_idx_to_req_ids[0] = req_id
  
      logits = self.model.forward(
          self.context.cache[req_id]["encoded"].view(1, -1), 0
      )
  
      if self.temperature > 0:
          probs = torch.softmax(
              logits[:, -1] / self.temperature, dim=-1
          )
          next_token = sample_top_p(probs, self.top_p)
      else:
          next_token = torch.argmax(logits[:, -1], dim=-1)
  
      self.context.cache[req_id]["encoded"] = torch.concat(
          (self.context.cache[req_id]["encoded"], next_token.view(1)),
          dim=-1,
      )
  
      current_text = self.tokenizer.decode(
          self.context.cache[req_id]["encoded"].view(-1).tolist()
      )
      prev_text_len = len(self.context.cache[req_id]["text"])
      new_text = current_text[prev_text_len:]
      self.context.cache[req_id]["text"] = current_text
  
      result = {
          req_id: {
              "text": new_text,
              "ids": next_token.view(-1).tolist(),
          }
      }
  
      self.context.cache[req_id]["padding"] = 0
  
      return result
  
imports: import json
  import logging
  import os
  import sys
  import time
  from abc import ABC
  from typing import List
  import packaging.version
  import requests
  import torch
  import transformers
  from ts.torch_handler.base_handler import BaseHandler
  from generate import sample_top_p
  from llama2 import Llama
label_desc: Use image node_id 8 for calling the LlamaHandler obj's underlying member method code with example usage: obj._run_prefill(req_id) and returns: result