class_docstr: Transformers handler class for sequence, token classification and question answering.
class_docstr_tok: ["Transformers","handler","class","for","sequence",",","token","classification","and","question","answering","."]
docstr: Generate tokens based on prompts
  Args:
      prefil : a batch of req ids for prefill
      decode : a batch of req ids for decode
  Returns:
      list: A list of dicts with predicted values for each input text in the batch.
docstr_tok: ["Generate","tokens","based","on","prompts","Args",":","prefil",":","a","batch","of","req","ids","for","prefill","decode",":","a","batch","of","req","ids","for","decode","Returns",":","list",":","A","list","of","dicts","with","predicted","values","for","each","input","text","in","the","batch","."]
code: def inference(self, *args):
      """
      Generate tokens based on prompts
      Args:
          prefil : a batch of req ids for prefill
          decode : a batch of req ids for decode
      Returns:
          list: A list of dicts with predicted values for each input text in the batch.
      """
      prefill, decode_ids = args[0]
  
      # Prefill requests
      results = {}
      for req_id in prefill:
          results.update(self._run_prefill(req_id))
  
      # Decode the rest
      decode_result = self._run_decode(decode_ids) if decode_ids else {}
      results.update(decode_result)
      return [results[i] for i in self.context.request_ids.values()]
  
imports: import json
  import logging
  import os
  import sys
  import time
  from abc import ABC
  from typing import List
  import packaging.version
  import requests
  import torch
  import transformers
  from ts.torch_handler.base_handler import BaseHandler
  from generate import sample_top_p
  from llama2 import Llama
label_desc: Use image node_id 4 for calling the LlamaHandler obj's underlying member method code with example usage: obj.inference() and returns: unknown