class_docstr: Custom Handler for integrating vLLM
class_docstr_tok: ["Custom","Handler","for","integrating","vLLM"]
docstr: Generates the model response for the given prompt
  Args:
      input_batch : List of input text prompts as returned by the preprocess function.
  Returns:
      list: A list of strings with the predicted values for each input text in the batch.
docstr_tok: ["Generates","the","model","response","for","the","given","prompt","Args",":","input_batch",":","List","of","input","text","prompts","as","returned","by","the","preprocess","function",".","Returns",":","list",":","A","list","of","strings","with","the","predicted","values","for","each","input","text","in","the","batch","."]
code: def inference(self, input_batch):
      """
      Generates the model response for the given prompt
      Args:
          input_batch : List of input text prompts as returned by the preprocess function.
      Returns:
          list: A list of strings with the predicted values for each input text in the batch.
      """
      logger.info(f"Input text is {input_batch}")
      sampling_params = SamplingParams(max_tokens=self.max_new_tokens)
      outputs = self.model.generate(
          input_batch, sampling_params=sampling_params
      )
  
      logger.info("Generated text: %s", outputs)
      return outputs
  
file_path: serve/examples/large_models/vllm/mistral/custom_handler.py
imports: import logging
  import torch
  import vllm
  from vllm import LLM, SamplingParams
  from ts.context import Context
  from ts.torch_handler.base_handler import BaseHandler
label_desc: Use image node_id 4 for calling the CustomHandler obj's underlying member method code with example usage: obj.inference(input_batch) and returns: outputs