class_docstr: Transformers handler class for sequence, token classification and question answering.
class_docstr_tok: ["Transformers","handler","class","for","sequence",",","token","classification","and","question","answering","."]
docstr: Predicts the class (or classes) of the received text using the serialized transformers
  checkpoint.
  Args:
      input_batch (tuple): A tuple with two tensors: the batch of input ids and the batch
                          of attention masks, as returned by the preprocess function.
  Returns:
      list: A list of strings with the predicted values for each input text in the batch.
docstr_tok: ["Predicts","the","class","(","or","classes",")","of","the","received","text","using","the","serialized","transformers","checkpoint",".","Args",":","input_batch","(","tuple",")",":","A","tuple","with","two","tensors",":","the","batch","of","input","ids","and","the","batch","of","attention","masks",",","as","returned","by","the","preprocess","function",".","Returns",":","list",":","A","list","of","strings","with","the","predicted","values","for","each","input","text","in","the","batch","."]
code: def inference(self, input_batch):
      """
      Predicts the class (or classes) of the received text using the serialized transformers
      checkpoint.
      Args:
          input_batch (tuple): A tuple with two tensors: the batch of input ids and the batch
                              of attention masks, as returned by the preprocess function.
      Returns:
          list: A list of strings with the predicted values for each input text in the batch.
      """
      input_ids_batch = input_batch[0]
  
      # insert padding if a partial batch was received
      num_inferences = len(input_ids_batch)
      logger.info(
          "Number of inference requests in batch: %s", num_inferences
      )
      logger.info("Model batch size: %s", self.batch_size)
      padding = self.batch_size - num_inferences
      if padding > 0:
          logger.info(
              "Padding input batch with %s padding inputs", padding
          )
          pad = torch.nn.ConstantPad1d((0, 0, 0, padding), value=0)
          input_ids_batch = pad(input_ids_batch)
  
      outputs = self.model.sample(
          input_ids_batch,
          self.max_length,
      )
  
      inferences = self.tokenizer.batch_decode(
          outputs,
          skip_special_tokens=True,
          clean_up_tokenization_spaces=False,
      )
      inferences = inferences[:num_inferences]
  
      logger.info("Generated text: %s", inferences)
      return inferences
  
file_path: serve/examples/large_models/inferentia2/opt/inf2_handler.py
imports: import logging
  import os
  from abc import ABC
  import torch
  import torch_neuronx
  import transformers
  from transformers import AutoTokenizer
  from transformers_neuronx.opt.model import OPTForSampling
  from ts.torch_handler.base_handler import BaseHandler
label_desc: Use image node_id 5 for calling the LLMHandler obj's underlying member method code with example usage: obj.inference(input_batch) and returns: inferences