class_docstr: Model wrapper around HuggingFace AutoModel models.
  
  Args:
      path: The path to a HuggingFace model.
      model_max_length: The maximum sequence length of the model.
      tokenizer_path: The path to the tokenizer.
      tokenizer_kwargs: Keyword arguments for the tokenizer.
      peft_path: The name or path to the HuggingFace's PEFT model.
      model_kwargs: Keyword arguments for the model.
      prompt_template: The model's prompt template.
      batch_size: Batch size for inference.
      logger: Logger for the model.
      shard_config: Shard config for tensor parallel.
class_docstr_tok: ["Model","wrapper","around","HuggingFace","AutoModel","models",".","Args",":","path",":","The","path","to","a","HuggingFace","model",".","model_max_length",":","The","maximum","sequence","length","of","the","model",".","tokenizer_path",":","The","path","to","the","tokenizer",".","tokenizer_kwargs",":","Keyword","arguments","for","the","tokenizer",".","peft_path",":","The","name","or","path","to","the","HuggingFace","'s","PEFT","model",".","model_kwargs",":","Keyword","arguments","for","the","model",".","prompt_template",":","The","model","'s","prompt","template",".","batch_size",":","Batch","size","for","inference",".","logger",":","Logger","for","the","model",".","shard_config",":","Shard","config","for","tensor","parallel","."]
docstr: Infer the given data.
  This function will call self.generate() to get model outputs and also self.model() to get logits.
  
  Args:
      data: The data for inference.
      inference_kwargs: Arguments for inference.
      debug: Whether to display generated prompt for debugging.
  
  Returns:
      Inference results.
docstr_tok: ["Infer","the","given","data",".","This","function","will","call","self.generate","(",")","to","get","model","outputs","and","also","self.model","(",")","to","get","logits",".","Args",":","data",":","The","data","for","inference",".","inference_kwargs",":","Arguments","for","inference",".","debug",":","Whether","to","display","generated","prompt","for","debugging",".","Returns",":","Inference","results","."]
code: def inference(
      self,
      data: List[Dict],
      inference_kwargs: Dict[str, Any],
      debug: bool = False,
  ) -> List[Dict]:
      """
      Infer the given data.
      This function will call self.generate() to get model outputs and also self.model() to get logits.
  
      Args:
          data: The data for inference.
          inference_kwargs: Arguments for inference.
          debug: Whether to display generated prompt for debugging.
  
      Returns:
          Inference results.
  
      """
      calculate_loss = inference_kwargs["calculate_loss"]
      classes = inference_kwargs["all_classes"]
      language = inference_kwargs["language"]
      pretrain = inference_kwargs["pretrain"]
      max_new_tokens = inference_kwargs["max_new_tokens"]
      few_shot_data = inference_kwargs.get("few_shot_data", None)
  
      # Some classification questions' options are texts not a single letter such as A, B, C and D.
      # If the text length is greater than 1, we won't calculate loss over choices.
      if classes is not None and any(len(c) > 1 for c in classes):
          classes = None
  
      self.choices = classes
      self.indices_for_choices = None
      if self.choices:
          # Get indices for each choice
          self._get_choices_indices(language)
  
          self.str_label_map = {
              choice: idx for idx, choice in enumerate(self.choices)
          }
  
      turn = (
          0
          if not isinstance(data[0]["output"], list)
          else len(data[0]["output"]) + 1
      )
      turn_desc = "" if turn == 0 else f"-turn{turn}"
  
      bar = tqdm(
          range(math.ceil(len(data) / self.batch_size)),
          desc=f"{data[0]['dataset']}-{data[0]['category']}{turn_desc} Inference steps",
          disable=not is_rank_0(),
      )
      loss_fct = torch.nn.CrossEntropyLoss(reduction="none")
  
      answers = copy.deepcopy(data)
      for i in range(0, len(data), self.batch_size):
          batch = data[i : i + self.batch_size]
          batch_prompt, batch_target = get_batch_prompt(
              self.prompt_template,
              batch,
              few_shot_data,
              self.tokenizer,
              language,
              self.model_max_length,
          )
  
          if is_rank_0() and debug and i == 0:
              self.logger.info(
                  f"Inference arguments for dataset {data[0]['dataset']} category {data[0]['category']} is:\n{inference_kwargs}"
              )
              self.logger.info("-" * 120)
              self.logger.info(
                  "An example prompt and prompt with target is:"
              )
              self.logger.info("-" * 120)
              self.logger.info(batch_prompt[0])
              self.logger.info("-" * 120)
              self.logger.info(batch_prompt[0] + batch_target[0][0])
  
          if not pretrain:
              batch_decodes, scores = self.generate(
                  batch_prompt, max_new_tokens
              )
  
          if calculate_loss:
              (
                  batch_losses,
                  batch_target_token_nums,
                  batch_bytes_nums,
              ) = self.get_loss(batch_prompt, batch_target, pretrain)
  
          probs = []
          if self.indices_for_choices:
              scores = scores.to(torch.float32)
              # If we have indices_for_choices(must be single-choice question), there will be only one target answer for one data sample.
              # Otherwise this will violate the single-choice setting.
  
              if calculate_loss:
                  labels = [
                      self.str_label_map[answers[i + j]["target"]]
                      for j in range(len(batch_decodes))
                  ]
  
                  loss_over_choices = (
                      loss_fct(
                          scores, torch.tensor(labels, dtype=torch.long)
                      )
                      .numpy()
                      .tolist()
                  )
  
              probs = scores.numpy().tolist()
              probs = [
                  {
                      choice: probs[i][self.str_label_map[choice]]
                      for choice in self.choices
                  }
                  for i in range(len(probs))
              ]
  
          for j in range(len(batch_prompt)):
              if not pretrain:
                  if isinstance(answers[i + j]["output"], list):
                      answers[i + j]["output"].append(
                          batch_decodes[j].strip()
                      )
                  else:
                      answers[i + j]["output"] = batch_decodes[
                          j
                      ].strip()
  
                  if isinstance(scores, torch.Tensor):
                      answers[i + j]["logits_over_choices"] = probs[j]
  
                      if calculate_loss:
                          answers[i + j][
                              "loss_over_choices"
                          ] = loss_over_choices[j]
  
              if calculate_loss:
                  answers[i + j]["loss"] = (
                      np.array(batch_losses[j])
                      / np.array(batch_target_token_nums[j])
                  ).tolist()
  
                  # loss_sum is specially used for pertrain dataset for calculating per-byte-perplexity.
                  # However, loss (which is per sample loss) suffices for most cases.
                  answers[i + j]["loss_sum"] = batch_losses[j]
                  answers[i + j]["token_num"] = batch_target_token_nums[
                      j
                  ]
  
                  if batch_bytes_nums:
                      answers[i + j]["byte_num"] = batch_bytes_nums[j]
  
          bar.update()
  
      return answers
  
imports: import copy
  import math
  from typing import Any, Dict, List, Optional, Tuple
  import numpy
  import torch
  from colossal_eval.utils import Conversation, get_batch_prompt, is_rank_0
  from peft import PeftModel
  from tqdm import tqdm
  from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoTokenizer
  from colossalai.logging import DistributedLogger
  from colossalai.shardformer import ShardConfig, ShardFormer
  from .base import BaseModel
label_desc: Use image node_id 9 for calling the HuggingFaceModel obj's underlying member method code with example usage: obj.inference(data, inference_kwargs, debug) and returns: answers