code: def inference(options):
      print(options)
      prompter = Prompter(options.prompt_template_name)
      model_name = get_model_name(options.model_name)
      device_setting = {"": 0} if model_name == "cpm-bee" else "auto"
      model_class, tokenizer_class, _ = get_model_tokenizer(model_name)
  
      tokenizer = tokenizer_class.from_pretrained(
          options.model_name_or_path,
          trust_remote_code=options.trust_remote_code,
      )
      compute_dtype = (
          torch.float16
          if options.fp16
          else (torch.bfloat16 if options.bf16 else torch.float32)
      )
      base_model = model_class.from_pretrained(
          options.model_name_or_path,
          load_in_8bit=False,
          torch_dtype=compute_dtype,
          device_map=device_setting,
      )
  
      if model_name == "llama":
          base_model.config.pad_token_id = (
              tokenizer.pad_token_id
          ) = 0  # unk
          base_model.config.bos_token_id = tokenizer.bos_token_id = 1
          base_model.config.eos_token_id = tokenizer.eos_token_id = 2
          tokenizer.padding_side = "left"
      elif model_name == "moss":
          base_model.config.pad_token_id = (
              tokenizer.pad_token_id
          ) = tokenizer.eos_token_id
      print(
          f"BOS:{tokenizer.bos_token_id},{tokenizer.bos_token}\tEOS:{tokenizer.eos_token_id},{tokenizer.eos_token}\tPAD:{tokenizer.pad_token_id},{tokenizer.pad_token}"
      )
  
      state_dict = None
      if options.lora_weights is not None:
          state_dict = merge_model(options, base_model)
      else:
          state_dict = base_model.state_dict()
      llm, sampling_params = load_llm(options, state_dict)
  
      records = []
      prompt_token_ids = []
      with open(options.input_file, "r") as reader:
          for line in reader:
              record = json.loads(line)
              records.append(record)
              prompt_ = prompter.generate_prompt(
                  instruction=record["instruction"],
                  input=record.get("input", None),
              )
              inputs = tokenizer(prompt_)
              input_ids = inputs["input_ids"][
                  : options.max_source_length
              ]
              prompt_token_ids.append(input_ids)
  
      outputs = llm.generate(
          sampling_params=sampling_params,
          prompt_token_ids=prompt_token_ids,
          use_tqdm=True,
      )
  
      with open(options.output_file, "w") as writer:
          for record, output in zip(records, outputs):
              record["output"] = output.outputs[0].text
              writer.write(
                  json.dumps(record, ensure_ascii=False) + "\n"
              )
  
file_path: DeepKE/example/llm/InstructKGC/src/inference_vllm.py
imports: import os
  import json
  import torch
  from dataclasses import dataclass, field
  from transformers import HfArgumentParser
  from peft import PeftModel
  from vllm.sampling_params import SamplingParams
  from utils.prompter import Prompter
  from utils.general_utils import get_model_tokenizer, get_model_name
  from utils.vllm_llm import LLM
label_desc: Use image node_id 3 for calling a global function with example usage: inference(options) without return types