code: def inference(options):
      print(options)
      prompter = Prompter(options.prompt_template_name)
      model_name = get_model_name(options.model_name)
      device_setting = "auto"
      model_class, tokenizer_class, _ = get_model_tokenizer(model_name)
      tokenizer = tokenizer_class.from_pretrained(
          options.model_name_or_path,
          trust_remote_code=options.trust_remote_code,
      )
  
      compute_dtype = (
          torch.float16
          if options.fp16
          else (torch.bfloat16 if options.bf16 else torch.float32)
      )
      model = model_class.from_pretrained(
          options.model_name_or_path,
          load_in_4bit=options.bits == 4,
          load_in_8bit=options.bits == 8,
          device_map=device_setting,
          offload_folder="./cache",  # If the `device_map` contains any value `"disk"`, the folder where we will offload weights.
          quantization_config=BitsAndBytesConfig(
              load_in_4bit=options.bits == 4,
              load_in_8bit=options.bits == 8,
              llm_int8_threshold=6.0,
              llm_int8_has_fp16_weight=False,
              bnb_4bit_compute_dtype=compute_dtype,
              bnb_4bit_use_double_quant=options.double_quant,
              bnb_4bit_quant_type=options.quant_type,
          ),
          torch_dtype=compute_dtype,
          trust_remote_code=options.trust_remote_code,
      )
      model.config.torch_dtype = compute_dtype
      if options.lora_weights is not None:
          if options.bits <= 8:
              config = LoraConfig.from_pretrained(options.lora_weights)
              model = get_peft_model(model, config)
              adapters_weights = torch.load(
                  os.path.join(options.lora_weights, WEIGHTS_NAME),
                  map_location=model.device,
              )
              set_peft_model_state_dict(model, adapters_weights)
          else:
              model = PeftModel.from_pretrained(
                  model,
                  options.lora_weights,
              )
  
      if model_name == "llama":
          model.config.pad_token_id = tokenizer.pad_token_id = 0  # unk
          model.config.bos_token_id = tokenizer.bos_token_id = 1
          model.config.eos_token_id = tokenizer.eos_token_id = 2
          tokenizer.padding_side = "left"
      elif model_name == "moss":
          model.config.pad_token_id = (
              tokenizer.pad_token_id
          ) = tokenizer.eos_token_id
      print(
          f"BOS:{tokenizer.bos_token_id},{tokenizer.bos_token}\tEOS:{tokenizer.eos_token_id},{tokenizer.eos_token}\tPAD:{tokenizer.pad_token_id},{tokenizer.pad_token}"
      )
  
      model.eval()
      if torch.__version__ >= "2" and sys.platform != "win32":
          model = torch.compile(model)
  
      def evaluate(
          instruction,
          input=None,
          options=None,
          **kwargs,
      ):
          prompt = prompter.generate_prompt(instruction, input)
          inputs = tokenizer(prompt, return_tensors="pt")
          input_ids = inputs["input_ids"].to(device)
  
          generation_config = GenerationConfig(
              temperature=options.temperature,
              top_p=options.top_p,
              top_k=options.top_k,
              num_beams=options.num_beams,
              max_new_tokens=options.max_new_tokens,
              pad_token_id=tokenizer.pad_token_id,
              return_dict_in_generate=True,
              output_scores=True,
              **kwargs,
          )
  
          with torch.no_grad():
              generation_output = model.generate(
                  input_ids=input_ids,
                  generation_config=generation_config,
                  **kwargs,
              )
          generation_output = generation_output.sequences[0]
          output = tokenizer.decode(
              generation_output, skip_special_tokens=True
          )
          output = prompter.get_response(output)
          return output
  
      records = []
      with open(options.input_file, "r") as reader:
          for line in reader:
              data = json.loads(line)
              records.append(data)
  
      with open(options.output_file, "w") as writer:
          for i, record in enumerate(records):
              result = evaluate(
                  instruction=record["instruction"],
                  input=record.get("input", None),
                  options=options,
              )
              print(i, result)
              record["output"] = result
              writer.write(
                  json.dumps(record, ensure_ascii=False) + "\n"
              )
  
file_path: DeepKE/example/llm/InstructKGC/src/inference.py
imports: import os
  import sys
  import json
  import argparse
  import torch
  from dataclasses import dataclass, field
  from transformers import HfArgumentParser, GenerationConfig, BitsAndBytesConfig
  from peft import LoraConfig, PeftModel, get_peft_model, set_peft_model_state_dict
  from peft.utils import WEIGHTS_NAME
  from utils.prompter import Prompter
  from utils.general_utils import get_model_tokenizer, get_model_name
label_desc: Use image node_id 1 for calling a global function with example usage: inference(options) and returns: output