code: def main(args):
      if args.flash_attn:
          replace_llama_attn(inference=True)
  
      # Set RoPE scaling factor
      config = transformers.AutoConfig.from_pretrained(
          args.base_model,
          cache_dir=args.cache_dir,
      )
  
      orig_ctx_len = getattr(config, "max_position_embeddings", None)
      if orig_ctx_len and args.context_size > orig_ctx_len:
          scaling_factor = float(
              math.ceil(args.context_size / orig_ctx_len)
          )
          config.rope_scaling = {
              "type": "linear",
              "factor": scaling_factor,
          }
  
      # Load model and tokenizer
      model = transformers.AutoModelForCausalLM.from_pretrained(
          args.base_model,
          config=config,
          cache_dir=args.cache_dir,
          torch_dtype=torch.float16,
          device_map="auto",
          quantization_config=BitsAndBytesConfig(
              load_in_4bit=True,
              bnb_4bit_use_double_quant=True,
              bnb_4bit_quant_type="nf4",
              bnb_4bit_compute_dtype=torch.bfloat16,
          ),
      )
      model.resize_token_embeddings(32001)
  
      tokenizer = transformers.AutoTokenizer.from_pretrained(
          args.base_model,
          cache_dir=args.cache_dir,
          model_max_length=args.context_size
          if args.context_size > orig_ctx_len
          else orig_ctx_len,
          padding_side="right",
          use_fast=False,
      )
  
      model.eval()
      if torch.__version__ >= "2" and sys.platform != "win32":
          model = torch.compile(model)
      respond = build_generator(
          model,
          tokenizer,
          temperature=args.temperature,
          top_p=args.top_p,
          max_gen_len=args.max_gen_len,
          use_cache=True,
      )
  
      material = read_txt_file(args.material)
      prompt_no_input = PROMPT_DICT["prompt_llama2"]
      prompt = prompt_no_input.format_map(
          {"instruction": material + "\n%s" % args.question}
      )
  
      output = respond(prompt=prompt)
  
file_path: long-lora/inference-qlora.py
imports: import os
  import sys
  import math
  import torch
  import argparse
  import textwrap
  import transformers
  from peft import PeftModel
  from transformers import GenerationConfig, TextStreamer, BitsAndBytesConfig
  from llama_attn_replace import replace_llama_attn
label_desc: Use image node_id 4 for calling a global function with example usage: main(args) without return types