code: def run_generation(
      model_name,
      batch_size,
      prompt_len,
      gen_len,
      cut_gen_len,
      cpu_offload,
      disk_offload,
      offload_dir,
      use_int8,
      num_nodes,
      num_gpus_per_node,
      use_deepspeed,
      dummy,
      output_file,
      pkl_file,
      no_log,
      verbose,
  ):
      # Load tokenizer
      tokenizer = AutoTokenizer.from_pretrained(
          model_name.replace("175b", "66b"), padding_side="left"
      )
  
      # Load model
      if use_int8:
          dtype = torch.int8
      else:
          dtype = torch.float16
  
      if dummy:
          config = get_model_config(model_name)
          filename = os.path.join(
              offload_dir, f"{model_name.replace('/', '-')}-hf-weights/"
          )
          if not os.path.exists(filename):
              print("create dummy weights")
              with init_empty_weights():
                  model = OPTForCausalLM(config)
              model.save_pretrained(
                  filename,
                  state_dict=meta_to_cpu(
                      model.state_dict(), torch.float16
                  ),
              )
          dummy_weights = filename
      else:
          dummy_weights = None
  
      print("load model")
      if use_deepspeed:
          model = get_ds_opt_model(
              model_name,
              dtype,
              cpu_offload,
              disk_offload,
              offload_dir,
              dummy_weights,
          )
      else:
          model = get_hf_opt_model(
              model_name,
              dtype,
              cpu_offload,
              disk_offload,
              offload_dir,
              num_gpus_per_node,
              dummy_weights,
          )
  
      # Run generation
      execute_gen_len = cut_gen_len if cut_gen_len else gen_len
      if use_deepspeed:
          prompts = ["Paris is the capital city of"] * (
              batch_size // WORLD_SIZE
          )
      else:
          prompts = ["Paris is the capital city of"] * batch_size
      input_ids = tokenizer(
          prompts,
          return_tensors="pt",
          padding="max_length",
          max_length=prompt_len,
      ).input_ids.cuda()
  
      # Warmup
      print("wamup")
      generate_kwargs_warmup = dict(max_new_tokens=1, do_sample=False)
      with torch.no_grad():
          output_ids = model.generate(
              input_ids=input_ids, **generate_kwargs_warmup
          )
  
      # Run
      print("benchmark")
      timers("generate-forward").reset()
      generate_kwargs = dict(
          max_new_tokens=execute_gen_len, do_sample=False
      )
      with torch.no_grad():
          output_ids = model.generate(
              input_ids=input_ids, **generate_kwargs
          )
      costs = timers("generate-forward").costs
  
      if use_deepspeed and args.local_rank != 0:
          return
  
      # Log output
      prefill_latency = costs[0]
      prefill_throughput = batch_size * prompt_len / prefill_latency
      if cut_gen_len:  # project latency of cut_gen_len to gen_len
          decode_latency = project_decode_latency(
              costs, prompt_len, gen_len
          )
      else:
          decode_latency = sum(costs[1:])
      decode_throughput = (
          batch_size * (gen_len - 1) / max(decode_latency, 1e-10)
      )
      num_generated_tokens = batch_size * gen_len
      total_latency = prefill_latency + decode_latency
      total_throughput = num_generated_tokens / total_latency
      gpu_peak_mem = torch.cuda.max_memory_allocated(
          torch.device("cuda")
      )
      out_str = ""
  
      if verbose >= 2:
          outputs = tokenizer.batch_decode(
              output_ids, skip_special_tokens=True
          )
          show_str = "Outputs:\n" + 70 * "-" + "\n"
          for i in [0, len(outputs) - 1]:
              show_str += f"{i}: {outputs[i]}\n"
              show_str += 70 * "-" + "\n"
          print(show_str)
  
          # Check lengths
          input_lens = [len(x) for x in input_ids]
          output_lens = [len(x) for x in output_ids]
          assert all(x == prompt_len for x in input_lens)
          assert all(
              x == prompt_len + execute_gen_len for x in output_lens
          )
  
      if args.log_file == "auto":
          filename = (
              get_filename(
                  model_name,
                  batch_size,
                  prompt_len,
                  gen_len,
                  cpu_offload,
                  disk_offload,
                  num_nodes,
                  num_gpus_per_node,
                  use_deepspeed,
              )
              + ".log"
          )
      else:
          filename = args.log_file
  
      projected = bool(cut_gen_len)
      opt_config = get_opt_config(args.model)
      cache_size = opt_config.cache_bytes(
          batch_size, prompt_len + gen_len
      )
      hidden_size = opt_config.hidden_bytes(
          batch_size, prompt_len + gen_len
      )
      log_str = write_benchmark_log(
          filename,
          opt_config.model_bytes(),
          cache_size,
          hidden_size,
          gpu_peak_mem,
          projected,
          prefill_latency,
          prefill_throughput,
          decode_latency,
          decode_throughput,
          total_latency,
          total_throughput,
      )
      if verbose >= 1:
          print(log_str)
  
file_path: H2O/h2o_flexgen/benchmark/hf_ds/hf_opt.py
imports: import argparse
  import multiprocessing
  import os
  import pickle
  import time
  import numpy
  from accelerate import infer_auto_device_map, init_empty_weights, load_checkpoint_and_dispatch
  from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM
  from transformers import OPTForCausalLM
  import torch
  from flexgen.timer import timers
  from flexgen.utils import GB, project_decode_latency, write_benchmark_log
  from flexgen.opt_config import get_opt_config, disable_torch_init, disable_hf_opt_init
label_desc: Use image node_id 7 for calling a global function with example usage: run_generation(model_name, batch_size, prompt_len, gen_len, cut_gen_len, cpu_offload, disk_offload, offload_dir, use_int8, num_nodes, num_gpus_per_node, use_deepspeed, dummy, output_file, pkl_file, no_log, verbose) without return types