code: def run_inference(model_name, requests):
      assert model_name == "opt-6.7b" or model_name == "opt-30b"
  
      env = ExecutionEnv.create("~/flexgen_offload_dir")
  
      policy = get_policy(model_name)
      batches, last_policy = get_batches(requests, policy)
  
      print(f"Init weights begin.")
      tic = time.time()
      model = OptLM(model_name, env, "~/opt_weights", policy)
      print(
          f"Init weights end. Elapsed: {time.time() - tic:.2f} s",
          flush=True,
      )
  
      # Generate
      print(f"Generate begin.")
      tic = time.time()
      gen_tokens = 0
      for i, batch in tqdm(
          enumerate(batches[:-1]), total=len(batches[:-1])
      ):
          input_ids = batch["input_ids"]
          print("batch size:", len(input_ids))
          print("prompt len:", len(input_ids[0]))
          print("max new tokens:", batch["max_new_tokens"])
          print("eos id:", batch["eos_token_id"])
          output_ids = model.generate(
              input_ids,
              do_sample=True,
              temperature=1e-7,
              max_new_tokens=batch["max_new_tokens"],
              stop=batch["eos_token_id"],
          )
          gen_tokens += len(input_ids) * batch["max_new_tokens"]
          total_time = time.time() - tic
          throughput = gen_tokens / total_time
          print(
              f"Generation throughput: {throughput:.2f} token/s",
              flush=True,
          )
  
      # last batch
      del model
      model = OptLM(model_name, env, "~/opt_weights", last_policy)
  
      tic = time.time()
      batch = batches[-1]
      input_ids = batch["input_ids"]
      output_ids = model.generate(
          input_ids,
          do_sample=True,
          temperature=1e-7,
          max_new_tokens=batch["max_new_tokens"],
          stop=batch["eos_token_id"],
      )
      gen_tokens += len(input_ids) * batch["max_new_tokens"]
      total_time += time.time() - tic
      throughput = gen_tokens / total_time
  
      print(f"Generate end. Elapsed: {total_time:.2f} s", flush=True)
      print(
          f"Generation throughput: {throughput:.2f} token/s", flush=True
      )
  
      return total_time, throughput
  
file_path: H2O/h2o_flexgen/benchmark/xsum/run_xsum_flexgen.py
imports: import argparse
  import numpy
  import torch
  import json
  import time
  from tqdm import tqdm
  import copy
  from flexgen.flex_opt import Policy, OptLM, ExecutionEnv, CompressionConfig
  from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
label_desc: Use image node_id 5 for calling a global function with example usage: run_inference(model_name, requests) and returns: total_time,  throughput