code: def get_batches(requests, policy):
      batches = []
      cpu_batch_size = policy.gpu_batch_size * policy.num_gpu_batches
      num_batch = ((len(requests) - 1) // cpu_batch_size) + 1
      print("number of batches", num_batch)
      tokenizer = AutoTokenizer.from_pretrained(
          "facebook/opt-30b", padding_side="left"
      )
  
      prompt_len = 0
      for i in range(num_batch):
          batch = {
              "max_new_tokens": 0,
              "input_ids": [],
              "eos_token_id": None,
          }
  
          start_idx = i * cpu_batch_size
          if i == num_batch - 1:
              end_idx = len(requests)
              num_gpu_batches = (
                  end_idx - start_idx - 1
              ) // policy.gpu_batch_size + 1
              last_policy = Policy(
                  policy.gpu_batch_size,
                  num_gpu_batches,
                  policy.w_gpu_percent,
                  policy.w_cpu_percent,
                  policy.cache_gpu_percent,
                  policy.cache_cpu_percent,
                  policy.act_gpu_percent,
                  policy.act_cpu_percent,
                  overlap=True,
                  sep_layer=True,
                  pin_weight=True,
                  cpu_cache_compute=policy.cpu_cache_compute,
                  attn_sparsity=1.0,
                  compress_weight=False,
                  comp_weight_config=CompressionConfig(
                      num_bits=4,
                      group_size=64,
                      group_dim=0,
                      symmetric=False,
                  ),
                  compress_cache=False,
                  comp_cache_config=CompressionConfig(
                      num_bits=4,
                      group_size=64,
                      group_dim=2,
                      symmetric=False,
                  ),
                  hh_ratio=policy.hh_ratio,
                  hh_all=True,
              )
              new_batch_size = (
                  last_policy.gpu_batch_size
                  * last_policy.num_gpu_batches
              )
              for j in range(cpu_batch_size):
                  requests.append(requests[end_idx - 1])
              end_idx = start_idx + new_batch_size
              print("last batch index range", start_idx, end_idx)
          else:
              end_idx = (i + 1) * cpu_batch_size
          assert (end_idx - start_idx) % policy.gpu_batch_size == 0
  
          for req in requests[start_idx:end_idx]:
              max_tokens = req["request"]["max_tokens"]
              prompt = req["request"]["prompt"]
              stop = req["request"]["stop"]
              input_ids = tokenizer(
                  [prompt], add_special_tokens=False
              ).input_ids[0]
              # eos_token_id = tokenizer(stop).input_ids[0]
  
              prompt_len = max(prompt_len, len(input_ids))
              batch["max_new_tokens"] = max(
                  batch["max_new_tokens"], max_tokens
              )
              batch["input_ids"].append(input_ids)
              # assert batch["eos_token_id"] is None or batch["eos_token_id"] == eos_token_id
              # batch["eos_token_id"] = eos_token_id
  
          batches.append(batch)
  
      # manual padding
      for batch in batches:
          for j in range(len(batch["input_ids"])):
              cnt = prompt_len - len(batch["input_ids"][j])
              batch["input_ids"][j] = np.pad(
                  batch["input_ids"][j],
                  (cnt, 0),
                  "constant",
                  constant_values=(0),
              )
  
      return batches, last_policy
  
file_path: H2O/h2o_flexgen/benchmark/xsum/run_xsum_h2o.py
imports: import argparse
  import numpy
  import torch
  import json
  import time
  from tqdm import tqdm
  import copy
  from flexgen.flex_opt import Policy, OptLM, ExecutionEnv, CompressionConfig
  from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
label_desc: Use image node_id 4 for calling a global function with example usage: get_batches(requests, policy) and returns: batches,  last_policy