code: def streaming_inference_heavy_hitter(
      model, tokenizer, prompts, kv_cache=None, max_gen_len=1000
  ):
      past_key_values = None
      for idx, prompt in enumerate(prompts):
          prompt = "USER: " + prompt + "\n\nASSISTANT: "
          print("\n" + prompt, end="")
          input_ids = tokenizer(prompt, return_tensors="pt").input_ids
          input_ids = input_ids.to(model.device)
          seq_len = input_ids.shape[1]
          if kv_cache is not None:
              space_needed = seq_len + max_gen_len
  
              for name, m in model.named_modules():
                  if isinstance(m, H2OLlamaAttention):
                      layer_idx = int(name.split(".")[2])
                      past_key_values[
                          layer_idx
                      ] = m.kv_cache.evict_for_space(
                          past_key_values[layer_idx], space_needed
                      )
  
          past_key_values = greedy_generate(
              model,
              tokenizer,
              input_ids,
              past_key_values,
              max_gen_len=max_gen_len,
          )
  
file_path: H2O/h2o_hf/run_streaming.py
imports: import warnings
  import torch
  import argparse
  import json
  import os
  import time
  import re
  import sys
  from tqdm import tqdm
  from streaming_llm.utils import load, download_url, load_jsonl
  from transformers.models.llama.modeling_llama import LlamaAttention
  from utils_real_drop.modify_llama import H2OLlamaAttention_streaming, H2OLlamaForCausalLM_streaming
label_desc: Use image node_id 3 for calling a global function with example usage: streaming_inference_heavy_hitter(model, tokenizer, prompts, kv_cache, max_gen_len) without return types