code: def passk_rel_drop(task2bvs_old, task2bvs_new):
      # old_rate:
      # dim0: problems
      # dim1: each experiment (model@temperature)
      # dim2: pass/fail booleans for each sample
      # this fn computes the relative drop in pass@k averaged over experiments
  
      passk_old = {}
      passk_new = {}
      # sample size => k => List[pass@k]
  
      for exp_i in range(len(task2bvs_old[0])):
          ntotal = []
          npass_old = []
          npass_new = []
          nsamples = None
          for task_i in range(len(task2bvs_old)):
              bv_old = task2bvs_old[task_i][exp_i]
              bv_new = task2bvs_new[task_i][exp_i]
              ntotal.append(len(bv_old))
              npass_old.append(bv_old.sum())
              npass_new.append(bv_new.sum())
              if nsamples is None:
                  nsamples = len(bv_old)
              assert len(bv_old) == len(bv_new) == nsamples
  
          d_old = passk_old.setdefault(nsamples, {})
          d_new = passk_new.setdefault(nsamples, {})
          for k in [1, 10, 100]:
              if nsamples >= k:
                  pass_at_k_old = (
                      estimate_pass_at_k(ntotal, npass_old, k).mean()
                      * 100
                  )
                  pass_at_k_new = (
                      estimate_pass_at_k(ntotal, npass_new, k).mean()
                      * 100
                  )
                  d_old.setdefault(k, []).append(pass_at_k_old)
                  d_new.setdefault(k, []).append(pass_at_k_new)
  
      for nsamples in passk_old:
          print("=====================================")
          print(f"{nsamples = }")
          do = passk_old[nsamples]
          dn = passk_new[nsamples]
          drops = []
          for k in [1, 10, 100]:
              if k in do:
                  pko = np.array(do[k])
                  pkn = np.array(dn[k])
                  drop = 100 * (pko - pkn) / pko
                  drops.append(drop)
                  print(
                      f"pass@{k}: \t{pko.mean():.1f}% -> {pkn.mean():.1f}% (drop {drop.mean():.1f}%)"
                  )
          drops = np.array(drops)
          print(f"+++ {drops.mean() = :.1f}%")
          print(f"+++ {drops.max() = :.1f}%")
          print("=====================================")
  
file_path: evalplus/tools/viz_passrate.py
imports: import json
  import os
  import pickle
  from os import PathLike
  from typing import List
  import matplotlib.pyplot
  import numpy
  from tqdm import tqdm
  from evalplus.data import get_human_eval_plus
  from evalplus.eval import estimate_pass_at_k
label_desc: Use image node_id 1 for calling a global function with example usage: passk_rel_drop(task2bvs_old, task2bvs_new) without return types