code: def evaluate(flags):
      if flags.parallel is None:
          n_workers = max(1, multiprocessing.cpu_count() // 2)
      else:
          n_workers = flags.parallel
  
      if os.path.isdir(flags.samples):
          result_path = os.path.join(flags.samples, "eval_results.json")
      else:
          assert flags.samples.endswith(".jsonl")
          result_path = flags.samples.replace(
              ".jsonl", "_eval_results.json"
          )
  
      if os.path.isfile(result_path) and not flags.i_just_wanna_run:
          print(f"Load from previous results from {result_path}")
          with open(result_path, "r") as f:
              results = json.load(f)
  
          results = compatible_eval_result(results)
      else:
          if flags.dataset == "humaneval":
              problems = get_human_eval_plus(mini=flags.mini)
              dataset_hash = get_human_eval_plus_hash()
              expected_output = get_groundtruth(
                  problems, dataset_hash, []
              )
          elif flags.dataset == "mbpp":
              problems = get_mbpp_plus(mini=flags.mini)
              dataset_hash = get_mbpp_plus_hash()
              expected_output = get_groundtruth(
                  problems,
                  dataset_hash,
                  MBPP_OUTPUT_NOT_NONE_TASKS,
              )
  
          results = {
              "date": datetime.now().strftime("%Y-%m-%d %H:%M"),
              "hash": dataset_hash,
              "eval": {},
          }
  
          with ProcessPoolExecutor(max_workers=n_workers) as executor:
              futures = []
              completion_id = Counter()
              n_samples = 0
              eval_results = defaultdict(list)
              remainings = set()
  
              print("Reading samples...")
              for sample in tqdm(load_solutions(flags.samples)):
                  task_id = sample["task_id"]
                  solution = (
                      sample["solution"]
                      if "solution" in sample
                      else problems[task_id]["prompt"]
                      + sample["completion"]
                  )
                  remainings.add(sample["_identifier"])
                  args = (
                      flags.dataset,
                      completion_id[task_id],
                      problems[task_id],
                      solution,
                      expected_output[task_id],
                      flags.base_only,
                      not flags.test_details,  # fast_check
                      sample["_identifier"],
                      flags.min_time_limit,
                      flags.gt_time_limit_factor,
                  )
                  futures.append(
                      executor.submit(check_correctness, *args)
                  )
                  completion_id[task_id] += 1
                  n_samples += 1
  
              assert n_samples == len(
                  remainings
              ), "Missing problems in unfinished"
              assert len(completion_id) == len(
                  problems
              ), "Missing problems in samples"
  
              def stucking_checker():
                  while remainings:
                      last_size = len(remainings)
                      time.sleep(20)
                      if (
                          last_size != len(remainings)
                          or len(remainings) == 0
                      ):
                          continue
                      # Potential stucking
                      warn(
                          "No samples had finished testing in the last 20s"
                      )
                      warn(
                          f"{len(remainings)} samples to be tested: {remainings}"
                      )
  
              threading.Thread(target=stucking_checker).start()
  
              for future in tqdm(
                  as_completed(futures), total=n_samples
              ):
                  result = future.result()
                  remainings.remove(result["_identifier"])
                  eval_results[result["task_id"]].append(result)
  
          # sort the results for each problem by completion_id
          for task_id, task_results in eval_results.items():
              task_results.sort(key=lambda x: x["completion_id"])
              results["eval"][task_id] = {
                  "nfiles": len(task_results),
                  "base": [x["base"] for x in task_results],
                  "plus": [x["plus"] for x in task_results]
                  if not flags.base_only
                  else [],
              }
  
      if os.path.isfile(result_path) and flags.i_just_wanna_run:
          decision = ""
          while decision.lower() not in ["y", "n"]:
              print(
                  f"{result_path} already exists. Press [Y/N] to overwrite or exit..."
              )
              decision = input()
  
          if decision.lower() == "y":
              # mv the file to a backup
              new_path = result_path + ".bak"
              while os.path.isfile(new_path):
                  new_path += ".bak"
              os.rename(result_path, new_path)
              print(f"Backup {result_path} to {new_path}")
  
      if not os.path.isfile(result_path):
          with open(result_path, "w") as f:
              json.dump(results, f)
  
      # Calculate pass@k.
      total = np.array([r["nfiles"] for r in results["eval"].values()])
      base_correct = []
      new_correct = []
  
      for res in results["eval"].values():
          bc = sum([r[0] == SUCCESS for r in res["base"]])
          base_correct.append(bc)
          if res["plus"]:
              new_correct.append(
                  sum(
                      [
                          res["plus"][i][0]
                          == res["base"][i][0]
                          == SUCCESS
                          for i in range(len(res["plus"]))
                      ]
                  )
              )
      base_correct = np.array(base_correct)
  
      pass_at_k = {
          f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean()
          for k in [1, 10, 100]
          if total.min() >= k
      }
      cprint(f"{flags.dataset} (base tests)", "red")
      for k, v in pass_at_k.items():
          cprint(f"{k}:\t{v:.3f}", "red")
  
      if new_correct:
          cprint(f"{flags.dataset}+ (base + extra tests)", "green")
          pass_at_k = {
              f"pass@{k}": estimate_pass_at_k(
                  total, np.array(new_correct), k
              ).mean()
              for k in [1, 10, 100]
              if (total >= k).all()
          }
          for k, v in pass_at_k.items():
              cprint(f"{k}:\t{v:.3f}", "green")
  
imports: import argparse
  import json
  import multiprocessing
  import os
  import pickle
  import threading
  import time
  from collections import Counter, defaultdict
  from concurrent.futures import ProcessPoolExecutor, as_completed
  from datetime import datetime
  from typing import Any, Dict, List, Optional, Tuple, Union
  from warnings import warn
  import numpy
  from termcolor import cprint
  from tqdm import tqdm
  from evalplus.data import get_human_eval_plus, get_human_eval_plus_hash, get_mbpp_plus, get_mbpp_plus_hash, load_solutions
  from evalplus.data.utils import CACHE_DIR
  from evalplus.eval import SUCCESS, compatible_eval_result, estimate_pass_at_k, untrusted_check
  from evalplus.eval._special_oracle import MBPP_OUTPUT_NOT_NONE_TASKS
  from evalplus.gen.util import trusted_exec
label_desc: Use image node_id 3 for calling a global function with example usage: evaluate(flags) without return types