code: def calculate_consistency(files: Sequence[Path]):
      """
      Calculate the consistency over all runs. We take all pairs of runs and
      for each pair, calculate the proportion of questions that had the same
      answer. We average this over all pairs
      """
      all_consistencies = []
      for idx, i in enumerate(files):
          i_log = load_jsonl_log(i)
          for j in files[idx + 1 :]:
              j_log = load_jsonl_log(j)
              assert len(i_log) == len(j_log)
  
              # Get all answers for each question. Inefficient
              joined = []
              for q1, c1 in i_log:
                  for q2, c2 in j_log:
                      if q1 == q2:
                          joined.append((q1, c1, c2))
  
              assert len(joined) == len(
                  i_log
              ), f"Len joined: {len(joined)}, Len j_log: {len(j_log)}"
              consistency = sum(
                  [c1 == c2 for _, c1, c2 in joined]
              ) / len(joined)
              all_consistencies.append(consistency)
  
      consistency = sum(all_consistencies) / len(all_consistencies)
      print(f"Consistency: {consistency}")
  
file_path: evals/evals/elsuite/sandbagging/scripts/consistency_plots.py
imports: import argparse
  import glob
  import json
  import os
  from pathlib import Path
  from typing import Sequence
  import matplotlib.pyplot
label_desc: Use image node_id 5 for calling a global function with example usage: calculate_consistency(files) without return types