code: def get_correlations(
      dataset_name: str,
      data_path: str,
      scores_path: str,
      out_path: str,
      sentinel: bool,
  ):
      score_file_prefix = f"scores_{dataset_name}_synthetic_"
      stats_output = os.path.join(
          out_path, f"{dataset_name}_statistics/"
      )
      final_output_file = os.path.join(
          out_path, f"final/{dataset_name}.csv"
      )
      summary_output_file = os.path.join(
          out_path, f"final/{dataset_name}_summary.csv"
      )
      base_df_output = os.path.join(stats_output, "base_df")
      correlations = Analyzer(reference_column=PERTURBATION_COLUMN)
  
      os.makedirs(stats_output, exist_ok=True)
      os.makedirs(os.path.dirname(final_output_file), exist_ok=True)
      os.makedirs(base_df_output, exist_ok=True)
  
      ### Grab all of scores and dump them into one big pandas file
      for file_name in glob(
          scores_path + "/**/" + score_file_prefix + "*"
      ):
          tune_type = file_name.split("/")[-2]
  
          perturbation_name = file_name.split("/")[-1][
              len(score_file_prefix) : -len(".tsv")
          ]
  
          names = perturbation_name.split("_")
  
          print(f"Reading scores: {tune_type} {perturbation_name}")
  
          if is_single_perturbation(perturbation_name) and not sentinel:
              percent = int(names[0][:-1])
              fold = names[-1]
              data_name = f"{percent}%_{names[1]}_{fold}"
          elif is_sentinel(perturbation_name) and sentinel:
              percent = int(names[0][:-1])
              fold = names[-1]
              data_name = f"{percent}%_{'_'.join(names[1:-1])}_{fold}"
          else:
              print(f"Skipping {file_name}")
              continue
  
          df = fetch_to_df(file_name)
  
          valid_data_file = os.path.join(
              data_path,
              dataset_name + "_synthetic",
              data_name + ".jsonl",
          )
  
          if not os.path.isfile(valid_data_file):
              print(f"{data_name}.jsonl not found. Skipping.")
              continue
  
          df = fetch_parturbation_mark(
              scores=df, path_to_data=valid_data_file
          )
  
          df.to_csv(
              os.path.join(base_df_output, perturbation_name + ".csv")
          )
  
          stats = correlations.get_statistics(df)
  
          for corr in [correlations.statistic, correlations.pval]:
              s = stats[corr]
  
              # needs to be synced with glob for merging to final below
              output_path = os.path.join(
                  stats_output,
                  dataset_name,
                  tune_type,
                  corr,
                  perturbation_name + ".csv",
              )
              os.makedirs(os.path.dirname(output_path), exist_ok=True)
              s[[PERTURBATION_COLUMN]].to_csv(output_path)
  
      # Get the stats into one table
      results_list = []
  
      # Needs to be kept in sync w/ output_path above
      print("Getting summary")
      for file_name in glob(
          os.path.join(stats_output, dataset_name, "*/*/*.csv")
      ):
          df = pandas.read_csv(file_name)
          df = df.transpose()
          df.columns = df.iloc[0]
          df.drop(df.index[0], inplace=True)
          df.index = [file_name.split("/")[-1][: -len(".csv")]]
          df["fold"] = pathlib.Path(file_name).stem.split("_")[-1]
          df["corr_type"] = file_name.split("/")[-2]
          df["tune_type"] = file_name.split("/")[-3]
          results_list.append(df)
      res = pandas.concat(results_list)
      rearrange_cols_and_save_to_csv(
          final_dataframe=res, out_f=final_output_file
      )
  
      subdirs = unique_subdirectories(
          scores_path,
          filter_fn=lambda d: d
          not in EMBEDDING_MODEL_NAMES + SCORE_GROUPS,
      )
      get_summary(
          model_names=subdirs,
          final_out=final_output_file,
          summary_out=summary_output_file,
          stat_name=correlations.statistic,
          pval_name=correlations.pval,
      )
  
imports: import csv
  from glob import glob
  import json
  import os
  import pathlib
  import numpy
  import pandas
  from parlai.core.params import ParlaiParser
  from projects.roscoe.meta_evaluation.correlations import Analyzer
  from projects.roscoe.score import EMB_MODEL_SCORES, EMBEDDING_MODEL_NAMES, GRAMMAR_MODEL_SCORES, LANGUAGE_MODEL_SCORES, NLI_MODEL_SCORES, REASONING_SCORES, SCORE_GROUPS, SUPERVISED_SCORES, UNSUPERVISED_SCORES
  from typing import Callable, List
label_desc: Use image node_id 12 for calling a global function with example usage: get_correlations(dataset_name, data_path, scores_path, out_path, sentinel) without return types