code: def generate_scores_and_correls_across_datasets(
      correlation_type: str,
      out_dir: str,
      score_set: str,
      reference_option: str,
      adjust_ranges: bool,
  ):
      """
      Finds correlations between ROSCOE and baseline scores with annotations, where the
      data is taken from all datasets.
  
      Correlations are still split by sentence embedding model.
      """
      all_datasets = list(DATASET_TO_CONFIG.keys())
  
      all_configs_per_model = defaultdict(list)
      for dataset in all_datasets:
          config = DATASET_TO_CONFIG[dataset]
          for alignment_scores_filename in config["alignment_scores"][
              score_set
          ]:
              model_name = alignment_scores_filename.split("/")[-2]
              all_configs_per_model[model_name].append(
                  {
                      "labels": config["human_labels"]["labels"],
                      "baseline_scores": config["baseline_scores"],
                      "alignment_scores": alignment_scores_filename,
                      "skip_indices": config["human_labels"][
                          "skip_indices"
                      ],
                  }
              )
  
      correl_tables = {}
      pval_tables = {}
      sample_size_tables = {}
      for model, model_configs in all_configs_per_model.items():
          all_values_by_title = defaultdict(list)
          all_agg_stepwise_score_names = set()
          all_agg_stepwise_label_names = set()
          for config in model_configs:
              values_by_title = read_and_process_data(
                  human_annotation_filename=config["labels"],
                  baseline_scores_filename=config["baseline_scores"],
                  roscoe_scores_filename=config["alignment_scores"],
                  removed_ids=config["skip_indices"],
                  reference_option=reference_option,
                  adjust_ranges=adjust_ranges,
              )
              if len(all_values_by_title.keys()) == 0:
                  for t in values_by_title.keys():
                      all_values_by_title[t] = values_by_title[t]
              else:
                  if len(all_values_by_title.keys()) != len(
                      values_by_title.keys()
                  ):
                      # remove labels that aren't in all datasets
                      diff = set(all_values_by_title.keys()) ^ set(
                          values_by_title.keys()
                      )
                      for l in diff:
                          if l in all_values_by_title:
                              del all_values_by_title[l]
                  for t in all_values_by_title.keys():
                      all_values_by_title[t].extend(values_by_title[t])
  
          (
              somersd_table,
              pval_table,
              sizes_table,
          ) = make_correlations_table(
              values_by_title=all_values_by_title,
              agg_stepwise_label_names=all_agg_stepwise_label_names,
              agg_stepwise_score_names=all_agg_stepwise_score_names,
              correlation_type=correlation_type,
          )
  
          # format for display
          somersd_table[0] = rename_columns_for_display(
              somersd_table[0]
          )
          pval_table[0] = rename_columns_for_display(pval_table[0])
          sizes_table[0] = rename_columns_for_display(sizes_table[0])
  
          # write this model's correlation table to a file
          base_file_name = f"{model}_across_{'_'.join(all_datasets)}"
          write_correls_and_sizes_tables(
              out_dir=out_dir,
              base_file_name=base_file_name,
              correl_tables={model: somersd_table},
          )
  
          correl_tables[model] = somersd_table
          pval_tables[model] = pval_table
          sample_size_tables[model] = sizes_table
  
      correl_tables_df = {
          m: labeled_table_rows_to_dataframe(t)
          for m, t in correl_tables.items()
      }
      pval_tables_df = {
          m: labeled_table_rows_to_dataframe(t)
          for m, t in pval_tables.items()
      }
      granular_sections, _ = granular_summary(
          correls_by_model=correl_tables_df,
          pvals_by_model=pval_tables_df,
      )
      write_granular_summary(
          dataset_name="_".join(all_datasets),
          granular_sections=granular_sections,
          out_dir=out_dir,
      )
  
      # Write a massive correlation table file with all the model tables from above concatenated together.
      base_file_name = f"all_scores_across_{'_'.join(all_datasets)}"
      write_correls_and_sizes_tables(
          out_dir=out_dir,
          base_file_name=base_file_name,
          correl_tables=correl_tables,
      )
  
imports: import argparse
  from collections import defaultdict
  import csv
  import math
  import os
  import pathlib
  from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
  import pandas
  from projects.roscoe.meta_evaluation.correlations import Analyzer
  from projects.roscoe.score import ROSCOE_SA, ROSCOE_SS, ROSCOE_LI, ROSCOE_LC, SUPERVISED_SCORES, UNSUPERVISED_SCORES
  from projects.roscoe.utils import assert_all_elements_same_length, ordered_union, split_list
  from projects.roscoe.meta_evaluation.table_file_writing import ANNOTATION_COLUMN_ORDER, COLUMN_RENAMES, MODELS, ROSCOE_SECTIONS, STEP_COLUMN_RENAMES, section_rows, write_every_dataset_summary_table_tex, write_every_dataset_final_summary_table_tex, write_granular_summary_tex
label_desc: Use image node_id 36 for calling a global function with example usage: generate_scores_and_correls_across_datasets(correlation_type, out_dir, score_set, reference_option, adjust_ranges) without return types