code: def evaluate_jaccard(
      bin_predictions,
      uncertainty_pairs,
      num_bins,
      targets,
      num_folds=8,
      combine_middle_bins=False,
  ):
      """
          Evaluate uncertainty estimation's ability to predict true error quantiles.
          For each bin, we calculate the jaccard index (JI) between the pred bins and GT error quantiles.
          We calculate the JI for each dictionary in the bin_predictions dict. For each bin, we calculate: a) the mean and
          std over all folds and all targets b) the mean and std for each target over all folds.
  
      Args:
          bin_predictions (Dict): dict of Pandas Dataframes where each dataframe has errors, predicted bins for all uncertainty measures for a model,
          uncertainty_pairs ([list]): list of lists describing the different uncert combinations to test,
          num_bins (int): Number of quantile bins,
          targets (list) list of targets to measure uncertainty estimation,
          num_folds (int): Number of folds,
  
  
      Returns:
          [Dict]: Dicts with JI for all targets combined and targets seperated.
      """
  
      # If combining middle bins, we need the original number of bins to calcualate the true quantiles of the error.
      # Then, we combine all the middle bins of the true quantiles, giving us 3 bins.
      # There are only 3 bins that have been predicted, so set num_bins to 3.
  
      if combine_middle_bins:
          num_bins_for_quantiles = num_bins
          num_bins = 3
      else:
          num_bins_for_quantiles = num_bins
      # initialize empty dicts
      all_jaccard_data = {}
      all_jaccard_bins_targets_sep = {}
  
      all_recall_data = {}
      all_recall_bins_targets_sep = {}
  
      all_precision_data = {}
      all_precision__bins_targets_sep = {}
  
      all_concat_jacc_bins_target_sep_foldwise = [
          {} for x in range(len(targets))
      ]
      all_concat_jacc_bins_target_sep_all = [
          {} for x in range(len(targets))
      ]
      # Loop over models (model) and uncertainty methods (uncert_pair)
      for i, (model, data_structs) in enumerate(
          bin_predictions.items()
      ):
          for (
              uncert_pair
          ) in (
              uncertainty_pairs
          ):  # uncert_pair = [pair name, error name , uncertainty name]
              uncertainty_type = uncert_pair[0]
  
              fold_mean_targets = []
              fold_mean_bins = [[] for x in range(num_bins)]
              fold_all_bins = [[] for x in range(num_bins)]
  
              fold_mean_targets_recall = []
              fold_mean_bins_recall = [[] for x in range(num_bins)]
              fold_all_bins_recall = [[] for x in range(num_bins)]
  
              fold_mean_targets_precision = []
              fold_mean_bins_precision = [[] for x in range(num_bins)]
              fold_all_bins_precision = [[] for x in range(num_bins)]
  
              fold_all_bins_concat_targets_sep_foldwise = [
                  [[] for y in range(num_bins)]
                  for x in range(len(targets))
              ]
              fold_all_bins_concat_targets_sep_all = [
                  [[] for y in range(num_bins)]
                  for x in range(len(targets))
              ]
  
              for fold in range(num_folds):
                  # Get the errors and predicted bins for this fold
                  fold_errors = data_structs[
                      (data_structs["Testing Fold"] == fold)
                  ][["uid", "target_idx", uncertainty_type + " Error"]]
                  fold_bins = data_structs[
                      (data_structs["Testing Fold"] == fold)
                  ][
                      [
                          "uid",
                          "target_idx",
                          uncertainty_type + " Uncertainty bins",
                      ]
                  ]
  
                  return_dict = bin_wise_jaccard(
                      fold_errors,
                      fold_bins,
                      num_bins,
                      num_bins_for_quantiles,
                      targets,
                      uncertainty_type,
                      combine_middle_bins,
                  )
  
                  fold_mean_targets.append(
                      return_dict["mean all targets"]
                  )
                  fold_mean_targets_recall.append(
                      return_dict["mean all targets recall"]
                  )
                  fold_mean_targets_precision.append(
                      return_dict["mean all targets precision"]
                  )
  
                  for idx_bin in range(
                      len(return_dict["mean all bins"])
                  ):
                      fold_mean_bins[idx_bin].append(
                          return_dict["mean all bins"][idx_bin]
                      )
                      fold_all_bins[idx_bin] = (
                          fold_all_bins[idx_bin]
                          + return_dict["all bins"][idx_bin]
                      )
  
                      fold_mean_bins_recall[idx_bin].append(
                          return_dict["mean all bins recall"][idx_bin]
                      )
                      fold_all_bins_recall[idx_bin] = (
                          fold_all_bins_recall[idx_bin]
                          + return_dict["all bins recall"][idx_bin]
                      )
  
                      fold_mean_bins_precision[idx_bin].append(
                          return_dict["mean all bins precision"][
                              idx_bin
                          ]
                      )
                      fold_all_bins_precision[idx_bin] = (
                          fold_all_bins_precision[idx_bin]
                          + return_dict["all bins precision"][idx_bin]
                      )
  
                      # Get the jaccard saved for the individual targets, flattening the folds and also not flattening the folds
                      for target_idx in range(len(targets)):
                          fold_all_bins_concat_targets_sep_foldwise[
                              target_idx
                          ][idx_bin] = (
                              fold_all_bins_concat_targets_sep_foldwise[
                                  target_idx
                              ][idx_bin]
                              + return_dict[
                                  "all bins concatenated targets seperated"
                              ][target_idx][idx_bin]
                          )
                          combined = (
                              fold_all_bins_concat_targets_sep_all[
                                  target_idx
                              ][idx_bin]
                              + return_dict[
                                  "all bins concatenated targets seperated"
                              ][target_idx][idx_bin]
                          )
  
                          fold_all_bins_concat_targets_sep_all[
                              target_idx
                          ][idx_bin] = combined
  
              all_jaccard_data[
                  model + " " + uncertainty_type
              ] = fold_mean_bins
              all_jaccard_bins_targets_sep[
                  model + " " + uncertainty_type
              ] = fold_all_bins
  
              all_recall_data[
                  model + " " + uncertainty_type
              ] = fold_mean_bins_recall
              all_recall_bins_targets_sep[
                  model + " " + uncertainty_type
              ] = fold_all_bins_recall
  
              all_precision_data[
                  model + " " + uncertainty_type
              ] = fold_mean_bins_precision
              all_precision__bins_targets_sep[
                  model + " " + uncertainty_type
              ] = fold_all_bins_precision
  
              for target_idx in range(
                  len(all_concat_jacc_bins_target_sep_foldwise)
              ):
                  all_concat_jacc_bins_target_sep_foldwise[target_idx][
                      model + " " + uncertainty_type
                  ] = fold_all_bins_concat_targets_sep_foldwise[
                      target_idx
                  ]
                  all_concat_jacc_bins_target_sep_all[target_idx][
                      model + " " + uncertainty_type
                  ] = fold_all_bins_concat_targets_sep_all[target_idx]
  
      return {
          "Jaccard All": all_jaccard_data,
          "Jaccard targets seperated": all_jaccard_bins_targets_sep,
          "Recall All": all_recall_data,
          "Recall targets seperated": all_recall_bins_targets_sep,
          "Precision All": all_precision_data,
          "Precision targets seperated": all_precision__bins_targets_sep,
          "all jacc concat bins targets sep foldwise": all_concat_jacc_bins_target_sep_foldwise,
          "all jacc concat bins targets sep all": all_concat_jacc_bins_target_sep_all,
      }
  
file_path: pykale/kale/evaluate/uncertainty_metrics.py
imports: from typing import Dict, List
  import numpy
  import pandas
  from kale.evaluate.similarity_metrics import jaccard_similarity
  from kale.prepdata.string_transform import strip_for_bound
label_desc: Use image node_id 4 for calling a global function with example usage: evaluate_jaccard(bin_predictions, uncertainty_pairs, num_bins, targets, num_folds, combine_middle_bins) and returns: dict