code: def get_per_attribute_predictions_freq(
      output_metadata,
      label_to_id_map,
      label_association_map,
      disparate_labels_list=None,
      disparate_threshold=0.0,
      class_to_label_name_map=None,
      topk=1,
  ):
      disparate_labels_list = disparate_labels_list or []
      to_predict_attributes = list(
          list(output_metadata.values())[0].keys()
      )
      to_predict_attributes.remove("prediction")
      to_predict_attributes.remove("path")
      to_predict_attributes.remove("orig_path")
      if "confidence_scores" in to_predict_attributes:
          to_predict_attributes.remove("confidence_scores")
      all_classes_names = list(label_to_id_map.keys())
      preds_values = [
          item["prediction"] for item in list(output_metadata.values())
      ]
      media_ids = [
          item["path"] for item in list(output_metadata.values())
      ]
      to_predict_associations = list(label_association_map.keys())
      confidence_score_values = []
      if "confidence_scores" in list(
          list(output_metadata.values())[0].keys()
      ):
          confidence_score_values = [
              item["confidence_scores"]
              for item in list(output_metadata.values())
          ]
  
      unique_associated_labels = []
      for _, value in label_association_map.items():
          unique_associated_labels.extend(value)
      unique_associated_labels = list(set(unique_associated_labels))
      print(f"Unique associated labels: {unique_associated_labels}")
  
      output_attribute_disparate_label_map = {}
      if (
          len(confidence_score_values) > 0
          and len(disparate_labels_list) > 0
      ):
          for attribute in to_predict_attributes:
              attribute_values = [
                  item[attribute]
                  for item in list(output_metadata.values())
              ]
              unique_attribute_values = list(set(attribute_values))
              # print(f"{attribute}: {unique_attribute_values}")
              num_images = len(preds_values)
              attribute_disparate_label_map = {}
              for idx in range(num_images):
                  attribute_value = attribute_values[
                      idx
                  ]  # like male, female etc
                  img_ids = media_ids[idx]
                  img_predictions = preds_values[idx][:topk]
                  img_confidence_scores = []
                  img_confidence_scores = confidence_score_values[idx][
                      :topk
                  ]
                  if class_to_label_name_map:
                      img_predictions = [
                          class_to_label_name_map[item]
                          for item in img_predictions
                      ]
                  if (
                      attribute_value
                      not in attribute_disparate_label_map
                  ):
                      attribute_disparate_label_map[
                          attribute_value
                      ] = {}
                      attribute_disparate_label_map[attribute_value][
                          "total"
                      ] = 1
                      attribute_disparate_label_map[attribute_value][
                          "above_disparate_threshold_images"
                      ] = 0
                      for disp_label in disparate_labels_list:
                          attribute_disparate_label_map[
                              attribute_value
                          ][disp_label] = 0
                  else:
                      attribute_disparate_label_map[attribute_value][
                          "total"
                      ] = (
                          attribute_disparate_label_map[
                              attribute_value
                          ]["total"]
                          + 1
                      )
                  # if the top-1 label score is above threshold, count it
                  if len(img_confidence_scores) > 0:
                      if (
                          float(img_confidence_scores[0])
                          >= disparate_threshold
                      ):
                          attribute_disparate_label_map[
                              attribute_value
                          ]["above_disparate_threshold_images"] = (
                              attribute_disparate_label_map[
                                  attribute_value
                              ]["above_disparate_threshold_images"]
                              + 1
                          )
                  found_intersection_disparate_preds = list(
                      set(disparate_labels_list).intersection(
                          set(img_predictions)
                      )
                  )
                  if len(found_intersection_disparate_preds) > 0:
                      for pred in found_intersection_disparate_preds:
                          score = float(
                              img_confidence_scores[
                                  img_predictions.index(pred)
                              ]
                          )
                          if score >= disparate_threshold:
                              attribute_disparate_label_map[
                                  attribute_value
                              ][pred] = (
                                  attribute_disparate_label_map[
                                      attribute_value
                                  ][pred]
                                  + 1
                              )
              for attr_val in unique_attribute_values:
                  attribute_disparate_label_map[attr_val][
                      "above_disparate_threshold_images"
                  ] = round(
                      attribute_disparate_label_map[attr_val][
                          "above_disparate_threshold_images"
                      ]
                      / attribute_disparate_label_map[attr_val][
                          "total"
                      ],
                      5,
                  )
                  for lbl_name in disparate_labels_list:
                      attribute_disparate_label_map[attr_val][
                          lbl_name
                      ] = round(
                          attribute_disparate_label_map[attr_val][
                              lbl_name
                          ]
                          / attribute_disparate_label_map[attr_val][
                              "total"
                          ],
                          5,
                      )
              output_attribute_disparate_label_map[
                  attribute
              ] = attribute_disparate_label_map
  
      # given the label association map, we want to predict:
      # per attribute: male, female, different race, different, different age etc
      # the attributes are defined in the output_metadata.
      # basically loop over the data, maintain a map:
      # attribute_type -> attribute_value : {total: val, association_name: count}
      print("Building the label association map...")
      (
          output_attributes_label_assoc_map,
          output_attributes_label_assoc_conf_scores_map,
      ) = (
          {},
          {},
      )
      for attribute in to_predict_attributes:
          attribute_values = [
              item[attribute] for item in list(output_metadata.values())
          ]
          unique_attribute_values = list(set(attribute_values))
          num_images = len(preds_values)
          (
              attribute_label_assoc_map,
              attributes_label_assoc_conf_scores_map,
          ) = ({}, {})
          # loop through all images and their predictions now
          for idx in range(num_images):
              attribute_value = attribute_values[
                  idx
              ]  # like male, female etc
              img_ids = media_ids[idx]
              img_predictions = preds_values[idx][:topk]
              img_confidence_scores = []
              if len(confidence_score_values) > 0:
                  img_confidence_scores = confidence_score_values[idx][
                      :topk
                  ]
              if class_to_label_name_map:
                  img_predictions = [
                      class_to_label_name_map[item]
                      for item in img_predictions
                  ]
              if attribute_value not in attribute_label_assoc_map:
                  attribute_label_assoc_map[attribute_value] = {}
                  attribute_label_assoc_map[attribute_value][
                      "total"
                  ] = 1
                  if len(confidence_score_values) > 0:
                      attributes_label_assoc_conf_scores_map[
                          attribute_value
                      ] = {}
                      for lbl_name in unique_associated_labels:
                          attributes_label_assoc_conf_scores_map[
                              attribute_value
                          ][lbl_name] = []
                  if "sexism" in label_association_map:
                      attribute_label_assoc_map[attribute_value][
                          "sexism_freq"
                      ] = {}
                      for sexist_label in label_association_map[
                          "sexism"
                      ]:
                          attribute_label_assoc_map[attribute_value][
                              "sexism_freq"
                          ][sexist_label] = 0
                  for assoc in to_predict_associations:
                      attribute_label_assoc_map[attribute_value][
                          assoc
                      ] = 0
              else:
                  attribute_label_assoc_map[attribute_value][
                      "total"
                  ] = (
                      attribute_label_assoc_map[attribute_value][
                          "total"
                      ]
                      + 1
                  )
              for assoc_name in to_predict_associations:
                  assoc_labels = label_association_map[assoc_name]
                  found_intersection_preds = list(
                      set(assoc_labels).intersection(
                          set(img_predictions)
                      )
                  )
                  if len(found_intersection_preds) > 0:
                      attribute_label_assoc_map[attribute_value][
                          assoc_name
                      ] = (
                          attribute_label_assoc_map[attribute_value][
                              assoc_name
                          ]
                          + 1
                      )
                      if assoc_name == "sexism":
                          for pred in found_intersection_preds:
                              attribute_label_assoc_map[
                                  attribute_value
                              ]["sexism_freq"][pred] = (
                                  attribute_label_assoc_map[
                                      attribute_value
                                  ]["sexism_freq"][pred]
                                  + 1
                              )
                      if len(confidence_score_values) > 0:
                          for pred in found_intersection_preds:
                              attributes_label_assoc_conf_scores_map[
                                  attribute_value
                              ][pred].append(
                                  float(
                                      img_confidence_scores[
                                          img_predictions.index(pred)
                                      ]
                                  )
                              )
          # compute the percentages now
          for attr_val in unique_attribute_values:
              for assoc_name in to_predict_associations:
                  attribute_label_assoc_map[attr_val][
                      assoc_name
                  ] = round(
                      100.0
                      * attribute_label_assoc_map[attr_val][assoc_name]
                      / attribute_label_assoc_map[attr_val]["total"],
                      3,
                  )
              if "sexism" in label_association_map:
                  for sexist_label in label_association_map["sexism"]:
                      attribute_label_assoc_map[attr_val][
                          "sexism_freq"
                      ][sexist_label] = round(
                          100.0
                          * attribute_label_assoc_map[attr_val][
                              "sexism_freq"
                          ][sexist_label]
                          / attribute_label_assoc_map[attr_val][
                              "total"
                          ],
                          3,
                      )
              if len(confidence_score_values) > 0:
                  for lbl_name in unique_associated_labels:
                      if (
                          len(
                              attributes_label_assoc_conf_scores_map[
                                  attr_val
                              ][lbl_name]
                          )
                          > 0
                      ):
                          attributes_label_assoc_conf_scores_map[
                              attr_val
                          ][lbl_name] = round(
                              np.mean(
                                  np.array(
                                      attributes_label_assoc_conf_scores_map[
                                          attr_val
                                      ][
                                          lbl_name
                                      ]
                                  )
                              ),
                              5,
                          )
          output_attributes_label_assoc_map[
              attribute
          ] = attribute_label_assoc_map
          output_attributes_label_assoc_conf_scores_map[
              attribute
          ] = attributes_label_assoc_conf_scores_map
  
      # now we calculate the label prediction rate and then the "absolute" difference
      # of label prediction label for one attribute value to the average label prediction rate
      # in remaining attribute values.
      (
          output_attributes_pred_freq_map,
          output_attributes_count_map,
          output_attributes_img_map,
          output_attributes_confidence_score_map,
      ) = ({}, {}, {}, {})
      for attribute in to_predict_attributes:
          attribute_values = [
              item[attribute] for item in list(output_metadata.values())
          ]
          unique_attribute_values = list(set(attribute_values))
          (
              attribute_preds,
              attribute_count,
              attributes_img_map,
              attribute_confidence_map,
          ) = (
              {},
              {},
              {},
              {},
          )
          num_images = len(preds_values)
          # loop through all images and their predictions now
          for idx in range(num_images):
              attribute_value = attribute_values[
                  idx
              ]  # like male, female etc
              img_predictions = preds_values[idx][:topk]
              img_ids = media_ids[idx]
              img_confidence_scores = []
              if len(confidence_score_values) > 0:
                  img_confidence_scores = confidence_score_values[idx][
                      :topk
                  ]
              if attribute_value not in attribute_preds:
                  attribute_preds[attribute_value] = {}
                  attributes_img_map[attribute_value] = {}
                  # attribute_confidence_map[attribute_value] = {}
                  for cls_name in all_classes_names:
                      attribute_preds[attribute_value][cls_name] = 0
                      attributes_img_map[attribute_value][cls_name] = []
              if attribute_value not in attribute_count:
                  attribute_count[attribute_value] = 1
              else:
                  attribute_count[attribute_value] = (
                      attribute_count[attribute_value] + 1
                  )
  
              if len(img_confidence_scores) > 0:
                  if attribute_value not in attribute_confidence_map:
                      attribute_confidence_map[attribute_value] = [
                          float(img_confidence_scores[0])
                      ]
                  else:
                      attribute_confidence_map[attribute_value].append(
                          float(img_confidence_scores[0])
                      )
  
              for lbl_pred in img_predictions:
                  attribute_preds[attribute_value][lbl_pred] = (
                      attribute_preds[attribute_value][lbl_pred] + 1
                  )
                  attributes_img_map[attribute_value][lbl_pred].append(
                      img_ids
                  )
          output_attributes_pred_freq_map[attribute] = attribute_preds
          output_attributes_count_map[attribute] = attribute_count
          output_attributes_img_map[attribute] = attributes_img_map
          output_attributes_confidence_score_map[
              attribute
          ] = attribute_confidence_map
  
      # now, if we have the confidence scores given, we have captured the best prediction score
      # for each attribute and attribute value. We want to calculate the average best score
      output_mean_attributes_confidence_score_map = {}
      if len(confidence_score_values) > 0:
          for (
              _attribute_type,
              attribute_map,
          ) in output_attributes_confidence_score_map.items():
              # output_mean_attributes_confidence_score_map[attribute_type] = {}
              for (
                  attribute_key_name,
                  scores_list,
              ) in attribute_map.items():
                  # print(scores_list)
                  mean_score = round(np.mean(np.array(scores_list)), 4)
                  # output_mean_attributes_confidence_score_map[attribute_type][attribute_key_name] = mean_score
                  if (
                      "n/a" not in attribute_key_name
                      and "unknown" not in attribute_key_name
                  ):
                      output_mean_attributes_confidence_score_map[
                          attribute_key_name
                      ] = mean_score
  
      # now we sort the dictionaries based on the frequency
      sorted_output_attributes_pred_freq_map = {}
      for (
          attribute_type,
          attribute_map,
      ) in output_attributes_pred_freq_map.items():
          sorted_output_attributes_pred_freq_map[attribute_type] = {}
          for attribute_key_name, pred_map in attribute_map.items():
              sorted_pred_map = dict(
                  sorted(
                      pred_map.items(),
                      key=lambda item: item[1],
                      reverse=True,
                  )
              )
              sorted_output_attributes_pred_freq_map[attribute_type][
                  attribute_key_name
              ] = sorted_pred_map
  
      # we also calculate the label prediction rate for each respective attribute.
      # As an example, for attribute = gender, the process is:
      # for female gender, we calculate the prediction rate of the predicted hashtags
      output_attributes_pred_rate_map = {}
      for (
          attribute_type,
          attribute_map,
      ) in output_attributes_pred_freq_map.items():
          output_attributes_pred_rate_map[attribute_type] = {}
          for attribute_key_name, pred_map in attribute_map.items():
              total_predictions = np.sum(
                  np.array(list(pred_map.values()))
              )
              pred_rate_map = {}
              for key, val in pred_map.items():
                  pred_rate_map[key] = round(
                      100.0 * (val / total_predictions), 3
                  )
              output_attributes_pred_rate_map[attribute_type][
                  attribute_key_name
              ] = pred_rate_map
  
      # now we have calculated the prediction rate map for the
      # labels in the respective attribute value. We want to now
      # do the comparisons of prediction rate across the
      # different possible attribute values (like across all genders)
      output_attributes_avg_pred_rate_map = {}
      all_classes_names = list(label_to_id_map.keys())
      for (
          attribute_type,
          attribute_map,
      ) in output_attributes_pred_rate_map.items():
          output_attributes_avg_pred_rate_map[attribute_type] = {}
          distinct_attribute_names = list(attribute_map.keys())
          for cls_name in all_classes_names:
              avg_cls_pred_rate = np.mean(
                  np.array(
                      [
                          attribute_map[attr_name][cls_name]
                          for attr_name in distinct_attribute_names
                      ]
                  )
              )
              output_attributes_avg_pred_rate_map[attribute_type][
                  cls_name
              ] = avg_cls_pred_rate
  
      # now we have the average prediction rate across all,
      # we want to compute the difference
      output_attributes_pred_rate_difference_map = {}
      for (
          attribute_type,
          attribute_map,
      ) in output_attributes_pred_rate_map.items():
          output_attributes_pred_rate_difference_map[
              attribute_type
          ] = {}
          num_distinct_attribute_names = len(list(attribute_map.keys()))
          for (
              attribute_key_name,
              rate_pred_map,
          ) in attribute_map.items():
              rate_pred_diff_map = {}
              for cls_name, local_pred_rate in rate_pred_map.items():
                  # we don't take the absolute value as otherwise the predictions
                  # that are lowest predicted and result in negative difference
                  # will count positive towards the cls_name incorrectly.
                  # Example: female and beard
                  rate_pred_diff_map[cls_name] = local_pred_rate - (
                      (
                          (
                              output_attributes_avg_pred_rate_map[
                                  attribute_type
                              ][cls_name]
                              * num_distinct_attribute_names
                          )
                          - local_pred_rate
                      )
                      / (num_distinct_attribute_names - 1)
                  )
              sorted_rate_pred_diff_map = dict(
                  sorted(
                      rate_pred_diff_map.items(),
                      key=lambda item: item[1],
                      reverse=True,
                  )
              )
              output_attributes_pred_rate_difference_map[
                  attribute_type
              ][attribute_key_name] = sorted_rate_pred_diff_map
      return (
          sorted_output_attributes_pred_freq_map,
          output_attributes_count_map,
          output_attributes_pred_rate_difference_map,
          output_attributes_img_map,
          output_attributes_label_assoc_map,
          output_attributes_confidence_score_map,
          output_mean_attributes_confidence_score_map,
          output_attributes_label_assoc_conf_scores_map,
          output_attribute_disparate_label_map,
      )
  
file_path: vissl/projects/fairness_indicators/harmful_label_associations/inference_label_assoc_casual_conversations.py
imports: import numpy
  import pandas
  from fvcore.common.file_io import PathManager
  from PIL import Image
  from vissl.utils.io import load_file
label_desc: Use image node_id 6 for calling a global function with example usage: get_per_attribute_predictions_freq(output_metadata, label_to_id_map, label_association_map, disparate_labels_list, disparate_threshold, class_to_label_name_map, topk) and returns: sorted_output_attributes_pred_freq_map,  output_attributes_count_map,  output_attributes_pred_rate_difference_map,  output_attributes_img_map,  output_attributes_label_assoc_map,  output_attributes_confidence_score_map,  output_mean_attributes_confidence_score_map,  output_attributes_label_assoc_conf_scores_map,  output_attribute_disparate_label_map