"""
Code adapted from calculate_mean_ap.py
author: Timothy C. Arlen
date: 28 Feb 2018
"""
import sys
sys.path.append('/deep/u/joycech/aicc-working/videollava')
from collections import defaultdict
import numpy as np
import json
import ast
import re
import cv2
from shapely import wkt, Polygon, box
from infer_utils import create_mask, create_mask_s2looking
def calc_iou_individual(pred_box, gt_box):
"""Calculate IoU of single predicted and ground truth box
Args:
pred_box (list of floats): location of predicted object as
[xmin, ymin, xmax, ymax]
gt_box (list of floats): location of ground truth object as
[xmin, ymin, xmax, ymax]
Returns:
float: value of the IoU for the two boxes.
Raises:
AssertionError: if the box is obviously malformed
"""
x1_t, y1_t, x2_t, y2_t = gt_box
try:
x1_p, y1_p, x2_p, y2_p = pred_box
except:
print("Prediction box is malformed? pred box: {}".format(pred_box))
return 0.0
if (x1_p > x2_p) or (y1_p > y2_p):
print("Prediction box is malformed? pred box: {}".format(pred_box))
return 0.0
if (x1_t > x2_t) or (y1_t > y2_t):
raise AssertionError(
"Ground Truth box is malformed? true box: {}".format(gt_box))
if (x2_t < x1_p or x2_p < x1_t or y2_t < y1_p or y2_p < y1_t):
return 0.0
far_x = np.min([x2_t, x2_p])
near_x = np.max([x1_t, x1_p])
far_y = np.min([y2_t, y2_p])
near_y = np.max([y1_t, y1_p])
inter_area = (far_x - near_x + 1) * (far_y - near_y + 1)
true_box_area = (x2_t - x1_t + 1) * (y2_t - y1_t + 1)
pred_box_area = (x2_p - x1_p + 1) * (y2_p - y1_p + 1)
iou = inter_area / (true_box_area + pred_box_area - inter_area)
return iou
def get_single_image_bound_results(gt_wkts, pred_boxes, img_size=256, dataset=None, id=None, predicted_mask=None, split=None, question=None):
"""
Calculates upper bound and lower bound number of true_pos, false_pos, false_neg from single batch of boxes.
Args:
gt_wkts (list of strs): list of wkt strings of input polygons, scaled to raw pixel value
pred_boxes (list of lists): list of list of boxes, where each box is formatted
as [x_min, y_min, x_max, y_max] on scale from 0-100
img_size (int): dimensions of the image. defaults to 256.
Returns:
tuple of dicts: true positives (int), false positives (int), false negatives (int)
"""
lb_preds = [[num * img_size / 100 for num in box] for box in pred_boxes]
# add error handling for this type of outputs: [0, 10, 12, 22], [0, 6, 12, 19], [0, 0], [31, 0]
try:
lb_preds = [box(*pred_box) for pred_box in lb_preds]
except:
lb_preds = []
for pred_box in pred_boxes:
if len(pred_box) == 4:
lb_preds.append(box(*pred_box))
if isinstance(gt_wkts, str):
gt_polygons = [wkt.loads(gt_wkts)]
elif gt_wkts is None:
gt_polygons = []
else:
gt_polygons = [wkt.loads(gt_wkt) for gt_wkt in gt_wkts]
# get mask of all gt_polygons and lb_preds
if dataset == None:
gt_mask = create_mask(gt_polygons, (img_size, img_size))
else:
gt_mask = create_mask_s2looking(id, split=split, question=question)
#gt_mask = create_mask(gt_polygons, (img_size, img_size))
if dataset != "geochat_s2looking":
lb_preds_mask = create_mask(lb_preds, (img_size, img_size))
else:
lb_preds_mask = predicted_mask
# get lower bound intersection and union masks
intersection = np.logical_and(gt_mask, lb_preds_mask)
union = np.logical_or(gt_mask, lb_preds_mask)
# compute lb metrics
lower_bound_iou = np.sum(intersection) / np.sum(union)
if np.sum(intersection) == 0 and np.sum(union) == 0:
return None, None
if np.isnan(lower_bound_iou):
lower_bound_iou = 0
fp = np.sum(np.logical_and(lb_preds_mask, np.logical_not(gt_mask)))
tp = np.sum(np.logical_and(lb_preds_mask, gt_mask))
fn = np.sum(np.logical_and(np.logical_not(lb_preds_mask), gt_mask))
lb_stats = {'true_pos': tp,
'false_pos': fp,
'false_neg': fn,
'intersection': np.sum(intersection),
'union': np.sum(union)}
return lb_stats
def get_single_image_results(gt_boxes, pred_boxes, iou_thr):
"""Calculates number of true_pos, false_pos, false_neg from single batch of boxes.
Args:
gt_boxes (list of list of floats): list of locations of ground truth
objects as [xmin, ymin, xmax, ymax]
pred_boxes (dict): dict of dicts of 'boxes' (formatted like `gt_boxes`)
and 'scores'
iou_thr (float): value of IoU to consider as threshold for a
true prediction.
Returns:
dict: true positives (int), false positives (int), false negatives (int)
"""
all_pred_indices = range(len(pred_boxes))
all_gt_indices = range(len(gt_boxes))
if len(all_pred_indices) == 0:
tp = 0
fp = 0
fn = len(gt_boxes)
return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
if len(all_gt_indices) == 0:
tp = 0
fp = len(pred_boxes)
fn = 0
return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
gt_idx_thr = []
pred_idx_thr = []
ious = []
for ipb, pred_box in enumerate(pred_boxes):
for igb, gt_box in enumerate(gt_boxes):
iou = calc_iou_individual(pred_box, gt_box)
if iou > iou_thr:
gt_idx_thr.append(igb)
pred_idx_thr.append(ipb)
ious.append(iou)
args_desc = np.argsort(ious)[::-1]
if len(args_desc) == 0:
# No matches
tp = 0
fp = len(pred_boxes)
fn = len(gt_boxes)
else:
gt_match_idx = []
pred_match_idx = []
for idx in args_desc:
gt_idx = gt_idx_thr[idx]
pr_idx = pred_idx_thr[idx]
# If the boxes are unmatched, add them to matches
if (gt_idx not in gt_match_idx) and (pr_idx not in pred_match_idx):
gt_match_idx.append(gt_idx)
pred_match_idx.append(pr_idx)
tp = len(gt_match_idx)
fp = len(pred_boxes) - len(pred_match_idx)
fn = len(gt_boxes) - len(gt_match_idx)
return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
def calc_precision_recall(img_results):
"""Calculates precision and recall from the set of images
Args:
img_results (dict): dictionary formatted like:
{
'img_id1': {'true_pos': int, 'false_pos': int, 'false_neg': int},
'img_id2': ...
...
}
Returns:
tuple: of floats of (precision, recall)
"""
true_pos = 0; false_pos = 0; false_neg = 0
for _, res in img_results.items():
true_pos += res['true_pos']
false_pos += res['false_pos']
false_neg += res['false_neg']
try:
precision = true_pos/(true_pos + false_pos)
except ZeroDivisionError:
precision = 0.0
print(true_pos, "true_pos", false_pos, "false_pos", false_neg, "false_neg")
try:
recall = true_pos/(true_pos + false_neg)
except ZeroDivisionError:
recall = 0.0
return (precision, recall)
def extract_bboxes(input_string):
"""
Takes as an input a string like in the image, there are two buildings that have been changed. the first building is located at [0.0, 0.69, 0.45, 0.9] and the second building is located at [0.46, 0.69, 0.99, 0.91]
Returns a list of bounding boxes in the format [x_min, y_min, x_max, y_max]
Input:
input_string (str): string containing the bounding boxes
Returns:
list of lists: list of bounding boxes
"""
matches = re.findall(r'\[\[.*?\]\]', input_string)
return [ast.literal_eval(match) for match in matches]
def referring_expression(answer_path, dataset, verbose=False, saving_path_root=None, img_size=256, split=None):
if type(answer_path) == dict:
results = answer_path
else:
with open(answer_path) as json_data:
results = json.load(json_data)
img_results = {}
lb_results = {}
# Loop over results and get precision, recall overall
for id, result in results.items():
if 'temporal_referring_expression' in result['task']:
if not "s2looking" in dataset:
continue # no bounding box outputs for temporal_referring_expression
# for the geochat s2looking predictions, we work directly with the predicted mask instead of the bounding boxes
if dataset == 'geochat_s2looking':
if 'referring_expression' in result['task'] or 'localization' in result['task']:
lb_res = get_single_image_bound_results(result['original_input_polygon'], [], dataset=dataset, id=id, predicted_mask=result['predicted_mask'], split=split, question=result["question"])
if lb_res != None:
lb_results[id] = lb_res
continue
elif 'question_answering' in result['task']:
continue
if 'referring_expression' in result['task'] or 'largest building' in result['task'] or "canonical" in result['task'] or 'localization' in result['task'] \
or 'geochat_referring' in result['task']:
# No bounding boxes in predicted string
if "[" not in result["predicted"]:
# Ground truth has no bounding boxes
if result["ground_truth"].startswith("There are no") or "no" in result["ground_truth"] or "No" in result["ground_truth"]:
# Discard true negatives
continue
# Ground truth has bounding boxes, not identified by the model --> all false negatives
else:
false_neg = "[" + result["ground_truth"] + "]"
false_neg = false_neg.replace(".", "")
try:
false_neg = len(ast.literal_eval(false_neg))
except:
# count the number of opening '[' in the string
false_neg = false_neg.count('[') - 1
if not "s2looking" in dataset:
gt_mask = create_mask(wkt.loads(result['original_input_polygon']), (img_size, img_size))
else:
gt_mask = create_mask_s2looking(id, split=split, question=result['question'])
# gt_mask = create_mask(wkt.loads(result['original_input_polygon']), (img_size, img_size))
img_results[id] = {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
false_neg = np.sum(gt_mask)
lb_results[id] = {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
# Bounding boxes in predicted and output string --> compare bounding boxes
else:
# To deal with cases where the model outputs an incomplete bounding box (e.g. "[24, 76,")
first_open_bracket_ind = result["predicted"].find("[")
last_close_bracket_ind = result["predicted"].rfind("]")
if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
else:
parsed_predicted = ""
# Load list of predicted bounding boxes
try:
predicted_boxes = ast.literal_eval("[" + parsed_predicted + "]")
except:
match = re.search(r'\[\[.*\]\]', result["predicted"])
if match:
predicted_boxes = ast.literal_eval(match.group())
else:
predicted_boxes = []
predicted_boxes = [[coord * 100 if coord < 1 else coord for coord in box] for box in predicted_boxes]
# Load list of ground truth bounding boxes
if result["ground_truth"].startswith("There are no") or "no" in result["ground_truth"].lower():
# If ground truth contains no boxes
ground_truth_boxes = []
first_open_bracket_ind = result["ground_truth"].find("[")
last_close_bracket_ind = result["ground_truth"].rfind("]")
if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
parsed_gt = result["ground_truth"][first_open_bracket_ind:last_close_bracket_ind+1]
else:
parsed_gt = ""
try:
ground_truth_boxes = ast.literal_eval("[" + parsed_gt + "]")
except:
match = re.search(r'\[\[.*\]\]', result["ground_truth"])
if match:
ground_truth_boxes = ast.literal_eval(match.group())
else:
ground_truth_boxes = []
# Get mask results from the two previous parsings
gt_wkts = result['original_input_polygon']
img_results[id] = get_single_image_results(ground_truth_boxes, predicted_boxes, iou_thr=0.5) ######
if 'referring_expression' in result['task'] or 'largest building' in result['task'] or "canonical" in result['task'] or 'localization' in result['task']:
if not "s2looking" in dataset:
lb_results[id] = get_single_image_bound_results(gt_wkts, predicted_boxes)
elif dataset=="s2looking":
lb_results[id] = get_single_image_bound_results(gt_wkts, predicted_boxes, dataset=dataset, id=id, split=split, question=result["question"])
else:
lb_results[id] = get_single_image_bound_results(gt_wkts, predicted_boxes, predicted_mask=result['predicted_mask'], split=split, question=result["question"])
precision, recall = calc_precision_recall(img_results)
print("Referring expression results (precision, recall): ", precision, recall)
print("Acc@0.5: ", np.sum([res['true_pos'] for res in img_results.values()]) / len(results.keys()))
if len(lb_results) != 0:
lb_intersection = np.sum([res['intersection'] for res in lb_results.values()])
lb_union = np.sum([res['union'] for res in lb_results.values()])
print("Lower bound IOU: ", lb_intersection / lb_union if lb_union != 0 else 0)
lb_precision, lb_recall = calc_precision_recall(lb_results)
print('Lower bound precision: ', lb_precision)
print('Lower bound recall: ', lb_recall)
print("Lower bound F1: ", 2 * (lb_precision * lb_recall) / (lb_precision + lb_recall) if (lb_precision + lb_recall) != 0 else 0)
if saving_path_root:
with open(f"{saving_path_root}/referring_expression_scores.json", 'w') as f:
json.dump(img_results, f)
if __name__ == '__main__':
answer_path = "scripts/geovlm/eval/xBD/answers/ckpt14000-old-aux-xbd-test-canon-auxiliary_interleave.json"
referring_expression(answer_path, dataset="xbd")