prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Add metrics for counting to eval.py
  We should add a new metric to the detection evaluation script so that it computes how close the counts are compared to ground truth.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `src/rv/detection/commands/eval_predictions.py`
  Content:
  ```
  1 import json
  2 from os.path import join
  3 
  4 import numpy as np
  5 import rasterio
  6 import click
  7 
  8 from object_detection.utils import object_detection_evaluation, label_map_util
  9 
  10 from rv.utils import (
  11     download_if_needed, make_empty_dir, get_local_path, upload_if_needed,
  12     get_boxes_from_geojson, download_and_build_vrt)
  13 from rv.detection.commands.settings import max_num_classes, temp_root_dir
  14 
  15 
  16 def get_eval_result(ground_truth_path, predictions_path, image_dataset):
  17     gt_boxes, gt_classes, _ = \
  18         get_boxes_from_geojson(ground_truth_path, image_dataset)
  19     # Subtract one because class id's start at 1, but evaluation api assumes
  20     # the start at 0. You might think we could just write the label_map.pbtxt
  21     # so the class ids start at 0, but that throws an exception.
  22     gt_classes -= 1
  23 
  24     pred_boxes, pred_classes, pred_scores = \
  25         get_boxes_from_geojson(predictions_path, image_dataset)
  26     pred_classes -= 1
  27 
  28     nb_gt_classes = len(set(gt_classes))
  29     od_eval = object_detection_evaluation.ObjectDetectionEvaluation(
  30         nb_gt_classes, matching_iou_threshold=0.1)
  31     image_key = 'image'
  32     od_eval.add_single_ground_truth_image_info(
  33         image_key, gt_boxes, gt_classes)
  34     od_eval.add_single_detected_image_info(
  35         image_key, pred_boxes, pred_scores, pred_classes)
  36 
  37     od_eval.evaluate()
  38     return od_eval.get_eval_result()
  39 
  40 
  41 def write_results(output_path, label_map_path, eval_result):
  42     label_map = label_map_util.load_labelmap(label_map_path)
  43     categories = label_map_util.convert_label_map_to_categories(
  44         label_map, max_num_classes=max_num_classes, use_display_name=True)
  45     category_index = label_map_util.create_category_index(categories)
  46 
  47     results = []
  48     for class_id in range(1, len(category_index) + 1):
  49         class_name = category_index[class_id]['name']
  50         # Subtract one to account for fact that class id's start at 1.
  51         # precisions and recalls are lists with one element for each
  52         # predicted box, assuming they are sorted by score. Each element is
  53         # the precision or recall assuming that all predicted boxes with that
  54         # score or above are used. So, the last element is the value assuming
  55         # that all predictions are used.
  56 
  57         precisions = eval_result.precisions[class_id - 1]
  58         recalls = eval_result.recalls[class_id - 1]
  59         # Get precision and recall assuming all predicted boxes are used.
  60         class_results = {
  61             'name': class_name,
  62             'precision': precisions[-1],
  63             'recall': recalls[-1]
  64         }
  65         results.append(class_results)
  66 
  67     with open(output_path, 'w') as output_file:
  68         output_file.write(json.dumps(results, indent=4))
  69 
  70 
  71 def _eval_predictions(image_uris, label_map_uri, ground_truth_uri,
  72                       predictions_uri, output_uri):
  73     temp_dir = join(temp_root_dir, 'eval_predictions')
  74     make_empty_dir(temp_dir)
  75 
  76     image_path = download_and_build_vrt(temp_dir, image_uris)
  77     image_dataset = rasterio.open(image_path)
  78 
  79     ground_truth_path = download_if_needed(temp_dir, ground_truth_uri)
  80     predictions_path = download_if_needed(temp_dir, predictions_uri)
  81     label_map_path = download_if_needed(temp_dir, label_map_uri)
  82 
  83     eval_result = get_eval_result(
  84         ground_truth_path, predictions_path, image_dataset)
  85 
  86     output_path = get_local_path(temp_dir, output_uri)
  87     write_results(output_path, label_map_path, eval_result)
  88     upload_if_needed(output_path, output_uri)
  89 
  90 
  91 @click.command()
  92 @click.argument('image_uris', nargs=-1)
  93 @click.argument('label_map_uri')
  94 @click.argument('ground_truth_uri')
  95 @click.argument('predictions_uri')
  96 @click.argument('output_uri')
  97 def eval_predictions(image_uris, label_map_uri, ground_truth_uri,
  98                      predictions_uri, output_uri):
  99     """Evaluate predictions against ground truth for a single predictions file.
  100 
  101     Args:
  102         ground_truth_uri: GeoJSON file with ground truth bounding boxes
  103         predictions_uri: GeoJSON file with predicted bounding boxes
  104         output_uri: JSON file with metrics
  105     """
  106     _eval_predictions(image_uris, label_map_uri, ground_truth_uri,
  107                       predictions_uri, output_uri)
  108 
  109 
  110 if __name__ == '__main__':
  111     eval_predictions()
  112
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/src/rv/detection/commands/eval_predictions.py b/src/rv/detection/commands/eval_predictions.py
  --- a/src/rv/detection/commands/eval_predictions.py
  +++ b/src/rv/detection/commands/eval_predictions.py
  @@ -1,7 +1,6 @@
   import json
  -from os.path import join
  +from os.path import join, dirname
   
  -import numpy as np
   import rasterio
   import click
   
  @@ -13,7 +12,7 @@
   from rv.detection.commands.settings import max_num_classes, temp_root_dir
   
   
  -def get_eval_result(ground_truth_path, predictions_path, image_dataset):
  +def get_od_eval(ground_truth_path, predictions_path, image_dataset):
       gt_boxes, gt_classes, _ = \
           get_boxes_from_geojson(ground_truth_path, image_dataset)
       # Subtract one because class id's start at 1, but evaluation api assumes
  @@ -35,10 +34,12 @@
           image_key, pred_boxes, pred_scores, pred_classes)
   
       od_eval.evaluate()
  -    return od_eval.get_eval_result()
  +    return od_eval
   
   
  -def write_results(output_path, label_map_path, eval_result):
  +def write_results(output_path, label_map_path, od_eval):
  +    make_empty_dir(dirname(output_path), empty_dir=False)
  +
       label_map = label_map_util.load_labelmap(label_map_path)
       categories = label_map_util.convert_label_map_to_categories(
           label_map, max_num_classes=max_num_classes, use_display_name=True)
  @@ -53,19 +54,30 @@
           # the precision or recall assuming that all predicted boxes with that
           # score or above are used. So, the last element is the value assuming
           # that all predictions are used.
  -
  +        eval_result = od_eval.get_eval_result()
           precisions = eval_result.precisions[class_id - 1]
           recalls = eval_result.recalls[class_id - 1]
           # Get precision and recall assuming all predicted boxes are used.
  +        precision = precisions[-1]
  +        recall = recalls[-1]
  +        f1 = (2 * precision * recall) / (precision + recall)
  +
  +        gt_count = od_eval.num_gt_instances_per_class[class_id -1]
  +        pred_count = len(recalls)
  +        count_error = pred_count - gt_count
  +        norm_count_error = count_error / gt_count
  +
           class_results = {
               'name': class_name,
  -            'precision': precisions[-1],
  -            'recall': recalls[-1]
  +            'precision': precision,
  +            'recall': recall,
  +            'f1': f1,
  +            'norm_count_error': norm_count_error
           }
           results.append(class_results)
   
       with open(output_path, 'w') as output_file:
  -        output_file.write(json.dumps(results, indent=4))
  +        output_file.write(json.dumps(results, indent=4, sort_keys=True))
   
   
   def _eval_predictions(image_uris, label_map_uri, ground_truth_uri,
  @@ -80,11 +92,11 @@
       predictions_path = download_if_needed(temp_dir, predictions_uri)
       label_map_path = download_if_needed(temp_dir, label_map_uri)
   
  -    eval_result = get_eval_result(
  +    od_eval = get_od_eval(
           ground_truth_path, predictions_path, image_dataset)
   
       output_path = get_local_path(temp_dir, output_uri)
  -    write_results(output_path, label_map_path, eval_result)
  +    write_results(output_path, label_map_path, od_eval)
       upload_if_needed(output_path, output_uri)
  
verification_info: {"golden_diff": "diff --git a/src/rv/detection/commands/eval_predictions.py b/src/rv/detection/commands/eval_predictions.py\n--- a/src/rv/detection/commands/eval_predictions.py\n+++ b/src/rv/detection/commands/eval_predictions.py\n@@ -1,7 +1,6 @@\n import json\n-from os.path import join\n+from os.path import join, dirname\n \n-import numpy as np\n import rasterio\n import click\n \n@@ -13,7 +12,7 @@\n from rv.detection.commands.settings import max_num_classes, temp_root_dir\n \n \n-def get_eval_result(ground_truth_path, predictions_path, image_dataset):\n+def get_od_eval(ground_truth_path, predictions_path, image_dataset):\n     gt_boxes, gt_classes, _ = \\\n         get_boxes_from_geojson(ground_truth_path, image_dataset)\n     # Subtract one because class id's start at 1, but evaluation api assumes\n@@ -35,10 +34,12 @@\n         image_key, pred_boxes, pred_scores, pred_classes)\n \n     od_eval.evaluate()\n-    return od_eval.get_eval_result()\n+    return od_eval\n \n \n-def write_results(output_path, label_map_path, eval_result):\n+def write_results(output_path, label_map_path, od_eval):\n+    make_empty_dir(dirname(output_path), empty_dir=False)\n+\n     label_map = label_map_util.load_labelmap(label_map_path)\n     categories = label_map_util.convert_label_map_to_categories(\n         label_map, max_num_classes=max_num_classes, use_display_name=True)\n@@ -53,19 +54,30 @@\n         # the precision or recall assuming that all predicted boxes with that\n         # score or above are used. So, the last element is the value assuming\n         # that all predictions are used.\n-\n+        eval_result = od_eval.get_eval_result()\n         precisions = eval_result.precisions[class_id - 1]\n         recalls = eval_result.recalls[class_id - 1]\n         # Get precision and recall assuming all predicted boxes are used.\n+        precision = precisions[-1]\n+        recall = recalls[-1]\n+        f1 = (2 * precision * recall) / (precision + recall)\n+\n+        gt_count = od_eval.num_gt_instances_per_class[class_id -1]\n+        pred_count = len(recalls)\n+        count_error = pred_count - gt_count\n+        norm_count_error = count_error / gt_count\n+\n         class_results = {\n             'name': class_name,\n-            'precision': precisions[-1],\n-            'recall': recalls[-1]\n+            'precision': precision,\n+            'recall': recall,\n+            'f1': f1,\n+            'norm_count_error': norm_count_error\n         }\n         results.append(class_results)\n \n     with open(output_path, 'w') as output_file:\n-        output_file.write(json.dumps(results, indent=4))\n+        output_file.write(json.dumps(results, indent=4, sort_keys=True))\n \n \n def _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n@@ -80,11 +92,11 @@\n     predictions_path = download_if_needed(temp_dir, predictions_uri)\n     label_map_path = download_if_needed(temp_dir, label_map_uri)\n \n-    eval_result = get_eval_result(\n+    od_eval = get_od_eval(\n         ground_truth_path, predictions_path, image_dataset)\n \n     output_path = get_local_path(temp_dir, output_uri)\n-    write_results(output_path, label_map_path, eval_result)\n+    write_results(output_path, label_map_path, od_eval)\n     upload_if_needed(output_path, output_uri)\n", "issue": "Add metrics for counting to eval.py\nWe should add a new metric to the detection evaluation script so that it computes how close the counts are compared to ground truth.\n", "before_files": [{"content": "import json\nfrom os.path import join\n\nimport numpy as np\nimport rasterio\nimport click\n\nfrom object_detection.utils import object_detection_evaluation, label_map_util\n\nfrom rv.utils import (\n    download_if_needed, make_empty_dir, get_local_path, upload_if_needed,\n    get_boxes_from_geojson, download_and_build_vrt)\nfrom rv.detection.commands.settings import max_num_classes, temp_root_dir\n\n\ndef get_eval_result(ground_truth_path, predictions_path, image_dataset):\n    gt_boxes, gt_classes, _ = \\\n        get_boxes_from_geojson(ground_truth_path, image_dataset)\n    # Subtract one because class id's start at 1, but evaluation api assumes\n    # the start at 0. You might think we could just write the label_map.pbtxt\n    # so the class ids start at 0, but that throws an exception.\n    gt_classes -= 1\n\n    pred_boxes, pred_classes, pred_scores = \\\n        get_boxes_from_geojson(predictions_path, image_dataset)\n    pred_classes -= 1\n\n    nb_gt_classes = len(set(gt_classes))\n    od_eval = object_detection_evaluation.ObjectDetectionEvaluation(\n        nb_gt_classes, matching_iou_threshold=0.1)\n    image_key = 'image'\n    od_eval.add_single_ground_truth_image_info(\n        image_key, gt_boxes, gt_classes)\n    od_eval.add_single_detected_image_info(\n        image_key, pred_boxes, pred_scores, pred_classes)\n\n    od_eval.evaluate()\n    return od_eval.get_eval_result()\n\n\ndef write_results(output_path, label_map_path, eval_result):\n    label_map = label_map_util.load_labelmap(label_map_path)\n    categories = label_map_util.convert_label_map_to_categories(\n        label_map, max_num_classes=max_num_classes, use_display_name=True)\n    category_index = label_map_util.create_category_index(categories)\n\n    results = []\n    for class_id in range(1, len(category_index) + 1):\n        class_name = category_index[class_id]['name']\n        # Subtract one to account for fact that class id's start at 1.\n        # precisions and recalls are lists with one element for each\n        # predicted box, assuming they are sorted by score. Each element is\n        # the precision or recall assuming that all predicted boxes with that\n        # score or above are used. So, the last element is the value assuming\n        # that all predictions are used.\n\n        precisions = eval_result.precisions[class_id - 1]\n        recalls = eval_result.recalls[class_id - 1]\n        # Get precision and recall assuming all predicted boxes are used.\n        class_results = {\n            'name': class_name,\n            'precision': precisions[-1],\n            'recall': recalls[-1]\n        }\n        results.append(class_results)\n\n    with open(output_path, 'w') as output_file:\n        output_file.write(json.dumps(results, indent=4))\n\n\ndef _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                      predictions_uri, output_uri):\n    temp_dir = join(temp_root_dir, 'eval_predictions')\n    make_empty_dir(temp_dir)\n\n    image_path = download_and_build_vrt(temp_dir, image_uris)\n    image_dataset = rasterio.open(image_path)\n\n    ground_truth_path = download_if_needed(temp_dir, ground_truth_uri)\n    predictions_path = download_if_needed(temp_dir, predictions_uri)\n    label_map_path = download_if_needed(temp_dir, label_map_uri)\n\n    eval_result = get_eval_result(\n        ground_truth_path, predictions_path, image_dataset)\n\n    output_path = get_local_path(temp_dir, output_uri)\n    write_results(output_path, label_map_path, eval_result)\n    upload_if_needed(output_path, output_uri)\n\n\n@click.command()\n@click.argument('image_uris', nargs=-1)\n@click.argument('label_map_uri')\n@click.argument('ground_truth_uri')\n@click.argument('predictions_uri')\n@click.argument('output_uri')\ndef eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                     predictions_uri, output_uri):\n    \"\"\"Evaluate predictions against ground truth for a single predictions file.\n\n    Args:\n        ground_truth_uri: GeoJSON file with ground truth bounding boxes\n        predictions_uri: GeoJSON file with predicted bounding boxes\n        output_uri: JSON file with metrics\n    \"\"\"\n    _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                      predictions_uri, output_uri)\n\n\nif __name__ == '__main__':\n    eval_predictions()\n", "path": "src/rv/detection/commands/eval_predictions.py"}], "after_files": [{"content": "import json\nfrom os.path import join, dirname\n\nimport rasterio\nimport click\n\nfrom object_detection.utils import object_detection_evaluation, label_map_util\n\nfrom rv.utils import (\n    download_if_needed, make_empty_dir, get_local_path, upload_if_needed,\n    get_boxes_from_geojson, download_and_build_vrt)\nfrom rv.detection.commands.settings import max_num_classes, temp_root_dir\n\n\ndef get_od_eval(ground_truth_path, predictions_path, image_dataset):\n    gt_boxes, gt_classes, _ = \\\n        get_boxes_from_geojson(ground_truth_path, image_dataset)\n    # Subtract one because class id's start at 1, but evaluation api assumes\n    # the start at 0. You might think we could just write the label_map.pbtxt\n    # so the class ids start at 0, but that throws an exception.\n    gt_classes -= 1\n\n    pred_boxes, pred_classes, pred_scores = \\\n        get_boxes_from_geojson(predictions_path, image_dataset)\n    pred_classes -= 1\n\n    nb_gt_classes = len(set(gt_classes))\n    od_eval = object_detection_evaluation.ObjectDetectionEvaluation(\n        nb_gt_classes, matching_iou_threshold=0.1)\n    image_key = 'image'\n    od_eval.add_single_ground_truth_image_info(\n        image_key, gt_boxes, gt_classes)\n    od_eval.add_single_detected_image_info(\n        image_key, pred_boxes, pred_scores, pred_classes)\n\n    od_eval.evaluate()\n    return od_eval\n\n\ndef write_results(output_path, label_map_path, od_eval):\n    make_empty_dir(dirname(output_path), empty_dir=False)\n\n    label_map = label_map_util.load_labelmap(label_map_path)\n    categories = label_map_util.convert_label_map_to_categories(\n        label_map, max_num_classes=max_num_classes, use_display_name=True)\n    category_index = label_map_util.create_category_index(categories)\n\n    results = []\n    for class_id in range(1, len(category_index) + 1):\n        class_name = category_index[class_id]['name']\n        # Subtract one to account for fact that class id's start at 1.\n        # precisions and recalls are lists with one element for each\n        # predicted box, assuming they are sorted by score. Each element is\n        # the precision or recall assuming that all predicted boxes with that\n        # score or above are used. So, the last element is the value assuming\n        # that all predictions are used.\n        eval_result = od_eval.get_eval_result()\n        precisions = eval_result.precisions[class_id - 1]\n        recalls = eval_result.recalls[class_id - 1]\n        # Get precision and recall assuming all predicted boxes are used.\n        precision = precisions[-1]\n        recall = recalls[-1]\n        f1 = (2 * precision * recall) / (precision + recall)\n\n        gt_count = od_eval.num_gt_instances_per_class[class_id -1]\n        pred_count = len(recalls)\n        count_error = pred_count - gt_count\n        norm_count_error = count_error / gt_count\n\n        class_results = {\n            'name': class_name,\n            'precision': precision,\n            'recall': recall,\n            'f1': f1,\n            'norm_count_error': norm_count_error\n        }\n        results.append(class_results)\n\n    with open(output_path, 'w') as output_file:\n        output_file.write(json.dumps(results, indent=4, sort_keys=True))\n\n\ndef _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                      predictions_uri, output_uri):\n    temp_dir = join(temp_root_dir, 'eval_predictions')\n    make_empty_dir(temp_dir)\n\n    image_path = download_and_build_vrt(temp_dir, image_uris)\n    image_dataset = rasterio.open(image_path)\n\n    ground_truth_path = download_if_needed(temp_dir, ground_truth_uri)\n    predictions_path = download_if_needed(temp_dir, predictions_uri)\n    label_map_path = download_if_needed(temp_dir, label_map_uri)\n\n    od_eval = get_od_eval(\n        ground_truth_path, predictions_path, image_dataset)\n\n    output_path = get_local_path(temp_dir, output_uri)\n    write_results(output_path, label_map_path, od_eval)\n    upload_if_needed(output_path, output_uri)\n\n\n@click.command()\n@click.argument('image_uris', nargs=-1)\n@click.argument('label_map_uri')\n@click.argument('ground_truth_uri')\n@click.argument('predictions_uri')\n@click.argument('output_uri')\ndef eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                     predictions_uri, output_uri):\n    \"\"\"Evaluate predictions against ground truth for a single predictions file.\n\n    Args:\n        ground_truth_uri: GeoJSON file with ground truth bounding boxes\n        predictions_uri: GeoJSON file with predicted bounding boxes\n        output_uri: JSON file with metrics\n    \"\"\"\n    _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                      predictions_uri, output_uri)\n\n\nif __name__ == '__main__':\n    eval_predictions()\n", "path": "src/rv/detection/commands/eval_predictions.py"}]}