We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Add metrics for counting to eval.py
We should add a new metric to the detection evaluation script so that it computes how close the counts are compared to ground truth.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/rv/detection/commands/eval_predictions.py`
Content:
```
1 import json
2 from os.path import join
3 
4 import numpy as np
5 import rasterio
6 import click
7 
8 from object_detection.utils import object_detection_evaluation, label_map_util
9 
10 from rv.utils import (
11     download_if_needed, make_empty_dir, get_local_path, upload_if_needed,
12     get_boxes_from_geojson, download_and_build_vrt)
13 from rv.detection.commands.settings import max_num_classes, temp_root_dir
14 
15 
16 def get_eval_result(ground_truth_path, predictions_path, image_dataset):
17     gt_boxes, gt_classes, _ = \
18         get_boxes_from_geojson(ground_truth_path, image_dataset)
19     # Subtract one because class id's start at 1, but evaluation api assumes
20     # the start at 0. You might think we could just write the label_map.pbtxt
21     # so the class ids start at 0, but that throws an exception.
22     gt_classes -= 1
23 
24     pred_boxes, pred_classes, pred_scores = \
25         get_boxes_from_geojson(predictions_path, image_dataset)
26     pred_classes -= 1
27 
28     nb_gt_classes = len(set(gt_classes))
29     od_eval = object_detection_evaluation.ObjectDetectionEvaluation(
30         nb_gt_classes, matching_iou_threshold=0.1)
31     image_key = 'image'
32     od_eval.add_single_ground_truth_image_info(
33         image_key, gt_boxes, gt_classes)
34     od_eval.add_single_detected_image_info(
35         image_key, pred_boxes, pred_scores, pred_classes)
36 
37     od_eval.evaluate()
38     return od_eval.get_eval_result()
39 
40 
41 def write_results(output_path, label_map_path, eval_result):
42     label_map = label_map_util.load_labelmap(label_map_path)
43     categories = label_map_util.convert_label_map_to_categories(
44         label_map, max_num_classes=max_num_classes, use_display_name=True)
45     category_index = label_map_util.create_category_index(categories)
46 
47     results = []
48     for class_id in range(1, len(category_index) + 1):
49         class_name = category_index[class_id]['name']
50         # Subtract one to account for fact that class id's start at 1.
51         # precisions and recalls are lists with one element for each
52         # predicted box, assuming they are sorted by score. Each element is
53         # the precision or recall assuming that all predicted boxes with that
54         # score or above are used. So, the last element is the value assuming
55         # that all predictions are used.
56 
57         precisions = eval_result.precisions[class_id - 1]
58         recalls = eval_result.recalls[class_id - 1]
59         # Get precision and recall assuming all predicted boxes are used.
60         class_results = {
61             'name': class_name,
62             'precision': precisions[-1],
63             'recall': recalls[-1]
64         }
65         results.append(class_results)
66 
67     with open(output_path, 'w') as output_file:
68         output_file.write(json.dumps(results, indent=4))
69 
70 
71 def _eval_predictions(image_uris, label_map_uri, ground_truth_uri,
72                       predictions_uri, output_uri):
73     temp_dir = join(temp_root_dir, 'eval_predictions')
74     make_empty_dir(temp_dir)
75 
76     image_path = download_and_build_vrt(temp_dir, image_uris)
77     image_dataset = rasterio.open(image_path)
78 
79     ground_truth_path = download_if_needed(temp_dir, ground_truth_uri)
80     predictions_path = download_if_needed(temp_dir, predictions_uri)
81     label_map_path = download_if_needed(temp_dir, label_map_uri)
82 
83     eval_result = get_eval_result(
84         ground_truth_path, predictions_path, image_dataset)
85 
86     output_path = get_local_path(temp_dir, output_uri)
87     write_results(output_path, label_map_path, eval_result)
88     upload_if_needed(output_path, output_uri)
89 
90 
91 @click.command()
92 @click.argument('image_uris', nargs=-1)
93 @click.argument('label_map_uri')
94 @click.argument('ground_truth_uri')
95 @click.argument('predictions_uri')
96 @click.argument('output_uri')
97 def eval_predictions(image_uris, label_map_uri, ground_truth_uri,
98                      predictions_uri, output_uri):
99     """Evaluate predictions against ground truth for a single predictions file.
100 
101     Args:
102         ground_truth_uri: GeoJSON file with ground truth bounding boxes
103         predictions_uri: GeoJSON file with predicted bounding boxes
104         output_uri: JSON file with metrics
105     """
106     _eval_predictions(image_uris, label_map_uri, ground_truth_uri,
107                       predictions_uri, output_uri)
108 
109 
110 if __name__ == '__main__':
111     eval_predictions()
112
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/rv/detection/commands/eval_predictions.py b/src/rv/detection/commands/eval_predictions.py
--- a/src/rv/detection/commands/eval_predictions.py
+++ b/src/rv/detection/commands/eval_predictions.py
@@ -1,7 +1,6 @@
 import json
-from os.path import join
+from os.path import join, dirname
 
-import numpy as np
 import rasterio
 import click
 
@@ -13,7 +12,7 @@
 from rv.detection.commands.settings import max_num_classes, temp_root_dir
 
 
-def get_eval_result(ground_truth_path, predictions_path, image_dataset):
+def get_od_eval(ground_truth_path, predictions_path, image_dataset):
     gt_boxes, gt_classes, _ = \
         get_boxes_from_geojson(ground_truth_path, image_dataset)
     # Subtract one because class id's start at 1, but evaluation api assumes
@@ -35,10 +34,12 @@
         image_key, pred_boxes, pred_scores, pred_classes)
 
     od_eval.evaluate()
-    return od_eval.get_eval_result()
+    return od_eval
 
 
-def write_results(output_path, label_map_path, eval_result):
+def write_results(output_path, label_map_path, od_eval):
+    make_empty_dir(dirname(output_path), empty_dir=False)
+
     label_map = label_map_util.load_labelmap(label_map_path)
     categories = label_map_util.convert_label_map_to_categories(
         label_map, max_num_classes=max_num_classes, use_display_name=True)
@@ -53,19 +54,30 @@
         # the precision or recall assuming that all predicted boxes with that
         # score or above are used. So, the last element is the value assuming
         # that all predictions are used.
-
+        eval_result = od_eval.get_eval_result()
         precisions = eval_result.precisions[class_id - 1]
         recalls = eval_result.recalls[class_id - 1]
         # Get precision and recall assuming all predicted boxes are used.
+        precision = precisions[-1]
+        recall = recalls[-1]
+        f1 = (2 * precision * recall) / (precision + recall)
+
+        gt_count = od_eval.num_gt_instances_per_class[class_id -1]
+        pred_count = len(recalls)
+        count_error = pred_count - gt_count
+        norm_count_error = count_error / gt_count
+
         class_results = {
             'name': class_name,
-            'precision': precisions[-1],
-            'recall': recalls[-1]
+            'precision': precision,
+            'recall': recall,
+            'f1': f1,
+            'norm_count_error': norm_count_error
         }
         results.append(class_results)
 
     with open(output_path, 'w') as output_file:
-        output_file.write(json.dumps(results, indent=4))
+        output_file.write(json.dumps(results, indent=4, sort_keys=True))
 
 
 def _eval_predictions(image_uris, label_map_uri, ground_truth_uri,
@@ -80,11 +92,11 @@
     predictions_path = download_if_needed(temp_dir, predictions_uri)
     label_map_path = download_if_needed(temp_dir, label_map_uri)
 
-    eval_result = get_eval_result(
+    od_eval = get_od_eval(
         ground_truth_path, predictions_path, image_dataset)
 
     output_path = get_local_path(temp_dir, output_uri)
-    write_results(output_path, label_map_path, eval_result)
+    write_results(output_path, label_map_path, od_eval)
     upload_if_needed(output_path, output_uri)

{"golden_diff": "diff --git a/src/rv/detection/commands/eval_predictions.py b/src/rv/detection/commands/eval_predictions.py\n--- a/src/rv/detection/commands/eval_predictions.py\n+++ b/src/rv/detection/commands/eval_predictions.py\n@@ -1,7 +1,6 @@\n import json\n-from os.path import join\n+from os.path import join, dirname\n \n-import numpy as np\n import rasterio\n import click\n \n@@ -13,7 +12,7 @@\n from rv.detection.commands.settings import max_num_classes, temp_root_dir\n \n \n-def get_eval_result(ground_truth_path, predictions_path, image_dataset):\n+def get_od_eval(ground_truth_path, predictions_path, image_dataset):\n     gt_boxes, gt_classes, _ = \\\n         get_boxes_from_geojson(ground_truth_path, image_dataset)\n     # Subtract one because class id's start at 1, but evaluation api assumes\n@@ -35,10 +34,12 @@\n         image_key, pred_boxes, pred_scores, pred_classes)\n \n     od_eval.evaluate()\n-    return od_eval.get_eval_result()\n+    return od_eval\n \n \n-def write_results(output_path, label_map_path, eval_result):\n+def write_results(output_path, label_map_path, od_eval):\n+    make_empty_dir(dirname(output_path), empty_dir=False)\n+\n     label_map = label_map_util.load_labelmap(label_map_path)\n     categories = label_map_util.convert_label_map_to_categories(\n         label_map, max_num_classes=max_num_classes, use_display_name=True)\n@@ -53,19 +54,30 @@\n         # the precision or recall assuming that all predicted boxes with that\n         # score or above are used. So, the last element is the value assuming\n         # that all predictions are used.\n-\n+        eval_result = od_eval.get_eval_result()\n         precisions = eval_result.precisions[class_id - 1]\n         recalls = eval_result.recalls[class_id - 1]\n         # Get precision and recall assuming all predicted boxes are used.\n+        precision = precisions[-1]\n+        recall = recalls[-1]\n+        f1 = (2 * precision * recall) / (precision + recall)\n+\n+        gt_count = od_eval.num_gt_instances_per_class[class_id -1]\n+        pred_count = len(recalls)\n+        count_error = pred_count - gt_count\n+        norm_count_error = count_error / gt_count\n+\n         class_results = {\n             'name': class_name,\n-            'precision': precisions[-1],\n-            'recall': recalls[-1]\n+            'precision': precision,\n+            'recall': recall,\n+            'f1': f1,\n+            'norm_count_error': norm_count_error\n         }\n         results.append(class_results)\n \n     with open(output_path, 'w') as output_file:\n-        output_file.write(json.dumps(results, indent=4))\n+        output_file.write(json.dumps(results, indent=4, sort_keys=True))\n \n \n def _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n@@ -80,11 +92,11 @@\n     predictions_path = download_if_needed(temp_dir, predictions_uri)\n     label_map_path = download_if_needed(temp_dir, label_map_uri)\n \n-    eval_result = get_eval_result(\n+    od_eval = get_od_eval(\n         ground_truth_path, predictions_path, image_dataset)\n \n     output_path = get_local_path(temp_dir, output_uri)\n-    write_results(output_path, label_map_path, eval_result)\n+    write_results(output_path, label_map_path, od_eval)\n     upload_if_needed(output_path, output_uri)\n", "issue": "Add metrics for counting to eval.py\nWe should add a new metric to the detection evaluation script so that it computes how close the counts are compared to ground truth.\n", "before_files": [{"content": "import json\nfrom os.path import join\n\nimport numpy as np\nimport rasterio\nimport click\n\nfrom object_detection.utils import object_detection_evaluation, label_map_util\n\nfrom rv.utils import (\n    download_if_needed, make_empty_dir, get_local_path, upload_if_needed,\n    get_boxes_from_geojson, download_and_build_vrt)\nfrom rv.detection.commands.settings import max_num_classes, temp_root_dir\n\n\ndef get_eval_result(ground_truth_path, predictions_path, image_dataset):\n    gt_boxes, gt_classes, _ = \\\n        get_boxes_from_geojson(ground_truth_path, image_dataset)\n    # Subtract one because class id's start at 1, but evaluation api assumes\n    # the start at 0. You might think we could just write the label_map.pbtxt\n    # so the class ids start at 0, but that throws an exception.\n    gt_classes -= 1\n\n    pred_boxes, pred_classes, pred_scores = \\\n        get_boxes_from_geojson(predictions_path, image_dataset)\n    pred_classes -= 1\n\n    nb_gt_classes = len(set(gt_classes))\n    od_eval = object_detection_evaluation.ObjectDetectionEvaluation(\n        nb_gt_classes, matching_iou_threshold=0.1)\n    image_key = 'image'\n    od_eval.add_single_ground_truth_image_info(\n        image_key, gt_boxes, gt_classes)\n    od_eval.add_single_detected_image_info(\n        image_key, pred_boxes, pred_scores, pred_classes)\n\n    od_eval.evaluate()\n    return od_eval.get_eval_result()\n\n\ndef write_results(output_path, label_map_path, eval_result):\n    label_map = label_map_util.load_labelmap(label_map_path)\n    categories = label_map_util.convert_label_map_to_categories(\n        label_map, max_num_classes=max_num_classes, use_display_name=True)\n    category_index = label_map_util.create_category_index(categories)\n\n    results = []\n    for class_id in range(1, len(category_index) + 1):\n        class_name = category_index[class_id]['name']\n        # Subtract one to account for fact that class id's start at 1.\n        # precisions and recalls are lists with one element for each\n        # predicted box, assuming they are sorted by score. Each element is\n        # the precision or recall assuming that all predicted boxes with that\n        # score or above are used. So, the last element is the value assuming\n        # that all predictions are used.\n\n        precisions = eval_result.precisions[class_id - 1]\n        recalls = eval_result.recalls[class_id - 1]\n        # Get precision and recall assuming all predicted boxes are used.\n        class_results = {\n            'name': class_name,\n            'precision': precisions[-1],\n            'recall': recalls[-1]\n        }\n        results.append(class_results)\n\n    with open(output_path, 'w') as output_file:\n        output_file.write(json.dumps(results, indent=4))\n\n\ndef _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                      predictions_uri, output_uri):\n    temp_dir = join(temp_root_dir, 'eval_predictions')\n    make_empty_dir(temp_dir)\n\n    image_path = download_and_build_vrt(temp_dir, image_uris)\n    image_dataset = rasterio.open(image_path)\n\n    ground_truth_path = download_if_needed(temp_dir, ground_truth_uri)\n    predictions_path = download_if_needed(temp_dir, predictions_uri)\n    label_map_path = download_if_needed(temp_dir, label_map_uri)\n\n    eval_result = get_eval_result(\n        ground_truth_path, predictions_path, image_dataset)\n\n    output_path = get_local_path(temp_dir, output_uri)\n    write_results(output_path, label_map_path, eval_result)\n    upload_if_needed(output_path, output_uri)\n\n\n@click.command()\n@click.argument('image_uris', nargs=-1)\n@click.argument('label_map_uri')\n@click.argument('ground_truth_uri')\n@click.argument('predictions_uri')\n@click.argument('output_uri')\ndef eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                     predictions_uri, output_uri):\n    \"\"\"Evaluate predictions against ground truth for a single predictions file.\n\n    Args:\n        ground_truth_uri: GeoJSON file with ground truth bounding boxes\n        predictions_uri: GeoJSON file with predicted bounding boxes\n        output_uri: JSON file with metrics\n    \"\"\"\n    _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                      predictions_uri, output_uri)\n\n\nif __name__ == '__main__':\n    eval_predictions()\n", "path": "src/rv/detection/commands/eval_predictions.py"}], "after_files": [{"content": "import json\nfrom os.path import join, dirname\n\nimport rasterio\nimport click\n\nfrom object_detection.utils import object_detection_evaluation, label_map_util\n\nfrom rv.utils import (\n    download_if_needed, make_empty_dir, get_local_path, upload_if_needed,\n    get_boxes_from_geojson, download_and_build_vrt)\nfrom rv.detection.commands.settings import max_num_classes, temp_root_dir\n\n\ndef get_od_eval(ground_truth_path, predictions_path, image_dataset):\n    gt_boxes, gt_classes, _ = \\\n        get_boxes_from_geojson(ground_truth_path, image_dataset)\n    # Subtract one because class id's start at 1, but evaluation api assumes\n    # the start at 0. You might think we could just write the label_map.pbtxt\n    # so the class ids start at 0, but that throws an exception.\n    gt_classes -= 1\n\n    pred_boxes, pred_classes, pred_scores = \\\n        get_boxes_from_geojson(predictions_path, image_dataset)\n    pred_classes -= 1\n\n    nb_gt_classes = len(set(gt_classes))\n    od_eval = object_detection_evaluation.ObjectDetectionEvaluation(\n        nb_gt_classes, matching_iou_threshold=0.1)\n    image_key = 'image'\n    od_eval.add_single_ground_truth_image_info(\n        image_key, gt_boxes, gt_classes)\n    od_eval.add_single_detected_image_info(\n        image_key, pred_boxes, pred_scores, pred_classes)\n\n    od_eval.evaluate()\n    return od_eval\n\n\ndef write_results(output_path, label_map_path, od_eval):\n    make_empty_dir(dirname(output_path), empty_dir=False)\n\n    label_map = label_map_util.load_labelmap(label_map_path)\n    categories = label_map_util.convert_label_map_to_categories(\n        label_map, max_num_classes=max_num_classes, use_display_name=True)\n    category_index = label_map_util.create_category_index(categories)\n\n    results = []\n    for class_id in range(1, len(category_index) + 1):\n        class_name = category_index[class_id]['name']\n        # Subtract one to account for fact that class id's start at 1.\n        # precisions and recalls are lists with one element for each\n        # predicted box, assuming they are sorted by score. Each element is\n        # the precision or recall assuming that all predicted boxes with that\n        # score or above are used. So, the last element is the value assuming\n        # that all predictions are used.\n        eval_result = od_eval.get_eval_result()\n        precisions = eval_result.precisions[class_id - 1]\n        recalls = eval_result.recalls[class_id - 1]\n        # Get precision and recall assuming all predicted boxes are used.\n        precision = precisions[-1]\n        recall = recalls[-1]\n        f1 = (2 * precision * recall) / (precision + recall)\n\n        gt_count = od_eval.num_gt_instances_per_class[class_id -1]\n        pred_count = len(recalls)\n        count_error = pred_count - gt_count\n        norm_count_error = count_error / gt_count\n\n        class_results = {\n            'name': class_name,\n            'precision': precision,\n            'recall': recall,\n            'f1': f1,\n            'norm_count_error': norm_count_error\n        }\n        results.append(class_results)\n\n    with open(output_path, 'w') as output_file:\n        output_file.write(json.dumps(results, indent=4, sort_keys=True))\n\n\ndef _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                      predictions_uri, output_uri):\n    temp_dir = join(temp_root_dir, 'eval_predictions')\n    make_empty_dir(temp_dir)\n\n    image_path = download_and_build_vrt(temp_dir, image_uris)\n    image_dataset = rasterio.open(image_path)\n\n    ground_truth_path = download_if_needed(temp_dir, ground_truth_uri)\n    predictions_path = download_if_needed(temp_dir, predictions_uri)\n    label_map_path = download_if_needed(temp_dir, label_map_uri)\n\n    od_eval = get_od_eval(\n        ground_truth_path, predictions_path, image_dataset)\n\n    output_path = get_local_path(temp_dir, output_uri)\n    write_results(output_path, label_map_path, od_eval)\n    upload_if_needed(output_path, output_uri)\n\n\n@click.command()\n@click.argument('image_uris', nargs=-1)\n@click.argument('label_map_uri')\n@click.argument('ground_truth_uri')\n@click.argument('predictions_uri')\n@click.argument('output_uri')\ndef eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                     predictions_uri, output_uri):\n    \"\"\"Evaluate predictions against ground truth for a single predictions file.\n\n    Args:\n        ground_truth_uri: GeoJSON file with ground truth bounding boxes\n        predictions_uri: GeoJSON file with predicted bounding boxes\n        output_uri: JSON file with metrics\n    \"\"\"\n    _eval_predictions(image_uris, label_map_uri, ground_truth_uri,\n                      predictions_uri, output_uri)\n\n\nif __name__ == '__main__':\n    eval_predictions()\n", "path": "src/rv/detection/commands/eval_predictions.py"}]}