prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Include per-scene metrics in eval.json
  It would be useful to see metrics for each scene in addition to metrics averaged over all scenes. 
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `rastervision/evaluation/classification_evaluation.py`
  Content:
  ```
  1 from abc import (ABC, abstractmethod)
  2 
  3 import json
  4 
  5 from rastervision.evaluation import ClassEvaluationItem
  6 from rastervision.utils.files import str_to_file
  7 
  8 
  9 class ClassificationEvaluation(ABC):
  10     """Base class for evaluating predictions for tasks that have classes.
  11 
  12     Evaluations can be keyed, for instance, if evaluations happen per class.
  13     """
  14 
  15     def __init__(self):
  16         self.clear()
  17 
  18     def clear(self):
  19         """Clear the Evaluation."""
  20         self.class_to_eval_item = {}
  21         self.avg_item = None
  22 
  23     def set_class_to_eval_item(self, class_to_eval_item):
  24         self.class_to_eval_item = class_to_eval_item
  25 
  26     def get_by_id(self, key):
  27         """Gets the evaluation for a particular EvaluationItem key"""
  28         return self.class_to_eval_item[key]
  29 
  30     def has_id(self, key):
  31         """Answers whether or not the EvaluationItem key is represented"""
  32         return key in self.class_to_eval_item
  33 
  34     def to_json(self):
  35         json_rep = []
  36         for eval_item in self.class_to_eval_item.values():
  37             json_rep.append(eval_item.to_json())
  38         json_rep.append(self.avg_item.to_json())
  39         return json_rep
  40 
  41     def save(self, output_uri):
  42         """Save this Evaluation to a file.
  43 
  44         Args:
  45             output_uri: string URI for the file to write.
  46         """
  47         json_str = json.dumps(self.to_json(), indent=4)
  48         str_to_file(json_str, output_uri)
  49 
  50     def merge(self, evaluation):
  51         """Merge Evaluation for another Scene into this one.
  52 
  53         This is useful for computing the average metrics of a set of scenes.
  54         The results of the averaging are stored in this Evaluation.
  55 
  56         Args:
  57             evaluation: Evaluation to merge into this one
  58         """
  59         if len(self.class_to_eval_item) == 0:
  60             self.class_to_eval_item = evaluation.class_to_eval_item
  61         else:
  62             for key, other_eval_item in \
  63                     evaluation.class_to_eval_item.items():
  64                 if self.has_id(key):
  65                     self.get_by_id(key).merge(other_eval_item)
  66                 else:
  67                     self.class_to_eval_item[key] = other_eval_item
  68 
  69         self.compute_avg()
  70 
  71     def compute_avg(self):
  72         """Compute average metrics over all keys."""
  73         self.avg_item = ClassEvaluationItem(class_name='average')
  74         for eval_item in self.class_to_eval_item.values():
  75             self.avg_item.merge(eval_item)
  76 
  77     @abstractmethod
  78     def compute(self, ground_truth_labels, prediction_labels):
  79         """Compute metrics for a single scene.
  80 
  81         Args:
  82             ground_truth_labels: Ground Truth labels to evaluate against.
  83             prediction_labels: The predicted labels to evaluate.
  84         """
  85         pass
  86
  ```
  
  Path: `rastervision/evaluation/semantic_segmentation_evaluator.py`
  Content:
  ```
  1 import logging
  2 
  3 from rastervision.data import ActivateMixin
  4 from rastervision.rv_config import RVConfig
  5 from rastervision.utils.files import (download_if_needed)
  6 from rastervision.evaluation import (ClassificationEvaluator,
  7                                      SemanticSegmentationEvaluation)
  8 
  9 log = logging.getLogger(__name__)
  10 
  11 
  12 class SemanticSegmentationEvaluator(ClassificationEvaluator):
  13     """Evaluates predictions for a set of scenes.
  14     """
  15 
  16     def __init__(self, class_map, output_uri):
  17         super().__init__(class_map, output_uri)
  18 
  19     def create_evaluation(self):
  20         return SemanticSegmentationEvaluation(self.class_map)
  21 
  22     def process(self, scenes, tmp_dir):
  23         evaluation = self.create_evaluation()
  24         for scene in scenes:
  25             log.info('Computing evaluation for scene {}...'.format(scene.id))
  26             label_source = scene.ground_truth_label_source
  27             label_store = scene.prediction_label_store
  28             with ActivateMixin.compose(label_source, label_store):
  29                 ground_truth = label_source.get_labels()
  30                 predictions = label_store.get_labels()
  31 
  32                 if scene.aoi_polygons:
  33                     # Filter labels based on AOI.
  34                     ground_truth = ground_truth.filter_by_aoi(
  35                         scene.aoi_polygons)
  36                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)
  37                 scene_evaluation = self.create_evaluation()
  38                 scene_evaluation.compute(ground_truth, predictions)
  39                 evaluation.merge(scene_evaluation)
  40 
  41             if hasattr(label_source, 'source') and hasattr(
  42                     label_source.source, 'vector_source') and hasattr(
  43                         label_store, 'vector_output'):
  44                 tmp_dir = RVConfig.get_tmp_dir().name
  45                 gt_geojson = label_source.source.vector_source.get_geojson()
  46                 for vo in label_store.vector_output:
  47                     pred_geojson = vo['uri']
  48                     mode = vo['mode']
  49                     class_id = vo['class_id']
  50                     pred_geojson_local = download_if_needed(
  51                         pred_geojson, tmp_dir)
  52                     scene_evaluation = self.create_evaluation()
  53                     scene_evaluation.compute_vector(
  54                         gt_geojson, pred_geojson_local, mode, class_id)
  55                     evaluation.merge(scene_evaluation)
  56 
  57         evaluation.save(self.output_uri)
  58
  ```
  
  Path: `rastervision/evaluation/classification_evaluator.py`
  Content:
  ```
  1 from abc import (abstractmethod)
  2 import logging
  3 
  4 from rastervision.evaluation import Evaluator
  5 from rastervision.data import ActivateMixin
  6 
  7 log = logging.getLogger(__name__)
  8 
  9 
  10 class ClassificationEvaluator(Evaluator):
  11     """Evaluates predictions for a set of scenes.
  12     """
  13 
  14     def __init__(self, class_map, output_uri):
  15         self.class_map = class_map
  16         self.output_uri = output_uri
  17 
  18     @abstractmethod
  19     def create_evaluation(self):
  20         pass
  21 
  22     def process(self, scenes, tmp_dir):
  23         evaluation = self.create_evaluation()
  24         for scene in scenes:
  25             log.info('Computing evaluation for scene {}...'.format(scene.id))
  26             label_source = scene.ground_truth_label_source
  27             label_store = scene.prediction_label_store
  28             with ActivateMixin.compose(label_source, label_store):
  29                 ground_truth = label_source.get_labels()
  30                 predictions = label_store.get_labels()
  31 
  32                 if scene.aoi_polygons:
  33                     # Filter labels based on AOI.
  34                     ground_truth = ground_truth.filter_by_aoi(
  35                         scene.aoi_polygons)
  36                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)
  37                 scene_evaluation = self.create_evaluation()
  38                 scene_evaluation.compute(ground_truth, predictions)
  39                 evaluation.merge(scene_evaluation)
  40 
  41         evaluation.save(self.output_uri)
  42
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/rastervision/evaluation/classification_evaluation.py b/rastervision/evaluation/classification_evaluation.py
  --- a/rastervision/evaluation/classification_evaluation.py
  +++ b/rastervision/evaluation/classification_evaluation.py
  @@ -1,4 +1,5 @@
   from abc import (ABC, abstractmethod)
  +import copy
   
   import json
   
  @@ -18,6 +19,7 @@
       def clear(self):
           """Clear the Evaluation."""
           self.class_to_eval_item = {}
  +        self.scene_to_eval = {}
           self.avg_item = None
   
       def set_class_to_eval_item(self, class_to_eval_item):
  @@ -36,6 +38,14 @@
           for eval_item in self.class_to_eval_item.values():
               json_rep.append(eval_item.to_json())
           json_rep.append(self.avg_item.to_json())
  +
  +        if self.scene_to_eval:
  +            json_rep = {'overall': json_rep}
  +            scene_to_eval_json = {}
  +            for scene_id, eval in self.scene_to_eval.items():
  +                scene_to_eval_json[scene_id] = eval.to_json()
  +            json_rep['per_scene'] = scene_to_eval_json
  +
           return json_rep
   
       def save(self, output_uri):
  @@ -47,7 +57,7 @@
           json_str = json.dumps(self.to_json(), indent=4)
           str_to_file(json_str, output_uri)
   
  -    def merge(self, evaluation):
  +    def merge(self, evaluation, scene_id=None):
           """Merge Evaluation for another Scene into this one.
   
           This is useful for computing the average metrics of a set of scenes.
  @@ -68,6 +78,9 @@
   
           self.compute_avg()
   
  +        if scene_id is not None:
  +            self.scene_to_eval[scene_id] = copy.deepcopy(evaluation)
  +
       def compute_avg(self):
           """Compute average metrics over all keys."""
           self.avg_item = ClassEvaluationItem(class_name='average')
  diff --git a/rastervision/evaluation/classification_evaluator.py b/rastervision/evaluation/classification_evaluator.py
  --- a/rastervision/evaluation/classification_evaluator.py
  +++ b/rastervision/evaluation/classification_evaluator.py
  @@ -36,6 +36,5 @@
                       predictions = predictions.filter_by_aoi(scene.aoi_polygons)
                   scene_evaluation = self.create_evaluation()
                   scene_evaluation.compute(ground_truth, predictions)
  -                evaluation.merge(scene_evaluation)
  -
  +                evaluation.merge(scene_evaluation, scene_id=scene.id)
           evaluation.save(self.output_uri)
  diff --git a/rastervision/evaluation/semantic_segmentation_evaluator.py b/rastervision/evaluation/semantic_segmentation_evaluator.py
  --- a/rastervision/evaluation/semantic_segmentation_evaluator.py
  +++ b/rastervision/evaluation/semantic_segmentation_evaluator.py
  @@ -36,7 +36,7 @@
                       predictions = predictions.filter_by_aoi(scene.aoi_polygons)
                   scene_evaluation = self.create_evaluation()
                   scene_evaluation.compute(ground_truth, predictions)
  -                evaluation.merge(scene_evaluation)
  +                evaluation.merge(scene_evaluation, scene_id=scene.id)
   
               if hasattr(label_source, 'source') and hasattr(
                       label_source.source, 'vector_source') and hasattr(
  @@ -52,6 +52,6 @@
                       scene_evaluation = self.create_evaluation()
                       scene_evaluation.compute_vector(
                           gt_geojson, pred_geojson_local, mode, class_id)
  -                    evaluation.merge(scene_evaluation)
  +                    evaluation.merge(scene_evaluation, scene_id=scene.id)
   
           evaluation.save(self.output_uri)
  
verification_info: {"golden_diff": "diff --git a/rastervision/evaluation/classification_evaluation.py b/rastervision/evaluation/classification_evaluation.py\n--- a/rastervision/evaluation/classification_evaluation.py\n+++ b/rastervision/evaluation/classification_evaluation.py\n@@ -1,4 +1,5 @@\n from abc import (ABC, abstractmethod)\n+import copy\n \n import json\n \n@@ -18,6 +19,7 @@\n     def clear(self):\n         \"\"\"Clear the Evaluation.\"\"\"\n         self.class_to_eval_item = {}\n+        self.scene_to_eval = {}\n         self.avg_item = None\n \n     def set_class_to_eval_item(self, class_to_eval_item):\n@@ -36,6 +38,14 @@\n         for eval_item in self.class_to_eval_item.values():\n             json_rep.append(eval_item.to_json())\n         json_rep.append(self.avg_item.to_json())\n+\n+        if self.scene_to_eval:\n+            json_rep = {'overall': json_rep}\n+            scene_to_eval_json = {}\n+            for scene_id, eval in self.scene_to_eval.items():\n+                scene_to_eval_json[scene_id] = eval.to_json()\n+            json_rep['per_scene'] = scene_to_eval_json\n+\n         return json_rep\n \n     def save(self, output_uri):\n@@ -47,7 +57,7 @@\n         json_str = json.dumps(self.to_json(), indent=4)\n         str_to_file(json_str, output_uri)\n \n-    def merge(self, evaluation):\n+    def merge(self, evaluation, scene_id=None):\n         \"\"\"Merge Evaluation for another Scene into this one.\n \n         This is useful for computing the average metrics of a set of scenes.\n@@ -68,6 +78,9 @@\n \n         self.compute_avg()\n \n+        if scene_id is not None:\n+            self.scene_to_eval[scene_id] = copy.deepcopy(evaluation)\n+\n     def compute_avg(self):\n         \"\"\"Compute average metrics over all keys.\"\"\"\n         self.avg_item = ClassEvaluationItem(class_name='average')\ndiff --git a/rastervision/evaluation/classification_evaluator.py b/rastervision/evaluation/classification_evaluator.py\n--- a/rastervision/evaluation/classification_evaluator.py\n+++ b/rastervision/evaluation/classification_evaluator.py\n@@ -36,6 +36,5 @@\n                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                 scene_evaluation = self.create_evaluation()\n                 scene_evaluation.compute(ground_truth, predictions)\n-                evaluation.merge(scene_evaluation)\n-\n+                evaluation.merge(scene_evaluation, scene_id=scene.id)\n         evaluation.save(self.output_uri)\ndiff --git a/rastervision/evaluation/semantic_segmentation_evaluator.py b/rastervision/evaluation/semantic_segmentation_evaluator.py\n--- a/rastervision/evaluation/semantic_segmentation_evaluator.py\n+++ b/rastervision/evaluation/semantic_segmentation_evaluator.py\n@@ -36,7 +36,7 @@\n                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                 scene_evaluation = self.create_evaluation()\n                 scene_evaluation.compute(ground_truth, predictions)\n-                evaluation.merge(scene_evaluation)\n+                evaluation.merge(scene_evaluation, scene_id=scene.id)\n \n             if hasattr(label_source, 'source') and hasattr(\n                     label_source.source, 'vector_source') and hasattr(\n@@ -52,6 +52,6 @@\n                     scene_evaluation = self.create_evaluation()\n                     scene_evaluation.compute_vector(\n                         gt_geojson, pred_geojson_local, mode, class_id)\n-                    evaluation.merge(scene_evaluation)\n+                    evaluation.merge(scene_evaluation, scene_id=scene.id)\n \n         evaluation.save(self.output_uri)\n", "issue": "Include per-scene metrics in eval.json\nIt would be useful to see metrics for each scene in addition to metrics averaged over all scenes. \n", "before_files": [{"content": "from abc import (ABC, abstractmethod)\n\nimport json\n\nfrom rastervision.evaluation import ClassEvaluationItem\nfrom rastervision.utils.files import str_to_file\n\n\nclass ClassificationEvaluation(ABC):\n    \"\"\"Base class for evaluating predictions for tasks that have classes.\n\n    Evaluations can be keyed, for instance, if evaluations happen per class.\n    \"\"\"\n\n    def __init__(self):\n        self.clear()\n\n    def clear(self):\n        \"\"\"Clear the Evaluation.\"\"\"\n        self.class_to_eval_item = {}\n        self.avg_item = None\n\n    def set_class_to_eval_item(self, class_to_eval_item):\n        self.class_to_eval_item = class_to_eval_item\n\n    def get_by_id(self, key):\n        \"\"\"Gets the evaluation for a particular EvaluationItem key\"\"\"\n        return self.class_to_eval_item[key]\n\n    def has_id(self, key):\n        \"\"\"Answers whether or not the EvaluationItem key is represented\"\"\"\n        return key in self.class_to_eval_item\n\n    def to_json(self):\n        json_rep = []\n        for eval_item in self.class_to_eval_item.values():\n            json_rep.append(eval_item.to_json())\n        json_rep.append(self.avg_item.to_json())\n        return json_rep\n\n    def save(self, output_uri):\n        \"\"\"Save this Evaluation to a file.\n\n        Args:\n            output_uri: string URI for the file to write.\n        \"\"\"\n        json_str = json.dumps(self.to_json(), indent=4)\n        str_to_file(json_str, output_uri)\n\n    def merge(self, evaluation):\n        \"\"\"Merge Evaluation for another Scene into this one.\n\n        This is useful for computing the average metrics of a set of scenes.\n        The results of the averaging are stored in this Evaluation.\n\n        Args:\n            evaluation: Evaluation to merge into this one\n        \"\"\"\n        if len(self.class_to_eval_item) == 0:\n            self.class_to_eval_item = evaluation.class_to_eval_item\n        else:\n            for key, other_eval_item in \\\n                    evaluation.class_to_eval_item.items():\n                if self.has_id(key):\n                    self.get_by_id(key).merge(other_eval_item)\n                else:\n                    self.class_to_eval_item[key] = other_eval_item\n\n        self.compute_avg()\n\n    def compute_avg(self):\n        \"\"\"Compute average metrics over all keys.\"\"\"\n        self.avg_item = ClassEvaluationItem(class_name='average')\n        for eval_item in self.class_to_eval_item.values():\n            self.avg_item.merge(eval_item)\n\n    @abstractmethod\n    def compute(self, ground_truth_labels, prediction_labels):\n        \"\"\"Compute metrics for a single scene.\n\n        Args:\n            ground_truth_labels: Ground Truth labels to evaluate against.\n            prediction_labels: The predicted labels to evaluate.\n        \"\"\"\n        pass\n", "path": "rastervision/evaluation/classification_evaluation.py"}, {"content": "import logging\n\nfrom rastervision.data import ActivateMixin\nfrom rastervision.rv_config import RVConfig\nfrom rastervision.utils.files import (download_if_needed)\nfrom rastervision.evaluation import (ClassificationEvaluator,\n                                     SemanticSegmentationEvaluation)\n\nlog = logging.getLogger(__name__)\n\n\nclass SemanticSegmentationEvaluator(ClassificationEvaluator):\n    \"\"\"Evaluates predictions for a set of scenes.\n    \"\"\"\n\n    def __init__(self, class_map, output_uri):\n        super().__init__(class_map, output_uri)\n\n    def create_evaluation(self):\n        return SemanticSegmentationEvaluation(self.class_map)\n\n    def process(self, scenes, tmp_dir):\n        evaluation = self.create_evaluation()\n        for scene in scenes:\n            log.info('Computing evaluation for scene {}...'.format(scene.id))\n            label_source = scene.ground_truth_label_source\n            label_store = scene.prediction_label_store\n            with ActivateMixin.compose(label_source, label_store):\n                ground_truth = label_source.get_labels()\n                predictions = label_store.get_labels()\n\n                if scene.aoi_polygons:\n                    # Filter labels based on AOI.\n                    ground_truth = ground_truth.filter_by_aoi(\n                        scene.aoi_polygons)\n                    predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                scene_evaluation = self.create_evaluation()\n                scene_evaluation.compute(ground_truth, predictions)\n                evaluation.merge(scene_evaluation)\n\n            if hasattr(label_source, 'source') and hasattr(\n                    label_source.source, 'vector_source') and hasattr(\n                        label_store, 'vector_output'):\n                tmp_dir = RVConfig.get_tmp_dir().name\n                gt_geojson = label_source.source.vector_source.get_geojson()\n                for vo in label_store.vector_output:\n                    pred_geojson = vo['uri']\n                    mode = vo['mode']\n                    class_id = vo['class_id']\n                    pred_geojson_local = download_if_needed(\n                        pred_geojson, tmp_dir)\n                    scene_evaluation = self.create_evaluation()\n                    scene_evaluation.compute_vector(\n                        gt_geojson, pred_geojson_local, mode, class_id)\n                    evaluation.merge(scene_evaluation)\n\n        evaluation.save(self.output_uri)\n", "path": "rastervision/evaluation/semantic_segmentation_evaluator.py"}, {"content": "from abc import (abstractmethod)\nimport logging\n\nfrom rastervision.evaluation import Evaluator\nfrom rastervision.data import ActivateMixin\n\nlog = logging.getLogger(__name__)\n\n\nclass ClassificationEvaluator(Evaluator):\n    \"\"\"Evaluates predictions for a set of scenes.\n    \"\"\"\n\n    def __init__(self, class_map, output_uri):\n        self.class_map = class_map\n        self.output_uri = output_uri\n\n    @abstractmethod\n    def create_evaluation(self):\n        pass\n\n    def process(self, scenes, tmp_dir):\n        evaluation = self.create_evaluation()\n        for scene in scenes:\n            log.info('Computing evaluation for scene {}...'.format(scene.id))\n            label_source = scene.ground_truth_label_source\n            label_store = scene.prediction_label_store\n            with ActivateMixin.compose(label_source, label_store):\n                ground_truth = label_source.get_labels()\n                predictions = label_store.get_labels()\n\n                if scene.aoi_polygons:\n                    # Filter labels based on AOI.\n                    ground_truth = ground_truth.filter_by_aoi(\n                        scene.aoi_polygons)\n                    predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                scene_evaluation = self.create_evaluation()\n                scene_evaluation.compute(ground_truth, predictions)\n                evaluation.merge(scene_evaluation)\n\n        evaluation.save(self.output_uri)\n", "path": "rastervision/evaluation/classification_evaluator.py"}], "after_files": [{"content": "from abc import (ABC, abstractmethod)\nimport copy\n\nimport json\n\nfrom rastervision.evaluation import ClassEvaluationItem\nfrom rastervision.utils.files import str_to_file\n\n\nclass ClassificationEvaluation(ABC):\n    \"\"\"Base class for evaluating predictions for tasks that have classes.\n\n    Evaluations can be keyed, for instance, if evaluations happen per class.\n    \"\"\"\n\n    def __init__(self):\n        self.clear()\n\n    def clear(self):\n        \"\"\"Clear the Evaluation.\"\"\"\n        self.class_to_eval_item = {}\n        self.scene_to_eval = {}\n        self.avg_item = None\n\n    def set_class_to_eval_item(self, class_to_eval_item):\n        self.class_to_eval_item = class_to_eval_item\n\n    def get_by_id(self, key):\n        \"\"\"Gets the evaluation for a particular EvaluationItem key\"\"\"\n        return self.class_to_eval_item[key]\n\n    def has_id(self, key):\n        \"\"\"Answers whether or not the EvaluationItem key is represented\"\"\"\n        return key in self.class_to_eval_item\n\n    def to_json(self):\n        json_rep = []\n        for eval_item in self.class_to_eval_item.values():\n            json_rep.append(eval_item.to_json())\n        json_rep.append(self.avg_item.to_json())\n\n        if self.scene_to_eval:\n            json_rep = {'overall': json_rep}\n            scene_to_eval_json = {}\n            for scene_id, eval in self.scene_to_eval.items():\n                scene_to_eval_json[scene_id] = eval.to_json()\n            json_rep['per_scene'] = scene_to_eval_json\n\n        return json_rep\n\n    def save(self, output_uri):\n        \"\"\"Save this Evaluation to a file.\n\n        Args:\n            output_uri: string URI for the file to write.\n        \"\"\"\n        json_str = json.dumps(self.to_json(), indent=4)\n        str_to_file(json_str, output_uri)\n\n    def merge(self, evaluation, scene_id=None):\n        \"\"\"Merge Evaluation for another Scene into this one.\n\n        This is useful for computing the average metrics of a set of scenes.\n        The results of the averaging are stored in this Evaluation.\n\n        Args:\n            evaluation: Evaluation to merge into this one\n        \"\"\"\n        if len(self.class_to_eval_item) == 0:\n            self.class_to_eval_item = evaluation.class_to_eval_item\n        else:\n            for key, other_eval_item in \\\n                    evaluation.class_to_eval_item.items():\n                if self.has_id(key):\n                    self.get_by_id(key).merge(other_eval_item)\n                else:\n                    self.class_to_eval_item[key] = other_eval_item\n\n        self.compute_avg()\n\n        if scene_id is not None:\n            self.scene_to_eval[scene_id] = copy.deepcopy(evaluation)\n\n    def compute_avg(self):\n        \"\"\"Compute average metrics over all keys.\"\"\"\n        self.avg_item = ClassEvaluationItem(class_name='average')\n        for eval_item in self.class_to_eval_item.values():\n            self.avg_item.merge(eval_item)\n\n    @abstractmethod\n    def compute(self, ground_truth_labels, prediction_labels):\n        \"\"\"Compute metrics for a single scene.\n\n        Args:\n            ground_truth_labels: Ground Truth labels to evaluate against.\n            prediction_labels: The predicted labels to evaluate.\n        \"\"\"\n        pass\n", "path": "rastervision/evaluation/classification_evaluation.py"}, {"content": "import logging\n\nfrom rastervision.data import ActivateMixin\nfrom rastervision.rv_config import RVConfig\nfrom rastervision.utils.files import (download_if_needed)\nfrom rastervision.evaluation import (ClassificationEvaluator,\n                                     SemanticSegmentationEvaluation)\n\nlog = logging.getLogger(__name__)\n\n\nclass SemanticSegmentationEvaluator(ClassificationEvaluator):\n    \"\"\"Evaluates predictions for a set of scenes.\n    \"\"\"\n\n    def __init__(self, class_map, output_uri):\n        super().__init__(class_map, output_uri)\n\n    def create_evaluation(self):\n        return SemanticSegmentationEvaluation(self.class_map)\n\n    def process(self, scenes, tmp_dir):\n        evaluation = self.create_evaluation()\n        for scene in scenes:\n            log.info('Computing evaluation for scene {}...'.format(scene.id))\n            label_source = scene.ground_truth_label_source\n            label_store = scene.prediction_label_store\n            with ActivateMixin.compose(label_source, label_store):\n                ground_truth = label_source.get_labels()\n                predictions = label_store.get_labels()\n\n                if scene.aoi_polygons:\n                    # Filter labels based on AOI.\n                    ground_truth = ground_truth.filter_by_aoi(\n                        scene.aoi_polygons)\n                    predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                scene_evaluation = self.create_evaluation()\n                scene_evaluation.compute(ground_truth, predictions)\n                evaluation.merge(scene_evaluation, scene_id=scene.id)\n\n            if hasattr(label_source, 'source') and hasattr(\n                    label_source.source, 'vector_source') and hasattr(\n                        label_store, 'vector_output'):\n                tmp_dir = RVConfig.get_tmp_dir().name\n                gt_geojson = label_source.source.vector_source.get_geojson()\n                for vo in label_store.vector_output:\n                    pred_geojson = vo['uri']\n                    mode = vo['mode']\n                    class_id = vo['class_id']\n                    pred_geojson_local = download_if_needed(\n                        pred_geojson, tmp_dir)\n                    scene_evaluation = self.create_evaluation()\n                    scene_evaluation.compute_vector(\n                        gt_geojson, pred_geojson_local, mode, class_id)\n                    evaluation.merge(scene_evaluation, scene_id=scene.id)\n\n        evaluation.save(self.output_uri)\n", "path": "rastervision/evaluation/semantic_segmentation_evaluator.py"}, {"content": "from abc import (abstractmethod)\nimport logging\n\nfrom rastervision.evaluation import Evaluator\nfrom rastervision.data import ActivateMixin\n\nlog = logging.getLogger(__name__)\n\n\nclass ClassificationEvaluator(Evaluator):\n    \"\"\"Evaluates predictions for a set of scenes.\n    \"\"\"\n\n    def __init__(self, class_map, output_uri):\n        self.class_map = class_map\n        self.output_uri = output_uri\n\n    @abstractmethod\n    def create_evaluation(self):\n        pass\n\n    def process(self, scenes, tmp_dir):\n        evaluation = self.create_evaluation()\n        for scene in scenes:\n            log.info('Computing evaluation for scene {}...'.format(scene.id))\n            label_source = scene.ground_truth_label_source\n            label_store = scene.prediction_label_store\n            with ActivateMixin.compose(label_source, label_store):\n                ground_truth = label_source.get_labels()\n                predictions = label_store.get_labels()\n\n                if scene.aoi_polygons:\n                    # Filter labels based on AOI.\n                    ground_truth = ground_truth.filter_by_aoi(\n                        scene.aoi_polygons)\n                    predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                scene_evaluation = self.create_evaluation()\n                scene_evaluation.compute(ground_truth, predictions)\n                evaluation.merge(scene_evaluation, scene_id=scene.id)\n        evaluation.save(self.output_uri)\n", "path": "rastervision/evaluation/classification_evaluator.py"}]}