We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Interpretation of the FuzzBench Reports
The available fuzzer configurations and the results might be confusing for someone who looks at a random FuzzBench report for the first time. To avoid misinterpretation of the results, maybe there could be
1. a description of the intent of the evaluation and/or an explanation of the evaluated fuzzer configurations, and
2. a distinction between "master/official/representative" and "developer" evaluations and fuzzer variants. 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `analysis/experiment_results.py`
Content:
```
1 # Copyright 2020 Google LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #      http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """ExperimentResults class."""
15 
16 import functools
17 import os
18 
19 from analysis import benchmark_results
20 from analysis import coverage_data_utils
21 from analysis import data_utils
22 from analysis import stat_tests
23 
24 
25 class ExperimentResults:  # pylint: disable=too-many-instance-attributes
26     """Provides the main interface for getting various analysis results and
27     plots about an experiment, represented by |experiment_df|.
28 
29     Can be used as the context of template based report generation. Each
30     result is a property, which is lazily computed and memorized when
31     needed multiple times. Therefore, when used as a context of a report
32     template, only the properties needed for the given report will be computed.
33     """
34 
35     def __init__(  # pylint: disable=too-many-arguments
36             self,
37             experiment_df,
38             coverage_dict,
39             output_directory,
40             plotter,
41             experiment_name=None):
42         if experiment_name:
43             self.name = experiment_name
44         else:
45             # Take name from first row.
46             self.name = experiment_df.experiment.iloc[0]
47 
48         # FuzzBench repo commit hash.
49         self.git_hash = None
50         if 'git_hash' in experiment_df.columns:
51             # Not possible to represent hashes for multiple experiments.
52             if len(experiment_df.experiment.unique()) == 1:
53                 self.git_hash = experiment_df.git_hash.iloc[0]
54 
55         # Earliest trial start time.
56         self.started = experiment_df.time_started.dropna().min()
57         # Latest trial end time.
58         self.ended = experiment_df.time_ended.dropna().max()
59 
60         # Keep data frame without non-interesting columns.
61         self._experiment_df = data_utils.drop_uninteresting_columns(
62             experiment_df)
63 
64         # Directory where the rendered plots are written to.
65         self._output_directory = output_directory
66 
67         self._plotter = plotter
68 
69         # Dictionary to store the full coverage data.
70         self._coverage_dict = coverage_dict
71 
72     def _get_full_path(self, filename):
73         return os.path.join(self._output_directory, filename)
74 
75     @property
76     @functools.lru_cache()
77     # TODO(lszekeres): With python3.8+, replace above two decorators with:
78     # @functools.cached_property
79     def _experiment_snapshots_df(self):
80         """Data frame containing only the time snapshots, for each benchmark,
81         based on which we do further analysis, i.e., statistical tests and
82         ranking."""
83         return data_utils.get_experiment_snapshots(self._experiment_df)
84 
85     @property
86     @functools.lru_cache()
87     def benchmarks(self):
88         """Returns the list of BenchmarkResults.
89 
90         This is cheap as no computation is done on the benchmark data,
91         until a property is evaluated.
92         """
93         benchmark_names = self._experiment_df.benchmark.unique()
94         return [
95             benchmark_results.BenchmarkResults(name, self._experiment_df,
96                                                self._coverage_dict,
97                                                self._output_directory,
98                                                self._plotter)
99             for name in sorted(benchmark_names)
100         ]
101 
102     @property
103     @functools.lru_cache()
104     def summary_table(self):
105         """A pivot table of medians for each fuzzer on each benchmark."""
106         return data_utils.experiment_pivot_table(
107             self._experiment_snapshots_df, data_utils.benchmark_rank_by_median)
108 
109     @property
110     def rank_by_unique_coverage_average_normalized_score(self):
111         """Rank fuzzers using average normalized score on unique coverage across
112         benchmarks."""
113         benchmarks_unique_coverage_list = [
114             benchmark.unique_region_cov_df for benchmark in self.benchmarks
115         ]
116         return coverage_data_utils.rank_by_average_normalized_score(
117             benchmarks_unique_coverage_list)
118 
119     @property
120     def rank_by_average_rank_and_average_rank(self):
121         """Rank fuzzers using average rank per benchmark and average rank
122         across benchmarks."""
123         return data_utils.experiment_level_ranking(
124             self._experiment_snapshots_df,
125             data_utils.benchmark_rank_by_average_rank,
126             data_utils.experiment_rank_by_average_rank)
127 
128     @property
129     def rank_by_mean_and_average_rank(self):
130         """Rank fuzzers using mean coverage per benchmark and average rank
131         across benchmarks."""
132         return data_utils.experiment_level_ranking(
133             self._experiment_snapshots_df, data_utils.benchmark_rank_by_mean,
134             data_utils.experiment_rank_by_average_rank)
135 
136     @property
137     def rank_by_median_and_average_rank(self):
138         """Rank fuzzers using median coverage per benchmark and average rank
139         across benchmarks."""
140         return data_utils.experiment_level_ranking(
141             self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,
142             data_utils.experiment_rank_by_average_rank)
143 
144     @property
145     def rank_by_median_and_average_normalized_score(self):
146         """Rank fuzzers using median coverage per benchmark and average
147         normalized score across benchmarks."""
148         return data_utils.experiment_level_ranking(
149             self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,
150             data_utils.experiment_rank_by_average_normalized_score)
151 
152     @property
153     def rank_by_median_and_number_of_firsts(self):
154         """Rank fuzzers using median coverage per benchmark and number of first
155         places across benchmarks."""
156         return data_utils.experiment_level_ranking(
157             self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,
158             data_utils.experiment_rank_by_num_firsts)
159 
160     @property
161     def rank_by_stat_test_wins_and_average_rank(self):
162         """Rank fuzzers using statistical test wins per benchmark and average
163         rank across benchmarks."""
164         return data_utils.experiment_level_ranking(
165             self._experiment_snapshots_df,
166             data_utils.benchmark_rank_by_stat_test_wins,
167             data_utils.experiment_rank_by_average_rank)
168 
169     @property
170     def friedman_p_value(self):
171         """Friedman test result."""
172         return stat_tests.friedman_test(self.summary_table)
173 
174     @property
175     @functools.lru_cache()
176     def friedman_posthoc_p_values(self):
177         """Friedman posthoc test results."""
178         return stat_tests.friedman_posthoc_tests(self.summary_table)
179 
180     @property
181     def friedman_conover_plot(self):
182         """Friedman/Conover posthoc test result plot."""
183         plot_filename = 'experiment_friedman_conover_plot.svg'
184         self._plotter.write_heatmap_plot(
185             self.friedman_posthoc_p_values['conover'],
186             self._get_full_path(plot_filename),
187             symmetric=True)
188         return plot_filename
189 
190     @property
191     def friedman_nemenyi_plot(self):
192         """Friedman/Nemenyi posthoc test result plot."""
193         plot_filename = 'experiment_friedman_nemenyi_plot.svg'
194         self._plotter.write_heatmap_plot(
195             self.friedman_posthoc_p_values['nemenyi'],
196             self._get_full_path(plot_filename),
197             symmetric=True)
198         return plot_filename
199 
200     @property
201     def critical_difference_plot(self):
202         """Critical difference diagram.
203 
204         Represents average ranks of fuzzers across all benchmarks,
205         considering medians on final coverage.
206         """
207         average_ranks = self.rank_by_median_and_average_rank
208         num_of_benchmarks = self.summary_table.shape[0]
209 
210         plot_filename = 'experiment_critical_difference_plot.svg'
211         self._plotter.write_critical_difference_plot(
212             average_ranks, num_of_benchmarks,
213             self._get_full_path(plot_filename))
214         return plot_filename
215
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/analysis/experiment_results.py b/analysis/experiment_results.py
--- a/analysis/experiment_results.py
+++ b/analysis/experiment_results.py
@@ -72,6 +72,19 @@
     def _get_full_path(self, filename):
         return os.path.join(self._output_directory, filename)
 
+    def linkify_names(self, df):
+        """For any DataFrame which is indexed by fuzzer names, turns the fuzzer
+        names into links to their directory with a description on GitHub."""
+        assert df.index.name == 'fuzzer'
+
+        def description_link(commit, fuzzer):
+            return (f'<a href="https://github.com/google/fuzzbench/blob/'
+                    f'{commit}/fuzzers/{fuzzer}">{fuzzer}</a>')
+
+        commit = self.git_hash if self.git_hash else 'master'
+        df.index = df.index.map(lambda fuzzer: description_link(commit, fuzzer))
+        return df
+
     @property
     @functools.lru_cache()
     # TODO(lszekeres): With python3.8+, replace above two decorators with:

{"golden_diff": "diff --git a/analysis/experiment_results.py b/analysis/experiment_results.py\n--- a/analysis/experiment_results.py\n+++ b/analysis/experiment_results.py\n@@ -72,6 +72,19 @@\n     def _get_full_path(self, filename):\n         return os.path.join(self._output_directory, filename)\n \n+    def linkify_names(self, df):\n+        \"\"\"For any DataFrame which is indexed by fuzzer names, turns the fuzzer\n+        names into links to their directory with a description on GitHub.\"\"\"\n+        assert df.index.name == 'fuzzer'\n+\n+        def description_link(commit, fuzzer):\n+            return (f'<a href=\"https://github.com/google/fuzzbench/blob/'\n+                    f'{commit}/fuzzers/{fuzzer}\">{fuzzer}</a>')\n+\n+        commit = self.git_hash if self.git_hash else 'master'\n+        df.index = df.index.map(lambda fuzzer: description_link(commit, fuzzer))\n+        return df\n+\n     @property\n     @functools.lru_cache()\n     # TODO(lszekeres): With python3.8+, replace above two decorators with:\n", "issue": "Interpretation of the FuzzBench Reports\nThe available fuzzer configurations and the results might be confusing for someone who looks at a random FuzzBench report for the first time. To avoid misinterpretation of the results, maybe there could be\r\n1. a description of the intent of the evaluation and/or an explanation of the evaluated fuzzer configurations, and\r\n2. a distinction between \"master/official/representative\" and \"developer\" evaluations and fuzzer variants. \n", "before_files": [{"content": "# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"ExperimentResults class.\"\"\"\n\nimport functools\nimport os\n\nfrom analysis import benchmark_results\nfrom analysis import coverage_data_utils\nfrom analysis import data_utils\nfrom analysis import stat_tests\n\n\nclass ExperimentResults:  # pylint: disable=too-many-instance-attributes\n    \"\"\"Provides the main interface for getting various analysis results and\n    plots about an experiment, represented by |experiment_df|.\n\n    Can be used as the context of template based report generation. Each\n    result is a property, which is lazily computed and memorized when\n    needed multiple times. Therefore, when used as a context of a report\n    template, only the properties needed for the given report will be computed.\n    \"\"\"\n\n    def __init__(  # pylint: disable=too-many-arguments\n            self,\n            experiment_df,\n            coverage_dict,\n            output_directory,\n            plotter,\n            experiment_name=None):\n        if experiment_name:\n            self.name = experiment_name\n        else:\n            # Take name from first row.\n            self.name = experiment_df.experiment.iloc[0]\n\n        # FuzzBench repo commit hash.\n        self.git_hash = None\n        if 'git_hash' in experiment_df.columns:\n            # Not possible to represent hashes for multiple experiments.\n            if len(experiment_df.experiment.unique()) == 1:\n                self.git_hash = experiment_df.git_hash.iloc[0]\n\n        # Earliest trial start time.\n        self.started = experiment_df.time_started.dropna().min()\n        # Latest trial end time.\n        self.ended = experiment_df.time_ended.dropna().max()\n\n        # Keep data frame without non-interesting columns.\n        self._experiment_df = data_utils.drop_uninteresting_columns(\n            experiment_df)\n\n        # Directory where the rendered plots are written to.\n        self._output_directory = output_directory\n\n        self._plotter = plotter\n\n        # Dictionary to store the full coverage data.\n        self._coverage_dict = coverage_dict\n\n    def _get_full_path(self, filename):\n        return os.path.join(self._output_directory, filename)\n\n    @property\n    @functools.lru_cache()\n    # TODO(lszekeres): With python3.8+, replace above two decorators with:\n    # @functools.cached_property\n    def _experiment_snapshots_df(self):\n        \"\"\"Data frame containing only the time snapshots, for each benchmark,\n        based on which we do further analysis, i.e., statistical tests and\n        ranking.\"\"\"\n        return data_utils.get_experiment_snapshots(self._experiment_df)\n\n    @property\n    @functools.lru_cache()\n    def benchmarks(self):\n        \"\"\"Returns the list of BenchmarkResults.\n\n        This is cheap as no computation is done on the benchmark data,\n        until a property is evaluated.\n        \"\"\"\n        benchmark_names = self._experiment_df.benchmark.unique()\n        return [\n            benchmark_results.BenchmarkResults(name, self._experiment_df,\n                                               self._coverage_dict,\n                                               self._output_directory,\n                                               self._plotter)\n            for name in sorted(benchmark_names)\n        ]\n\n    @property\n    @functools.lru_cache()\n    def summary_table(self):\n        \"\"\"A pivot table of medians for each fuzzer on each benchmark.\"\"\"\n        return data_utils.experiment_pivot_table(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_median)\n\n    @property\n    def rank_by_unique_coverage_average_normalized_score(self):\n        \"\"\"Rank fuzzers using average normalized score on unique coverage across\n        benchmarks.\"\"\"\n        benchmarks_unique_coverage_list = [\n            benchmark.unique_region_cov_df for benchmark in self.benchmarks\n        ]\n        return coverage_data_utils.rank_by_average_normalized_score(\n            benchmarks_unique_coverage_list)\n\n    @property\n    def rank_by_average_rank_and_average_rank(self):\n        \"\"\"Rank fuzzers using average rank per benchmark and average rank\n        across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df,\n            data_utils.benchmark_rank_by_average_rank,\n            data_utils.experiment_rank_by_average_rank)\n\n    @property\n    def rank_by_mean_and_average_rank(self):\n        \"\"\"Rank fuzzers using mean coverage per benchmark and average rank\n        across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_mean,\n            data_utils.experiment_rank_by_average_rank)\n\n    @property\n    def rank_by_median_and_average_rank(self):\n        \"\"\"Rank fuzzers using median coverage per benchmark and average rank\n        across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,\n            data_utils.experiment_rank_by_average_rank)\n\n    @property\n    def rank_by_median_and_average_normalized_score(self):\n        \"\"\"Rank fuzzers using median coverage per benchmark and average\n        normalized score across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,\n            data_utils.experiment_rank_by_average_normalized_score)\n\n    @property\n    def rank_by_median_and_number_of_firsts(self):\n        \"\"\"Rank fuzzers using median coverage per benchmark and number of first\n        places across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,\n            data_utils.experiment_rank_by_num_firsts)\n\n    @property\n    def rank_by_stat_test_wins_and_average_rank(self):\n        \"\"\"Rank fuzzers using statistical test wins per benchmark and average\n        rank across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df,\n            data_utils.benchmark_rank_by_stat_test_wins,\n            data_utils.experiment_rank_by_average_rank)\n\n    @property\n    def friedman_p_value(self):\n        \"\"\"Friedman test result.\"\"\"\n        return stat_tests.friedman_test(self.summary_table)\n\n    @property\n    @functools.lru_cache()\n    def friedman_posthoc_p_values(self):\n        \"\"\"Friedman posthoc test results.\"\"\"\n        return stat_tests.friedman_posthoc_tests(self.summary_table)\n\n    @property\n    def friedman_conover_plot(self):\n        \"\"\"Friedman/Conover posthoc test result plot.\"\"\"\n        plot_filename = 'experiment_friedman_conover_plot.svg'\n        self._plotter.write_heatmap_plot(\n            self.friedman_posthoc_p_values['conover'],\n            self._get_full_path(plot_filename),\n            symmetric=True)\n        return plot_filename\n\n    @property\n    def friedman_nemenyi_plot(self):\n        \"\"\"Friedman/Nemenyi posthoc test result plot.\"\"\"\n        plot_filename = 'experiment_friedman_nemenyi_plot.svg'\n        self._plotter.write_heatmap_plot(\n            self.friedman_posthoc_p_values['nemenyi'],\n            self._get_full_path(plot_filename),\n            symmetric=True)\n        return plot_filename\n\n    @property\n    def critical_difference_plot(self):\n        \"\"\"Critical difference diagram.\n\n        Represents average ranks of fuzzers across all benchmarks,\n        considering medians on final coverage.\n        \"\"\"\n        average_ranks = self.rank_by_median_and_average_rank\n        num_of_benchmarks = self.summary_table.shape[0]\n\n        plot_filename = 'experiment_critical_difference_plot.svg'\n        self._plotter.write_critical_difference_plot(\n            average_ranks, num_of_benchmarks,\n            self._get_full_path(plot_filename))\n        return plot_filename\n", "path": "analysis/experiment_results.py"}], "after_files": [{"content": "# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"ExperimentResults class.\"\"\"\n\nimport functools\nimport os\n\nfrom analysis import benchmark_results\nfrom analysis import coverage_data_utils\nfrom analysis import data_utils\nfrom analysis import stat_tests\n\n\nclass ExperimentResults:  # pylint: disable=too-many-instance-attributes\n    \"\"\"Provides the main interface for getting various analysis results and\n    plots about an experiment, represented by |experiment_df|.\n\n    Can be used as the context of template based report generation. Each\n    result is a property, which is lazily computed and memorized when\n    needed multiple times. Therefore, when used as a context of a report\n    template, only the properties needed for the given report will be computed.\n    \"\"\"\n\n    def __init__(  # pylint: disable=too-many-arguments\n            self,\n            experiment_df,\n            coverage_dict,\n            output_directory,\n            plotter,\n            experiment_name=None):\n        if experiment_name:\n            self.name = experiment_name\n        else:\n            # Take name from first row.\n            self.name = experiment_df.experiment.iloc[0]\n\n        # FuzzBench repo commit hash.\n        self.git_hash = None\n        if 'git_hash' in experiment_df.columns:\n            # Not possible to represent hashes for multiple experiments.\n            if len(experiment_df.experiment.unique()) == 1:\n                self.git_hash = experiment_df.git_hash.iloc[0]\n\n        # Earliest trial start time.\n        self.started = experiment_df.time_started.dropna().min()\n        # Latest trial end time.\n        self.ended = experiment_df.time_ended.dropna().max()\n\n        # Keep data frame without non-interesting columns.\n        self._experiment_df = data_utils.drop_uninteresting_columns(\n            experiment_df)\n\n        # Directory where the rendered plots are written to.\n        self._output_directory = output_directory\n\n        self._plotter = plotter\n\n        # Dictionary to store the full coverage data.\n        self._coverage_dict = coverage_dict\n\n    def _get_full_path(self, filename):\n        return os.path.join(self._output_directory, filename)\n\n    def linkify_names(self, df):\n        \"\"\"For any DataFrame which is indexed by fuzzer names, turns the fuzzer\n        names into links to their directory with a description on GitHub.\"\"\"\n        assert df.index.name == 'fuzzer'\n\n        def description_link(commit, fuzzer):\n            return (f'<a href=\"https://github.com/google/fuzzbench/blob/'\n                    f'{commit}/fuzzers/{fuzzer}\">{fuzzer}</a>')\n\n        commit = self.git_hash if self.git_hash else 'master'\n        df.index = df.index.map(lambda fuzzer: description_link(commit, fuzzer))\n        return df\n\n    @property\n    @functools.lru_cache()\n    # TODO(lszekeres): With python3.8+, replace above two decorators with:\n    # @functools.cached_property\n    def _experiment_snapshots_df(self):\n        \"\"\"Data frame containing only the time snapshots, for each benchmark,\n        based on which we do further analysis, i.e., statistical tests and\n        ranking.\"\"\"\n        return data_utils.get_experiment_snapshots(self._experiment_df)\n\n    @property\n    @functools.lru_cache()\n    def benchmarks(self):\n        \"\"\"Returns the list of BenchmarkResults.\n\n        This is cheap as no computation is done on the benchmark data,\n        until a property is evaluated.\n        \"\"\"\n        benchmark_names = self._experiment_df.benchmark.unique()\n        return [\n            benchmark_results.BenchmarkResults(name, self._experiment_df,\n                                               self._coverage_dict,\n                                               self._output_directory,\n                                               self._plotter)\n            for name in sorted(benchmark_names)\n        ]\n\n    @property\n    @functools.lru_cache()\n    def summary_table(self):\n        \"\"\"A pivot table of medians for each fuzzer on each benchmark.\"\"\"\n        return data_utils.experiment_pivot_table(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_median)\n\n    @property\n    def rank_by_unique_coverage_average_normalized_score(self):\n        \"\"\"Rank fuzzers using average normalized score on unique coverage across\n        benchmarks.\"\"\"\n        benchmarks_unique_coverage_list = [\n            benchmark.unique_region_cov_df for benchmark in self.benchmarks\n        ]\n        return coverage_data_utils.rank_by_average_normalized_score(\n            benchmarks_unique_coverage_list)\n\n    @property\n    def rank_by_average_rank_and_average_rank(self):\n        \"\"\"Rank fuzzers using average rank per benchmark and average rank\n        across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df,\n            data_utils.benchmark_rank_by_average_rank,\n            data_utils.experiment_rank_by_average_rank)\n\n    @property\n    def rank_by_mean_and_average_rank(self):\n        \"\"\"Rank fuzzers using mean coverage per benchmark and average rank\n        across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_mean,\n            data_utils.experiment_rank_by_average_rank)\n\n    @property\n    def rank_by_median_and_average_rank(self):\n        \"\"\"Rank fuzzers using median coverage per benchmark and average rank\n        across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,\n            data_utils.experiment_rank_by_average_rank)\n\n    @property\n    def rank_by_median_and_average_normalized_score(self):\n        \"\"\"Rank fuzzers using median coverage per benchmark and average\n        normalized score across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,\n            data_utils.experiment_rank_by_average_normalized_score)\n\n    @property\n    def rank_by_median_and_number_of_firsts(self):\n        \"\"\"Rank fuzzers using median coverage per benchmark and number of first\n        places across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,\n            data_utils.experiment_rank_by_num_firsts)\n\n    @property\n    def rank_by_stat_test_wins_and_average_rank(self):\n        \"\"\"Rank fuzzers using statistical test wins per benchmark and average\n        rank across benchmarks.\"\"\"\n        return data_utils.experiment_level_ranking(\n            self._experiment_snapshots_df,\n            data_utils.benchmark_rank_by_stat_test_wins,\n            data_utils.experiment_rank_by_average_rank)\n\n    @property\n    def friedman_p_value(self):\n        \"\"\"Friedman test result.\"\"\"\n        return stat_tests.friedman_test(self.summary_table)\n\n    @property\n    @functools.lru_cache()\n    def friedman_posthoc_p_values(self):\n        \"\"\"Friedman posthoc test results.\"\"\"\n        return stat_tests.friedman_posthoc_tests(self.summary_table)\n\n    @property\n    def friedman_conover_plot(self):\n        \"\"\"Friedman/Conover posthoc test result plot.\"\"\"\n        plot_filename = 'experiment_friedman_conover_plot.svg'\n        self._plotter.write_heatmap_plot(\n            self.friedman_posthoc_p_values['conover'],\n            self._get_full_path(plot_filename),\n            symmetric=True)\n        return plot_filename\n\n    @property\n    def friedman_nemenyi_plot(self):\n        \"\"\"Friedman/Nemenyi posthoc test result plot.\"\"\"\n        plot_filename = 'experiment_friedman_nemenyi_plot.svg'\n        self._plotter.write_heatmap_plot(\n            self.friedman_posthoc_p_values['nemenyi'],\n            self._get_full_path(plot_filename),\n            symmetric=True)\n        return plot_filename\n\n    @property\n    def critical_difference_plot(self):\n        \"\"\"Critical difference diagram.\n\n        Represents average ranks of fuzzers across all benchmarks,\n        considering medians on final coverage.\n        \"\"\"\n        average_ranks = self.rank_by_median_and_average_rank\n        num_of_benchmarks = self.summary_table.shape[0]\n\n        plot_filename = 'experiment_critical_difference_plot.svg'\n        self._plotter.write_critical_difference_plot(\n            average_ranks, num_of_benchmarks,\n            self._get_full_path(plot_filename))\n        return plot_filename\n", "path": "analysis/experiment_results.py"}]}