We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Dynamic report generation
Though we already provide things like ipython notebooks to generate reports, I think we should lower the bar so that people can modify the report from the experiment from the report page.
I can think of two ways to do this:
1. Allow dynamic report generation using some kind of webapp.
2. Having a link to an ipython notebook that uses the data for the report on each report page.

Option 1 is a bit of a challenge because we generate reports server side and they take a while to do.
Option 2 is not ideal because I assume there is startup time for ipython notebooks and because it puts a greater burden on users.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `analysis/data_utils.py`
Content:
```
1 # Copyright 2020 Google LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #      http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """Utility functions for data (frame) transformations."""
15 from analysis import stat_tests
16 from common import environment
17 
18 
19 class EmptyDataError(ValueError):
20     """An exception for when the data is empty."""
21 
22 
23 def validate_data(experiment_df):
24     """Checks if the experiment data is valid."""
25     if experiment_df.empty:
26         raise EmptyDataError('Empty experiment data.')
27 
28     expected_columns = {
29         'experiment', 'benchmark', 'fuzzer', 'trial_id', 'time_started',
30         'time_ended', 'time', 'edges_covered'
31     }
32     missing_columns = expected_columns.difference(experiment_df.columns)
33     if missing_columns:
34         raise ValueError(
35             'Missing columns in experiment data: {}'.format(missing_columns))
36 
37 
38 def drop_uninteresting_columns(experiment_df):
39     """Returns table with only interesting columns."""
40     return experiment_df[[
41         'benchmark', 'fuzzer', 'trial_id', 'time', 'edges_covered'
42     ]]
43 
44 
45 def clobber_experiments_data(df, experiments):
46     """Clobber experiment data that is part of lower priority (generally
47     earlier) versions of the same trials in |df|. For example in experiment-1 we
48     may test fuzzer-a on benchmark-1. In experiment-2 we may again test fuzzer-a
49     on benchmark-1 because fuzzer-a was updated. This function will remove the
50     snapshots from fuzzer-a,benchmark-1,experiment-1 from |df| because we want
51     the report to only contain the up-to-date data. Experiment priority is
52     determined by order of each experiment in |experiments| with the highest
53     priority experiment coming last in that list."""
54     # We don't call |df| "experiment_df" because it is a misnomer and leads to
55     # confusion in this case where it contains data from multiple experiments.
56 
57     # Include everything from the last experiment.
58     experiments = experiments.copy()  # Copy so we dont mutate experiments.
59     experiments.reverse()
60     highest_rank_experiment = experiments[0]
61     result = df[df.experiment == highest_rank_experiment]
62 
63     for experiment in experiments[1:]:
64         # Include data for not yet covered benchmark/fuzzer pairs.
65         covered_pairs = result[['benchmark', 'fuzzer']].drop_duplicates()
66         covered_pairs = covered_pairs.apply(tuple, axis=1)
67         experiment_data = df[df.experiment == experiment]
68         experiment_pairs = experiment_data[['benchmark',
69                                             'fuzzer']].apply(tuple, axis=1)
70         to_include = experiment_data[~experiment_pairs.isin(covered_pairs)]
71         result = result.append(to_include)
72     return result
73 
74 
75 def filter_fuzzers(experiment_df, included_fuzzers):
76     """Returns table with only rows where fuzzer is in |included_fuzzers|."""
77     return experiment_df[experiment_df['fuzzer'].isin(included_fuzzers)]
78 
79 
80 def filter_benchmarks(experiment_df, included_benchmarks):
81     """Returns table with only rows where benchmark is in
82     |included_benchmarks|."""
83     return experiment_df[experiment_df['benchmark'].isin(included_benchmarks)]
84 
85 
86 def label_fuzzers_by_experiment(experiment_df):
87     """Returns table where every fuzzer is labeled by the experiment it
88     was run in."""
89     experiment_df['fuzzer'] = (experiment_df['fuzzer'] + '-' +
90                                experiment_df['experiment'])
91 
92     return experiment_df
93 
94 
95 def filter_max_time(experiment_df, max_time):
96     """Returns table with snapshots that have time less than or equal to
97     |max_time|."""
98     return experiment_df[experiment_df['time'] <= max_time]
99 
100 
101 # Creating "snapshots" (see README.md for definition).
102 
103 _DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD = 1.0
104 
105 
106 def get_benchmark_snapshot(benchmark_df,
107                            threshold=_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD):
108     """Finds the latest time where all trials were still running
109     (or |threshold| * trials were still running). In most cases, this is the end
110     of the experiment. However, if some trials (or more than (1 - |threshold|) *
111     trials) ended earlier, we will use an earlier "snapshot" time for comparing
112     results.
113 
114     Returns data frame that only contains the measurements of the picked
115     snapshot time.
116     """
117     # Allow overriding threshold with environment variable as well.
118     threshold = environment.get('BENCHMARK_SAMPLE_NUM_THRESHOLD', threshold)
119 
120     num_trials = benchmark_df.trial_id.nunique()
121     trials_running_at_time = benchmark_df.time.value_counts()
122     criteria = trials_running_at_time >= threshold * num_trials
123     ok_times = trials_running_at_time[criteria]
124     latest_ok_time = ok_times.index.max()
125     benchmark_snapshot_df = benchmark_df[benchmark_df.time == latest_ok_time]
126     return benchmark_snapshot_df
127 
128 
129 _DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD = 0.8
130 
131 
132 def get_fuzzers_with_not_enough_samples(
133         benchmark_snapshot_df, threshold=_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD):
134     """Returns fuzzers that didn't have enough trials running at snapshot time.
135     It takes a benchmark snapshot and finds the fuzzers that have a sample size
136     smaller than 80% of the largest sample size. Default threshold can be
137     overridden.
138     """
139     # Allow overriding threshold with environment variable as well.
140     threshold = environment.get('FUZZER_SAMPLE_NUM_THRESHOLD', threshold)
141 
142     samples_per_fuzzer = benchmark_snapshot_df.fuzzer.value_counts()
143     max_samples = samples_per_fuzzer.max()
144     few_sample_criteria = samples_per_fuzzer < threshold * max_samples
145     few_sample_fuzzers = samples_per_fuzzer[few_sample_criteria].index
146     return few_sample_fuzzers.tolist()
147 
148 
149 def get_experiment_snapshots(experiment_df):
150     """Finds a good snapshot time for each benchmark in the experiment data.
151 
152     Returns the data frame that only contains the measurements made at these
153     snapshot times.
154     """
155     benchmark_groups = experiment_df.groupby('benchmark')
156     experiment_snapshots = benchmark_groups.apply(get_benchmark_snapshot)
157     # We don't need the extra index added by the groupby('benchmark').
158     experiment_snapshots.reset_index(drop=True, inplace=True)
159     return experiment_snapshots
160 
161 
162 # Summary tables containing statistics on the samples.
163 
164 
165 def benchmark_summary(benchmark_snapshot_df):
166     """Creates summary table for a benchmark snapshot with columns:
167     |fuzzer|time||count|mean|std|min|25%|median|75%|max|
168     """
169     groups = benchmark_snapshot_df.groupby(['fuzzer', 'time'])
170     summary = groups['edges_covered'].describe()
171     summary.rename(columns={'50%': 'median'}, inplace=True)
172     return summary.sort_values(('median'), ascending=False)
173 
174 
175 def experiment_summary(experiment_snapshots_df):
176     """Creates summary table for all benchmarks in experiment, i.e. table like:
177     |benchmark|| < benchmark level summary >
178     """
179     groups = experiment_snapshots_df.groupby('benchmark')
180     summaries = groups.apply(benchmark_summary)
181     return summaries
182 
183 
184 # Per-benchmark fuzzer ranking options.
185 
186 
187 def benchmark_rank_by_mean(benchmark_snapshot_df):
188     """Returns ranking of fuzzers based on mean coverage."""
189     assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
190     means = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].mean()
191     means.rename('mean cov', inplace=True)
192     return means.sort_values(ascending=False)
193 
194 
195 def benchmark_rank_by_median(benchmark_snapshot_df):
196     """Returns ranking of fuzzers based on median coverage."""
197     assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
198     medians = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].median()
199     medians.rename('median cov', inplace=True)
200     return medians.sort_values(ascending=False)
201 
202 
203 def benchmark_rank_by_average_rank(benchmark_snapshot_df):
204     """Ranks all coverage measurements in the snapshot across fuzzers.
205 
206     Returns the average rank by fuzzer.
207     """
208     # Make a copy of the dataframe view, because we want to add a new column.
209     measurements = benchmark_snapshot_df[['fuzzer', 'edges_covered']].copy()
210     measurements['rank'] = measurements['edges_covered'].rank()
211     avg_rank = measurements.groupby('fuzzer').mean()
212     avg_rank.rename(columns={'rank': 'avg rank'}, inplace=True)
213     avg_rank.sort_values('avg rank', ascending=False, inplace=True)
214     return avg_rank['avg rank']
215 
216 
217 def benchmark_rank_by_stat_test_wins(benchmark_snapshot_df):
218     """Carries out one-tailed statistical tests for each fuzzer pair.
219 
220     Returns ranking according to the number of statistical test wins.
221     """
222     p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)
223 
224     # Turn "significant" p-values into 1-s.
225     better_than = p_values.applymap(
226         lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)
227     better_than = better_than.applymap(int)
228 
229     score = better_than.sum(axis=1).sort_values(ascending=False)
230     score.rename('stat wins', inplace=True)
231 
232     return score
233 
234 
235 def create_better_than_table(benchmark_snapshot_df):
236     """Creates table showing whether fuzzer in row is statistically
237     significantly better than the fuzzer in the column."""
238     p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)
239 
240     # Turn "significant" p-values into 1-s.
241     better_than = p_values.applymap(
242         lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)
243     better_than = better_than.applymap(int)
244 
245     # Order rows and columns of matrix according to score ranking.
246     score = better_than.sum(axis=1).sort_values(ascending=False)
247     better_than = better_than.reindex(index=score.index,
248                                       columns=score.index[::-1])
249     return better_than
250 
251 
252 # Experiment level ranking of fuzzers (across-benchmarks).
253 # Experiment level ranking depends on the per-benchmark ranking method.
254 
255 
256 def experiment_pivot_table(experiment_snapshots_df,
257                            benchmark_level_ranking_function):
258     """Creates a pivot table according to a given per benchmark ranking, where
259     the columns are the fuzzers, the rows are the benchmarks, and the values
260     are the scores according to the per benchmark ranking."""
261     benchmark_blocks = experiment_snapshots_df.groupby('benchmark')
262     groups_ranked = benchmark_blocks.apply(benchmark_level_ranking_function)
263     already_unstacked = groups_ranked.index.names == ['benchmark']
264     pivot_df = groups_ranked if already_unstacked else groups_ranked.unstack()
265     return pivot_df
266 
267 
268 def experiment_rank_by_average_rank(experiment_pivot_df):
269     """Creates experiment level ranking of fuzzers.
270 
271     Takes a pivot table representing per benchmark ranking scores. Ranks
272     fuzzers per benchmark, then takes the average rank across benchmarks
273     (smaller is better).
274     """
275     # Rank fuzzers in each benchmark block.
276     pivot_ranked = experiment_pivot_df.rank('columns',
277                                             na_option='keep',
278                                             ascending=False)
279     average_ranks = pivot_ranked.mean().sort_values()
280     return average_ranks.rename('average rank')
281 
282 
283 def experiment_rank_by_num_firsts(experiment_pivot_df):
284     """Creates experiment level ranking by number of first places in per
285     benchmark rankings (higher is better)."""
286     # Rank fuzzers in each benchmark block.
287     pivot_ranked = experiment_pivot_df.rank('columns',
288                                             na_option='keep',
289                                             ascending=False)
290     # Count first places for each fuzzer.
291     firsts = pivot_ranked[pivot_ranked == 1]
292     num_firsts = firsts.sum().sort_values(ascending=False)
293     return num_firsts.rename('number of wins')
294 
295 
296 def experiment_rank_by_average_normalized_score(experiment_pivot_df):
297     """Creates experiment level ranking by taking the average of normalized per
298     benchmark scores from 0 to 100, where 100 is the highest reach coverage."""
299     # Normalize coverage values.
300     benchmark_maximum = experiment_pivot_df.max(axis='columns')
301     normalized_score = experiment_pivot_df.div(benchmark_maximum,
302                                                axis='index').mul(100)
303 
304     average_score = normalized_score.mean().sort_values(ascending=False)
305     return average_score.rename('average normalized score')
306 
307 
308 def experiment_level_ranking(experiment_snapshots_df,
309                              benchmark_level_ranking_function,
310                              experiment_level_ranking_function):
311     """Returns an aggregate ranking of fuzzers across all benchmarks."""
312     pivot_table = experiment_pivot_table(experiment_snapshots_df,
313                                          benchmark_level_ranking_function)
314     return experiment_level_ranking_function(pivot_table)
315
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/analysis/data_utils.py b/analysis/data_utils.py
--- a/analysis/data_utils.py
+++ b/analysis/data_utils.py
@@ -100,18 +100,17 @@
 
 # Creating "snapshots" (see README.md for definition).
 
-_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD = 1.0
+_MIN_FRACTION_OF_ALIVE_TRIALS_AT_SNAPSHOT = 0.5
 
 
 def get_benchmark_snapshot(benchmark_df,
-                           threshold=_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD):
-    """Finds the latest time where all trials were still running
-    (or |threshold| * trials were still running). In most cases, this is the end
-    of the experiment. However, if some trials (or more than (1 - |threshold|) *
-    trials) ended earlier, we will use an earlier "snapshot" time for comparing
-    results.
-
-    Returns data frame that only contains the measurements of the picked
+                           threshold=_MIN_FRACTION_OF_ALIVE_TRIALS_AT_SNAPSHOT):
+    """Finds the latest time where |threshold| fraction of the trials were still
+    running. In most cases, this is the end of the experiment. However, if less
+    than |threshold| fraction of the trials reached the end of the experiment,
+    then we will use an earlier "snapshot" time for comparing results.
+
+    Returns a data frame that only contains the measurements of the picked
     snapshot time.
     """
     # Allow overriding threshold with environment variable as well.

{"golden_diff": "diff --git a/analysis/data_utils.py b/analysis/data_utils.py\n--- a/analysis/data_utils.py\n+++ b/analysis/data_utils.py\n@@ -100,18 +100,17 @@\n \n # Creating \"snapshots\" (see README.md for definition).\n \n-_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD = 1.0\n+_MIN_FRACTION_OF_ALIVE_TRIALS_AT_SNAPSHOT = 0.5\n \n \n def get_benchmark_snapshot(benchmark_df,\n-                           threshold=_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD):\n-    \"\"\"Finds the latest time where all trials were still running\n-    (or |threshold| * trials were still running). In most cases, this is the end\n-    of the experiment. However, if some trials (or more than (1 - |threshold|) *\n-    trials) ended earlier, we will use an earlier \"snapshot\" time for comparing\n-    results.\n-\n-    Returns data frame that only contains the measurements of the picked\n+                           threshold=_MIN_FRACTION_OF_ALIVE_TRIALS_AT_SNAPSHOT):\n+    \"\"\"Finds the latest time where |threshold| fraction of the trials were still\n+    running. In most cases, this is the end of the experiment. However, if less\n+    than |threshold| fraction of the trials reached the end of the experiment,\n+    then we will use an earlier \"snapshot\" time for comparing results.\n+\n+    Returns a data frame that only contains the measurements of the picked\n     snapshot time.\n     \"\"\"\n     # Allow overriding threshold with environment variable as well.\n", "issue": "Dynamic report generation\nThough we already provide things like ipython notebooks to generate reports, I think we should lower the bar so that people can modify the report from the experiment from the report page.\r\nI can think of two ways to do this:\r\n1. Allow dynamic report generation using some kind of webapp.\r\n2. Having a link to an ipython notebook that uses the data for the report on each report page.\r\n\r\nOption 1 is a bit of a challenge because we generate reports server side and they take a while to do.\r\nOption 2 is not ideal because I assume there is startup time for ipython notebooks and because it puts a greater burden on users.\n", "before_files": [{"content": "# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Utility functions for data (frame) transformations.\"\"\"\nfrom analysis import stat_tests\nfrom common import environment\n\n\nclass EmptyDataError(ValueError):\n    \"\"\"An exception for when the data is empty.\"\"\"\n\n\ndef validate_data(experiment_df):\n    \"\"\"Checks if the experiment data is valid.\"\"\"\n    if experiment_df.empty:\n        raise EmptyDataError('Empty experiment data.')\n\n    expected_columns = {\n        'experiment', 'benchmark', 'fuzzer', 'trial_id', 'time_started',\n        'time_ended', 'time', 'edges_covered'\n    }\n    missing_columns = expected_columns.difference(experiment_df.columns)\n    if missing_columns:\n        raise ValueError(\n            'Missing columns in experiment data: {}'.format(missing_columns))\n\n\ndef drop_uninteresting_columns(experiment_df):\n    \"\"\"Returns table with only interesting columns.\"\"\"\n    return experiment_df[[\n        'benchmark', 'fuzzer', 'trial_id', 'time', 'edges_covered'\n    ]]\n\n\ndef clobber_experiments_data(df, experiments):\n    \"\"\"Clobber experiment data that is part of lower priority (generally\n    earlier) versions of the same trials in |df|. For example in experiment-1 we\n    may test fuzzer-a on benchmark-1. In experiment-2 we may again test fuzzer-a\n    on benchmark-1 because fuzzer-a was updated. This function will remove the\n    snapshots from fuzzer-a,benchmark-1,experiment-1 from |df| because we want\n    the report to only contain the up-to-date data. Experiment priority is\n    determined by order of each experiment in |experiments| with the highest\n    priority experiment coming last in that list.\"\"\"\n    # We don't call |df| \"experiment_df\" because it is a misnomer and leads to\n    # confusion in this case where it contains data from multiple experiments.\n\n    # Include everything from the last experiment.\n    experiments = experiments.copy()  # Copy so we dont mutate experiments.\n    experiments.reverse()\n    highest_rank_experiment = experiments[0]\n    result = df[df.experiment == highest_rank_experiment]\n\n    for experiment in experiments[1:]:\n        # Include data for not yet covered benchmark/fuzzer pairs.\n        covered_pairs = result[['benchmark', 'fuzzer']].drop_duplicates()\n        covered_pairs = covered_pairs.apply(tuple, axis=1)\n        experiment_data = df[df.experiment == experiment]\n        experiment_pairs = experiment_data[['benchmark',\n                                            'fuzzer']].apply(tuple, axis=1)\n        to_include = experiment_data[~experiment_pairs.isin(covered_pairs)]\n        result = result.append(to_include)\n    return result\n\n\ndef filter_fuzzers(experiment_df, included_fuzzers):\n    \"\"\"Returns table with only rows where fuzzer is in |included_fuzzers|.\"\"\"\n    return experiment_df[experiment_df['fuzzer'].isin(included_fuzzers)]\n\n\ndef filter_benchmarks(experiment_df, included_benchmarks):\n    \"\"\"Returns table with only rows where benchmark is in\n    |included_benchmarks|.\"\"\"\n    return experiment_df[experiment_df['benchmark'].isin(included_benchmarks)]\n\n\ndef label_fuzzers_by_experiment(experiment_df):\n    \"\"\"Returns table where every fuzzer is labeled by the experiment it\n    was run in.\"\"\"\n    experiment_df['fuzzer'] = (experiment_df['fuzzer'] + '-' +\n                               experiment_df['experiment'])\n\n    return experiment_df\n\n\ndef filter_max_time(experiment_df, max_time):\n    \"\"\"Returns table with snapshots that have time less than or equal to\n    |max_time|.\"\"\"\n    return experiment_df[experiment_df['time'] <= max_time]\n\n\n# Creating \"snapshots\" (see README.md for definition).\n\n_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD = 1.0\n\n\ndef get_benchmark_snapshot(benchmark_df,\n                           threshold=_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD):\n    \"\"\"Finds the latest time where all trials were still running\n    (or |threshold| * trials were still running). In most cases, this is the end\n    of the experiment. However, if some trials (or more than (1 - |threshold|) *\n    trials) ended earlier, we will use an earlier \"snapshot\" time for comparing\n    results.\n\n    Returns data frame that only contains the measurements of the picked\n    snapshot time.\n    \"\"\"\n    # Allow overriding threshold with environment variable as well.\n    threshold = environment.get('BENCHMARK_SAMPLE_NUM_THRESHOLD', threshold)\n\n    num_trials = benchmark_df.trial_id.nunique()\n    trials_running_at_time = benchmark_df.time.value_counts()\n    criteria = trials_running_at_time >= threshold * num_trials\n    ok_times = trials_running_at_time[criteria]\n    latest_ok_time = ok_times.index.max()\n    benchmark_snapshot_df = benchmark_df[benchmark_df.time == latest_ok_time]\n    return benchmark_snapshot_df\n\n\n_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD = 0.8\n\n\ndef get_fuzzers_with_not_enough_samples(\n        benchmark_snapshot_df, threshold=_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD):\n    \"\"\"Returns fuzzers that didn't have enough trials running at snapshot time.\n    It takes a benchmark snapshot and finds the fuzzers that have a sample size\n    smaller than 80% of the largest sample size. Default threshold can be\n    overridden.\n    \"\"\"\n    # Allow overriding threshold with environment variable as well.\n    threshold = environment.get('FUZZER_SAMPLE_NUM_THRESHOLD', threshold)\n\n    samples_per_fuzzer = benchmark_snapshot_df.fuzzer.value_counts()\n    max_samples = samples_per_fuzzer.max()\n    few_sample_criteria = samples_per_fuzzer < threshold * max_samples\n    few_sample_fuzzers = samples_per_fuzzer[few_sample_criteria].index\n    return few_sample_fuzzers.tolist()\n\n\ndef get_experiment_snapshots(experiment_df):\n    \"\"\"Finds a good snapshot time for each benchmark in the experiment data.\n\n    Returns the data frame that only contains the measurements made at these\n    snapshot times.\n    \"\"\"\n    benchmark_groups = experiment_df.groupby('benchmark')\n    experiment_snapshots = benchmark_groups.apply(get_benchmark_snapshot)\n    # We don't need the extra index added by the groupby('benchmark').\n    experiment_snapshots.reset_index(drop=True, inplace=True)\n    return experiment_snapshots\n\n\n# Summary tables containing statistics on the samples.\n\n\ndef benchmark_summary(benchmark_snapshot_df):\n    \"\"\"Creates summary table for a benchmark snapshot with columns:\n    |fuzzer|time||count|mean|std|min|25%|median|75%|max|\n    \"\"\"\n    groups = benchmark_snapshot_df.groupby(['fuzzer', 'time'])\n    summary = groups['edges_covered'].describe()\n    summary.rename(columns={'50%': 'median'}, inplace=True)\n    return summary.sort_values(('median'), ascending=False)\n\n\ndef experiment_summary(experiment_snapshots_df):\n    \"\"\"Creates summary table for all benchmarks in experiment, i.e. table like:\n    |benchmark|| < benchmark level summary >\n    \"\"\"\n    groups = experiment_snapshots_df.groupby('benchmark')\n    summaries = groups.apply(benchmark_summary)\n    return summaries\n\n\n# Per-benchmark fuzzer ranking options.\n\n\ndef benchmark_rank_by_mean(benchmark_snapshot_df):\n    \"\"\"Returns ranking of fuzzers based on mean coverage.\"\"\"\n    assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'\n    means = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].mean()\n    means.rename('mean cov', inplace=True)\n    return means.sort_values(ascending=False)\n\n\ndef benchmark_rank_by_median(benchmark_snapshot_df):\n    \"\"\"Returns ranking of fuzzers based on median coverage.\"\"\"\n    assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'\n    medians = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].median()\n    medians.rename('median cov', inplace=True)\n    return medians.sort_values(ascending=False)\n\n\ndef benchmark_rank_by_average_rank(benchmark_snapshot_df):\n    \"\"\"Ranks all coverage measurements in the snapshot across fuzzers.\n\n    Returns the average rank by fuzzer.\n    \"\"\"\n    # Make a copy of the dataframe view, because we want to add a new column.\n    measurements = benchmark_snapshot_df[['fuzzer', 'edges_covered']].copy()\n    measurements['rank'] = measurements['edges_covered'].rank()\n    avg_rank = measurements.groupby('fuzzer').mean()\n    avg_rank.rename(columns={'rank': 'avg rank'}, inplace=True)\n    avg_rank.sort_values('avg rank', ascending=False, inplace=True)\n    return avg_rank['avg rank']\n\n\ndef benchmark_rank_by_stat_test_wins(benchmark_snapshot_df):\n    \"\"\"Carries out one-tailed statistical tests for each fuzzer pair.\n\n    Returns ranking according to the number of statistical test wins.\n    \"\"\"\n    p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)\n\n    # Turn \"significant\" p-values into 1-s.\n    better_than = p_values.applymap(\n        lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)\n    better_than = better_than.applymap(int)\n\n    score = better_than.sum(axis=1).sort_values(ascending=False)\n    score.rename('stat wins', inplace=True)\n\n    return score\n\n\ndef create_better_than_table(benchmark_snapshot_df):\n    \"\"\"Creates table showing whether fuzzer in row is statistically\n    significantly better than the fuzzer in the column.\"\"\"\n    p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)\n\n    # Turn \"significant\" p-values into 1-s.\n    better_than = p_values.applymap(\n        lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)\n    better_than = better_than.applymap(int)\n\n    # Order rows and columns of matrix according to score ranking.\n    score = better_than.sum(axis=1).sort_values(ascending=False)\n    better_than = better_than.reindex(index=score.index,\n                                      columns=score.index[::-1])\n    return better_than\n\n\n# Experiment level ranking of fuzzers (across-benchmarks).\n# Experiment level ranking depends on the per-benchmark ranking method.\n\n\ndef experiment_pivot_table(experiment_snapshots_df,\n                           benchmark_level_ranking_function):\n    \"\"\"Creates a pivot table according to a given per benchmark ranking, where\n    the columns are the fuzzers, the rows are the benchmarks, and the values\n    are the scores according to the per benchmark ranking.\"\"\"\n    benchmark_blocks = experiment_snapshots_df.groupby('benchmark')\n    groups_ranked = benchmark_blocks.apply(benchmark_level_ranking_function)\n    already_unstacked = groups_ranked.index.names == ['benchmark']\n    pivot_df = groups_ranked if already_unstacked else groups_ranked.unstack()\n    return pivot_df\n\n\ndef experiment_rank_by_average_rank(experiment_pivot_df):\n    \"\"\"Creates experiment level ranking of fuzzers.\n\n    Takes a pivot table representing per benchmark ranking scores. Ranks\n    fuzzers per benchmark, then takes the average rank across benchmarks\n    (smaller is better).\n    \"\"\"\n    # Rank fuzzers in each benchmark block.\n    pivot_ranked = experiment_pivot_df.rank('columns',\n                                            na_option='keep',\n                                            ascending=False)\n    average_ranks = pivot_ranked.mean().sort_values()\n    return average_ranks.rename('average rank')\n\n\ndef experiment_rank_by_num_firsts(experiment_pivot_df):\n    \"\"\"Creates experiment level ranking by number of first places in per\n    benchmark rankings (higher is better).\"\"\"\n    # Rank fuzzers in each benchmark block.\n    pivot_ranked = experiment_pivot_df.rank('columns',\n                                            na_option='keep',\n                                            ascending=False)\n    # Count first places for each fuzzer.\n    firsts = pivot_ranked[pivot_ranked == 1]\n    num_firsts = firsts.sum().sort_values(ascending=False)\n    return num_firsts.rename('number of wins')\n\n\ndef experiment_rank_by_average_normalized_score(experiment_pivot_df):\n    \"\"\"Creates experiment level ranking by taking the average of normalized per\n    benchmark scores from 0 to 100, where 100 is the highest reach coverage.\"\"\"\n    # Normalize coverage values.\n    benchmark_maximum = experiment_pivot_df.max(axis='columns')\n    normalized_score = experiment_pivot_df.div(benchmark_maximum,\n                                               axis='index').mul(100)\n\n    average_score = normalized_score.mean().sort_values(ascending=False)\n    return average_score.rename('average normalized score')\n\n\ndef experiment_level_ranking(experiment_snapshots_df,\n                             benchmark_level_ranking_function,\n                             experiment_level_ranking_function):\n    \"\"\"Returns an aggregate ranking of fuzzers across all benchmarks.\"\"\"\n    pivot_table = experiment_pivot_table(experiment_snapshots_df,\n                                         benchmark_level_ranking_function)\n    return experiment_level_ranking_function(pivot_table)\n", "path": "analysis/data_utils.py"}], "after_files": [{"content": "# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Utility functions for data (frame) transformations.\"\"\"\nfrom analysis import stat_tests\nfrom common import environment\n\n\nclass EmptyDataError(ValueError):\n    \"\"\"An exception for when the data is empty.\"\"\"\n\n\ndef validate_data(experiment_df):\n    \"\"\"Checks if the experiment data is valid.\"\"\"\n    if experiment_df.empty:\n        raise EmptyDataError('Empty experiment data.')\n\n    expected_columns = {\n        'experiment', 'benchmark', 'fuzzer', 'trial_id', 'time_started',\n        'time_ended', 'time', 'edges_covered'\n    }\n    missing_columns = expected_columns.difference(experiment_df.columns)\n    if missing_columns:\n        raise ValueError(\n            'Missing columns in experiment data: {}'.format(missing_columns))\n\n\ndef drop_uninteresting_columns(experiment_df):\n    \"\"\"Returns table with only interesting columns.\"\"\"\n    return experiment_df[[\n        'benchmark', 'fuzzer', 'trial_id', 'time', 'edges_covered'\n    ]]\n\n\ndef clobber_experiments_data(df, experiments):\n    \"\"\"Clobber experiment data that is part of lower priority (generally\n    earlier) versions of the same trials in |df|. For example in experiment-1 we\n    may test fuzzer-a on benchmark-1. In experiment-2 we may again test fuzzer-a\n    on benchmark-1 because fuzzer-a was updated. This function will remove the\n    snapshots from fuzzer-a,benchmark-1,experiment-1 from |df| because we want\n    the report to only contain the up-to-date data. Experiment priority is\n    determined by order of each experiment in |experiments| with the highest\n    priority experiment coming last in that list.\"\"\"\n    # We don't call |df| \"experiment_df\" because it is a misnomer and leads to\n    # confusion in this case where it contains data from multiple experiments.\n\n    # Include everything from the last experiment.\n    experiments = experiments.copy()  # Copy so we dont mutate experiments.\n    experiments.reverse()\n    highest_rank_experiment = experiments[0]\n    result = df[df.experiment == highest_rank_experiment]\n\n    for experiment in experiments[1:]:\n        # Include data for not yet covered benchmark/fuzzer pairs.\n        covered_pairs = result[['benchmark', 'fuzzer']].drop_duplicates()\n        covered_pairs = covered_pairs.apply(tuple, axis=1)\n        experiment_data = df[df.experiment == experiment]\n        experiment_pairs = experiment_data[['benchmark',\n                                            'fuzzer']].apply(tuple, axis=1)\n        to_include = experiment_data[~experiment_pairs.isin(covered_pairs)]\n        result = result.append(to_include)\n    return result\n\n\ndef filter_fuzzers(experiment_df, included_fuzzers):\n    \"\"\"Returns table with only rows where fuzzer is in |included_fuzzers|.\"\"\"\n    return experiment_df[experiment_df['fuzzer'].isin(included_fuzzers)]\n\n\ndef filter_benchmarks(experiment_df, included_benchmarks):\n    \"\"\"Returns table with only rows where benchmark is in\n    |included_benchmarks|.\"\"\"\n    return experiment_df[experiment_df['benchmark'].isin(included_benchmarks)]\n\n\ndef label_fuzzers_by_experiment(experiment_df):\n    \"\"\"Returns table where every fuzzer is labeled by the experiment it\n    was run in.\"\"\"\n    experiment_df['fuzzer'] = (experiment_df['fuzzer'] + '-' +\n                               experiment_df['experiment'])\n\n    return experiment_df\n\n\ndef filter_max_time(experiment_df, max_time):\n    \"\"\"Returns table with snapshots that have time less than or equal to\n    |max_time|.\"\"\"\n    return experiment_df[experiment_df['time'] <= max_time]\n\n\n# Creating \"snapshots\" (see README.md for definition).\n\n_MIN_FRACTION_OF_ALIVE_TRIALS_AT_SNAPSHOT = 0.5\n\n\ndef get_benchmark_snapshot(benchmark_df,\n                           threshold=_MIN_FRACTION_OF_ALIVE_TRIALS_AT_SNAPSHOT):\n    \"\"\"Finds the latest time where |threshold| fraction of the trials were still\n    running. In most cases, this is the end of the experiment. However, if less\n    than |threshold| fraction of the trials reached the end of the experiment,\n    then we will use an earlier \"snapshot\" time for comparing results.\n\n    Returns a data frame that only contains the measurements of the picked\n    snapshot time.\n    \"\"\"\n    # Allow overriding threshold with environment variable as well.\n    threshold = environment.get('BENCHMARK_SAMPLE_NUM_THRESHOLD', threshold)\n\n    num_trials = benchmark_df.trial_id.nunique()\n    trials_running_at_time = benchmark_df.time.value_counts()\n    criteria = trials_running_at_time >= threshold * num_trials\n    ok_times = trials_running_at_time[criteria]\n    latest_ok_time = ok_times.index.max()\n    benchmark_snapshot_df = benchmark_df[benchmark_df.time == latest_ok_time]\n    return benchmark_snapshot_df\n\n\n_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD = 0.8\n\n\ndef get_fuzzers_with_not_enough_samples(\n        benchmark_snapshot_df, threshold=_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD):\n    \"\"\"Returns fuzzers that didn't have enough trials running at snapshot time.\n    It takes a benchmark snapshot and finds the fuzzers that have a sample size\n    smaller than 80% of the largest sample size. Default threshold can be\n    overridden.\n    \"\"\"\n    # Allow overriding threshold with environment variable as well.\n    threshold = environment.get('FUZZER_SAMPLE_NUM_THRESHOLD', threshold)\n\n    samples_per_fuzzer = benchmark_snapshot_df.fuzzer.value_counts()\n    max_samples = samples_per_fuzzer.max()\n    few_sample_criteria = samples_per_fuzzer < threshold * max_samples\n    few_sample_fuzzers = samples_per_fuzzer[few_sample_criteria].index\n    return few_sample_fuzzers.tolist()\n\n\ndef get_experiment_snapshots(experiment_df):\n    \"\"\"Finds a good snapshot time for each benchmark in the experiment data.\n\n    Returns the data frame that only contains the measurements made at these\n    snapshot times.\n    \"\"\"\n    benchmark_groups = experiment_df.groupby('benchmark')\n    experiment_snapshots = benchmark_groups.apply(get_benchmark_snapshot)\n    # We don't need the extra index added by the groupby('benchmark').\n    experiment_snapshots.reset_index(drop=True, inplace=True)\n    return experiment_snapshots\n\n\n# Summary tables containing statistics on the samples.\n\n\ndef benchmark_summary(benchmark_snapshot_df):\n    \"\"\"Creates summary table for a benchmark snapshot with columns:\n    |fuzzer|time||count|mean|std|min|25%|median|75%|max|\n    \"\"\"\n    groups = benchmark_snapshot_df.groupby(['fuzzer', 'time'])\n    summary = groups['edges_covered'].describe()\n    summary.rename(columns={'50%': 'median'}, inplace=True)\n    return summary.sort_values(('median'), ascending=False)\n\n\ndef experiment_summary(experiment_snapshots_df):\n    \"\"\"Creates summary table for all benchmarks in experiment, i.e. table like:\n    |benchmark|| < benchmark level summary >\n    \"\"\"\n    groups = experiment_snapshots_df.groupby('benchmark')\n    summaries = groups.apply(benchmark_summary)\n    return summaries\n\n\n# Per-benchmark fuzzer ranking options.\n\n\ndef benchmark_rank_by_mean(benchmark_snapshot_df):\n    \"\"\"Returns ranking of fuzzers based on mean coverage.\"\"\"\n    assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'\n    means = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].mean()\n    means.rename('mean cov', inplace=True)\n    return means.sort_values(ascending=False)\n\n\ndef benchmark_rank_by_median(benchmark_snapshot_df):\n    \"\"\"Returns ranking of fuzzers based on median coverage.\"\"\"\n    assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'\n    medians = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].median()\n    medians.rename('median cov', inplace=True)\n    return medians.sort_values(ascending=False)\n\n\ndef benchmark_rank_by_average_rank(benchmark_snapshot_df):\n    \"\"\"Ranks all coverage measurements in the snapshot across fuzzers.\n\n    Returns the average rank by fuzzer.\n    \"\"\"\n    # Make a copy of the dataframe view, because we want to add a new column.\n    measurements = benchmark_snapshot_df[['fuzzer', 'edges_covered']].copy()\n    measurements['rank'] = measurements['edges_covered'].rank()\n    avg_rank = measurements.groupby('fuzzer').mean()\n    avg_rank.rename(columns={'rank': 'avg rank'}, inplace=True)\n    avg_rank.sort_values('avg rank', ascending=False, inplace=True)\n    return avg_rank['avg rank']\n\n\ndef benchmark_rank_by_stat_test_wins(benchmark_snapshot_df):\n    \"\"\"Carries out one-tailed statistical tests for each fuzzer pair.\n\n    Returns ranking according to the number of statistical test wins.\n    \"\"\"\n    p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)\n\n    # Turn \"significant\" p-values into 1-s.\n    better_than = p_values.applymap(\n        lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)\n    better_than = better_than.applymap(int)\n\n    score = better_than.sum(axis=1).sort_values(ascending=False)\n    score.rename('stat wins', inplace=True)\n\n    return score\n\n\ndef create_better_than_table(benchmark_snapshot_df):\n    \"\"\"Creates table showing whether fuzzer in row is statistically\n    significantly better than the fuzzer in the column.\"\"\"\n    p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)\n\n    # Turn \"significant\" p-values into 1-s.\n    better_than = p_values.applymap(\n        lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)\n    better_than = better_than.applymap(int)\n\n    # Order rows and columns of matrix according to score ranking.\n    score = better_than.sum(axis=1).sort_values(ascending=False)\n    better_than = better_than.reindex(index=score.index,\n                                      columns=score.index[::-1])\n    return better_than\n\n\n# Experiment level ranking of fuzzers (across-benchmarks).\n# Experiment level ranking depends on the per-benchmark ranking method.\n\n\ndef experiment_pivot_table(experiment_snapshots_df,\n                           benchmark_level_ranking_function):\n    \"\"\"Creates a pivot table according to a given per benchmark ranking, where\n    the columns are the fuzzers, the rows are the benchmarks, and the values\n    are the scores according to the per benchmark ranking.\"\"\"\n    benchmark_blocks = experiment_snapshots_df.groupby('benchmark')\n    groups_ranked = benchmark_blocks.apply(benchmark_level_ranking_function)\n    already_unstacked = groups_ranked.index.names == ['benchmark']\n    pivot_df = groups_ranked if already_unstacked else groups_ranked.unstack()\n    return pivot_df\n\n\ndef experiment_rank_by_average_rank(experiment_pivot_df):\n    \"\"\"Creates experiment level ranking of fuzzers.\n\n    Takes a pivot table representing per benchmark ranking scores. Ranks\n    fuzzers per benchmark, then takes the average rank across benchmarks\n    (smaller is better).\n    \"\"\"\n    # Rank fuzzers in each benchmark block.\n    pivot_ranked = experiment_pivot_df.rank('columns',\n                                            na_option='keep',\n                                            ascending=False)\n    average_ranks = pivot_ranked.mean().sort_values()\n    return average_ranks.rename('average rank')\n\n\ndef experiment_rank_by_num_firsts(experiment_pivot_df):\n    \"\"\"Creates experiment level ranking by number of first places in per\n    benchmark rankings (higher is better).\"\"\"\n    # Rank fuzzers in each benchmark block.\n    pivot_ranked = experiment_pivot_df.rank('columns',\n                                            na_option='keep',\n                                            ascending=False)\n    # Count first places for each fuzzer.\n    firsts = pivot_ranked[pivot_ranked == 1]\n    num_firsts = firsts.sum().sort_values(ascending=False)\n    return num_firsts.rename('number of wins')\n\n\ndef experiment_rank_by_average_normalized_score(experiment_pivot_df):\n    \"\"\"Creates experiment level ranking by taking the average of normalized per\n    benchmark scores from 0 to 100, where 100 is the highest reach coverage.\"\"\"\n    # Normalize coverage values.\n    benchmark_maximum = experiment_pivot_df.max(axis='columns')\n    normalized_score = experiment_pivot_df.div(benchmark_maximum,\n                                               axis='index').mul(100)\n\n    average_score = normalized_score.mean().sort_values(ascending=False)\n    return average_score.rename('average normalized score')\n\n\ndef experiment_level_ranking(experiment_snapshots_df,\n                             benchmark_level_ranking_function,\n                             experiment_level_ranking_function):\n    \"\"\"Returns an aggregate ranking of fuzzers across all benchmarks.\"\"\"\n    pivot_table = experiment_pivot_table(experiment_snapshots_df,\n                                         benchmark_level_ranking_function)\n    return experiment_level_ranking_function(pivot_table)\n", "path": "analysis/data_utils.py"}]}