We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Write a validator to prevent issues requesting bug experiments
https://github.com/google/fuzzbench/commit/da5fab4dfcabcd856e0247b90122bb577f0c734e#diff-c89bd86bcf4447f199973358896b7a69d69d92f258653f28e77583ee8967dd13R49 has happened twice now, time to prevent it.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `service/automatic_run_experiment.py`
Content:
```
1 #!/usr/bin/env python3
2 # Copyright 2020 Google LLC
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #      http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 """Reads experiment-requests.yaml and determines if there is a new experiment
16 and runs it if needed. Note that this code uses a config file for experiments
17 that is specific to the FuzzBench service. Therefore this code will break if
18 others try to run it."""
19 import argparse
20 import collections
21 import os
22 import re
23 import sys
24 from typing import Optional
25 
26 from common import benchmark_utils
27 from common import logs
28 from common import utils
29 from common import yaml_utils
30 from database import models
31 from database import utils as db_utils
32 from experiment import run_experiment
33 
34 logger = logs.Logger('automatic_run_experiment')  # pylint: disable=invalid-name
35 
36 EXPERIMENT_CONFIG_FILE = os.path.join(utils.ROOT_DIR, 'service',
37                                       'experiment-config.yaml')
38 
39 REQUESTED_EXPERIMENTS_PATH = os.path.join(utils.ROOT_DIR, 'service',
40                                           'experiment-requests.yaml')
41 
42 # Don't run an experiment if we have a "request" just containing this keyword.
43 # TODO(metzman): Look into replacing this mechanism for pausing the service.
44 PAUSE_SERVICE_KEYWORD = 'PAUSE_SERVICE'
45 
46 EXPERIMENT_NAME_REGEX = re.compile(r'^\d{4}-\d{2}-\d{2}.*')
47 
48 
49 def _get_experiment_name(experiment_config: dict) -> str:
50     """Returns the name of the experiment described by |experiment_config| as a
51     string."""
52     # Use str because the yaml parser will parse things like `2020-05-06` as
53     # a datetime if not included in quotes.
54     return str(experiment_config['experiment'])
55 
56 
57 def _get_description(experiment_config: dict) -> Optional[str]:
58     """Returns the description of the experiment described by
59     |experiment_config| as a string."""
60     return experiment_config.get('description')
61 
62 
63 def _use_oss_fuzz_corpus(experiment_config: dict) -> bool:
64     """Returns the oss_fuzz_corpus flag of the experiment described by
65     |experiment_config| as a bool."""
66     return bool(experiment_config.get('oss_fuzz_corpus'))
67 
68 
69 def _get_requested_experiments():
70     """Return requested experiments."""
71     return yaml_utils.read(REQUESTED_EXPERIMENTS_PATH)
72 
73 
74 def validate_experiment_name(experiment_name):
75     """Returns True if |experiment_name| is valid."""
76     if EXPERIMENT_NAME_REGEX.match(experiment_name) is None:
77         logger.error('Experiment name: %s is not valid.', experiment_name)
78         return False
79     try:
80         run_experiment.validate_experiment_name(experiment_name)
81         return True
82     except run_experiment.ValidationError:
83         logger.error('Experiment name: %s is not valid.', experiment_name)
84         return False
85 
86 
87 def _validate_individual_experiment_requests(experiment_requests):
88     """Returns True if all requests in |experiment_request| are valid in
89     isolation. Does not account for PAUSE_SERVICE_KEYWORD or duplicates."""
90     valid = True
91     # Validate format.
92     for request in experiment_requests:
93         if not isinstance(request, dict):
94             logger.error('Request: %s is not a dict.', request)
95             experiment_requests.remove(request)
96             valid = False
97             continue
98 
99         if 'experiment' not in request:
100             logger.error('Request: %s does not have field "experiment".',
101                          request)
102             valid = False
103             continue
104 
105         experiment = _get_experiment_name(request)
106         if not validate_experiment_name(experiment):
107             valid = False
108             # Request isn't so malformed that we can find other issues, if
109             # present.
110 
111         fuzzers = request.get('fuzzers')
112         if not fuzzers:
113             logger.error('Request: %s does not specify any fuzzers.', request)
114             valid = False
115             continue
116 
117         for fuzzer in fuzzers:
118             try:
119                 run_experiment.validate_fuzzer(fuzzer)
120             except run_experiment.ValidationError:
121                 logger.error('Fuzzer: %s is invalid.', fuzzer)
122                 valid = False
123 
124         description = request.get('description')
125         if description is not None and not isinstance(description, str):
126             logger.error(
127                 'Request: %s "description" attribute is not a valid string.',
128                 request)
129             valid = False
130 
131         oss_fuzz_corpus = request.get('oss_fuzz_corpus')
132         if oss_fuzz_corpus is not None and not isinstance(
133                 oss_fuzz_corpus, bool):
134             logger.error(
135                 'Request: %s "oss_fuzz_corpus" attribute is not a valid bool.',
136                 request)
137             valid = False
138 
139         experiment_type = request.get('type',
140                                       benchmark_utils.BenchmarkType.CODE.value)
141         if experiment_type not in benchmark_utils.BENCHMARK_TYPE_STRS:
142             logger.error('Type: %s is invalid, must be one of %s',
143                          experiment_type, benchmark_utils.BENCHMARK_TYPE_STRS)
144             valid = False
145 
146     return valid
147 
148 
149 def validate_experiment_requests(experiment_requests):
150     """Returns True if all requests in |experiment_requests| are valid."""
151     # This function tries to find as many requests as possible.
152     if PAUSE_SERVICE_KEYWORD in experiment_requests:
153         # This is a special case where a string is used instead of an experiment
154         # to tell the service not to run experiments automatically. Remove it
155         # from the list because it fails validation.
156         experiment_requests = experiment_requests[:]  # Don't mutate input.
157         experiment_requests.remove(PAUSE_SERVICE_KEYWORD)
158 
159     if not _validate_individual_experiment_requests(experiment_requests):
160         # Don't try the next validation step if the previous failed, we might
161         # exception.
162         return False
163 
164     # Make sure experiment requests have a unique name, we can't run the same
165     # experiment twice.
166     counts = collections.Counter(
167         [request['experiment'] for request in experiment_requests])
168 
169     valid = True
170     for experiment_name, count in counts.items():
171         if count != 1:
172             logger.error('Experiment: "%s" appears %d times.',
173                          str(experiment_name), count)
174             valid = False
175 
176     return valid
177 
178 
179 def run_requested_experiment(dry_run):
180     """Run the oldest requested experiment that hasn't been run yet in
181     experiment-requests.yaml."""
182     requested_experiments = _get_requested_experiments()
183 
184     # TODO(metzman): Look into supporting benchmarks as an optional parameter so
185     # that people can add fuzzers that don't support everything.
186 
187     if PAUSE_SERVICE_KEYWORD in requested_experiments:
188         # Check if automated experiment service is paused.
189         logs.warning('Pause service requested, not running experiment.')
190         return
191 
192     requested_experiment = None
193     for experiment_config in reversed(requested_experiments):
194         experiment_name = _get_experiment_name(experiment_config)
195         with db_utils.session_scope() as session:
196             is_new_experiment = session.query(models.Experiment).filter(
197                 models.Experiment.name == experiment_name).first() is None
198         if is_new_experiment:
199             requested_experiment = experiment_config
200             break
201 
202     if requested_experiment is None:
203         logs.info('No new experiment to run. Exiting.')
204         return
205 
206     experiment_name = _get_experiment_name(requested_experiment)
207     if not validate_experiment_requests([requested_experiment]):
208         logs.error('Requested experiment: %s in %s is not valid.',
209                    requested_experiment, REQUESTED_EXPERIMENTS_PATH)
210         return
211     fuzzers = requested_experiment['fuzzers']
212 
213     benchmark_type = requested_experiment.get('type')
214     if benchmark_type == benchmark_utils.BenchmarkType.BUG.value:
215         valid_benchmarks = benchmark_utils.exclude_non_cpp(
216             benchmark_utils.get_bug_benchmarks())
217     else:
218         valid_benchmarks = benchmark_utils.exclude_non_cpp(
219             benchmark_utils.get_coverage_benchmarks())
220 
221     benchmarks = requested_experiment.get('benchmarks')
222     if benchmarks is None:
223         benchmarks = valid_benchmarks
224     else:
225         errors = False
226         for benchmark in benchmarks:
227             if benchmark not in valid_benchmarks:
228                 logs.error(
229                     'Requested experiment:'
230                     ' in %s, %s is not a valid %s benchmark.',
231                     requested_experiment, benchmark, benchmark_type)
232                 errors = True
233         if errors:
234             return
235 
236     logs.info('Running experiment: %s with fuzzers: %s.', experiment_name,
237               ' '.join(fuzzers))
238     description = _get_description(requested_experiment)
239     oss_fuzz_corpus = _use_oss_fuzz_corpus(requested_experiment)
240     _run_experiment(experiment_name, fuzzers, benchmarks, description,
241                     oss_fuzz_corpus, dry_run)
242 
243 
244 def _run_experiment(  # pylint: disable=too-many-arguments
245         experiment_name,
246         fuzzers,
247         benchmarks,
248         description,
249         oss_fuzz_corpus,
250         dry_run=False):
251     """Run an experiment named |experiment_name| on |fuzzer_configs| and shut it
252     down once it terminates."""
253     logs.info('Starting experiment: %s.', experiment_name)
254     if dry_run:
255         logs.info('Dry run. Not actually running experiment.')
256         return
257     run_experiment.start_experiment(experiment_name,
258                                     EXPERIMENT_CONFIG_FILE,
259                                     benchmarks,
260                                     fuzzers,
261                                     description=description,
262                                     oss_fuzz_corpus=oss_fuzz_corpus)
263 
264 
265 def main():
266     """Run an experiment."""
267     logs.initialize()
268     parser = argparse.ArgumentParser(description='Run a requested experiment.')
269     # TODO(metzman): Add a way to exit immediately if there is already an
270     # experiment running. FuzzBench's scheduler isn't smart enough to deal with
271     # this properly.
272     parser.add_argument('-d',
273                         '--dry-run',
274                         help='Dry run, don\'t actually run the experiment',
275                         default=False,
276                         action='store_true')
277     args = parser.parse_args()
278     try:
279         run_requested_experiment(args.dry_run)
280     except Exception:  # pylint: disable=broad-except
281         logger.error('Error running requested experiment.')
282         return 1
283     return 0
284 
285 
286 if __name__ == '__main__':
287     sys.exit(main())
288
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/service/automatic_run_experiment.py b/service/automatic_run_experiment.py
--- a/service/automatic_run_experiment.py
+++ b/service/automatic_run_experiment.py
@@ -143,6 +143,18 @@
                          experiment_type, benchmark_utils.BENCHMARK_TYPE_STRS)
             valid = False
 
+        benchmarks = request.get('benchmarks', [])
+        for benchmark in benchmarks:
+            benchmark_type = benchmark_utils.get_type(benchmark)
+            if (benchmark_type == benchmark_utils.BenchmarkType.BUG.value and
+                    experiment_type != benchmark_utils.BenchmarkType.BUG.value):
+                logger.error(
+                    'Benchmark %s is "type: bug". '
+                    'Experiment %s must be "type: bug" as well.', benchmark,
+                    experiment)
+                valid = False
+                break
+
     return valid

{"golden_diff": "diff --git a/service/automatic_run_experiment.py b/service/automatic_run_experiment.py\n--- a/service/automatic_run_experiment.py\n+++ b/service/automatic_run_experiment.py\n@@ -143,6 +143,18 @@\n                          experiment_type, benchmark_utils.BENCHMARK_TYPE_STRS)\n             valid = False\n \n+        benchmarks = request.get('benchmarks', [])\n+        for benchmark in benchmarks:\n+            benchmark_type = benchmark_utils.get_type(benchmark)\n+            if (benchmark_type == benchmark_utils.BenchmarkType.BUG.value and\n+                    experiment_type != benchmark_utils.BenchmarkType.BUG.value):\n+                logger.error(\n+                    'Benchmark %s is \"type: bug\". '\n+                    'Experiment %s must be \"type: bug\" as well.', benchmark,\n+                    experiment)\n+                valid = False\n+                break\n+\n     return valid\n", "issue": "Write a validator to prevent issues requesting bug experiments\nhttps://github.com/google/fuzzbench/commit/da5fab4dfcabcd856e0247b90122bb577f0c734e#diff-c89bd86bcf4447f199973358896b7a69d69d92f258653f28e77583ee8967dd13R49 has happened twice now, time to prevent it.\n", "before_files": [{"content": "#!/usr/bin/env python3\n# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Reads experiment-requests.yaml and determines if there is a new experiment\nand runs it if needed. Note that this code uses a config file for experiments\nthat is specific to the FuzzBench service. Therefore this code will break if\nothers try to run it.\"\"\"\nimport argparse\nimport collections\nimport os\nimport re\nimport sys\nfrom typing import Optional\n\nfrom common import benchmark_utils\nfrom common import logs\nfrom common import utils\nfrom common import yaml_utils\nfrom database import models\nfrom database import utils as db_utils\nfrom experiment import run_experiment\n\nlogger = logs.Logger('automatic_run_experiment')  # pylint: disable=invalid-name\n\nEXPERIMENT_CONFIG_FILE = os.path.join(utils.ROOT_DIR, 'service',\n                                      'experiment-config.yaml')\n\nREQUESTED_EXPERIMENTS_PATH = os.path.join(utils.ROOT_DIR, 'service',\n                                          'experiment-requests.yaml')\n\n# Don't run an experiment if we have a \"request\" just containing this keyword.\n# TODO(metzman): Look into replacing this mechanism for pausing the service.\nPAUSE_SERVICE_KEYWORD = 'PAUSE_SERVICE'\n\nEXPERIMENT_NAME_REGEX = re.compile(r'^\\d{4}-\\d{2}-\\d{2}.*')\n\n\ndef _get_experiment_name(experiment_config: dict) -> str:\n    \"\"\"Returns the name of the experiment described by |experiment_config| as a\n    string.\"\"\"\n    # Use str because the yaml parser will parse things like `2020-05-06` as\n    # a datetime if not included in quotes.\n    return str(experiment_config['experiment'])\n\n\ndef _get_description(experiment_config: dict) -> Optional[str]:\n    \"\"\"Returns the description of the experiment described by\n    |experiment_config| as a string.\"\"\"\n    return experiment_config.get('description')\n\n\ndef _use_oss_fuzz_corpus(experiment_config: dict) -> bool:\n    \"\"\"Returns the oss_fuzz_corpus flag of the experiment described by\n    |experiment_config| as a bool.\"\"\"\n    return bool(experiment_config.get('oss_fuzz_corpus'))\n\n\ndef _get_requested_experiments():\n    \"\"\"Return requested experiments.\"\"\"\n    return yaml_utils.read(REQUESTED_EXPERIMENTS_PATH)\n\n\ndef validate_experiment_name(experiment_name):\n    \"\"\"Returns True if |experiment_name| is valid.\"\"\"\n    if EXPERIMENT_NAME_REGEX.match(experiment_name) is None:\n        logger.error('Experiment name: %s is not valid.', experiment_name)\n        return False\n    try:\n        run_experiment.validate_experiment_name(experiment_name)\n        return True\n    except run_experiment.ValidationError:\n        logger.error('Experiment name: %s is not valid.', experiment_name)\n        return False\n\n\ndef _validate_individual_experiment_requests(experiment_requests):\n    \"\"\"Returns True if all requests in |experiment_request| are valid in\n    isolation. Does not account for PAUSE_SERVICE_KEYWORD or duplicates.\"\"\"\n    valid = True\n    # Validate format.\n    for request in experiment_requests:\n        if not isinstance(request, dict):\n            logger.error('Request: %s is not a dict.', request)\n            experiment_requests.remove(request)\n            valid = False\n            continue\n\n        if 'experiment' not in request:\n            logger.error('Request: %s does not have field \"experiment\".',\n                         request)\n            valid = False\n            continue\n\n        experiment = _get_experiment_name(request)\n        if not validate_experiment_name(experiment):\n            valid = False\n            # Request isn't so malformed that we can find other issues, if\n            # present.\n\n        fuzzers = request.get('fuzzers')\n        if not fuzzers:\n            logger.error('Request: %s does not specify any fuzzers.', request)\n            valid = False\n            continue\n\n        for fuzzer in fuzzers:\n            try:\n                run_experiment.validate_fuzzer(fuzzer)\n            except run_experiment.ValidationError:\n                logger.error('Fuzzer: %s is invalid.', fuzzer)\n                valid = False\n\n        description = request.get('description')\n        if description is not None and not isinstance(description, str):\n            logger.error(\n                'Request: %s \"description\" attribute is not a valid string.',\n                request)\n            valid = False\n\n        oss_fuzz_corpus = request.get('oss_fuzz_corpus')\n        if oss_fuzz_corpus is not None and not isinstance(\n                oss_fuzz_corpus, bool):\n            logger.error(\n                'Request: %s \"oss_fuzz_corpus\" attribute is not a valid bool.',\n                request)\n            valid = False\n\n        experiment_type = request.get('type',\n                                      benchmark_utils.BenchmarkType.CODE.value)\n        if experiment_type not in benchmark_utils.BENCHMARK_TYPE_STRS:\n            logger.error('Type: %s is invalid, must be one of %s',\n                         experiment_type, benchmark_utils.BENCHMARK_TYPE_STRS)\n            valid = False\n\n    return valid\n\n\ndef validate_experiment_requests(experiment_requests):\n    \"\"\"Returns True if all requests in |experiment_requests| are valid.\"\"\"\n    # This function tries to find as many requests as possible.\n    if PAUSE_SERVICE_KEYWORD in experiment_requests:\n        # This is a special case where a string is used instead of an experiment\n        # to tell the service not to run experiments automatically. Remove it\n        # from the list because it fails validation.\n        experiment_requests = experiment_requests[:]  # Don't mutate input.\n        experiment_requests.remove(PAUSE_SERVICE_KEYWORD)\n\n    if not _validate_individual_experiment_requests(experiment_requests):\n        # Don't try the next validation step if the previous failed, we might\n        # exception.\n        return False\n\n    # Make sure experiment requests have a unique name, we can't run the same\n    # experiment twice.\n    counts = collections.Counter(\n        [request['experiment'] for request in experiment_requests])\n\n    valid = True\n    for experiment_name, count in counts.items():\n        if count != 1:\n            logger.error('Experiment: \"%s\" appears %d times.',\n                         str(experiment_name), count)\n            valid = False\n\n    return valid\n\n\ndef run_requested_experiment(dry_run):\n    \"\"\"Run the oldest requested experiment that hasn't been run yet in\n    experiment-requests.yaml.\"\"\"\n    requested_experiments = _get_requested_experiments()\n\n    # TODO(metzman): Look into supporting benchmarks as an optional parameter so\n    # that people can add fuzzers that don't support everything.\n\n    if PAUSE_SERVICE_KEYWORD in requested_experiments:\n        # Check if automated experiment service is paused.\n        logs.warning('Pause service requested, not running experiment.')\n        return\n\n    requested_experiment = None\n    for experiment_config in reversed(requested_experiments):\n        experiment_name = _get_experiment_name(experiment_config)\n        with db_utils.session_scope() as session:\n            is_new_experiment = session.query(models.Experiment).filter(\n                models.Experiment.name == experiment_name).first() is None\n        if is_new_experiment:\n            requested_experiment = experiment_config\n            break\n\n    if requested_experiment is None:\n        logs.info('No new experiment to run. Exiting.')\n        return\n\n    experiment_name = _get_experiment_name(requested_experiment)\n    if not validate_experiment_requests([requested_experiment]):\n        logs.error('Requested experiment: %s in %s is not valid.',\n                   requested_experiment, REQUESTED_EXPERIMENTS_PATH)\n        return\n    fuzzers = requested_experiment['fuzzers']\n\n    benchmark_type = requested_experiment.get('type')\n    if benchmark_type == benchmark_utils.BenchmarkType.BUG.value:\n        valid_benchmarks = benchmark_utils.exclude_non_cpp(\n            benchmark_utils.get_bug_benchmarks())\n    else:\n        valid_benchmarks = benchmark_utils.exclude_non_cpp(\n            benchmark_utils.get_coverage_benchmarks())\n\n    benchmarks = requested_experiment.get('benchmarks')\n    if benchmarks is None:\n        benchmarks = valid_benchmarks\n    else:\n        errors = False\n        for benchmark in benchmarks:\n            if benchmark not in valid_benchmarks:\n                logs.error(\n                    'Requested experiment:'\n                    ' in %s, %s is not a valid %s benchmark.',\n                    requested_experiment, benchmark, benchmark_type)\n                errors = True\n        if errors:\n            return\n\n    logs.info('Running experiment: %s with fuzzers: %s.', experiment_name,\n              ' '.join(fuzzers))\n    description = _get_description(requested_experiment)\n    oss_fuzz_corpus = _use_oss_fuzz_corpus(requested_experiment)\n    _run_experiment(experiment_name, fuzzers, benchmarks, description,\n                    oss_fuzz_corpus, dry_run)\n\n\ndef _run_experiment(  # pylint: disable=too-many-arguments\n        experiment_name,\n        fuzzers,\n        benchmarks,\n        description,\n        oss_fuzz_corpus,\n        dry_run=False):\n    \"\"\"Run an experiment named |experiment_name| on |fuzzer_configs| and shut it\n    down once it terminates.\"\"\"\n    logs.info('Starting experiment: %s.', experiment_name)\n    if dry_run:\n        logs.info('Dry run. Not actually running experiment.')\n        return\n    run_experiment.start_experiment(experiment_name,\n                                    EXPERIMENT_CONFIG_FILE,\n                                    benchmarks,\n                                    fuzzers,\n                                    description=description,\n                                    oss_fuzz_corpus=oss_fuzz_corpus)\n\n\ndef main():\n    \"\"\"Run an experiment.\"\"\"\n    logs.initialize()\n    parser = argparse.ArgumentParser(description='Run a requested experiment.')\n    # TODO(metzman): Add a way to exit immediately if there is already an\n    # experiment running. FuzzBench's scheduler isn't smart enough to deal with\n    # this properly.\n    parser.add_argument('-d',\n                        '--dry-run',\n                        help='Dry run, don\\'t actually run the experiment',\n                        default=False,\n                        action='store_true')\n    args = parser.parse_args()\n    try:\n        run_requested_experiment(args.dry_run)\n    except Exception:  # pylint: disable=broad-except\n        logger.error('Error running requested experiment.')\n        return 1\n    return 0\n\n\nif __name__ == '__main__':\n    sys.exit(main())\n", "path": "service/automatic_run_experiment.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Reads experiment-requests.yaml and determines if there is a new experiment\nand runs it if needed. Note that this code uses a config file for experiments\nthat is specific to the FuzzBench service. Therefore this code will break if\nothers try to run it.\"\"\"\nimport argparse\nimport collections\nimport os\nimport re\nimport sys\nfrom typing import Optional\n\nfrom common import benchmark_utils\nfrom common import logs\nfrom common import utils\nfrom common import yaml_utils\nfrom database import models\nfrom database import utils as db_utils\nfrom experiment import run_experiment\n\nlogger = logs.Logger('automatic_run_experiment')  # pylint: disable=invalid-name\n\nEXPERIMENT_CONFIG_FILE = os.path.join(utils.ROOT_DIR, 'service',\n                                      'experiment-config.yaml')\n\nREQUESTED_EXPERIMENTS_PATH = os.path.join(utils.ROOT_DIR, 'service',\n                                          'experiment-requests.yaml')\n\n# Don't run an experiment if we have a \"request\" just containing this keyword.\n# TODO(metzman): Look into replacing this mechanism for pausing the service.\nPAUSE_SERVICE_KEYWORD = 'PAUSE_SERVICE'\n\nEXPERIMENT_NAME_REGEX = re.compile(r'^\\d{4}-\\d{2}-\\d{2}.*')\n\n\ndef _get_experiment_name(experiment_config: dict) -> str:\n    \"\"\"Returns the name of the experiment described by |experiment_config| as a\n    string.\"\"\"\n    # Use str because the yaml parser will parse things like `2020-05-06` as\n    # a datetime if not included in quotes.\n    return str(experiment_config['experiment'])\n\n\ndef _get_description(experiment_config: dict) -> Optional[str]:\n    \"\"\"Returns the description of the experiment described by\n    |experiment_config| as a string.\"\"\"\n    return experiment_config.get('description')\n\n\ndef _use_oss_fuzz_corpus(experiment_config: dict) -> bool:\n    \"\"\"Returns the oss_fuzz_corpus flag of the experiment described by\n    |experiment_config| as a bool.\"\"\"\n    return bool(experiment_config.get('oss_fuzz_corpus'))\n\n\ndef _get_requested_experiments():\n    \"\"\"Return requested experiments.\"\"\"\n    return yaml_utils.read(REQUESTED_EXPERIMENTS_PATH)\n\n\ndef validate_experiment_name(experiment_name):\n    \"\"\"Returns True if |experiment_name| is valid.\"\"\"\n    if EXPERIMENT_NAME_REGEX.match(experiment_name) is None:\n        logger.error('Experiment name: %s is not valid.', experiment_name)\n        return False\n    try:\n        run_experiment.validate_experiment_name(experiment_name)\n        return True\n    except run_experiment.ValidationError:\n        logger.error('Experiment name: %s is not valid.', experiment_name)\n        return False\n\n\ndef _validate_individual_experiment_requests(experiment_requests):\n    \"\"\"Returns True if all requests in |experiment_request| are valid in\n    isolation. Does not account for PAUSE_SERVICE_KEYWORD or duplicates.\"\"\"\n    valid = True\n    # Validate format.\n    for request in experiment_requests:\n        if not isinstance(request, dict):\n            logger.error('Request: %s is not a dict.', request)\n            experiment_requests.remove(request)\n            valid = False\n            continue\n\n        if 'experiment' not in request:\n            logger.error('Request: %s does not have field \"experiment\".',\n                         request)\n            valid = False\n            continue\n\n        experiment = _get_experiment_name(request)\n        if not validate_experiment_name(experiment):\n            valid = False\n            # Request isn't so malformed that we can find other issues, if\n            # present.\n\n        fuzzers = request.get('fuzzers')\n        if not fuzzers:\n            logger.error('Request: %s does not specify any fuzzers.', request)\n            valid = False\n            continue\n\n        for fuzzer in fuzzers:\n            try:\n                run_experiment.validate_fuzzer(fuzzer)\n            except run_experiment.ValidationError:\n                logger.error('Fuzzer: %s is invalid.', fuzzer)\n                valid = False\n\n        description = request.get('description')\n        if description is not None and not isinstance(description, str):\n            logger.error(\n                'Request: %s \"description\" attribute is not a valid string.',\n                request)\n            valid = False\n\n        oss_fuzz_corpus = request.get('oss_fuzz_corpus')\n        if oss_fuzz_corpus is not None and not isinstance(\n                oss_fuzz_corpus, bool):\n            logger.error(\n                'Request: %s \"oss_fuzz_corpus\" attribute is not a valid bool.',\n                request)\n            valid = False\n\n        experiment_type = request.get('type',\n                                      benchmark_utils.BenchmarkType.CODE.value)\n        if experiment_type not in benchmark_utils.BENCHMARK_TYPE_STRS:\n            logger.error('Type: %s is invalid, must be one of %s',\n                         experiment_type, benchmark_utils.BENCHMARK_TYPE_STRS)\n            valid = False\n\n        benchmarks = request.get('benchmarks', [])\n        for benchmark in benchmarks:\n            benchmark_type = benchmark_utils.get_type(benchmark)\n            if (benchmark_type == benchmark_utils.BenchmarkType.BUG.value and\n                    experiment_type != benchmark_utils.BenchmarkType.BUG.value):\n                logger.error(\n                    'Benchmark %s is \"type: bug\". '\n                    'Experiment %s must be \"type: bug\" as well.', benchmark,\n                    experiment)\n                valid = False\n                break\n\n    return valid\n\n\ndef validate_experiment_requests(experiment_requests):\n    \"\"\"Returns True if all requests in |experiment_requests| are valid.\"\"\"\n    # This function tries to find as many requests as possible.\n    if PAUSE_SERVICE_KEYWORD in experiment_requests:\n        # This is a special case where a string is used instead of an experiment\n        # to tell the service not to run experiments automatically. Remove it\n        # from the list because it fails validation.\n        experiment_requests = experiment_requests[:]  # Don't mutate input.\n        experiment_requests.remove(PAUSE_SERVICE_KEYWORD)\n\n    if not _validate_individual_experiment_requests(experiment_requests):\n        # Don't try the next validation step if the previous failed, we might\n        # exception.\n        return False\n\n    # Make sure experiment requests have a unique name, we can't run the same\n    # experiment twice.\n    counts = collections.Counter(\n        [request['experiment'] for request in experiment_requests])\n\n    valid = True\n    for experiment_name, count in counts.items():\n        if count != 1:\n            logger.error('Experiment: \"%s\" appears %d times.',\n                         str(experiment_name), count)\n            valid = False\n\n    return valid\n\n\ndef run_requested_experiment(dry_run):\n    \"\"\"Run the oldest requested experiment that hasn't been run yet in\n    experiment-requests.yaml.\"\"\"\n    requested_experiments = _get_requested_experiments()\n\n    # TODO(metzman): Look into supporting benchmarks as an optional parameter so\n    # that people can add fuzzers that don't support everything.\n\n    if PAUSE_SERVICE_KEYWORD in requested_experiments:\n        # Check if automated experiment service is paused.\n        logs.warning('Pause service requested, not running experiment.')\n        return\n\n    requested_experiment = None\n    for experiment_config in reversed(requested_experiments):\n        experiment_name = _get_experiment_name(experiment_config)\n        with db_utils.session_scope() as session:\n            is_new_experiment = session.query(models.Experiment).filter(\n                models.Experiment.name == experiment_name).first() is None\n        if is_new_experiment:\n            requested_experiment = experiment_config\n            break\n\n    if requested_experiment is None:\n        logs.info('No new experiment to run. Exiting.')\n        return\n\n    experiment_name = _get_experiment_name(requested_experiment)\n    if not validate_experiment_requests([requested_experiment]):\n        logs.error('Requested experiment: %s in %s is not valid.',\n                   requested_experiment, REQUESTED_EXPERIMENTS_PATH)\n        return\n    fuzzers = requested_experiment['fuzzers']\n\n    benchmark_type = requested_experiment.get('type')\n    if benchmark_type == benchmark_utils.BenchmarkType.BUG.value:\n        valid_benchmarks = benchmark_utils.exclude_non_cpp(\n            benchmark_utils.get_bug_benchmarks())\n    else:\n        valid_benchmarks = benchmark_utils.exclude_non_cpp(\n            benchmark_utils.get_coverage_benchmarks())\n\n    benchmarks = requested_experiment.get('benchmarks')\n    if benchmarks is None:\n        benchmarks = valid_benchmarks\n    else:\n        errors = False\n        for benchmark in benchmarks:\n            if benchmark not in valid_benchmarks:\n                logs.error(\n                    'Requested experiment:'\n                    ' in %s, %s is not a valid %s benchmark.',\n                    requested_experiment, benchmark, benchmark_type)\n                errors = True\n        if errors:\n            return\n\n    logs.info('Running experiment: %s with fuzzers: %s.', experiment_name,\n              ' '.join(fuzzers))\n    description = _get_description(requested_experiment)\n    oss_fuzz_corpus = _use_oss_fuzz_corpus(requested_experiment)\n    _run_experiment(experiment_name, fuzzers, benchmarks, description,\n                    oss_fuzz_corpus, dry_run)\n\n\ndef _run_experiment(  # pylint: disable=too-many-arguments\n        experiment_name,\n        fuzzers,\n        benchmarks,\n        description,\n        oss_fuzz_corpus,\n        dry_run=False):\n    \"\"\"Run an experiment named |experiment_name| on |fuzzer_configs| and shut it\n    down once it terminates.\"\"\"\n    logs.info('Starting experiment: %s.', experiment_name)\n    if dry_run:\n        logs.info('Dry run. Not actually running experiment.')\n        return\n    run_experiment.start_experiment(experiment_name,\n                                    EXPERIMENT_CONFIG_FILE,\n                                    benchmarks,\n                                    fuzzers,\n                                    description=description,\n                                    oss_fuzz_corpus=oss_fuzz_corpus)\n\n\ndef main():\n    \"\"\"Run an experiment.\"\"\"\n    logs.initialize()\n    parser = argparse.ArgumentParser(description='Run a requested experiment.')\n    # TODO(metzman): Add a way to exit immediately if there is already an\n    # experiment running. FuzzBench's scheduler isn't smart enough to deal with\n    # this properly.\n    parser.add_argument('-d',\n                        '--dry-run',\n                        help='Dry run, don\\'t actually run the experiment',\n                        default=False,\n                        action='store_true')\n    args = parser.parse_args()\n    try:\n        run_requested_experiment(args.dry_run)\n    except Exception:  # pylint: disable=broad-except\n        logger.error('Error running requested experiment.')\n        return 1\n    return 0\n\n\nif __name__ == '__main__':\n    sys.exit(main())\n", "path": "service/automatic_run_experiment.py"}]}