We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Use round-robin ordering for leaderboard experiment execution
## 🚀 Feature

Don't repeat all `--n` runs of each benchmark in order, perform 1 run of each benchmark, then proceed to the 2nd run, etc, until all `n` runs have been completed.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `compiler_gym/leaderboard/llvm_instcount.py`
Content:
```
1 # Copyright (c) Facebook, Inc. and its affiliates.
2 #
3 # This source code is licensed under the MIT license found in the
4 # LICENSE file in the root directory of this source tree.
5 """LLVM is a popular open source compiler used widely in industry and research.
6 The :code:`llvm-ic-v0` environment exposes LLVM's optimizing passes as a set of
7 actions that can be applied to a particular program. The goal of the agent is to
8 select the sequence of optimizations that lead to the greatest reduction in
9 instruction count in the program being compiled. Reward is the reduction in
10 instruction count achieved scaled to the reduction achieved by LLVM's builtin
11 :code:`-Oz` pipeline.
12 
13 +--------------------+------------------------------------------------------+
14 | Property           | Value                                                |
15 +====================+======================================================+
16 | Environment        | :class:`LlvmEnv <compiler_gym.envs.LlvmEnv>`.        |
17 +--------------------+------------------------------------------------------+
18 | Observation Space  | Any.                                                 |
19 +--------------------+------------------------------------------------------+
20 | Reward Space       | Instruction count reduction relative to :code:`-Oz`. |
21 +--------------------+------------------------------------------------------+
22 | Test Dataset       | The 23 cBench benchmarks.                            |
23 +--------------------+------------------------------------------------------+
24 
25 Users who wish to create a submission for this leaderboard may use
26 :func:`eval_llvm_instcount_policy()
27 <compiler_gym.leaderboard.llvm_instcount.eval_llvm_instcount_policy>` to
28 automatically evaluate their agent on the test set.
29 """
30 import logging
31 import os
32 from itertools import islice
33 from pathlib import Path
34 from threading import Thread
35 from time import sleep
36 from typing import Callable, List
37 
38 import gym
39 import humanize
40 from absl import app, flags
41 
42 import compiler_gym.envs  # noqa Register environments.
43 from compiler_gym.bin.validate import main as validate
44 from compiler_gym.compiler_env_state import (
45     CompilerEnvState,
46     CompilerEnvStateReader,
47     CompilerEnvStateWriter,
48 )
49 from compiler_gym.envs import LlvmEnv
50 from compiler_gym.util.statistics import arithmetic_mean, geometric_mean
51 from compiler_gym.util.timer import Timer, humanize_duration_hms
52 
53 flags.DEFINE_string(
54     "leaderboard_results",
55     "llvm_instcount-results.csv",
56     "The path of the file to write results to.",
57 )
58 flags.DEFINE_string(
59     "leaderboard_logfile",
60     "llvm_instcount-results.log",
61     "The path of a file to stream CompilerGym logs to.",
62 )
63 flags.DEFINE_integer(
64     "max_benchmarks",
65     0,
66     "If > 0, use only the the first --max_benchmarks benchmarks from the "
67     "dataset, as determined by alphabetical sort. If not set, all benchmarks "
68     "from the dataset are used.",
69 )
70 flags.DEFINE_integer(
71     "n", 10, "The number of repetitions of the search to run for each benchmark."
72 )
73 flags.DEFINE_string("test_dataset", "cbench-v1", "The dataset to use for the search.")
74 flags.DEFINE_boolean("validate", True, "Run validation on the results.")
75 flags.DEFINE_boolean(
76     "resume",
77     False,
78     "If true, read the --leaderboard_results file first and run only the "
79     "evaluations not already in the results file.",
80 )
81 FLAGS = flags.FLAGS
82 
83 # A policy is a function that accepts as input an LLVM environment, and
84 # interacts with that environment with the goal of maximising cumulative reward.
85 Policy = Callable[[LlvmEnv], None]
86 
87 
88 class _EvalPolicyWorker(Thread):
89     """Worker thread to evaluate a policy."""
90 
91     def __init__(
92         self,
93         env: LlvmEnv,
94         benchmarks: List[str],
95         policy: Policy,
96         init_states: List[CompilerEnvState],
97     ):
98         super().__init__()
99         self.env = env
100         self.benchmarks = benchmarks
101         self.policy = policy
102         self.states: List[CompilerEnvState] = init_states
103         self.alive = True
104 
105     def run(self):
106         # Determine if we need to print a header.
107         header = (
108             not Path(FLAGS.leaderboard_results).is_file()
109             or os.stat(FLAGS.leaderboard_results).st_size == 0
110         )
111         with CompilerEnvStateWriter(
112             open(FLAGS.leaderboard_results, "a"), header=header
113         ) as writer:
114             for benchmark in self.benchmarks:
115                 self.env.reset(benchmark=benchmark)
116                 with Timer() as timer:
117                     self.policy(self.env)
118 
119                 # Sanity check that the policy didn't change the expected
120                 # experimental setup.
121                 assert self.env.in_episode, "Environment is no longer in an episode"
122                 assert self.env.benchmark and (
123                     self.env.benchmark == benchmark
124                 ), "Policy changed environment benchmark"
125                 assert self.env.reward_space, "Policy unset environment reward space"
126                 assert (
127                     self.env.reward_space.id == "IrInstructionCountOz"
128                 ), "Policy changed environment reward space"
129 
130                 # Override walltime in the generated state.
131                 state = self.env.state.copy()
132                 state.walltime = timer.time
133 
134                 writer.write_state(state, flush=True)
135                 self.states.append(state)
136 
137                 if not self.alive:
138                     return
139 
140 
141 def eval_llvm_instcount_policy(policy: Policy) -> None:
142     """Evaluate an LLVM codesize policy and generate results for a leaderboard
143     submission.
144 
145     To use it, you define your policy as a function that takes an
146     :class:`LlvmEnv <compiler_gym.envs.LlvmEnv>` instance as input and modifies
147     it in place. For example, for a trivial random policy:
148 
149         >>> from compiler_gym.envs import LlvmEnv
150         >>> def my_policy(env: LlvmEnv) -> None:
151         ....   # Defines a policy that takes 10 random steps.
152         ...    for _ in range(10):
153         ...        _, _, done, _ = env.step(env.action_space.sample())
154         ...        if done: break
155 
156     If your policy is stateful, you can use a class and override the
157     :code:`__call__()` method:
158 
159         >>> class MyPolicy:
160         ...     def __init__(self):
161         ...         self.my_stateful_vars = {}  # or similar
162         ...     def __call__(self, env: LlvmEnv) -> None:
163         ...         pass # ... do fun stuff!
164         >>> my_policy = MyPolicy()
165 
166     The role of your policy is to perform a sequence of actions on the supplied
167     environment so as to maximize cumulative reward. By default, no observation
168     space is set on the environment, so :meth:`env.step()
169     <compiler_gym.envs.CompilerEnv.step>` will return :code:`None` for the
170     observation. You may set a new observation space:
171 
172         >>> env.observation_space = "InstCount"  # Set a new space for env.step()
173         >>> env.observation["InstCount"]  # Calculate a one-off observation.
174 
175     However, the policy may not change the reward space of the environment, or
176     the benchmark.
177 
178     Once you have defined your policy, call the
179     :func:`eval_llvm_instcount_policy()
180     <compiler_gym.leaderboard.llvm_instcount.eval_llvm_instcount_policy>` helper
181     function, passing it your policy as its only argument:
182 
183     >>> eval_llvm_instcount_policy(my_policy)
184 
185     Put together as a complete example, a leaderboard submission script may look
186     like:
187 
188     .. code-block:: python
189 
190         # my_policy.py
191         from compiler_gym.leaderboard.llvm_instcount import eval_llvm_instcount_policy
192         from compiler_gym.envs import LlvmEnv
193 
194         def my_policy(env: LlvmEnv) -> None:
195             env.observation_space = "InstCount"  # we're going to use instcount space
196             pass # ... do fun stuff!
197 
198         if __name__ == "__main__":
199             eval_llvm_instcount_policy(my_policy)
200 
201     The :func:`eval_llvm_instcount_policy()
202     <compiler_gym.leaderboard.llvm_instcount.eval_llvm_instcount_policy>` helper
203     defines a number of commandline flags that can be overriden to control the
204     behavior of the evaluation. For example the flag :code:`--n` determines the
205     number of times the policy is run on each benchmark (default is 10), and
206     :code:`--leaderboard_results` determines the path of the generated results file:
207 
208     .. code-block::
209 
210         $ python my_policy.py --n=5 --leaderboard_results=my_policy_results.csv
211 
212     You can use :code:`--helpfull` flag to list all of the flags that are
213     defined:
214 
215     .. code-block::
216 
217         $ python my_policy.py --helpfull
218 
219     Once you are happy with your approach, see the `contributing guide
220     <https://github.com/facebookresearch/CompilerGym/blob/development/CONTRIBUTING.md#leaderboard-submissions>`_
221     for instructions on preparing a submission to the leaderboard.
222     """
223 
224     def main(argv):
225         assert len(argv) == 1, f"Unknown args: {argv[:1]}"
226         assert FLAGS.n > 0, "n must be > 0"
227 
228         with gym.make("llvm-ic-v0") as env:
229 
230             # Stream verbose CompilerGym logs to file.
231             logger = logging.getLogger("compiler_gym")
232             logger.setLevel(logging.DEBUG)
233             env.logger.setLevel(logging.DEBUG)
234             log_handler = logging.FileHandler(FLAGS.leaderboard_logfile)
235             logger.addHandler(log_handler)
236             logger.propagate = False
237 
238             print(f"Writing results to {FLAGS.leaderboard_results}")
239             print(f"Writing logs to {FLAGS.leaderboard_logfile}")
240 
241             # Build the list of benchmarks to evaluate.
242             benchmarks = env.datasets[FLAGS.test_dataset].benchmark_uris()
243             if FLAGS.max_benchmarks:
244                 benchmarks = islice(benchmarks, FLAGS.max_benchmarks)
245             benchmarks = list(benchmarks)
246 
247             # Repeat the searches for the requested number of iterations.
248             benchmarks *= FLAGS.n
249             benchmarks = sorted(benchmarks)
250             total_count = len(benchmarks)
251 
252             # If we are resuming from a previous job, read the states that have
253             # already been proccessed and remove those benchmarks from the list
254             # of benchmarks to evaluate.
255             init_states = []
256             if FLAGS.resume and Path(FLAGS.leaderboard_results).is_file():
257                 with CompilerEnvStateReader(open(FLAGS.leaderboard_results)) as reader:
258                     for state in reader:
259                         init_states.append(state)
260                         if state.benchmark in benchmarks:
261                             benchmarks.remove(state.benchmark)
262 
263             # Run the benchmark loop in background so that we can asynchronously
264             # log progress.
265             worker = _EvalPolicyWorker(env, benchmarks, policy, init_states)
266             worker.start()
267             timer = Timer().reset()
268             try:
269                 print(
270                     f"=== Evaluating policy on "
271                     f"{humanize.intcomma(total_count)} "
272                     f"{FLAGS.test_dataset} benchmarks ==="
273                     "\n\n"  # Blank lines will be filled below
274                 )
275                 while worker.is_alive():
276                     done_count = len(worker.states)
277                     remaining_count = total_count - done_count
278                     time = timer.time
279                     gmean_reward = geometric_mean([s.reward for s in worker.states])
280                     mean_walltime = (
281                         arithmetic_mean([s.walltime for s in worker.states]) or time
282                     )
283                     print(
284                         "\r\033[2A"
285                         "\033[K"
286                         f"Runtime: {humanize_duration_hms(time)}. "
287                         f"Estimated completion: {humanize_duration_hms(time + mean_walltime * remaining_count)}. "
288                         f"Completed: {humanize.intcomma(done_count)} / {humanize.intcomma(total_count)} "
289                         f"({done_count / total_count:.1%})."
290                         "\n\033[K"
291                         f"Current mean walltime: {mean_walltime:.3f}s / benchmark."
292                         "\n\033[K"
293                         f"Current geomean reward: {gmean_reward:.4f}.",
294                         flush=True,
295                         end="",
296                     )
297                     sleep(1)
298             except KeyboardInterrupt:
299                 print("\nkeyboard interrupt", flush=True)
300                 worker.alive = False
301                 # User interrupt, don't validate.
302                 FLAGS.validate = False
303 
304         if FLAGS.validate:
305             FLAGS.env = "llvm-ic-v0"
306             validate(["argv0", FLAGS.leaderboard_results])
307 
308     app.run(main)
309
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/compiler_gym/leaderboard/llvm_instcount.py b/compiler_gym/leaderboard/llvm_instcount.py
--- a/compiler_gym/leaderboard/llvm_instcount.py
+++ b/compiler_gym/leaderboard/llvm_instcount.py
@@ -246,7 +246,6 @@
 
             # Repeat the searches for the requested number of iterations.
             benchmarks *= FLAGS.n
-            benchmarks = sorted(benchmarks)
             total_count = len(benchmarks)
 
             # If we are resuming from a previous job, read the states that have

{"golden_diff": "diff --git a/compiler_gym/leaderboard/llvm_instcount.py b/compiler_gym/leaderboard/llvm_instcount.py\n--- a/compiler_gym/leaderboard/llvm_instcount.py\n+++ b/compiler_gym/leaderboard/llvm_instcount.py\n@@ -246,7 +246,6 @@\n \n             # Repeat the searches for the requested number of iterations.\n             benchmarks *= FLAGS.n\n-            benchmarks = sorted(benchmarks)\n             total_count = len(benchmarks)\n \n             # If we are resuming from a previous job, read the states that have\n", "issue": "Use round-robin ordering for leaderboard experiment execution\n## \ud83d\ude80 Feature\r\n\r\nDon't repeat all `--n` runs of each benchmark in order, perform 1 run of each benchmark, then proceed to the 2nd run, etc, until all `n` runs have been completed.\n", "before_files": [{"content": "# Copyright (c) Facebook, Inc. and its affiliates.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\"\"\"LLVM is a popular open source compiler used widely in industry and research.\nThe :code:`llvm-ic-v0` environment exposes LLVM's optimizing passes as a set of\nactions that can be applied to a particular program. The goal of the agent is to\nselect the sequence of optimizations that lead to the greatest reduction in\ninstruction count in the program being compiled. Reward is the reduction in\ninstruction count achieved scaled to the reduction achieved by LLVM's builtin\n:code:`-Oz` pipeline.\n\n+--------------------+------------------------------------------------------+\n| Property           | Value                                                |\n+====================+======================================================+\n| Environment        | :class:`LlvmEnv <compiler_gym.envs.LlvmEnv>`.        |\n+--------------------+------------------------------------------------------+\n| Observation Space  | Any.                                                 |\n+--------------------+------------------------------------------------------+\n| Reward Space       | Instruction count reduction relative to :code:`-Oz`. |\n+--------------------+------------------------------------------------------+\n| Test Dataset       | The 23 cBench benchmarks.                            |\n+--------------------+------------------------------------------------------+\n\nUsers who wish to create a submission for this leaderboard may use\n:func:`eval_llvm_instcount_policy()\n<compiler_gym.leaderboard.llvm_instcount.eval_llvm_instcount_policy>` to\nautomatically evaluate their agent on the test set.\n\"\"\"\nimport logging\nimport os\nfrom itertools import islice\nfrom pathlib import Path\nfrom threading import Thread\nfrom time import sleep\nfrom typing import Callable, List\n\nimport gym\nimport humanize\nfrom absl import app, flags\n\nimport compiler_gym.envs  # noqa Register environments.\nfrom compiler_gym.bin.validate import main as validate\nfrom compiler_gym.compiler_env_state import (\n    CompilerEnvState,\n    CompilerEnvStateReader,\n    CompilerEnvStateWriter,\n)\nfrom compiler_gym.envs import LlvmEnv\nfrom compiler_gym.util.statistics import arithmetic_mean, geometric_mean\nfrom compiler_gym.util.timer import Timer, humanize_duration_hms\n\nflags.DEFINE_string(\n    \"leaderboard_results\",\n    \"llvm_instcount-results.csv\",\n    \"The path of the file to write results to.\",\n)\nflags.DEFINE_string(\n    \"leaderboard_logfile\",\n    \"llvm_instcount-results.log\",\n    \"The path of a file to stream CompilerGym logs to.\",\n)\nflags.DEFINE_integer(\n    \"max_benchmarks\",\n    0,\n    \"If > 0, use only the the first --max_benchmarks benchmarks from the \"\n    \"dataset, as determined by alphabetical sort. If not set, all benchmarks \"\n    \"from the dataset are used.\",\n)\nflags.DEFINE_integer(\n    \"n\", 10, \"The number of repetitions of the search to run for each benchmark.\"\n)\nflags.DEFINE_string(\"test_dataset\", \"cbench-v1\", \"The dataset to use for the search.\")\nflags.DEFINE_boolean(\"validate\", True, \"Run validation on the results.\")\nflags.DEFINE_boolean(\n    \"resume\",\n    False,\n    \"If true, read the --leaderboard_results file first and run only the \"\n    \"evaluations not already in the results file.\",\n)\nFLAGS = flags.FLAGS\n\n# A policy is a function that accepts as input an LLVM environment, and\n# interacts with that environment with the goal of maximising cumulative reward.\nPolicy = Callable[[LlvmEnv], None]\n\n\nclass _EvalPolicyWorker(Thread):\n    \"\"\"Worker thread to evaluate a policy.\"\"\"\n\n    def __init__(\n        self,\n        env: LlvmEnv,\n        benchmarks: List[str],\n        policy: Policy,\n        init_states: List[CompilerEnvState],\n    ):\n        super().__init__()\n        self.env = env\n        self.benchmarks = benchmarks\n        self.policy = policy\n        self.states: List[CompilerEnvState] = init_states\n        self.alive = True\n\n    def run(self):\n        # Determine if we need to print a header.\n        header = (\n            not Path(FLAGS.leaderboard_results).is_file()\n            or os.stat(FLAGS.leaderboard_results).st_size == 0\n        )\n        with CompilerEnvStateWriter(\n            open(FLAGS.leaderboard_results, \"a\"), header=header\n        ) as writer:\n            for benchmark in self.benchmarks:\n                self.env.reset(benchmark=benchmark)\n                with Timer() as timer:\n                    self.policy(self.env)\n\n                # Sanity check that the policy didn't change the expected\n                # experimental setup.\n                assert self.env.in_episode, \"Environment is no longer in an episode\"\n                assert self.env.benchmark and (\n                    self.env.benchmark == benchmark\n                ), \"Policy changed environment benchmark\"\n                assert self.env.reward_space, \"Policy unset environment reward space\"\n                assert (\n                    self.env.reward_space.id == \"IrInstructionCountOz\"\n                ), \"Policy changed environment reward space\"\n\n                # Override walltime in the generated state.\n                state = self.env.state.copy()\n                state.walltime = timer.time\n\n                writer.write_state(state, flush=True)\n                self.states.append(state)\n\n                if not self.alive:\n                    return\n\n\ndef eval_llvm_instcount_policy(policy: Policy) -> None:\n    \"\"\"Evaluate an LLVM codesize policy and generate results for a leaderboard\n    submission.\n\n    To use it, you define your policy as a function that takes an\n    :class:`LlvmEnv <compiler_gym.envs.LlvmEnv>` instance as input and modifies\n    it in place. For example, for a trivial random policy:\n\n        >>> from compiler_gym.envs import LlvmEnv\n        >>> def my_policy(env: LlvmEnv) -> None:\n        ....   # Defines a policy that takes 10 random steps.\n        ...    for _ in range(10):\n        ...        _, _, done, _ = env.step(env.action_space.sample())\n        ...        if done: break\n\n    If your policy is stateful, you can use a class and override the\n    :code:`__call__()` method:\n\n        >>> class MyPolicy:\n        ...     def __init__(self):\n        ...         self.my_stateful_vars = {}  # or similar\n        ...     def __call__(self, env: LlvmEnv) -> None:\n        ...         pass # ... do fun stuff!\n        >>> my_policy = MyPolicy()\n\n    The role of your policy is to perform a sequence of actions on the supplied\n    environment so as to maximize cumulative reward. By default, no observation\n    space is set on the environment, so :meth:`env.step()\n    <compiler_gym.envs.CompilerEnv.step>` will return :code:`None` for the\n    observation. You may set a new observation space:\n\n        >>> env.observation_space = \"InstCount\"  # Set a new space for env.step()\n        >>> env.observation[\"InstCount\"]  # Calculate a one-off observation.\n\n    However, the policy may not change the reward space of the environment, or\n    the benchmark.\n\n    Once you have defined your policy, call the\n    :func:`eval_llvm_instcount_policy()\n    <compiler_gym.leaderboard.llvm_instcount.eval_llvm_instcount_policy>` helper\n    function, passing it your policy as its only argument:\n\n    >>> eval_llvm_instcount_policy(my_policy)\n\n    Put together as a complete example, a leaderboard submission script may look\n    like:\n\n    .. code-block:: python\n\n        # my_policy.py\n        from compiler_gym.leaderboard.llvm_instcount import eval_llvm_instcount_policy\n        from compiler_gym.envs import LlvmEnv\n\n        def my_policy(env: LlvmEnv) -> None:\n            env.observation_space = \"InstCount\"  # we're going to use instcount space\n            pass # ... do fun stuff!\n\n        if __name__ == \"__main__\":\n            eval_llvm_instcount_policy(my_policy)\n\n    The :func:`eval_llvm_instcount_policy()\n    <compiler_gym.leaderboard.llvm_instcount.eval_llvm_instcount_policy>` helper\n    defines a number of commandline flags that can be overriden to control the\n    behavior of the evaluation. For example the flag :code:`--n` determines the\n    number of times the policy is run on each benchmark (default is 10), and\n    :code:`--leaderboard_results` determines the path of the generated results file:\n\n    .. code-block::\n\n        $ python my_policy.py --n=5 --leaderboard_results=my_policy_results.csv\n\n    You can use :code:`--helpfull` flag to list all of the flags that are\n    defined:\n\n    .. code-block::\n\n        $ python my_policy.py --helpfull\n\n    Once you are happy with your approach, see the `contributing guide\n    <https://github.com/facebookresearch/CompilerGym/blob/development/CONTRIBUTING.md#leaderboard-submissions>`_\n    for instructions on preparing a submission to the leaderboard.\n    \"\"\"\n\n    def main(argv):\n        assert len(argv) == 1, f\"Unknown args: {argv[:1]}\"\n        assert FLAGS.n > 0, \"n must be > 0\"\n\n        with gym.make(\"llvm-ic-v0\") as env:\n\n            # Stream verbose CompilerGym logs to file.\n            logger = logging.getLogger(\"compiler_gym\")\n            logger.setLevel(logging.DEBUG)\n            env.logger.setLevel(logging.DEBUG)\n            log_handler = logging.FileHandler(FLAGS.leaderboard_logfile)\n            logger.addHandler(log_handler)\n            logger.propagate = False\n\n            print(f\"Writing results to {FLAGS.leaderboard_results}\")\n            print(f\"Writing logs to {FLAGS.leaderboard_logfile}\")\n\n            # Build the list of benchmarks to evaluate.\n            benchmarks = env.datasets[FLAGS.test_dataset].benchmark_uris()\n            if FLAGS.max_benchmarks:\n                benchmarks = islice(benchmarks, FLAGS.max_benchmarks)\n            benchmarks = list(benchmarks)\n\n            # Repeat the searches for the requested number of iterations.\n            benchmarks *= FLAGS.n\n            benchmarks = sorted(benchmarks)\n            total_count = len(benchmarks)\n\n            # If we are resuming from a previous job, read the states that have\n            # already been proccessed and remove those benchmarks from the list\n            # of benchmarks to evaluate.\n            init_states = []\n            if FLAGS.resume and Path(FLAGS.leaderboard_results).is_file():\n                with CompilerEnvStateReader(open(FLAGS.leaderboard_results)) as reader:\n                    for state in reader:\n                        init_states.append(state)\n                        if state.benchmark in benchmarks:\n                            benchmarks.remove(state.benchmark)\n\n            # Run the benchmark loop in background so that we can asynchronously\n            # log progress.\n            worker = _EvalPolicyWorker(env, benchmarks, policy, init_states)\n            worker.start()\n            timer = Timer().reset()\n            try:\n                print(\n                    f\"=== Evaluating policy on \"\n                    f\"{humanize.intcomma(total_count)} \"\n                    f\"{FLAGS.test_dataset} benchmarks ===\"\n                    \"\\n\\n\"  # Blank lines will be filled below\n                )\n                while worker.is_alive():\n                    done_count = len(worker.states)\n                    remaining_count = total_count - done_count\n                    time = timer.time\n                    gmean_reward = geometric_mean([s.reward for s in worker.states])\n                    mean_walltime = (\n                        arithmetic_mean([s.walltime for s in worker.states]) or time\n                    )\n                    print(\n                        \"\\r\\033[2A\"\n                        \"\\033[K\"\n                        f\"Runtime: {humanize_duration_hms(time)}. \"\n                        f\"Estimated completion: {humanize_duration_hms(time + mean_walltime * remaining_count)}. \"\n                        f\"Completed: {humanize.intcomma(done_count)} / {humanize.intcomma(total_count)} \"\n                        f\"({done_count / total_count:.1%}).\"\n                        \"\\n\\033[K\"\n                        f\"Current mean walltime: {mean_walltime:.3f}s / benchmark.\"\n                        \"\\n\\033[K\"\n                        f\"Current geomean reward: {gmean_reward:.4f}.\",\n                        flush=True,\n                        end=\"\",\n                    )\n                    sleep(1)\n            except KeyboardInterrupt:\n                print(\"\\nkeyboard interrupt\", flush=True)\n                worker.alive = False\n                # User interrupt, don't validate.\n                FLAGS.validate = False\n\n        if FLAGS.validate:\n            FLAGS.env = \"llvm-ic-v0\"\n            validate([\"argv0\", FLAGS.leaderboard_results])\n\n    app.run(main)\n", "path": "compiler_gym/leaderboard/llvm_instcount.py"}], "after_files": [{"content": "# Copyright (c) Facebook, Inc. and its affiliates.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\"\"\"LLVM is a popular open source compiler used widely in industry and research.\nThe :code:`llvm-ic-v0` environment exposes LLVM's optimizing passes as a set of\nactions that can be applied to a particular program. The goal of the agent is to\nselect the sequence of optimizations that lead to the greatest reduction in\ninstruction count in the program being compiled. Reward is the reduction in\ninstruction count achieved scaled to the reduction achieved by LLVM's builtin\n:code:`-Oz` pipeline.\n\n+--------------------+------------------------------------------------------+\n| Property           | Value                                                |\n+====================+======================================================+\n| Environment        | :class:`LlvmEnv <compiler_gym.envs.LlvmEnv>`.        |\n+--------------------+------------------------------------------------------+\n| Observation Space  | Any.                                                 |\n+--------------------+------------------------------------------------------+\n| Reward Space       | Instruction count reduction relative to :code:`-Oz`. |\n+--------------------+------------------------------------------------------+\n| Test Dataset       | The 23 cBench benchmarks.                            |\n+--------------------+------------------------------------------------------+\n\nUsers who wish to create a submission for this leaderboard may use\n:func:`eval_llvm_instcount_policy()\n<compiler_gym.leaderboard.llvm_instcount.eval_llvm_instcount_policy>` to\nautomatically evaluate their agent on the test set.\n\"\"\"\nimport logging\nimport os\nfrom itertools import islice\nfrom pathlib import Path\nfrom threading import Thread\nfrom time import sleep\nfrom typing import Callable, List\n\nimport gym\nimport humanize\nfrom absl import app, flags\n\nimport compiler_gym.envs  # noqa Register environments.\nfrom compiler_gym.bin.validate import main as validate\nfrom compiler_gym.compiler_env_state import (\n    CompilerEnvState,\n    CompilerEnvStateReader,\n    CompilerEnvStateWriter,\n)\nfrom compiler_gym.envs import LlvmEnv\nfrom compiler_gym.util.statistics import arithmetic_mean, geometric_mean\nfrom compiler_gym.util.timer import Timer, humanize_duration_hms\n\nflags.DEFINE_string(\n    \"leaderboard_results\",\n    \"llvm_instcount-results.csv\",\n    \"The path of the file to write results to.\",\n)\nflags.DEFINE_string(\n    \"leaderboard_logfile\",\n    \"llvm_instcount-results.log\",\n    \"The path of a file to stream CompilerGym logs to.\",\n)\nflags.DEFINE_integer(\n    \"max_benchmarks\",\n    0,\n    \"If > 0, use only the the first --max_benchmarks benchmarks from the \"\n    \"dataset, as determined by alphabetical sort. If not set, all benchmarks \"\n    \"from the dataset are used.\",\n)\nflags.DEFINE_integer(\n    \"n\", 10, \"The number of repetitions of the search to run for each benchmark.\"\n)\nflags.DEFINE_string(\"test_dataset\", \"cbench-v1\", \"The dataset to use for the search.\")\nflags.DEFINE_boolean(\"validate\", True, \"Run validation on the results.\")\nflags.DEFINE_boolean(\n    \"resume\",\n    False,\n    \"If true, read the --leaderboard_results file first and run only the \"\n    \"evaluations not already in the results file.\",\n)\nFLAGS = flags.FLAGS\n\n# A policy is a function that accepts as input an LLVM environment, and\n# interacts with that environment with the goal of maximising cumulative reward.\nPolicy = Callable[[LlvmEnv], None]\n\n\nclass _EvalPolicyWorker(Thread):\n    \"\"\"Worker thread to evaluate a policy.\"\"\"\n\n    def __init__(\n        self,\n        env: LlvmEnv,\n        benchmarks: List[str],\n        policy: Policy,\n        init_states: List[CompilerEnvState],\n    ):\n        super().__init__()\n        self.env = env\n        self.benchmarks = benchmarks\n        self.policy = policy\n        self.states: List[CompilerEnvState] = init_states\n        self.alive = True\n\n    def run(self):\n        # Determine if we need to print a header.\n        header = (\n            not Path(FLAGS.leaderboard_results).is_file()\n            or os.stat(FLAGS.leaderboard_results).st_size == 0\n        )\n        with CompilerEnvStateWriter(\n            open(FLAGS.leaderboard_results, \"a\"), header=header\n        ) as writer:\n            for benchmark in self.benchmarks:\n                self.env.reset(benchmark=benchmark)\n                with Timer() as timer:\n                    self.policy(self.env)\n\n                # Sanity check that the policy didn't change the expected\n                # experimental setup.\n                assert self.env.in_episode, \"Environment is no longer in an episode\"\n                assert self.env.benchmark and (\n                    self.env.benchmark == benchmark\n                ), \"Policy changed environment benchmark\"\n                assert self.env.reward_space, \"Policy unset environment reward space\"\n                assert (\n                    self.env.reward_space.id == \"IrInstructionCountOz\"\n                ), \"Policy changed environment reward space\"\n\n                # Override walltime in the generated state.\n                state = self.env.state.copy()\n                state.walltime = timer.time\n\n                writer.write_state(state, flush=True)\n                self.states.append(state)\n\n                if not self.alive:\n                    return\n\n\ndef eval_llvm_instcount_policy(policy: Policy) -> None:\n    \"\"\"Evaluate an LLVM codesize policy and generate results for a leaderboard\n    submission.\n\n    To use it, you define your policy as a function that takes an\n    :class:`LlvmEnv <compiler_gym.envs.LlvmEnv>` instance as input and modifies\n    it in place. For example, for a trivial random policy:\n\n        >>> from compiler_gym.envs import LlvmEnv\n        >>> def my_policy(env: LlvmEnv) -> None:\n        ....   # Defines a policy that takes 10 random steps.\n        ...    for _ in range(10):\n        ...        _, _, done, _ = env.step(env.action_space.sample())\n        ...        if done: break\n\n    If your policy is stateful, you can use a class and override the\n    :code:`__call__()` method:\n\n        >>> class MyPolicy:\n        ...     def __init__(self):\n        ...         self.my_stateful_vars = {}  # or similar\n        ...     def __call__(self, env: LlvmEnv) -> None:\n        ...         pass # ... do fun stuff!\n        >>> my_policy = MyPolicy()\n\n    The role of your policy is to perform a sequence of actions on the supplied\n    environment so as to maximize cumulative reward. By default, no observation\n    space is set on the environment, so :meth:`env.step()\n    <compiler_gym.envs.CompilerEnv.step>` will return :code:`None` for the\n    observation. You may set a new observation space:\n\n        >>> env.observation_space = \"InstCount\"  # Set a new space for env.step()\n        >>> env.observation[\"InstCount\"]  # Calculate a one-off observation.\n\n    However, the policy may not change the reward space of the environment, or\n    the benchmark.\n\n    Once you have defined your policy, call the\n    :func:`eval_llvm_instcount_policy()\n    <compiler_gym.leaderboard.llvm_instcount.eval_llvm_instcount_policy>` helper\n    function, passing it your policy as its only argument:\n\n    >>> eval_llvm_instcount_policy(my_policy)\n\n    Put together as a complete example, a leaderboard submission script may look\n    like:\n\n    .. code-block:: python\n\n        # my_policy.py\n        from compiler_gym.leaderboard.llvm_instcount import eval_llvm_instcount_policy\n        from compiler_gym.envs import LlvmEnv\n\n        def my_policy(env: LlvmEnv) -> None:\n            env.observation_space = \"InstCount\"  # we're going to use instcount space\n            pass # ... do fun stuff!\n\n        if __name__ == \"__main__\":\n            eval_llvm_instcount_policy(my_policy)\n\n    The :func:`eval_llvm_instcount_policy()\n    <compiler_gym.leaderboard.llvm_instcount.eval_llvm_instcount_policy>` helper\n    defines a number of commandline flags that can be overriden to control the\n    behavior of the evaluation. For example the flag :code:`--n` determines the\n    number of times the policy is run on each benchmark (default is 10), and\n    :code:`--leaderboard_results` determines the path of the generated results file:\n\n    .. code-block::\n\n        $ python my_policy.py --n=5 --leaderboard_results=my_policy_results.csv\n\n    You can use :code:`--helpfull` flag to list all of the flags that are\n    defined:\n\n    .. code-block::\n\n        $ python my_policy.py --helpfull\n\n    Once you are happy with your approach, see the `contributing guide\n    <https://github.com/facebookresearch/CompilerGym/blob/development/CONTRIBUTING.md#leaderboard-submissions>`_\n    for instructions on preparing a submission to the leaderboard.\n    \"\"\"\n\n    def main(argv):\n        assert len(argv) == 1, f\"Unknown args: {argv[:1]}\"\n        assert FLAGS.n > 0, \"n must be > 0\"\n\n        with gym.make(\"llvm-ic-v0\") as env:\n\n            # Stream verbose CompilerGym logs to file.\n            logger = logging.getLogger(\"compiler_gym\")\n            logger.setLevel(logging.DEBUG)\n            env.logger.setLevel(logging.DEBUG)\n            log_handler = logging.FileHandler(FLAGS.leaderboard_logfile)\n            logger.addHandler(log_handler)\n            logger.propagate = False\n\n            print(f\"Writing results to {FLAGS.leaderboard_results}\")\n            print(f\"Writing logs to {FLAGS.leaderboard_logfile}\")\n\n            # Build the list of benchmarks to evaluate.\n            benchmarks = env.datasets[FLAGS.test_dataset].benchmark_uris()\n            if FLAGS.max_benchmarks:\n                benchmarks = islice(benchmarks, FLAGS.max_benchmarks)\n            benchmarks = list(benchmarks)\n\n            # Repeat the searches for the requested number of iterations.\n            benchmarks *= FLAGS.n\n            total_count = len(benchmarks)\n\n            # If we are resuming from a previous job, read the states that have\n            # already been proccessed and remove those benchmarks from the list\n            # of benchmarks to evaluate.\n            init_states = []\n            if FLAGS.resume and Path(FLAGS.leaderboard_results).is_file():\n                with CompilerEnvStateReader(open(FLAGS.leaderboard_results)) as reader:\n                    for state in reader:\n                        init_states.append(state)\n                        if state.benchmark in benchmarks:\n                            benchmarks.remove(state.benchmark)\n\n            # Run the benchmark loop in background so that we can asynchronously\n            # log progress.\n            worker = _EvalPolicyWorker(env, benchmarks, policy, init_states)\n            worker.start()\n            timer = Timer().reset()\n            try:\n                print(\n                    f\"=== Evaluating policy on \"\n                    f\"{humanize.intcomma(total_count)} \"\n                    f\"{FLAGS.test_dataset} benchmarks ===\"\n                    \"\\n\\n\"  # Blank lines will be filled below\n                )\n                while worker.is_alive():\n                    done_count = len(worker.states)\n                    remaining_count = total_count - done_count\n                    time = timer.time\n                    gmean_reward = geometric_mean([s.reward for s in worker.states])\n                    mean_walltime = (\n                        arithmetic_mean([s.walltime for s in worker.states]) or time\n                    )\n                    print(\n                        \"\\r\\033[2A\"\n                        \"\\033[K\"\n                        f\"Runtime: {humanize_duration_hms(time)}. \"\n                        f\"Estimated completion: {humanize_duration_hms(time + mean_walltime * remaining_count)}. \"\n                        f\"Completed: {humanize.intcomma(done_count)} / {humanize.intcomma(total_count)} \"\n                        f\"({done_count / total_count:.1%}).\"\n                        \"\\n\\033[K\"\n                        f\"Current mean walltime: {mean_walltime:.3f}s / benchmark.\"\n                        \"\\n\\033[K\"\n                        f\"Current geomean reward: {gmean_reward:.4f}.\",\n                        flush=True,\n                        end=\"\",\n                    )\n                    sleep(1)\n            except KeyboardInterrupt:\n                print(\"\\nkeyboard interrupt\", flush=True)\n                worker.alive = False\n                # User interrupt, don't validate.\n                FLAGS.validate = False\n\n        if FLAGS.validate:\n            FLAGS.env = \"llvm-ic-v0\"\n            validate([\"argv0\", FLAGS.leaderboard_results])\n\n    app.run(main)\n", "path": "compiler_gym/leaderboard/llvm_instcount.py"}]}