prompt: You will be provided with a partial code base and an issue statement explaining a problem to resolve.
  <issue>
  Registry path CLI option for oaievalset
  ### Describe the feature or improvement you're requesting
  
  It should take an `--registry_path` like `oaieval` does
  
  ### Additional context
  
  Useful when using as a library, to run eval sets stored in my own project outside this repo.
  
  </issue>
  <code>
  [start of evals/cli/oaievalset.py]
  1 """
  2 This file defines the `oaievalset` CLI for running eval sets.
  3 """
  4 import argparse
  5 import json
  6 import logging
  7 import subprocess
  8 from pathlib import Path
  9 from typing import Optional, cast
  10 
  11 from evals.registry import Registry
  12 
  13 Task = list[str]
  14 logger = logging.getLogger(__name__)
  15 
  16 
  17 class Progress:
  18     def __init__(self, file: str) -> None:
  19         self.file = Path(file)
  20         self.completed: list[Task] = []
  21 
  22     def load(self) -> bool:
  23         if not self.file.exists():
  24             return False
  25 
  26         with self.file.open() as f:
  27             for line in f:
  28                 self.completed.append(json.loads(line))
  29         return len(self.completed) > 0
  30 
  31     def add(self, item: Task) -> None:
  32         self.completed.append(item)
  33         self.save()
  34 
  35     def save(self) -> None:
  36         self.file.parent.mkdir(parents=True, exist_ok=True)
  37         with self.file.open("w") as f:
  38             for item in self.completed:
  39                 f.write(json.dumps(item) + "\n")
  40             print(highlight(f"Saved progress to {self.file}"))
  41 
  42 
  43 def highlight(str: str) -> str:
  44     return f"\033[1;32m>>> {str}\033[0m"
  45 
  46 
  47 def get_parser() -> argparse.ArgumentParser:
  48     parser = argparse.ArgumentParser(description="Run eval sets through the API")
  49     parser.add_argument("model", type=str, help="Name of a completion model.")
  50     parser.add_argument("eval_set", type=str, help="Name of eval set. See registry.")
  51     parser.add_argument(
  52         "--resume",
  53         action=argparse.BooleanOptionalAction,
  54         default=True,
  55         help="Resume from last checkpoint.",
  56     )
  57     parser.add_argument(
  58         "--exit-on-error",
  59         action=argparse.BooleanOptionalAction,
  60         default=True,
  61         help="Exit if any oaieval command fails.",
  62     )
  63     return parser
  64 
  65 
  66 class OaiEvalSetArguments(argparse.Namespace):
  67     model: str
  68     eval_set: str
  69     resume: bool
  70     exit_on_error: bool
  71 
  72 
  73 def run(
  74     args: OaiEvalSetArguments,
  75     unknown_args: list[str],
  76     registry: Optional[Registry] = None,
  77     run_command: str = "oaieval",
  78 ) -> None:
  79     registry = registry or Registry()
  80     commands: list[Task] = []
  81     eval_set = registry.get_eval_set(args.eval_set) if args.eval_set else None
  82     if eval_set:
  83         for index, eval in enumerate(registry.get_evals(eval_set.evals)):
  84             if not eval or not eval.key:
  85                 logger.debug("The eval #%d in eval_set is not valid", index)
  86 
  87             command = [run_command, args.model, eval.key] + unknown_args
  88             if command in commands:
  89                 continue
  90             commands.append(command)
  91     else:
  92         logger.warning("No eval set found for %s", args.eval_set)
  93 
  94     num_evals = len(commands)
  95 
  96     progress = Progress(f"/tmp/oaievalset/{args.model}.{args.eval_set}.progress.txt")
  97     if args.resume and progress.load():
  98         print(f"Loaded progress from {progress.file}")
  99         print(f"{len(progress.completed)}/{len(commands)} evals already completed:")
  100         for item in progress.completed:
  101             print("  " + " ".join(item))
  102 
  103     commands = [c for c in commands if c not in progress.completed]
  104     command_strs = [" ".join(cmd) for cmd in commands]
  105     print("Going to run the following commands:")
  106     for command_str in command_strs:
  107         print("  " + command_str)
  108 
  109     num_already_completed = num_evals - len(commands)
  110     for idx, command in enumerate(commands):
  111         real_idx = idx + num_already_completed
  112         print(highlight("Running command: " + " ".join(command) + f" ({real_idx+1}/{num_evals})"))
  113         subprocess.run(command, stdout=subprocess.PIPE, check=args.exit_on_error)
  114         progress.add(command)
  115 
  116     print(highlight("All done!"))
  117 
  118 
  119 def main() -> None:
  120     parser = get_parser()
  121     args, unknown_args = parser.parse_known_args()
  122     run(cast(OaiEvalSetArguments, args), unknown_args)
  123 
  124 
  125 if __name__ == "__main__":
  126     main()
  127
  [end of evals/cli/oaievalset.py]
  </code>
  
  I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format.
  <patch>
  diff --git a/file.py b/file.py
  --- a/file.py
  +++ b/file.py
  @@ -1,27 +1,35 @@
   def euclidean(a, b):
  -    while b:
  -        a, b = b, a % b
  -    return a
  +    if b == 0:
  +        return a
  +    return euclidean(b, a % b)
   
   
   def bresenham(x0, y0, x1, y1):
       points = []
       dx = abs(x1 - x0)
       dy = abs(y1 - y0)
  -    sx = 1 if x0 < x1 else -1
  -    sy = 1 if y0 < y1 else -1
  -    err = dx - dy
  +    x, y = x0, y0
  +    sx = -1 if x0 > x1 else 1
  +    sy = -1 if y0 > y1 else 1
   
  -    while True:
  -        points.append((x0, y0))
  -        if x0 == x1 and y0 == y1:
  -            break
  -        e2 = 2 * err
  -        if e2 > -dy:
  -            err -= dy
  -            x0 += sx
  -        if e2 < dx:
  -            err += dx
  -            y0 += sy
  +    if dx > dy:
  +        err = dx / 2.0
  +        while x != x1:
  +            points.append((x, y))
  +            err -= dy
  +            if err < 0:
  +                y += sy
  +                err += dx
  +            x += sx
  +    else:
  +        err = dy / 2.0
  +        while y != y1:
  +            points.append((x, y))
  +            err -= dx
  +            if err < 0:
  +                x += sx
  +                err += dy
  +            y += sy
  +
  +    points.append((x, y))
       return points
  
  </patch>
golden_diff: diff --git a/evals/cli/oaievalset.py b/evals/cli/oaievalset.py
  --- a/evals/cli/oaievalset.py
  +++ b/evals/cli/oaievalset.py
  @@ -48,6 +48,13 @@
       parser = argparse.ArgumentParser(description="Run eval sets through the API")
       parser.add_argument("model", type=str, help="Name of a completion model.")
       parser.add_argument("eval_set", type=str, help="Name of eval set. See registry.")
  +    parser.add_argument(
  +        "--registry_path",
  +        type=str,
  +        default=None,
  +        action="append",
  +        help="Path to the registry",
  +    )
       parser.add_argument(
           "--resume",
           action=argparse.BooleanOptionalAction,
  @@ -66,6 +73,7 @@
   class OaiEvalSetArguments(argparse.Namespace):
       model: str
       eval_set: str
  +    registry_path: Optional[str]
       resume: bool
       exit_on_error: bool
   
  @@ -77,6 +85,9 @@
       run_command: str = "oaieval",
   ) -> None:
       registry = registry or Registry()
  +    if args.registry_path:
  +        registry.add_registry_paths(args.registry_path)
  +
       commands: list[Task] = []
       eval_set = registry.get_eval_set(args.eval_set) if args.eval_set else None
       if eval_set:
  @@ -85,6 +96,9 @@
                   logger.debug("The eval #%d in eval_set is not valid", index)
   
               command = [run_command, args.model, eval.key] + unknown_args
  +            if args.registry_path:
  +                command.append("--registry_path")
  +                command = command + args.registry_path
               if command in commands:
                   continue
               commands.append(command)
  
verification_info: {"golden_diff": "diff --git a/evals/cli/oaievalset.py b/evals/cli/oaievalset.py\n--- a/evals/cli/oaievalset.py\n+++ b/evals/cli/oaievalset.py\n@@ -48,6 +48,13 @@\n     parser = argparse.ArgumentParser(description=\"Run eval sets through the API\")\n     parser.add_argument(\"model\", type=str, help=\"Name of a completion model.\")\n     parser.add_argument(\"eval_set\", type=str, help=\"Name of eval set. See registry.\")\n+    parser.add_argument(\n+        \"--registry_path\",\n+        type=str,\n+        default=None,\n+        action=\"append\",\n+        help=\"Path to the registry\",\n+    )\n     parser.add_argument(\n         \"--resume\",\n         action=argparse.BooleanOptionalAction,\n@@ -66,6 +73,7 @@\n class OaiEvalSetArguments(argparse.Namespace):\n     model: str\n     eval_set: str\n+    registry_path: Optional[str]\n     resume: bool\n     exit_on_error: bool\n \n@@ -77,6 +85,9 @@\n     run_command: str = \"oaieval\",\n ) -> None:\n     registry = registry or Registry()\n+    if args.registry_path:\n+        registry.add_registry_paths(args.registry_path)\n+\n     commands: list[Task] = []\n     eval_set = registry.get_eval_set(args.eval_set) if args.eval_set else None\n     if eval_set:\n@@ -85,6 +96,9 @@\n                 logger.debug(\"The eval #%d in eval_set is not valid\", index)\n \n             command = [run_command, args.model, eval.key] + unknown_args\n+            if args.registry_path:\n+                command.append(\"--registry_path\")\n+                command = command + args.registry_path\n             if command in commands:\n                 continue\n             commands.append(command)\n", "issue": "Registry path CLI option for oaievalset\n### Describe the feature or improvement you're requesting\n\nIt should take an `--registry_path` like `oaieval` does\n\n### Additional context\n\nUseful when using as a library, to run eval sets stored in my own project outside this repo.\n", "before_files": [{"content": "\"\"\"\nThis file defines the `oaievalset` CLI for running eval sets.\n\"\"\"\nimport argparse\nimport json\nimport logging\nimport subprocess\nfrom pathlib import Path\nfrom typing import Optional, cast\n\nfrom evals.registry import Registry\n\nTask = list[str]\nlogger = logging.getLogger(__name__)\n\n\nclass Progress:\n    def __init__(self, file: str) -> None:\n        self.file = Path(file)\n        self.completed: list[Task] = []\n\n    def load(self) -> bool:\n        if not self.file.exists():\n            return False\n\n        with self.file.open() as f:\n            for line in f:\n                self.completed.append(json.loads(line))\n        return len(self.completed) > 0\n\n    def add(self, item: Task) -> None:\n        self.completed.append(item)\n        self.save()\n\n    def save(self) -> None:\n        self.file.parent.mkdir(parents=True, exist_ok=True)\n        with self.file.open(\"w\") as f:\n            for item in self.completed:\n                f.write(json.dumps(item) + \"\\n\")\n            print(highlight(f\"Saved progress to {self.file}\"))\n\n\ndef highlight(str: str) -> str:\n    return f\"\\033[1;32m>>> {str}\\033[0m\"\n\n\ndef get_parser() -> argparse.ArgumentParser:\n    parser = argparse.ArgumentParser(description=\"Run eval sets through the API\")\n    parser.add_argument(\"model\", type=str, help=\"Name of a completion model.\")\n    parser.add_argument(\"eval_set\", type=str, help=\"Name of eval set. See registry.\")\n    parser.add_argument(\n        \"--resume\",\n        action=argparse.BooleanOptionalAction,\n        default=True,\n        help=\"Resume from last checkpoint.\",\n    )\n    parser.add_argument(\n        \"--exit-on-error\",\n        action=argparse.BooleanOptionalAction,\n        default=True,\n        help=\"Exit if any oaieval command fails.\",\n    )\n    return parser\n\n\nclass OaiEvalSetArguments(argparse.Namespace):\n    model: str\n    eval_set: str\n    resume: bool\n    exit_on_error: bool\n\n\ndef run(\n    args: OaiEvalSetArguments,\n    unknown_args: list[str],\n    registry: Optional[Registry] = None,\n    run_command: str = \"oaieval\",\n) -> None:\n    registry = registry or Registry()\n    commands: list[Task] = []\n    eval_set = registry.get_eval_set(args.eval_set) if args.eval_set else None\n    if eval_set:\n        for index, eval in enumerate(registry.get_evals(eval_set.evals)):\n            if not eval or not eval.key:\n                logger.debug(\"The eval #%d in eval_set is not valid\", index)\n\n            command = [run_command, args.model, eval.key] + unknown_args\n            if command in commands:\n                continue\n            commands.append(command)\n    else:\n        logger.warning(\"No eval set found for %s\", args.eval_set)\n\n    num_evals = len(commands)\n\n    progress = Progress(f\"/tmp/oaievalset/{args.model}.{args.eval_set}.progress.txt\")\n    if args.resume and progress.load():\n        print(f\"Loaded progress from {progress.file}\")\n        print(f\"{len(progress.completed)}/{len(commands)} evals already completed:\")\n        for item in progress.completed:\n            print(\"  \" + \" \".join(item))\n\n    commands = [c for c in commands if c not in progress.completed]\n    command_strs = [\" \".join(cmd) for cmd in commands]\n    print(\"Going to run the following commands:\")\n    for command_str in command_strs:\n        print(\"  \" + command_str)\n\n    num_already_completed = num_evals - len(commands)\n    for idx, command in enumerate(commands):\n        real_idx = idx + num_already_completed\n        print(highlight(\"Running command: \" + \" \".join(command) + f\" ({real_idx+1}/{num_evals})\"))\n        subprocess.run(command, stdout=subprocess.PIPE, check=args.exit_on_error)\n        progress.add(command)\n\n    print(highlight(\"All done!\"))\n\n\ndef main() -> None:\n    parser = get_parser()\n    args, unknown_args = parser.parse_known_args()\n    run(cast(OaiEvalSetArguments, args), unknown_args)\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "evals/cli/oaievalset.py"}]}