We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Tune the check for training metrics
We currently set an arbitrary max 5% worse result, we might need to tune it

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/analyze_training_metrics.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 """ Given a directory containing training metrics, generate SVF graphs and check that the metrics are not getting worse than before.
6 """
7 
8 import argparse
9 import json
10 import logging
11 import sys
12 from collections import defaultdict
13 from datetime import datetime, timezone
14 from pathlib import Path
15 from typing import Dict
16 
17 import matplotlib.dates as mdates
18 import matplotlib.pyplot as plt
19 
20 LOGGER = logging.getLogger(__name__)
21 
22 logging.basicConfig(level=logging.INFO)
23 
24 # If the latest metric point is 5% lower than the previous one, show a warning and exit with 1.
25 WARNING_THRESHOLD = 0.95
26 
27 REPORT_METRICS = ["accuracy", "precision", "recall"]
28 
29 
30 def plot_graph(
31     model_name: str,
32     metric_name: str,
33     values_dict: Dict[datetime, float],
34     output_directory: Path,
35 ) -> bool:
36     sorted_metrics = sorted(values_dict.items())
37     x, y = zip(*sorted_metrics)
38 
39     # Compute the threshold
40     if len(y) >= 2:
41         before_last_value = y[-2]
42     else:
43         before_last_value = y[-1]
44     metric_threshold = before_last_value * WARNING_THRESHOLD
45 
46     figure = plt.figure()
47     axes = plt.axes()
48 
49     # Formatting of the figure
50     figure.autofmt_xdate()
51     axes.fmt_xdata = mdates.DateFormatter("%Y-%m-%d-%H-%M")
52     axes.set_title(f"{model_name} {metric_name}")
53 
54     # Display threshold
55     axes.axhline(y=metric_threshold, linestyle="--", color="red")
56     plt.annotate(
57         "{:.4f}".format(metric_threshold),
58         (x[-1], metric_threshold),
59         textcoords="offset points",  # how to position the text
60         xytext=(-10, 10),  # distance from text to points (x,y)
61         ha="center",
62         color="red",
63     )
64 
65     # Display point values
66     for single_x, single_y in zip(x, y):
67         label = "{:.4f}".format(single_y)
68 
69         plt.annotate(
70             label,
71             (single_x, single_y),
72             textcoords="offset points",
73             xytext=(0, 10),
74             ha="center",
75         )
76 
77     axes.plot_date(x, y, marker=".", fmt="-")
78 
79     output_file_path = output_directory.resolve() / f"{model_name}_{metric_name}.svg"
80     LOGGER.info("Saving %s figure", output_file_path)
81     plt.savefig(output_file_path)
82 
83     plt.close(figure)
84 
85     # Check if the threshold has been crossed
86     return y[-1] < metric_threshold
87 
88 
89 def analyze_metrics(metrics_directory: str, output_directory: str):
90     root = Path(metrics_directory)
91 
92     metrics: Dict[str, Dict[str, Dict[datetime, float]]] = defaultdict(
93         lambda: defaultdict(dict)
94     )
95 
96     threshold_ever_crossed = False
97 
98     for metric_file_path in root.glob("metric*.json"):
99 
100         # Load the metric
101         with open(metric_file_path, "r") as metric_file:
102             metric = json.load(metric_file)
103 
104         # Get the model, date and version from the file
105         # TODO: Might be better storing it in the file
106         file_path_parts = metric_file_path.stem.split("_")
107 
108         assert file_path_parts[:5] == ["metric", "project", "relman", "bugbug", "train"]
109         model_name = file_path_parts[5]
110         assert file_path_parts[6:8] == ["per", "date"]
111         date_parts = list(map(int, file_path_parts[8:14]))
112         date = datetime(
113             date_parts[0],
114             date_parts[1],
115             date_parts[2],
116             date_parts[3],
117             date_parts[4],
118             date_parts[5],
119             tzinfo=timezone.utc,
120         )
121         # version = file_path_parts[14:]  # TODO: Use version
122 
123         # Then process the report
124         for key, value in metric["report"]["average"].items():
125             if key not in REPORT_METRICS:
126                 continue
127 
128             metrics[model_name][key][date] = value
129 
130         # Also process the test_* metrics
131         for key, value in metric.items():
132             if not key.startswith("test_"):
133                 continue
134 
135             metrics[model_name][f"{key}_mean"][date] = value["mean"]
136             metrics[model_name][f"{key}_std"][date] = value["std"]
137 
138     for model_name in metrics:
139         for metric_name, values in metrics[model_name].items():
140             threshold_crossed = plot_graph(
141                 model_name, metric_name, values, Path(output_directory)
142             )
143 
144             if threshold_crossed:
145                 LOGGER.warning(
146                     "Last metric %r for model %s is 5%% worse than the previous one",
147                     metric_name,
148                     model_name,
149                 )
150 
151                 threshold_ever_crossed = threshold_ever_crossed or threshold_crossed
152 
153     if threshold_ever_crossed:
154         sys.exit(1)
155 
156 
157 def main():
158     parser = argparse.ArgumentParser(description=__doc__)
159 
160     parser.add_argument(
161         "metrics_directory",
162         metavar="metrics-directory",
163         help="In which directory the script can find the metrics JSON files",
164     )
165     parser.add_argument(
166         "output_directory",
167         metavar="output-directory",
168         help="In which directory the script will save the generated graphs",
169     )
170 
171     args = parser.parse_args()
172 
173     analyze_metrics(args.metrics_directory, args.output_directory)
174 
175 
176 if __name__ == "__main__":
177     main()
178
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/analyze_training_metrics.py b/scripts/analyze_training_metrics.py
--- a/scripts/analyze_training_metrics.py
+++ b/scripts/analyze_training_metrics.py
@@ -21,7 +21,8 @@
 
 logging.basicConfig(level=logging.INFO)
 
-# If the latest metric point is 5% lower than the previous one, show a warning and exit with 1.
+# By default, if the latest metric point is 5% lower than the previous one, show a warning and exit
+# with 1.
 WARNING_THRESHOLD = 0.95
 
 REPORT_METRICS = ["accuracy", "precision", "recall"]
@@ -32,6 +33,7 @@
     metric_name: str,
     values_dict: Dict[datetime, float],
     output_directory: Path,
+    warning_threshold: float,
 ) -> bool:
     sorted_metrics = sorted(values_dict.items())
     x, y = zip(*sorted_metrics)
@@ -41,7 +43,7 @@
         before_last_value = y[-2]
     else:
         before_last_value = y[-1]
-    metric_threshold = before_last_value * WARNING_THRESHOLD
+    metric_threshold = before_last_value * warning_threshold
 
     figure = plt.figure()
     axes = plt.axes()
@@ -86,7 +88,9 @@
     return y[-1] < metric_threshold
 
 
-def analyze_metrics(metrics_directory: str, output_directory: str):
+def analyze_metrics(
+    metrics_directory: str, output_directory: str, warning_threshold: float
+):
     root = Path(metrics_directory)
 
     metrics: Dict[str, Dict[str, Dict[datetime, float]]] = defaultdict(
@@ -138,14 +142,21 @@
     for model_name in metrics:
         for metric_name, values in metrics[model_name].items():
             threshold_crossed = plot_graph(
-                model_name, metric_name, values, Path(output_directory)
+                model_name,
+                metric_name,
+                values,
+                Path(output_directory),
+                warning_threshold,
             )
 
+            diff = (1 - warning_threshold) * 100
+
             if threshold_crossed:
                 LOGGER.warning(
-                    "Last metric %r for model %s is 5%% worse than the previous one",
+                    "Last metric %r for model %s is %f%% worse than the previous one",
                     metric_name,
                     model_name,
+                    diff,
                 )
 
                 threshold_ever_crossed = threshold_ever_crossed or threshold_crossed
@@ -167,10 +178,18 @@
         metavar="output-directory",
         help="In which directory the script will save the generated graphs",
     )
+    parser.add_argument(
+        "--warning_threshold",
+        default=WARNING_THRESHOLD,
+        type=float,
+        help="If the last metric value is below the previous one*warning_threshold, fails. Default to 0.95",
+    )
 
     args = parser.parse_args()
 
-    analyze_metrics(args.metrics_directory, args.output_directory)
+    analyze_metrics(
+        args.metrics_directory, args.output_directory, args.warning_threshold
+    )
 
 
 if __name__ == "__main__":

{"golden_diff": "diff --git a/scripts/analyze_training_metrics.py b/scripts/analyze_training_metrics.py\n--- a/scripts/analyze_training_metrics.py\n+++ b/scripts/analyze_training_metrics.py\n@@ -21,7 +21,8 @@\n \n logging.basicConfig(level=logging.INFO)\n \n-# If the latest metric point is 5% lower than the previous one, show a warning and exit with 1.\n+# By default, if the latest metric point is 5% lower than the previous one, show a warning and exit\n+# with 1.\n WARNING_THRESHOLD = 0.95\n \n REPORT_METRICS = [\"accuracy\", \"precision\", \"recall\"]\n@@ -32,6 +33,7 @@\n     metric_name: str,\n     values_dict: Dict[datetime, float],\n     output_directory: Path,\n+    warning_threshold: float,\n ) -> bool:\n     sorted_metrics = sorted(values_dict.items())\n     x, y = zip(*sorted_metrics)\n@@ -41,7 +43,7 @@\n         before_last_value = y[-2]\n     else:\n         before_last_value = y[-1]\n-    metric_threshold = before_last_value * WARNING_THRESHOLD\n+    metric_threshold = before_last_value * warning_threshold\n \n     figure = plt.figure()\n     axes = plt.axes()\n@@ -86,7 +88,9 @@\n     return y[-1] < metric_threshold\n \n \n-def analyze_metrics(metrics_directory: str, output_directory: str):\n+def analyze_metrics(\n+    metrics_directory: str, output_directory: str, warning_threshold: float\n+):\n     root = Path(metrics_directory)\n \n     metrics: Dict[str, Dict[str, Dict[datetime, float]]] = defaultdict(\n@@ -138,14 +142,21 @@\n     for model_name in metrics:\n         for metric_name, values in metrics[model_name].items():\n             threshold_crossed = plot_graph(\n-                model_name, metric_name, values, Path(output_directory)\n+                model_name,\n+                metric_name,\n+                values,\n+                Path(output_directory),\n+                warning_threshold,\n             )\n \n+            diff = (1 - warning_threshold) * 100\n+\n             if threshold_crossed:\n                 LOGGER.warning(\n-                    \"Last metric %r for model %s is 5%% worse than the previous one\",\n+                    \"Last metric %r for model %s is %f%% worse than the previous one\",\n                     metric_name,\n                     model_name,\n+                    diff,\n                 )\n \n                 threshold_ever_crossed = threshold_ever_crossed or threshold_crossed\n@@ -167,10 +178,18 @@\n         metavar=\"output-directory\",\n         help=\"In which directory the script will save the generated graphs\",\n     )\n+    parser.add_argument(\n+        \"--warning_threshold\",\n+        default=WARNING_THRESHOLD,\n+        type=float,\n+        help=\"If the last metric value is below the previous one*warning_threshold, fails. Default to 0.95\",\n+    )\n \n     args = parser.parse_args()\n \n-    analyze_metrics(args.metrics_directory, args.output_directory)\n+    analyze_metrics(\n+        args.metrics_directory, args.output_directory, args.warning_threshold\n+    )\n \n \n if __name__ == \"__main__\":\n", "issue": "Tune the check for training metrics\nWe currently set an arbitrary max 5% worse result, we might need to tune it\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\"\"\" Given a directory containing training metrics, generate SVF graphs and check that the metrics are not getting worse than before.\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport sys\nfrom collections import defaultdict\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom typing import Dict\n\nimport matplotlib.dates as mdates\nimport matplotlib.pyplot as plt\n\nLOGGER = logging.getLogger(__name__)\n\nlogging.basicConfig(level=logging.INFO)\n\n# If the latest metric point is 5% lower than the previous one, show a warning and exit with 1.\nWARNING_THRESHOLD = 0.95\n\nREPORT_METRICS = [\"accuracy\", \"precision\", \"recall\"]\n\n\ndef plot_graph(\n    model_name: str,\n    metric_name: str,\n    values_dict: Dict[datetime, float],\n    output_directory: Path,\n) -> bool:\n    sorted_metrics = sorted(values_dict.items())\n    x, y = zip(*sorted_metrics)\n\n    # Compute the threshold\n    if len(y) >= 2:\n        before_last_value = y[-2]\n    else:\n        before_last_value = y[-1]\n    metric_threshold = before_last_value * WARNING_THRESHOLD\n\n    figure = plt.figure()\n    axes = plt.axes()\n\n    # Formatting of the figure\n    figure.autofmt_xdate()\n    axes.fmt_xdata = mdates.DateFormatter(\"%Y-%m-%d-%H-%M\")\n    axes.set_title(f\"{model_name} {metric_name}\")\n\n    # Display threshold\n    axes.axhline(y=metric_threshold, linestyle=\"--\", color=\"red\")\n    plt.annotate(\n        \"{:.4f}\".format(metric_threshold),\n        (x[-1], metric_threshold),\n        textcoords=\"offset points\",  # how to position the text\n        xytext=(-10, 10),  # distance from text to points (x,y)\n        ha=\"center\",\n        color=\"red\",\n    )\n\n    # Display point values\n    for single_x, single_y in zip(x, y):\n        label = \"{:.4f}\".format(single_y)\n\n        plt.annotate(\n            label,\n            (single_x, single_y),\n            textcoords=\"offset points\",\n            xytext=(0, 10),\n            ha=\"center\",\n        )\n\n    axes.plot_date(x, y, marker=\".\", fmt=\"-\")\n\n    output_file_path = output_directory.resolve() / f\"{model_name}_{metric_name}.svg\"\n    LOGGER.info(\"Saving %s figure\", output_file_path)\n    plt.savefig(output_file_path)\n\n    plt.close(figure)\n\n    # Check if the threshold has been crossed\n    return y[-1] < metric_threshold\n\n\ndef analyze_metrics(metrics_directory: str, output_directory: str):\n    root = Path(metrics_directory)\n\n    metrics: Dict[str, Dict[str, Dict[datetime, float]]] = defaultdict(\n        lambda: defaultdict(dict)\n    )\n\n    threshold_ever_crossed = False\n\n    for metric_file_path in root.glob(\"metric*.json\"):\n\n        # Load the metric\n        with open(metric_file_path, \"r\") as metric_file:\n            metric = json.load(metric_file)\n\n        # Get the model, date and version from the file\n        # TODO: Might be better storing it in the file\n        file_path_parts = metric_file_path.stem.split(\"_\")\n\n        assert file_path_parts[:5] == [\"metric\", \"project\", \"relman\", \"bugbug\", \"train\"]\n        model_name = file_path_parts[5]\n        assert file_path_parts[6:8] == [\"per\", \"date\"]\n        date_parts = list(map(int, file_path_parts[8:14]))\n        date = datetime(\n            date_parts[0],\n            date_parts[1],\n            date_parts[2],\n            date_parts[3],\n            date_parts[4],\n            date_parts[5],\n            tzinfo=timezone.utc,\n        )\n        # version = file_path_parts[14:]  # TODO: Use version\n\n        # Then process the report\n        for key, value in metric[\"report\"][\"average\"].items():\n            if key not in REPORT_METRICS:\n                continue\n\n            metrics[model_name][key][date] = value\n\n        # Also process the test_* metrics\n        for key, value in metric.items():\n            if not key.startswith(\"test_\"):\n                continue\n\n            metrics[model_name][f\"{key}_mean\"][date] = value[\"mean\"]\n            metrics[model_name][f\"{key}_std\"][date] = value[\"std\"]\n\n    for model_name in metrics:\n        for metric_name, values in metrics[model_name].items():\n            threshold_crossed = plot_graph(\n                model_name, metric_name, values, Path(output_directory)\n            )\n\n            if threshold_crossed:\n                LOGGER.warning(\n                    \"Last metric %r for model %s is 5%% worse than the previous one\",\n                    metric_name,\n                    model_name,\n                )\n\n                threshold_ever_crossed = threshold_ever_crossed or threshold_crossed\n\n    if threshold_ever_crossed:\n        sys.exit(1)\n\n\ndef main():\n    parser = argparse.ArgumentParser(description=__doc__)\n\n    parser.add_argument(\n        \"metrics_directory\",\n        metavar=\"metrics-directory\",\n        help=\"In which directory the script can find the metrics JSON files\",\n    )\n    parser.add_argument(\n        \"output_directory\",\n        metavar=\"output-directory\",\n        help=\"In which directory the script will save the generated graphs\",\n    )\n\n    args = parser.parse_args()\n\n    analyze_metrics(args.metrics_directory, args.output_directory)\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "scripts/analyze_training_metrics.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\"\"\" Given a directory containing training metrics, generate SVF graphs and check that the metrics are not getting worse than before.\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport sys\nfrom collections import defaultdict\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom typing import Dict\n\nimport matplotlib.dates as mdates\nimport matplotlib.pyplot as plt\n\nLOGGER = logging.getLogger(__name__)\n\nlogging.basicConfig(level=logging.INFO)\n\n# By default, if the latest metric point is 5% lower than the previous one, show a warning and exit\n# with 1.\nWARNING_THRESHOLD = 0.95\n\nREPORT_METRICS = [\"accuracy\", \"precision\", \"recall\"]\n\n\ndef plot_graph(\n    model_name: str,\n    metric_name: str,\n    values_dict: Dict[datetime, float],\n    output_directory: Path,\n    warning_threshold: float,\n) -> bool:\n    sorted_metrics = sorted(values_dict.items())\n    x, y = zip(*sorted_metrics)\n\n    # Compute the threshold\n    if len(y) >= 2:\n        before_last_value = y[-2]\n    else:\n        before_last_value = y[-1]\n    metric_threshold = before_last_value * warning_threshold\n\n    figure = plt.figure()\n    axes = plt.axes()\n\n    # Formatting of the figure\n    figure.autofmt_xdate()\n    axes.fmt_xdata = mdates.DateFormatter(\"%Y-%m-%d-%H-%M\")\n    axes.set_title(f\"{model_name} {metric_name}\")\n\n    # Display threshold\n    axes.axhline(y=metric_threshold, linestyle=\"--\", color=\"red\")\n    plt.annotate(\n        \"{:.4f}\".format(metric_threshold),\n        (x[-1], metric_threshold),\n        textcoords=\"offset points\",  # how to position the text\n        xytext=(-10, 10),  # distance from text to points (x,y)\n        ha=\"center\",\n        color=\"red\",\n    )\n\n    # Display point values\n    for single_x, single_y in zip(x, y):\n        label = \"{:.4f}\".format(single_y)\n\n        plt.annotate(\n            label,\n            (single_x, single_y),\n            textcoords=\"offset points\",\n            xytext=(0, 10),\n            ha=\"center\",\n        )\n\n    axes.plot_date(x, y, marker=\".\", fmt=\"-\")\n\n    output_file_path = output_directory.resolve() / f\"{model_name}_{metric_name}.svg\"\n    LOGGER.info(\"Saving %s figure\", output_file_path)\n    plt.savefig(output_file_path)\n\n    plt.close(figure)\n\n    # Check if the threshold has been crossed\n    return y[-1] < metric_threshold\n\n\ndef analyze_metrics(\n    metrics_directory: str, output_directory: str, warning_threshold: float\n):\n    root = Path(metrics_directory)\n\n    metrics: Dict[str, Dict[str, Dict[datetime, float]]] = defaultdict(\n        lambda: defaultdict(dict)\n    )\n\n    threshold_ever_crossed = False\n\n    for metric_file_path in root.glob(\"metric*.json\"):\n\n        # Load the metric\n        with open(metric_file_path, \"r\") as metric_file:\n            metric = json.load(metric_file)\n\n        # Get the model, date and version from the file\n        # TODO: Might be better storing it in the file\n        file_path_parts = metric_file_path.stem.split(\"_\")\n\n        assert file_path_parts[:5] == [\"metric\", \"project\", \"relman\", \"bugbug\", \"train\"]\n        model_name = file_path_parts[5]\n        assert file_path_parts[6:8] == [\"per\", \"date\"]\n        date_parts = list(map(int, file_path_parts[8:14]))\n        date = datetime(\n            date_parts[0],\n            date_parts[1],\n            date_parts[2],\n            date_parts[3],\n            date_parts[4],\n            date_parts[5],\n            tzinfo=timezone.utc,\n        )\n        # version = file_path_parts[14:]  # TODO: Use version\n\n        # Then process the report\n        for key, value in metric[\"report\"][\"average\"].items():\n            if key not in REPORT_METRICS:\n                continue\n\n            metrics[model_name][key][date] = value\n\n        # Also process the test_* metrics\n        for key, value in metric.items():\n            if not key.startswith(\"test_\"):\n                continue\n\n            metrics[model_name][f\"{key}_mean\"][date] = value[\"mean\"]\n            metrics[model_name][f\"{key}_std\"][date] = value[\"std\"]\n\n    for model_name in metrics:\n        for metric_name, values in metrics[model_name].items():\n            threshold_crossed = plot_graph(\n                model_name,\n                metric_name,\n                values,\n                Path(output_directory),\n                warning_threshold,\n            )\n\n            diff = (1 - warning_threshold) * 100\n\n            if threshold_crossed:\n                LOGGER.warning(\n                    \"Last metric %r for model %s is %f%% worse than the previous one\",\n                    metric_name,\n                    model_name,\n                    diff,\n                )\n\n                threshold_ever_crossed = threshold_ever_crossed or threshold_crossed\n\n    if threshold_ever_crossed:\n        sys.exit(1)\n\n\ndef main():\n    parser = argparse.ArgumentParser(description=__doc__)\n\n    parser.add_argument(\n        \"metrics_directory\",\n        metavar=\"metrics-directory\",\n        help=\"In which directory the script can find the metrics JSON files\",\n    )\n    parser.add_argument(\n        \"output_directory\",\n        metavar=\"output-directory\",\n        help=\"In which directory the script will save the generated graphs\",\n    )\n    parser.add_argument(\n        \"--warning_threshold\",\n        default=WARNING_THRESHOLD,\n        type=float,\n        help=\"If the last metric value is below the previous one*warning_threshold, fails. Default to 0.95\",\n    )\n\n    args = parser.parse_args()\n\n    analyze_metrics(\n        args.metrics_directory, args.output_directory, args.warning_threshold\n    )\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "scripts/analyze_training_metrics.py"}]}