We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Issues with latest MIRACL 2CR
On my iMac Pro (Intel), I'm getting the following failures:

```
condition bm25-mdpr-tied-pft-msmarco-hybrid.bn:
  - split: dev
      nDCG@10: 0.6540 [OK]
      R@100  : 0.9321 [FAIL] expected 0.9100

condition bm25-mdpr-tied-pft-msmarco-hybrid.zh:
  - split: dev
      nDCG@10: 0.5255 [FAIL] expected 0.5254
      R@100  : 0.9587 [OK]
```


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/repro_matrix/run_all_miracl.py`
Content:
```
1 #
2 # Pyserini: Reproducible IR research with sparse and dense representations
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 #
16 
17 import argparse
18 import math
19 import os
20 import subprocess
21 import time
22 from collections import defaultdict
23 from string import Template
24 
25 import yaml
26 
27 from scripts.repro_matrix.defs_miracl import models, languages, trec_eval_metric_definitions
28 from scripts.repro_matrix.utils import run_eval_and_return_metric, ok_str, okish_str, fail_str
29 
30 
31 def print_results(metric, split):
32     print(f'Metric = {metric}, Split = {split}')
33     print(' ' * 32, end='')
34     for lang in languages:
35         print(f'{lang[0]:3}    ', end='')
36     print('')
37     for model in models:
38         print(f'{model:30}', end='')
39         for lang in languages:
40             key = f'{model}.{lang[0]}'
41             print(f'{table[key][split][metric]:7.3f}', end='')
42         print('')
43     print('')
44 
45 
46 def extract_topic_fn_from_cmd(cmd):
47     cmd = cmd.split()
48     topic_idx = cmd.index('--topics')
49     return cmd[topic_idx + 1]
50 
51 
52 if __name__ == '__main__':
53     parser = argparse.ArgumentParser(description='Generate regression matrix for MIRACL.')
54     parser.add_argument('--skip-eval', action='store_true', default=False, help='Skip running trec_eval.')
55     args = parser.parse_args()
56 
57     start = time.time()
58 
59     table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
60 
61     with open('pyserini/resources/miracl.yaml') as f:
62         yaml_data = yaml.safe_load(f)
63         for condition in yaml_data['conditions']:
64             name = condition['name']
65             eval_key = condition['eval_key']
66             cmd_template = condition['command']
67             cmd_lst = cmd_template.split()
68 
69             print(f'condition {name}:')
70             lang = name.split('.')[-1]
71             is_hybrid_run = 'hybrid' in name
72 
73             for splits in condition['splits']:
74                 split = splits['split']
75                 if is_hybrid_run:
76                     hits = int(cmd_lst[cmd_lst.index('--k') + 1])
77                 else:
78                     hits = int(cmd_lst[cmd_lst.index('--hits') + 1])
79 
80                 print(f'  - split: {split}')
81 
82                 runfile = f'runs/run.miracl.{name}.{split}.top{hits}.txt'
83                 if is_hybrid_run:
84                     bm25_output = f'runs/run.miracl.bm25.{lang}.{split}.top{hits}.txt'
85                     mdpr_output = f'runs/run.miracl.mdpr-tied-pft-msmarco.{lang}.{split}.top{hits}.txt'
86                     if not os.path.exists(bm25_output):
87                         print(f'Missing BM25 file: {bm25_output}')
88                         continue
89                     if not os.path.exists(mdpr_output):
90                         print(f'Missing mDPR file: {mdpr_output}')
91                         continue
92                     cmd = Template(cmd_template).substitute(split=split, output=runfile, bm25_output=bm25_output, mdpr_output=mdpr_output)
93                 else:
94                     cmd = Template(cmd_template).substitute(split=split, output=runfile)
95 
96                 # In the yaml file, the topics are written as something like '--topics miracl-v1.0-ar-${split}'
97                 # This works for the dev split because the topics are directly included in Anserini/Pyserini.
98                 # For this training split, we have to map the symbol into a file in tools/topics-and-qrels/
99                 # Here, we assume that the developer has cloned the miracl repo and placed the topics there.
100                 if split == 'train':
101                     cmd = cmd.replace(f'--topics miracl-v1.0-{lang}-{split}',
102                                       f'--topics tools/topics-and-qrels/topics.miracl-v1.0-{lang}-{split}.tsv')
103 
104                 if not os.path.exists(runfile):
105                     print(f'    Running: {cmd}')
106                     rtn = subprocess.run(cmd.split(), capture_output=True)
107                     stderr = rtn.stderr.decode()
108                     if '--topics' in cmd:
109                         topic_fn = extract_topic_fn_from_cmd(cmd)
110                         if f'ValueError: Topic {topic_fn} Not Found' in stderr:
111                             print(f'Skipping {topic_fn}: file not found.')
112                             continue
113 
114                 for expected in splits['scores']:
115                     for metric in expected:
116                         if not args.skip_eval:
117                             # We have the translate the training qrels into a file located in tools/topics-and-qrels/
118                             # because they are not included with Anserini/Pyserini by default.
119                             # Here, we assume that the developer has cloned the miracl repo and placed the qrels there.
120                             if split == 'train':
121                                 qrels = f'tools/topics-and-qrels/qrels.{eval_key}-train.tsv'
122                             else:
123                                 qrels = f'{eval_key}-{split}'
124                             score = float(run_eval_and_return_metric(metric, qrels,
125                                                                      trec_eval_metric_definitions[metric], runfile))
126                             if math.isclose(score, float(expected[metric])):
127                                 result_str = ok_str
128                             # Flaky test: small difference on Mac Studio (M1 chip)
129                             elif name == 'mdpr-tied-pft-msmarco.hi' and split == 'train' \
130                                     and math.isclose(score, float(expected[metric]), abs_tol=2e-4):
131                                 result_str = okish_str
132                             else:
133                                 result_str = fail_str + f' expected {expected[metric]:.4f}'
134                             print(f'      {metric:7}: {score:.4f} {result_str}')
135                             table[name][split][metric] = score
136                         else:
137                             table[name][split][metric] = expected[metric]
138 
139             print('')
140 
141     for metric in ['nDCG@10', 'R@100']:
142         for split in ['dev', 'train']:
143             print_results(metric, split)
144 
145     end = time.time()
146     print(f'Total elapsed time: {end - start:.0f}s')
147
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/repro_matrix/run_all_miracl.py b/scripts/repro_matrix/run_all_miracl.py
--- a/scripts/repro_matrix/run_all_miracl.py
+++ b/scripts/repro_matrix/run_all_miracl.py
@@ -30,12 +30,12 @@
 
 def print_results(metric, split):
     print(f'Metric = {metric}, Split = {split}')
-    print(' ' * 32, end='')
+    print(' ' * 35, end='')
     for lang in languages:
         print(f'{lang[0]:3}    ', end='')
     print('')
     for model in models:
-        print(f'{model:30}', end='')
+        print(f'{model:33}', end='')
         for lang in languages:
             key = f'{model}.{lang[0]}'
             print(f'{table[key][split][metric]:7.3f}', end='')
@@ -125,9 +125,18 @@
                                                                      trec_eval_metric_definitions[metric], runfile))
                             if math.isclose(score, float(expected[metric])):
                                 result_str = ok_str
-                            # Flaky test: small difference on Mac Studio (M1 chip)
-                            elif name == 'mdpr-tied-pft-msmarco.hi' and split == 'train' \
-                                    and math.isclose(score, float(expected[metric]), abs_tol=2e-4):
+                            # Flaky tests
+                            elif (name == 'mdpr-tied-pft-msmarco.hi' and split == 'train'
+                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \
+                                 (name == 'mdpr-tied-pft-msmarco-ft-all.ru'
+                                  and split == 'dev' and metric == 'nDCG@10'
+                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \
+                                 (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.te'
+                                  and split == 'train' and metric == 'nDCG@10'
+                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \
+                                 (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.zh'
+                                  and split == 'dev' and metric == 'nDCG@10'
+                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)):
                                 result_str = okish_str
                             else:
                                 result_str = fail_str + f' expected {expected[metric]:.4f}'

{"golden_diff": "diff --git a/scripts/repro_matrix/run_all_miracl.py b/scripts/repro_matrix/run_all_miracl.py\n--- a/scripts/repro_matrix/run_all_miracl.py\n+++ b/scripts/repro_matrix/run_all_miracl.py\n@@ -30,12 +30,12 @@\n \n def print_results(metric, split):\n     print(f'Metric = {metric}, Split = {split}')\n-    print(' ' * 32, end='')\n+    print(' ' * 35, end='')\n     for lang in languages:\n         print(f'{lang[0]:3}    ', end='')\n     print('')\n     for model in models:\n-        print(f'{model:30}', end='')\n+        print(f'{model:33}', end='')\n         for lang in languages:\n             key = f'{model}.{lang[0]}'\n             print(f'{table[key][split][metric]:7.3f}', end='')\n@@ -125,9 +125,18 @@\n                                                                      trec_eval_metric_definitions[metric], runfile))\n                             if math.isclose(score, float(expected[metric])):\n                                 result_str = ok_str\n-                            # Flaky test: small difference on Mac Studio (M1 chip)\n-                            elif name == 'mdpr-tied-pft-msmarco.hi' and split == 'train' \\\n-                                    and math.isclose(score, float(expected[metric]), abs_tol=2e-4):\n+                            # Flaky tests\n+                            elif (name == 'mdpr-tied-pft-msmarco.hi' and split == 'train'\n+                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \\\n+                                 (name == 'mdpr-tied-pft-msmarco-ft-all.ru'\n+                                  and split == 'dev' and metric == 'nDCG@10'\n+                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \\\n+                                 (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.te'\n+                                  and split == 'train' and metric == 'nDCG@10'\n+                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \\\n+                                 (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.zh'\n+                                  and split == 'dev' and metric == 'nDCG@10'\n+                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)):\n                                 result_str = okish_str\n                             else:\n                                 result_str = fail_str + f' expected {expected[metric]:.4f}'\n", "issue": "Issues with latest MIRACL 2CR\nOn my iMac Pro (Intel), I'm getting the following failures:\r\n\r\n```\r\ncondition bm25-mdpr-tied-pft-msmarco-hybrid.bn:\r\n  - split: dev\r\n      nDCG@10: 0.6540 [OK]\r\n      R@100  : 0.9321 [FAIL] expected 0.9100\r\n\r\ncondition bm25-mdpr-tied-pft-msmarco-hybrid.zh:\r\n  - split: dev\r\n      nDCG@10: 0.5255 [FAIL] expected 0.5254\r\n      R@100  : 0.9587 [OK]\r\n```\r\n\n", "before_files": [{"content": "#\n# Pyserini: Reproducible IR research with sparse and dense representations\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport argparse\nimport math\nimport os\nimport subprocess\nimport time\nfrom collections import defaultdict\nfrom string import Template\n\nimport yaml\n\nfrom scripts.repro_matrix.defs_miracl import models, languages, trec_eval_metric_definitions\nfrom scripts.repro_matrix.utils import run_eval_and_return_metric, ok_str, okish_str, fail_str\n\n\ndef print_results(metric, split):\n    print(f'Metric = {metric}, Split = {split}')\n    print(' ' * 32, end='')\n    for lang in languages:\n        print(f'{lang[0]:3}    ', end='')\n    print('')\n    for model in models:\n        print(f'{model:30}', end='')\n        for lang in languages:\n            key = f'{model}.{lang[0]}'\n            print(f'{table[key][split][metric]:7.3f}', end='')\n        print('')\n    print('')\n\n\ndef extract_topic_fn_from_cmd(cmd):\n    cmd = cmd.split()\n    topic_idx = cmd.index('--topics')\n    return cmd[topic_idx + 1]\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(description='Generate regression matrix for MIRACL.')\n    parser.add_argument('--skip-eval', action='store_true', default=False, help='Skip running trec_eval.')\n    args = parser.parse_args()\n\n    start = time.time()\n\n    table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))\n\n    with open('pyserini/resources/miracl.yaml') as f:\n        yaml_data = yaml.safe_load(f)\n        for condition in yaml_data['conditions']:\n            name = condition['name']\n            eval_key = condition['eval_key']\n            cmd_template = condition['command']\n            cmd_lst = cmd_template.split()\n\n            print(f'condition {name}:')\n            lang = name.split('.')[-1]\n            is_hybrid_run = 'hybrid' in name\n\n            for splits in condition['splits']:\n                split = splits['split']\n                if is_hybrid_run:\n                    hits = int(cmd_lst[cmd_lst.index('--k') + 1])\n                else:\n                    hits = int(cmd_lst[cmd_lst.index('--hits') + 1])\n\n                print(f'  - split: {split}')\n\n                runfile = f'runs/run.miracl.{name}.{split}.top{hits}.txt'\n                if is_hybrid_run:\n                    bm25_output = f'runs/run.miracl.bm25.{lang}.{split}.top{hits}.txt'\n                    mdpr_output = f'runs/run.miracl.mdpr-tied-pft-msmarco.{lang}.{split}.top{hits}.txt'\n                    if not os.path.exists(bm25_output):\n                        print(f'Missing BM25 file: {bm25_output}')\n                        continue\n                    if not os.path.exists(mdpr_output):\n                        print(f'Missing mDPR file: {mdpr_output}')\n                        continue\n                    cmd = Template(cmd_template).substitute(split=split, output=runfile, bm25_output=bm25_output, mdpr_output=mdpr_output)\n                else:\n                    cmd = Template(cmd_template).substitute(split=split, output=runfile)\n\n                # In the yaml file, the topics are written as something like '--topics miracl-v1.0-ar-${split}'\n                # This works for the dev split because the topics are directly included in Anserini/Pyserini.\n                # For this training split, we have to map the symbol into a file in tools/topics-and-qrels/\n                # Here, we assume that the developer has cloned the miracl repo and placed the topics there.\n                if split == 'train':\n                    cmd = cmd.replace(f'--topics miracl-v1.0-{lang}-{split}',\n                                      f'--topics tools/topics-and-qrels/topics.miracl-v1.0-{lang}-{split}.tsv')\n\n                if not os.path.exists(runfile):\n                    print(f'    Running: {cmd}')\n                    rtn = subprocess.run(cmd.split(), capture_output=True)\n                    stderr = rtn.stderr.decode()\n                    if '--topics' in cmd:\n                        topic_fn = extract_topic_fn_from_cmd(cmd)\n                        if f'ValueError: Topic {topic_fn} Not Found' in stderr:\n                            print(f'Skipping {topic_fn}: file not found.')\n                            continue\n\n                for expected in splits['scores']:\n                    for metric in expected:\n                        if not args.skip_eval:\n                            # We have the translate the training qrels into a file located in tools/topics-and-qrels/\n                            # because they are not included with Anserini/Pyserini by default.\n                            # Here, we assume that the developer has cloned the miracl repo and placed the qrels there.\n                            if split == 'train':\n                                qrels = f'tools/topics-and-qrels/qrels.{eval_key}-train.tsv'\n                            else:\n                                qrels = f'{eval_key}-{split}'\n                            score = float(run_eval_and_return_metric(metric, qrels,\n                                                                     trec_eval_metric_definitions[metric], runfile))\n                            if math.isclose(score, float(expected[metric])):\n                                result_str = ok_str\n                            # Flaky test: small difference on Mac Studio (M1 chip)\n                            elif name == 'mdpr-tied-pft-msmarco.hi' and split == 'train' \\\n                                    and math.isclose(score, float(expected[metric]), abs_tol=2e-4):\n                                result_str = okish_str\n                            else:\n                                result_str = fail_str + f' expected {expected[metric]:.4f}'\n                            print(f'      {metric:7}: {score:.4f} {result_str}')\n                            table[name][split][metric] = score\n                        else:\n                            table[name][split][metric] = expected[metric]\n\n            print('')\n\n    for metric in ['nDCG@10', 'R@100']:\n        for split in ['dev', 'train']:\n            print_results(metric, split)\n\n    end = time.time()\n    print(f'Total elapsed time: {end - start:.0f}s')\n", "path": "scripts/repro_matrix/run_all_miracl.py"}], "after_files": [{"content": "#\n# Pyserini: Reproducible IR research with sparse and dense representations\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport argparse\nimport math\nimport os\nimport subprocess\nimport time\nfrom collections import defaultdict\nfrom string import Template\n\nimport yaml\n\nfrom scripts.repro_matrix.defs_miracl import models, languages, trec_eval_metric_definitions\nfrom scripts.repro_matrix.utils import run_eval_and_return_metric, ok_str, okish_str, fail_str\n\n\ndef print_results(metric, split):\n    print(f'Metric = {metric}, Split = {split}')\n    print(' ' * 35, end='')\n    for lang in languages:\n        print(f'{lang[0]:3}    ', end='')\n    print('')\n    for model in models:\n        print(f'{model:33}', end='')\n        for lang in languages:\n            key = f'{model}.{lang[0]}'\n            print(f'{table[key][split][metric]:7.3f}', end='')\n        print('')\n    print('')\n\n\ndef extract_topic_fn_from_cmd(cmd):\n    cmd = cmd.split()\n    topic_idx = cmd.index('--topics')\n    return cmd[topic_idx + 1]\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(description='Generate regression matrix for MIRACL.')\n    parser.add_argument('--skip-eval', action='store_true', default=False, help='Skip running trec_eval.')\n    args = parser.parse_args()\n\n    start = time.time()\n\n    table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))\n\n    with open('pyserini/resources/miracl.yaml') as f:\n        yaml_data = yaml.safe_load(f)\n        for condition in yaml_data['conditions']:\n            name = condition['name']\n            eval_key = condition['eval_key']\n            cmd_template = condition['command']\n            cmd_lst = cmd_template.split()\n\n            print(f'condition {name}:')\n            lang = name.split('.')[-1]\n            is_hybrid_run = 'hybrid' in name\n\n            for splits in condition['splits']:\n                split = splits['split']\n                if is_hybrid_run:\n                    hits = int(cmd_lst[cmd_lst.index('--k') + 1])\n                else:\n                    hits = int(cmd_lst[cmd_lst.index('--hits') + 1])\n\n                print(f'  - split: {split}')\n\n                runfile = f'runs/run.miracl.{name}.{split}.top{hits}.txt'\n                if is_hybrid_run:\n                    bm25_output = f'runs/run.miracl.bm25.{lang}.{split}.top{hits}.txt'\n                    mdpr_output = f'runs/run.miracl.mdpr-tied-pft-msmarco.{lang}.{split}.top{hits}.txt'\n                    if not os.path.exists(bm25_output):\n                        print(f'Missing BM25 file: {bm25_output}')\n                        continue\n                    if not os.path.exists(mdpr_output):\n                        print(f'Missing mDPR file: {mdpr_output}')\n                        continue\n                    cmd = Template(cmd_template).substitute(split=split, output=runfile, bm25_output=bm25_output, mdpr_output=mdpr_output)\n                else:\n                    cmd = Template(cmd_template).substitute(split=split, output=runfile)\n\n                # In the yaml file, the topics are written as something like '--topics miracl-v1.0-ar-${split}'\n                # This works for the dev split because the topics are directly included in Anserini/Pyserini.\n                # For this training split, we have to map the symbol into a file in tools/topics-and-qrels/\n                # Here, we assume that the developer has cloned the miracl repo and placed the topics there.\n                if split == 'train':\n                    cmd = cmd.replace(f'--topics miracl-v1.0-{lang}-{split}',\n                                      f'--topics tools/topics-and-qrels/topics.miracl-v1.0-{lang}-{split}.tsv')\n\n                if not os.path.exists(runfile):\n                    print(f'    Running: {cmd}')\n                    rtn = subprocess.run(cmd.split(), capture_output=True)\n                    stderr = rtn.stderr.decode()\n                    if '--topics' in cmd:\n                        topic_fn = extract_topic_fn_from_cmd(cmd)\n                        if f'ValueError: Topic {topic_fn} Not Found' in stderr:\n                            print(f'Skipping {topic_fn}: file not found.')\n                            continue\n\n                for expected in splits['scores']:\n                    for metric in expected:\n                        if not args.skip_eval:\n                            # We have the translate the training qrels into a file located in tools/topics-and-qrels/\n                            # because they are not included with Anserini/Pyserini by default.\n                            # Here, we assume that the developer has cloned the miracl repo and placed the qrels there.\n                            if split == 'train':\n                                qrels = f'tools/topics-and-qrels/qrels.{eval_key}-train.tsv'\n                            else:\n                                qrels = f'{eval_key}-{split}'\n                            score = float(run_eval_and_return_metric(metric, qrels,\n                                                                     trec_eval_metric_definitions[metric], runfile))\n                            if math.isclose(score, float(expected[metric])):\n                                result_str = ok_str\n                            # Flaky tests\n                            elif (name == 'mdpr-tied-pft-msmarco.hi' and split == 'train'\n                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \\\n                                 (name == 'mdpr-tied-pft-msmarco-ft-all.ru'\n                                  and split == 'dev' and metric == 'nDCG@10'\n                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \\\n                                 (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.te'\n                                  and split == 'train' and metric == 'nDCG@10'\n                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \\\n                                 (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.zh'\n                                  and split == 'dev' and metric == 'nDCG@10'\n                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)):\n                                result_str = okish_str\n                            else:\n                                result_str = fail_str + f' expected {expected[metric]:.4f}'\n                            print(f'      {metric:7}: {score:.4f} {result_str}')\n                            table[name][split][metric] = score\n                        else:\n                            table[name][split][metric] = expected[metric]\n\n            print('')\n\n    for metric in ['nDCG@10', 'R@100']:\n        for split in ['dev', 'train']:\n            print_results(metric, split)\n\n    end = time.time()\n    print(f'Total elapsed time: {end - start:.0f}s')\n", "path": "scripts/repro_matrix/run_all_miracl.py"}]}