We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Using world-logs while evaluating a model with more than one task, only keeps the world log for the last task.
Trying to evaluate one mode with two tasks, while keeping the model outputs in world logs, I noticed that there was no result from one of the tasks in the world log. To reproduce this one may try running
```
 parlai eval_model -t wizard_of_wikipedia,babi \
--world-logs /some/path/world-log \
--num-examples 1 --model repeat_label
```
Running this, there is only a single line in the `world-log.json` file. Checking the file you can see `"id": "babi:Task1k:1"` which may mean that parlai is generating separate world log files for each task, but assumes same name for all of them and writes over the previous ones.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `parlai/scripts/eval_model.py`
Content:
```
1 #!/usr/bin/env python3
2 
3 # Copyright (c) Facebook, Inc. and its affiliates.
4 # This source code is licensed under the MIT license found in the
5 # LICENSE file in the root directory of this source tree.
6 
7 """
8 Basic example which iterates through the tasks specified and evaluates the given model
9 on them.
10 
11 ## Examples
12 
13 ```shell
14 parlai eval_model --task "babi:Task1k:2" -m "repeat_label"
15 parlai eval_model --task convai2 --model-file "/path/to/model_file"
16 ```
17 """
18 
19 from parlai.core.params import ParlaiParser, print_announcements
20 from parlai.core.agents import create_agent
21 from parlai.core.logs import TensorboardLogger
22 from parlai.core.metrics import (
23     aggregate_named_reports,
24     aggregate_unnamed_reports,
25     Metric,
26 )
27 from parlai.core.worlds import create_task
28 from parlai.utils.misc import TimeLogger, nice_report
29 from parlai.utils.world_logging import WorldLogger
30 from parlai.core.script import ParlaiScript, register_script
31 from parlai.utils.io import PathManager
32 import parlai.utils.logging as logging
33 
34 import json
35 import os
36 import random
37 
38 from parlai.utils.distributed import (
39     is_primary_worker,
40     all_gather_list,
41     is_distributed,
42     get_rank,
43 )
44 
45 
46 def setup_args(parser=None):
47     if parser is None:
48         parser = ParlaiParser(True, True, 'Evaluate a model')
49     # Get command line arguments
50     parser.add_argument(
51         '-rf',
52         '--report-filename',
53         type=str,
54         default='',
55         help='Saves a json file of the evaluation report either as an '
56         'extension to the model-file (if begins with a ".") or a whole '
57         'file path. Set to the empty string to not save at all.',
58     )
59     parser.add_argument(
60         '--world-logs',
61         type=str,
62         default='',
63         help='Saves a jsonl file of the world logs.'
64         'Set to the empty string to not save at all.',
65     )
66     parser.add_argument(
67         '--save-format',
68         type=str,
69         default='conversations',
70         choices=['conversations', 'parlai'],
71     )
72     parser.add_argument('-ne', '--num-examples', type=int, default=-1)
73     parser.add_argument('-d', '--display-examples', type='bool', default=False)
74     parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=10)
75     parser.add_argument(
76         '-mcs',
77         '--metrics',
78         type=str,
79         default='default',
80         help='list of metrics to show/compute, e.g. all, default,'
81         'or give a list split by , like '
82         'ppl,f1,accuracy,hits@1,rouge,bleu'
83         'the rouge metrics will be computed as rouge-1, rouge-2 and rouge-l',
84     )
85     parser.add_argument(
86         '-micro',
87         '--aggregate-micro',
88         type='bool',
89         default=False,
90         help='Report micro-averaged metrics instead of macro averaged metrics.',
91         recommended=False,
92     )
93     WorldLogger.add_cmdline_args(parser, partial_opt=None)
94     TensorboardLogger.add_cmdline_args(parser, partial_opt=None)
95     parser.set_params(datatype='valid')
96     return parser
97 
98 
99 def _save_eval_stats(opt, report):
100     if not is_primary_worker:
101         return
102     report_fname = opt['report_filename']
103     if report_fname == '':
104         return
105     if report_fname.startswith('.'):
106         report_fname = opt['model_file'] + report_fname
107 
108     json_serializable_report = report
109     for k, v in report.items():
110         if isinstance(v, Metric):
111             v = v.value()
112         json_serializable_report[k] = v
113 
114     # Save report
115     with PathManager.open(report_fname, 'w') as f:
116         logging.info(f'Saving model report to {report_fname}')
117         json.dump({'opt': opt, 'report': json_serializable_report}, f, indent=4)
118         f.write("\n")  # for jq
119 
120 
121 def _eval_single_world(opt, agent, task):
122     logging.info(f'Evaluating task {task} using datatype {opt.get("datatype")}.')
123     # set up world logger
124     world_logger = WorldLogger(opt) if opt['world_logs'] else None
125 
126     task_opt = opt.copy()  # copy opt since we're editing the task
127     task_opt['task'] = task
128     world = create_task(task_opt, agent)  # create worlds for tasks
129 
130     # set up logging
131     log_every_n_secs = opt.get('log_every_n_secs', -1)
132     if log_every_n_secs <= 0:
133         log_every_n_secs = float('inf')
134     log_time = TimeLogger()
135 
136     # max number of examples to evaluate
137     max_cnt = opt['num_examples'] if opt['num_examples'] > 0 else float('inf')
138     cnt = 0
139     total_cnt = world.num_examples()
140 
141     if is_distributed():
142         logging.warning('Progress bar is approximate in distributed mode.')
143 
144     while not world.epoch_done() and cnt < max_cnt:
145         cnt += opt.get('batchsize', 1)
146         world.parley()
147         if world_logger is not None:
148             world_logger.log(world)
149         if opt['display_examples']:
150             # display examples
151             print(world.display() + '\n~~')
152         if log_time.time() > log_every_n_secs:
153             report = world.report()
154             text, report = log_time.log(
155                 report.get('exs', 0), min(max_cnt, total_cnt), report
156             )
157             logging.info(text)
158 
159     if world_logger is not None:
160         # dump world acts to file
161         world_logger.reset()  # add final acts to logs
162         if is_distributed():
163             rank = get_rank()
164             base_outfile, extension = os.path.splitext(opt['world_logs'])
165             outfile = base_outfile + f'_{rank}' + extension
166         else:
167             outfile = opt['world_logs']
168         world_logger.write(outfile, world, file_format=opt['save_format'])
169 
170     report = aggregate_unnamed_reports(all_gather_list(world.report()))
171     world.reset()
172 
173     return report
174 
175 
176 def eval_model(opt):
177     """
178     Evaluates a model.
179 
180     :param opt: tells the evaluation function how to run
181     :return: the final result of calling report()
182     """
183     random.seed(42)
184     if 'train' in opt['datatype'] and 'evalmode' not in opt['datatype']:
185         raise ValueError(
186             'You should use --datatype train:evalmode if you want to evaluate on '
187             'the training set.'
188         )
189 
190     # load model and possibly print opt
191     agent = create_agent(opt, requireModelExists=True)
192     agent.opt.log()
193 
194     tasks = opt['task'].split(',')
195     reports = []
196     for task in tasks:
197         task_report = _eval_single_world(opt, agent, task)
198         reports.append(task_report)
199 
200     report = aggregate_named_reports(
201         dict(zip(tasks, reports)), micro_average=opt.get('aggregate_micro', False)
202     )
203 
204     # print announcments and report
205     print_announcements(opt)
206     logging.info(
207         f'Finished evaluating tasks {tasks} using datatype {opt.get("datatype")}'
208     )
209 
210     print(nice_report(report))
211     _save_eval_stats(opt, report)
212     return report
213 
214 
215 @register_script('eval_model', aliases=['em', 'eval'])
216 class EvalModel(ParlaiScript):
217     @classmethod
218     def setup_args(cls):
219         return setup_args()
220 
221     def run(self):
222         return eval_model(self.opt)
223 
224 
225 if __name__ == '__main__':
226     EvalModel.main()
227
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/parlai/scripts/eval_model.py b/parlai/scripts/eval_model.py
--- a/parlai/scripts/eval_model.py
+++ b/parlai/scripts/eval_model.py
@@ -118,13 +118,27 @@
         f.write("\n")  # for jq
 
 
+def get_task_world_logs(task, world_logs, is_multitask=False):
+    if not is_multitask:
+        return world_logs
+    else:
+        base_outfile, extension = os.path.splitext(world_logs)
+        return f'{base_outfile}_{task}{extension}'
+
+
 def _eval_single_world(opt, agent, task):
     logging.info(f'Evaluating task {task} using datatype {opt.get("datatype")}.')
     # set up world logger
-    world_logger = WorldLogger(opt) if opt['world_logs'] else None
-
     task_opt = opt.copy()  # copy opt since we're editing the task
     task_opt['task'] = task
+    # add task suffix in case of multi-tasking
+    if opt['world_logs']:
+        task_opt['world_logs'] = get_task_world_logs(
+            task, task_opt['world_logs'], is_multitask=len(opt['task'].split(',')) > 1
+        )
+
+    world_logger = WorldLogger(task_opt) if task_opt['world_logs'] else None
+
     world = create_task(task_opt, agent)  # create worlds for tasks
 
     # set up logging
@@ -161,10 +175,10 @@
         world_logger.reset()  # add final acts to logs
         if is_distributed():
             rank = get_rank()
-            base_outfile, extension = os.path.splitext(opt['world_logs'])
+            base_outfile, extension = os.path.splitext(task_opt['world_logs'])
             outfile = base_outfile + f'_{rank}' + extension
         else:
-            outfile = opt['world_logs']
+            outfile = task_opt['world_logs']
         world_logger.write(outfile, world, file_format=opt['save_format'])
 
     report = aggregate_unnamed_reports(all_gather_list(world.report()))

{"golden_diff": "diff --git a/parlai/scripts/eval_model.py b/parlai/scripts/eval_model.py\n--- a/parlai/scripts/eval_model.py\n+++ b/parlai/scripts/eval_model.py\n@@ -118,13 +118,27 @@\n         f.write(\"\\n\")  # for jq\n \n \n+def get_task_world_logs(task, world_logs, is_multitask=False):\n+    if not is_multitask:\n+        return world_logs\n+    else:\n+        base_outfile, extension = os.path.splitext(world_logs)\n+        return f'{base_outfile}_{task}{extension}'\n+\n+\n def _eval_single_world(opt, agent, task):\n     logging.info(f'Evaluating task {task} using datatype {opt.get(\"datatype\")}.')\n     # set up world logger\n-    world_logger = WorldLogger(opt) if opt['world_logs'] else None\n-\n     task_opt = opt.copy()  # copy opt since we're editing the task\n     task_opt['task'] = task\n+    # add task suffix in case of multi-tasking\n+    if opt['world_logs']:\n+        task_opt['world_logs'] = get_task_world_logs(\n+            task, task_opt['world_logs'], is_multitask=len(opt['task'].split(',')) > 1\n+        )\n+\n+    world_logger = WorldLogger(task_opt) if task_opt['world_logs'] else None\n+\n     world = create_task(task_opt, agent)  # create worlds for tasks\n \n     # set up logging\n@@ -161,10 +175,10 @@\n         world_logger.reset()  # add final acts to logs\n         if is_distributed():\n             rank = get_rank()\n-            base_outfile, extension = os.path.splitext(opt['world_logs'])\n+            base_outfile, extension = os.path.splitext(task_opt['world_logs'])\n             outfile = base_outfile + f'_{rank}' + extension\n         else:\n-            outfile = opt['world_logs']\n+            outfile = task_opt['world_logs']\n         world_logger.write(outfile, world, file_format=opt['save_format'])\n \n     report = aggregate_unnamed_reports(all_gather_list(world.report()))\n", "issue": "Using world-logs while evaluating a model with more than one task, only keeps the world log for the last task.\nTrying to evaluate one mode with two tasks, while keeping the model outputs in world logs, I noticed that there was no result from one of the tasks in the world log. To reproduce this one may try running\r\n```\r\n parlai eval_model -t wizard_of_wikipedia,babi \\\r\n--world-logs /some/path/world-log \\\r\n--num-examples 1 --model repeat_label\r\n```\r\nRunning this, there is only a single line in the `world-log.json` file. Checking the file you can see `\"id\": \"babi:Task1k:1\"` which may mean that parlai is generating separate world log files for each task, but assumes same name for all of them and writes over the previous ones.\n", "before_files": [{"content": "#!/usr/bin/env python3\n\n# Copyright (c) Facebook, Inc. and its affiliates.\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\n\"\"\"\nBasic example which iterates through the tasks specified and evaluates the given model\non them.\n\n## Examples\n\n```shell\nparlai eval_model --task \"babi:Task1k:2\" -m \"repeat_label\"\nparlai eval_model --task convai2 --model-file \"/path/to/model_file\"\n```\n\"\"\"\n\nfrom parlai.core.params import ParlaiParser, print_announcements\nfrom parlai.core.agents import create_agent\nfrom parlai.core.logs import TensorboardLogger\nfrom parlai.core.metrics import (\n    aggregate_named_reports,\n    aggregate_unnamed_reports,\n    Metric,\n)\nfrom parlai.core.worlds import create_task\nfrom parlai.utils.misc import TimeLogger, nice_report\nfrom parlai.utils.world_logging import WorldLogger\nfrom parlai.core.script import ParlaiScript, register_script\nfrom parlai.utils.io import PathManager\nimport parlai.utils.logging as logging\n\nimport json\nimport os\nimport random\n\nfrom parlai.utils.distributed import (\n    is_primary_worker,\n    all_gather_list,\n    is_distributed,\n    get_rank,\n)\n\n\ndef setup_args(parser=None):\n    if parser is None:\n        parser = ParlaiParser(True, True, 'Evaluate a model')\n    # Get command line arguments\n    parser.add_argument(\n        '-rf',\n        '--report-filename',\n        type=str,\n        default='',\n        help='Saves a json file of the evaluation report either as an '\n        'extension to the model-file (if begins with a \".\") or a whole '\n        'file path. Set to the empty string to not save at all.',\n    )\n    parser.add_argument(\n        '--world-logs',\n        type=str,\n        default='',\n        help='Saves a jsonl file of the world logs.'\n        'Set to the empty string to not save at all.',\n    )\n    parser.add_argument(\n        '--save-format',\n        type=str,\n        default='conversations',\n        choices=['conversations', 'parlai'],\n    )\n    parser.add_argument('-ne', '--num-examples', type=int, default=-1)\n    parser.add_argument('-d', '--display-examples', type='bool', default=False)\n    parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=10)\n    parser.add_argument(\n        '-mcs',\n        '--metrics',\n        type=str,\n        default='default',\n        help='list of metrics to show/compute, e.g. all, default,'\n        'or give a list split by , like '\n        'ppl,f1,accuracy,hits@1,rouge,bleu'\n        'the rouge metrics will be computed as rouge-1, rouge-2 and rouge-l',\n    )\n    parser.add_argument(\n        '-micro',\n        '--aggregate-micro',\n        type='bool',\n        default=False,\n        help='Report micro-averaged metrics instead of macro averaged metrics.',\n        recommended=False,\n    )\n    WorldLogger.add_cmdline_args(parser, partial_opt=None)\n    TensorboardLogger.add_cmdline_args(parser, partial_opt=None)\n    parser.set_params(datatype='valid')\n    return parser\n\n\ndef _save_eval_stats(opt, report):\n    if not is_primary_worker:\n        return\n    report_fname = opt['report_filename']\n    if report_fname == '':\n        return\n    if report_fname.startswith('.'):\n        report_fname = opt['model_file'] + report_fname\n\n    json_serializable_report = report\n    for k, v in report.items():\n        if isinstance(v, Metric):\n            v = v.value()\n        json_serializable_report[k] = v\n\n    # Save report\n    with PathManager.open(report_fname, 'w') as f:\n        logging.info(f'Saving model report to {report_fname}')\n        json.dump({'opt': opt, 'report': json_serializable_report}, f, indent=4)\n        f.write(\"\\n\")  # for jq\n\n\ndef _eval_single_world(opt, agent, task):\n    logging.info(f'Evaluating task {task} using datatype {opt.get(\"datatype\")}.')\n    # set up world logger\n    world_logger = WorldLogger(opt) if opt['world_logs'] else None\n\n    task_opt = opt.copy()  # copy opt since we're editing the task\n    task_opt['task'] = task\n    world = create_task(task_opt, agent)  # create worlds for tasks\n\n    # set up logging\n    log_every_n_secs = opt.get('log_every_n_secs', -1)\n    if log_every_n_secs <= 0:\n        log_every_n_secs = float('inf')\n    log_time = TimeLogger()\n\n    # max number of examples to evaluate\n    max_cnt = opt['num_examples'] if opt['num_examples'] > 0 else float('inf')\n    cnt = 0\n    total_cnt = world.num_examples()\n\n    if is_distributed():\n        logging.warning('Progress bar is approximate in distributed mode.')\n\n    while not world.epoch_done() and cnt < max_cnt:\n        cnt += opt.get('batchsize', 1)\n        world.parley()\n        if world_logger is not None:\n            world_logger.log(world)\n        if opt['display_examples']:\n            # display examples\n            print(world.display() + '\\n~~')\n        if log_time.time() > log_every_n_secs:\n            report = world.report()\n            text, report = log_time.log(\n                report.get('exs', 0), min(max_cnt, total_cnt), report\n            )\n            logging.info(text)\n\n    if world_logger is not None:\n        # dump world acts to file\n        world_logger.reset()  # add final acts to logs\n        if is_distributed():\n            rank = get_rank()\n            base_outfile, extension = os.path.splitext(opt['world_logs'])\n            outfile = base_outfile + f'_{rank}' + extension\n        else:\n            outfile = opt['world_logs']\n        world_logger.write(outfile, world, file_format=opt['save_format'])\n\n    report = aggregate_unnamed_reports(all_gather_list(world.report()))\n    world.reset()\n\n    return report\n\n\ndef eval_model(opt):\n    \"\"\"\n    Evaluates a model.\n\n    :param opt: tells the evaluation function how to run\n    :return: the final result of calling report()\n    \"\"\"\n    random.seed(42)\n    if 'train' in opt['datatype'] and 'evalmode' not in opt['datatype']:\n        raise ValueError(\n            'You should use --datatype train:evalmode if you want to evaluate on '\n            'the training set.'\n        )\n\n    # load model and possibly print opt\n    agent = create_agent(opt, requireModelExists=True)\n    agent.opt.log()\n\n    tasks = opt['task'].split(',')\n    reports = []\n    for task in tasks:\n        task_report = _eval_single_world(opt, agent, task)\n        reports.append(task_report)\n\n    report = aggregate_named_reports(\n        dict(zip(tasks, reports)), micro_average=opt.get('aggregate_micro', False)\n    )\n\n    # print announcments and report\n    print_announcements(opt)\n    logging.info(\n        f'Finished evaluating tasks {tasks} using datatype {opt.get(\"datatype\")}'\n    )\n\n    print(nice_report(report))\n    _save_eval_stats(opt, report)\n    return report\n\n\n@register_script('eval_model', aliases=['em', 'eval'])\nclass EvalModel(ParlaiScript):\n    @classmethod\n    def setup_args(cls):\n        return setup_args()\n\n    def run(self):\n        return eval_model(self.opt)\n\n\nif __name__ == '__main__':\n    EvalModel.main()\n", "path": "parlai/scripts/eval_model.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n\n# Copyright (c) Facebook, Inc. and its affiliates.\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\n\"\"\"\nBasic example which iterates through the tasks specified and evaluates the given model\non them.\n\n## Examples\n\n```shell\nparlai eval_model --task \"babi:Task1k:2\" -m \"repeat_label\"\nparlai eval_model --task convai2 --model-file \"/path/to/model_file\"\n```\n\"\"\"\n\nfrom parlai.core.params import ParlaiParser, print_announcements\nfrom parlai.core.agents import create_agent\nfrom parlai.core.logs import TensorboardLogger\nfrom parlai.core.metrics import (\n    aggregate_named_reports,\n    aggregate_unnamed_reports,\n    Metric,\n)\nfrom parlai.core.worlds import create_task\nfrom parlai.utils.misc import TimeLogger, nice_report\nfrom parlai.utils.world_logging import WorldLogger\nfrom parlai.core.script import ParlaiScript, register_script\nfrom parlai.utils.io import PathManager\nimport parlai.utils.logging as logging\n\nimport json\nimport os\nimport random\n\nfrom parlai.utils.distributed import (\n    is_primary_worker,\n    all_gather_list,\n    is_distributed,\n    get_rank,\n)\n\n\ndef setup_args(parser=None):\n    if parser is None:\n        parser = ParlaiParser(True, True, 'Evaluate a model')\n    # Get command line arguments\n    parser.add_argument(\n        '-rf',\n        '--report-filename',\n        type=str,\n        default='',\n        help='Saves a json file of the evaluation report either as an '\n        'extension to the model-file (if begins with a \".\") or a whole '\n        'file path. Set to the empty string to not save at all.',\n    )\n    parser.add_argument(\n        '--world-logs',\n        type=str,\n        default='',\n        help='Saves a jsonl file of the world logs.'\n        'Set to the empty string to not save at all.',\n    )\n    parser.add_argument(\n        '--save-format',\n        type=str,\n        default='conversations',\n        choices=['conversations', 'parlai'],\n    )\n    parser.add_argument('-ne', '--num-examples', type=int, default=-1)\n    parser.add_argument('-d', '--display-examples', type='bool', default=False)\n    parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=10)\n    parser.add_argument(\n        '-mcs',\n        '--metrics',\n        type=str,\n        default='default',\n        help='list of metrics to show/compute, e.g. all, default,'\n        'or give a list split by , like '\n        'ppl,f1,accuracy,hits@1,rouge,bleu'\n        'the rouge metrics will be computed as rouge-1, rouge-2 and rouge-l',\n    )\n    parser.add_argument(\n        '-micro',\n        '--aggregate-micro',\n        type='bool',\n        default=False,\n        help='Report micro-averaged metrics instead of macro averaged metrics.',\n        recommended=False,\n    )\n    WorldLogger.add_cmdline_args(parser, partial_opt=None)\n    TensorboardLogger.add_cmdline_args(parser, partial_opt=None)\n    parser.set_params(datatype='valid')\n    return parser\n\n\ndef _save_eval_stats(opt, report):\n    if not is_primary_worker:\n        return\n    report_fname = opt['report_filename']\n    if report_fname == '':\n        return\n    if report_fname.startswith('.'):\n        report_fname = opt['model_file'] + report_fname\n\n    json_serializable_report = report\n    for k, v in report.items():\n        if isinstance(v, Metric):\n            v = v.value()\n        json_serializable_report[k] = v\n\n    # Save report\n    with PathManager.open(report_fname, 'w') as f:\n        logging.info(f'Saving model report to {report_fname}')\n        json.dump({'opt': opt, 'report': json_serializable_report}, f, indent=4)\n        f.write(\"\\n\")  # for jq\n\n\ndef get_task_world_logs(task, world_logs, is_multitask=False):\n    if not is_multitask:\n        return world_logs\n    else:\n        base_outfile, extension = os.path.splitext(world_logs)\n        return f'{base_outfile}_{task}{extension}'\n\n\ndef _eval_single_world(opt, agent, task):\n    logging.info(f'Evaluating task {task} using datatype {opt.get(\"datatype\")}.')\n    # set up world logger\n    task_opt = opt.copy()  # copy opt since we're editing the task\n    task_opt['task'] = task\n    # add task suffix in case of multi-tasking\n    if opt['world_logs']:\n        task_opt['world_logs'] = get_task_world_logs(\n            task, task_opt['world_logs'], is_multitask=len(opt['task'].split(',')) > 1\n        )\n\n    world_logger = WorldLogger(task_opt) if task_opt['world_logs'] else None\n\n    world = create_task(task_opt, agent)  # create worlds for tasks\n\n    # set up logging\n    log_every_n_secs = opt.get('log_every_n_secs', -1)\n    if log_every_n_secs <= 0:\n        log_every_n_secs = float('inf')\n    log_time = TimeLogger()\n\n    # max number of examples to evaluate\n    max_cnt = opt['num_examples'] if opt['num_examples'] > 0 else float('inf')\n    cnt = 0\n    total_cnt = world.num_examples()\n\n    if is_distributed():\n        logging.warning('Progress bar is approximate in distributed mode.')\n\n    while not world.epoch_done() and cnt < max_cnt:\n        cnt += opt.get('batchsize', 1)\n        world.parley()\n        if world_logger is not None:\n            world_logger.log(world)\n        if opt['display_examples']:\n            # display examples\n            print(world.display() + '\\n~~')\n        if log_time.time() > log_every_n_secs:\n            report = world.report()\n            text, report = log_time.log(\n                report.get('exs', 0), min(max_cnt, total_cnt), report\n            )\n            logging.info(text)\n\n    if world_logger is not None:\n        # dump world acts to file\n        world_logger.reset()  # add final acts to logs\n        if is_distributed():\n            rank = get_rank()\n            base_outfile, extension = os.path.splitext(task_opt['world_logs'])\n            outfile = base_outfile + f'_{rank}' + extension\n        else:\n            outfile = task_opt['world_logs']\n        world_logger.write(outfile, world, file_format=opt['save_format'])\n\n    report = aggregate_unnamed_reports(all_gather_list(world.report()))\n    world.reset()\n\n    return report\n\n\ndef eval_model(opt):\n    \"\"\"\n    Evaluates a model.\n\n    :param opt: tells the evaluation function how to run\n    :return: the final result of calling report()\n    \"\"\"\n    random.seed(42)\n    if 'train' in opt['datatype'] and 'evalmode' not in opt['datatype']:\n        raise ValueError(\n            'You should use --datatype train:evalmode if you want to evaluate on '\n            'the training set.'\n        )\n\n    # load model and possibly print opt\n    agent = create_agent(opt, requireModelExists=True)\n    agent.opt.log()\n\n    tasks = opt['task'].split(',')\n    reports = []\n    for task in tasks:\n        task_report = _eval_single_world(opt, agent, task)\n        reports.append(task_report)\n\n    report = aggregate_named_reports(\n        dict(zip(tasks, reports)), micro_average=opt.get('aggregate_micro', False)\n    )\n\n    # print announcments and report\n    print_announcements(opt)\n    logging.info(\n        f'Finished evaluating tasks {tasks} using datatype {opt.get(\"datatype\")}'\n    )\n\n    print(nice_report(report))\n    _save_eval_stats(opt, report)\n    return report\n\n\n@register_script('eval_model', aliases=['em', 'eval'])\nclass EvalModel(ParlaiScript):\n    @classmethod\n    def setup_args(cls):\n        return setup_args()\n\n    def run(self):\n        return eval_model(self.opt)\n\n\nif __name__ == '__main__':\n    EvalModel.main()\n", "path": "parlai/scripts/eval_model.py"}]}