We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Add BLEU as a metric


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `parlai/core/metrics.py`
Content:
```
1 # Copyright (c) 2017-present, Facebook, Inc.
2 # All rights reserved.
3 # This source code is licensed under the BSD-style license found in the
4 # LICENSE file in the root directory of this source tree. An additional grant
5 # of patent rights can be found in the PATENTS file in the same directory.
6 """Provides standard metric evaluations for dialog.
7 Uses locking and shared memory when ``numthreads`` is set to >1 to share metrics
8 between processes.
9 """
10 
11 from parlai.core.thread_utils import SharedTable
12 from parlai.core.utils import round_sigfigs, no_lock
13 from collections import Counter
14 
15 import re
16 import math
17 
18 re_art = re.compile(r'\b(a|an|the)\b')
19 re_punc = re.compile(r'[!"#$%&()*+,-./:;<=>?@\[\]\\^`{|}~_\']')
20 def normalize_answer(s):
21     """Lower text and remove punctuation, articles and extra whitespace."""
22     def remove_articles(text):
23         return re_art.sub(' ', text)
24 
25     def white_space_fix(text):
26         return ' '.join(text.split())
27 
28     def remove_punc(text):
29         return re_punc.sub(' ', text)  # convert punctuation to spaces
30 
31     def lower(text):
32         return text.lower()
33 
34     return white_space_fix(remove_articles(remove_punc(lower(s))))
35 
36 
37 def _exact_match(guess, answers):
38     """Check if guess is a (normalized) exact match with any answer."""
39     if guess is None or answers is None:
40         return False
41     guess = normalize_answer(guess)
42     for a in answers:
43         if guess == normalize_answer(a):
44             return True
45     return False
46 
47 
48 def _f1_score(guess, answers):
49     """Return the max F1 score between the guess and any answer."""
50     def _score(g_tokens, a_tokens):
51         common = Counter(g_tokens) & Counter(a_tokens)
52         num_same = sum(common.values())
53         if num_same == 0:
54             return 0
55         precision = 1.0 * num_same / len(g_tokens)
56         recall = 1.0 * num_same / len(a_tokens)
57         f1 = (2 * precision * recall) / (precision + recall)
58         return f1
59 
60     if guess is None or answers is None:
61         return 0
62     g_tokens = normalize_answer(guess).split()
63     scores = [_score(g_tokens, normalize_answer(a).split()) for a in answers]
64     return max(scores)
65 
66 
67 def aggregate_metrics(reporters):
68     #reporters is a list of teachers or worlds
69     m = {}
70     m['tasks'] = {}
71     sums = {'accuracy': 0, 'f1': 0, 'loss': 0, 'ppl': 0}
72     num_tasks = 0
73     total = 0
74     for i in range(len(reporters)):
75         tid = reporters[i].getID()
76         mt = reporters[i].report()
77         while tid in m['tasks']:
78             # prevent name cloberring if using multiple tasks with same ID
79             tid += '_'
80         m['tasks'][tid] = mt
81         total += mt['exs']
82         found_any = False
83         for k in sums.keys():
84             if k in mt:
85                 sums[k] += mt[k]
86                 found_any = True
87         if found_any:
88             num_tasks += 1
89     m['exs'] = total
90     m['accuracy'] = 0
91     if num_tasks > 0:
92         for k in sums.keys():
93             m[k] = round_sigfigs(sums[k] / num_tasks, 4)
94     return m
95 
96 
97 def compute_time_metrics(world, max_time):
98     # Determine time_left and num_epochs
99     exs_per_epoch = world.num_examples() if world.num_examples() else 0
100     num_epochs = world.opt.get('num_epochs', 0)
101     max_exs = exs_per_epoch * num_epochs
102     total_exs = world.get_total_exs()
103 
104     m = {}
105     if (max_exs > 0 and total_exs > 0) or max_time > 0:
106         m = {}
107         time_left = None
108         time = world.get_time()
109         total_epochs = world.get_total_epochs()
110 
111         if (num_epochs > 0 and total_exs > 0 and max_exs > 0):
112             exs_per_sec = time / total_exs
113             time_left = (max_exs - total_exs) * exs_per_sec
114         if max_time > 0:
115             other_time_left = max_time - time
116             if time_left is not None:
117                 time_left = min(time_left, other_time_left)
118             else:
119                 time_left = other_time_left
120         if time_left is not None:
121             m['time_left'] = math.floor(time_left)
122         if num_epochs > 0:
123             if (total_exs > 0 and exs_per_epoch > 0):
124                 display_epochs = int(total_exs / exs_per_epoch)
125             else:
126                 display_epochs = total_epochs
127             m['num_epochs'] = display_epochs
128     return m
129 
130 
131 class Metrics(object):
132     """Class that maintains evaluation metrics over dialog."""
133 
134     def __init__(self, opt):
135         self.metrics = {}
136         self.metrics['cnt'] = 0
137         self.metrics_list = ['mean_rank', 'loss', 'correct', 'f1', 'ppl']
138         for k in self.metrics_list:
139             self.metrics[k] = 0.0
140             self.metrics[k + '_cnt'] = 0
141         self.eval_pr = [1, 5, 10, 100]
142         for k in self.eval_pr:
143             self.metrics['hits@' + str(k)] = 0
144         self.metrics['hits@_cnt'] = 0
145         self.flags = {'has_text_cands': False, 'print_prediction_metrics': False}
146         if opt.get('numthreads', 1) > 1:
147             self.metrics = SharedTable(self.metrics)
148             self.flags = SharedTable(self.flags)
149 
150     def __str__(self):
151         return str(self.metrics)
152 
153     def __repr__(self):
154         representation = super().__repr__()
155         return representation.replace('>', ': {}>'.format(repr(self.metrics)))
156 
157     def _lock(self):
158         if hasattr(self.metrics, 'get_lock'):
159             # use the shared_table's lock
160             return self.metrics.get_lock()
161         else:
162             # otherwise do nothing
163             return no_lock()
164 
165     def update_ranking_metrics(self, observation, labels):
166         text_cands = observation.get('text_candidates', None)
167         if text_cands is None:
168             return
169         else:
170             text = observation.get('text', None)
171 
172             # Now loop through text candidates, assuming they are sorted.
173             # If any of them is a label then score a point.
174             # maintain hits@1, 5, 10, 50, 100,  etc.
175             label_set = set(normalize_answer(l) for l in labels)
176             cnts = {k: 0 for k in self.eval_pr}
177             cnt = 0
178             for c in text_cands:
179                 cnt += 1
180                 if normalize_answer(c) in label_set:
181                     for k in self.eval_pr:
182                         if cnt <= k:
183                             cnts[k] += 1
184             # hits metric is 1 if cnts[k] > 0.
185             # (other metrics such as p@k and r@k take
186             # the value of cnt into account.)
187             with self._lock():
188                 self.flags['has_text_cands'] = True
189                 for k in self.eval_pr:
190                     if cnts[k] > 0:
191                         self.metrics['hits@' + str(k)] += 1
192                 self.metrics['hits@_cnt'] += 1
193 
194     def update(self, observation, labels):
195         with self._lock():
196             self.metrics['cnt'] += 1
197 
198         # Exact match metric.
199         correct = 0
200         prediction = observation.get('text', None)
201         if prediction is not None:
202             if _exact_match(prediction, labels):
203                 correct = 1
204             with self._lock():
205                 self.flags['print_prediction_metrics'] = True
206                 self.metrics['correct'] += correct
207                 self.metrics['correct_cnt'] += 1
208 
209             # F1 metric.
210             f1 = _f1_score(prediction, labels)
211             with self._lock():
212                 self.metrics['f1'] += f1
213                 self.metrics['f1_cnt'] += 1
214 
215         # Ranking metrics.
216         self.update_ranking_metrics(observation, labels)
217 
218         # User-reported metrics
219         if 'metrics' in observation:
220             for k, v in observation['metrics'].items():
221                 if k not in ['correct', 'f1', 'hits@k']:
222                     if k in self.metrics_list:
223                         with self._lock():
224                             self.metrics[k] += v
225                             self.metrics[k + '_cnt'] += 1
226                     else:
227                         if type(self.metrics) is SharedTable:
228                             # can't share custom metrics during hogwild
229                             pass
230                         else:
231                             # no need to lock because not SharedTable
232                             if k not in self.metrics:
233                                 self.metrics[k] = v
234                                 self.metrics_list.append(k)
235                                 self.metrics[k + '_cnt'] = 1.0
236                             else:
237                                 self.metrics[k] += v
238 
239         # Return a dict containing the metrics for this specific example.
240         # Metrics across all data is stored internally in the class, and
241         # can be accessed with the report method.
242         loss = {}
243         loss['correct'] = correct
244         return loss
245 
246     def report(self):
247         # Report the metrics over all data seen so far.
248         m = {}
249         total = self.metrics['cnt']
250         m['exs'] = total
251         if total > 0:
252             if self.flags['print_prediction_metrics']:
253                 m['accuracy'] = round_sigfigs(self.metrics['correct'] / max(1, self.metrics['correct_cnt']), 4)
254                 m['f1'] = round_sigfigs(self.metrics['f1'] / max(1, self.metrics['f1_cnt']), 4)
255                 if self.flags['has_text_cands']:
256                     for k in self.eval_pr:
257                         m['hits@' + str(k)] = round_sigfigs(
258                             self.metrics['hits@' + str(k)] / max(1, self.metrics['hits@_cnt']), 3)
259             for k in self.metrics_list:
260                 if self.metrics[k + '_cnt'] > 0 and k != 'correct' and k != 'f1':
261                     m[k] = round_sigfigs(self.metrics[k] / max(1, self.metrics[k + '_cnt']), 4)
262         return m
263 
264     def clear(self):
265         with self._lock():
266             self.metrics['cnt'] = 0
267             for k in self.metrics_list:
268                 v = self.metrics[k]
269                 v_typ = type(v)
270                 if 'Tensor' in str(v_typ):
271                     self.metrics[k].zero_()
272                 else:
273                     self.metrics[k] = 0.0
274                 self.metrics[k + '_cnt'] = 0
275             for k in self.eval_pr:
276                 self.metrics['hits@' + str(k)] = 0
277             self.metrics['hits@_cnt'] = 0
278
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/parlai/core/metrics.py b/parlai/core/metrics.py
--- a/parlai/core/metrics.py
+++ b/parlai/core/metrics.py
@@ -15,8 +15,17 @@
 import re
 import math
 
+try:
+    from nltk.translate import bleu_score as nltkbleu
+except ImportError:
+    # User doesn't have nltk installed, so we can't use it for bleu
+    # We'll just turn off things, but we might want to warn the user
+    nltkbleu = None
+
 re_art = re.compile(r'\b(a|an|the)\b')
 re_punc = re.compile(r'[!"#$%&()*+,-./:;<=>?@\[\]\\^`{|}~_\']')
+
+
 def normalize_answer(s):
     """Lower text and remove punctuation, articles and extra whitespace."""
     def remove_articles(text):
@@ -64,11 +73,29 @@
     return max(scores)
 
 
+def _bleu(guess, answers):
+    if nltkbleu is None:
+        # bleu library not installed, just return a default value
+        return None
+    # Warning: BLEU calculation *should* include proper tokenization and
+    # punctuation etc. We're using the normalize_answer for everything though,
+    # so we're over-estimating our BLEU scores.  Also note that NLTK's bleu is
+    # going to be slower than fairseq's (which is written in C), but fairseq's
+    # requires that everything be in arrays of ints (i.e. as tensors). NLTK's
+    # works with strings, which is better suited for this module.
+    return nltkbleu.sentence_bleu(
+        [normalize_answer(a).split(" ") for a in answers],
+        normalize_answer(guess).split(" ")
+    )
+
+
 def aggregate_metrics(reporters):
     #reporters is a list of teachers or worlds
     m = {}
     m['tasks'] = {}
     sums = {'accuracy': 0, 'f1': 0, 'loss': 0, 'ppl': 0}
+    if nltkbleu is not None:
+        sums['bleu'] = 0
     num_tasks = 0
     total = 0
     for i in range(len(reporters)):
@@ -135,6 +162,9 @@
         self.metrics = {}
         self.metrics['cnt'] = 0
         self.metrics_list = ['mean_rank', 'loss', 'correct', 'f1', 'ppl']
+        if nltkbleu is not None:
+            # only compute bleu if we can
+            self.metrics_list.append('bleu')
         for k in self.metrics_list:
             self.metrics[k] = 0.0
             self.metrics[k + '_cnt'] = 0
@@ -206,11 +236,15 @@
                 self.metrics['correct'] += correct
                 self.metrics['correct_cnt'] += 1
 
-            # F1 metric.
+            # F1 and BLEU metrics.
             f1 = _f1_score(prediction, labels)
+            bleu = _bleu(prediction, labels)
             with self._lock():
                 self.metrics['f1'] += f1
                 self.metrics['f1_cnt'] += 1
+                if bleu is not None:
+                    self.metrics['bleu'] += bleu
+                    self.metrics['bleu_cnt'] += 1
 
         # Ranking metrics.
         self.update_ranking_metrics(observation, labels)
@@ -218,7 +252,7 @@
         # User-reported metrics
         if 'metrics' in observation:
             for k, v in observation['metrics'].items():
-                if k not in ['correct', 'f1', 'hits@k']:
+                if k not in ['correct', 'f1', 'hits@k', 'bleu']:
                     if k in self.metrics_list:
                         with self._lock():
                             self.metrics[k] += v

{"golden_diff": "diff --git a/parlai/core/metrics.py b/parlai/core/metrics.py\n--- a/parlai/core/metrics.py\n+++ b/parlai/core/metrics.py\n@@ -15,8 +15,17 @@\n import re\n import math\n \n+try:\n+    from nltk.translate import bleu_score as nltkbleu\n+except ImportError:\n+    # User doesn't have nltk installed, so we can't use it for bleu\n+    # We'll just turn off things, but we might want to warn the user\n+    nltkbleu = None\n+\n re_art = re.compile(r'\\b(a|an|the)\\b')\n re_punc = re.compile(r'[!\"#$%&()*+,-./:;<=>?@\\[\\]\\\\^`{|}~_\\']')\n+\n+\n def normalize_answer(s):\n     \"\"\"Lower text and remove punctuation, articles and extra whitespace.\"\"\"\n     def remove_articles(text):\n@@ -64,11 +73,29 @@\n     return max(scores)\n \n \n+def _bleu(guess, answers):\n+    if nltkbleu is None:\n+        # bleu library not installed, just return a default value\n+        return None\n+    # Warning: BLEU calculation *should* include proper tokenization and\n+    # punctuation etc. We're using the normalize_answer for everything though,\n+    # so we're over-estimating our BLEU scores.  Also note that NLTK's bleu is\n+    # going to be slower than fairseq's (which is written in C), but fairseq's\n+    # requires that everything be in arrays of ints (i.e. as tensors). NLTK's\n+    # works with strings, which is better suited for this module.\n+    return nltkbleu.sentence_bleu(\n+        [normalize_answer(a).split(\" \") for a in answers],\n+        normalize_answer(guess).split(\" \")\n+    )\n+\n+\n def aggregate_metrics(reporters):\n     #reporters is a list of teachers or worlds\n     m = {}\n     m['tasks'] = {}\n     sums = {'accuracy': 0, 'f1': 0, 'loss': 0, 'ppl': 0}\n+    if nltkbleu is not None:\n+        sums['bleu'] = 0\n     num_tasks = 0\n     total = 0\n     for i in range(len(reporters)):\n@@ -135,6 +162,9 @@\n         self.metrics = {}\n         self.metrics['cnt'] = 0\n         self.metrics_list = ['mean_rank', 'loss', 'correct', 'f1', 'ppl']\n+        if nltkbleu is not None:\n+            # only compute bleu if we can\n+            self.metrics_list.append('bleu')\n         for k in self.metrics_list:\n             self.metrics[k] = 0.0\n             self.metrics[k + '_cnt'] = 0\n@@ -206,11 +236,15 @@\n                 self.metrics['correct'] += correct\n                 self.metrics['correct_cnt'] += 1\n \n-            # F1 metric.\n+            # F1 and BLEU metrics.\n             f1 = _f1_score(prediction, labels)\n+            bleu = _bleu(prediction, labels)\n             with self._lock():\n                 self.metrics['f1'] += f1\n                 self.metrics['f1_cnt'] += 1\n+                if bleu is not None:\n+                    self.metrics['bleu'] += bleu\n+                    self.metrics['bleu_cnt'] += 1\n \n         # Ranking metrics.\n         self.update_ranking_metrics(observation, labels)\n@@ -218,7 +252,7 @@\n         # User-reported metrics\n         if 'metrics' in observation:\n             for k, v in observation['metrics'].items():\n-                if k not in ['correct', 'f1', 'hits@k']:\n+                if k not in ['correct', 'f1', 'hits@k', 'bleu']:\n                     if k in self.metrics_list:\n                         with self._lock():\n                             self.metrics[k] += v\n", "issue": "Add BLEU as a metric\n\n", "before_files": [{"content": "# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n# This source code is licensed under the BSD-style license found in the\n# LICENSE file in the root directory of this source tree. An additional grant\n# of patent rights can be found in the PATENTS file in the same directory.\n\"\"\"Provides standard metric evaluations for dialog.\nUses locking and shared memory when ``numthreads`` is set to >1 to share metrics\nbetween processes.\n\"\"\"\n\nfrom parlai.core.thread_utils import SharedTable\nfrom parlai.core.utils import round_sigfigs, no_lock\nfrom collections import Counter\n\nimport re\nimport math\n\nre_art = re.compile(r'\\b(a|an|the)\\b')\nre_punc = re.compile(r'[!\"#$%&()*+,-./:;<=>?@\\[\\]\\\\^`{|}~_\\']')\ndef normalize_answer(s):\n    \"\"\"Lower text and remove punctuation, articles and extra whitespace.\"\"\"\n    def remove_articles(text):\n        return re_art.sub(' ', text)\n\n    def white_space_fix(text):\n        return ' '.join(text.split())\n\n    def remove_punc(text):\n        return re_punc.sub(' ', text)  # convert punctuation to spaces\n\n    def lower(text):\n        return text.lower()\n\n    return white_space_fix(remove_articles(remove_punc(lower(s))))\n\n\ndef _exact_match(guess, answers):\n    \"\"\"Check if guess is a (normalized) exact match with any answer.\"\"\"\n    if guess is None or answers is None:\n        return False\n    guess = normalize_answer(guess)\n    for a in answers:\n        if guess == normalize_answer(a):\n            return True\n    return False\n\n\ndef _f1_score(guess, answers):\n    \"\"\"Return the max F1 score between the guess and any answer.\"\"\"\n    def _score(g_tokens, a_tokens):\n        common = Counter(g_tokens) & Counter(a_tokens)\n        num_same = sum(common.values())\n        if num_same == 0:\n            return 0\n        precision = 1.0 * num_same / len(g_tokens)\n        recall = 1.0 * num_same / len(a_tokens)\n        f1 = (2 * precision * recall) / (precision + recall)\n        return f1\n\n    if guess is None or answers is None:\n        return 0\n    g_tokens = normalize_answer(guess).split()\n    scores = [_score(g_tokens, normalize_answer(a).split()) for a in answers]\n    return max(scores)\n\n\ndef aggregate_metrics(reporters):\n    #reporters is a list of teachers or worlds\n    m = {}\n    m['tasks'] = {}\n    sums = {'accuracy': 0, 'f1': 0, 'loss': 0, 'ppl': 0}\n    num_tasks = 0\n    total = 0\n    for i in range(len(reporters)):\n        tid = reporters[i].getID()\n        mt = reporters[i].report()\n        while tid in m['tasks']:\n            # prevent name cloberring if using multiple tasks with same ID\n            tid += '_'\n        m['tasks'][tid] = mt\n        total += mt['exs']\n        found_any = False\n        for k in sums.keys():\n            if k in mt:\n                sums[k] += mt[k]\n                found_any = True\n        if found_any:\n            num_tasks += 1\n    m['exs'] = total\n    m['accuracy'] = 0\n    if num_tasks > 0:\n        for k in sums.keys():\n            m[k] = round_sigfigs(sums[k] / num_tasks, 4)\n    return m\n\n\ndef compute_time_metrics(world, max_time):\n    # Determine time_left and num_epochs\n    exs_per_epoch = world.num_examples() if world.num_examples() else 0\n    num_epochs = world.opt.get('num_epochs', 0)\n    max_exs = exs_per_epoch * num_epochs\n    total_exs = world.get_total_exs()\n\n    m = {}\n    if (max_exs > 0 and total_exs > 0) or max_time > 0:\n        m = {}\n        time_left = None\n        time = world.get_time()\n        total_epochs = world.get_total_epochs()\n\n        if (num_epochs > 0 and total_exs > 0 and max_exs > 0):\n            exs_per_sec = time / total_exs\n            time_left = (max_exs - total_exs) * exs_per_sec\n        if max_time > 0:\n            other_time_left = max_time - time\n            if time_left is not None:\n                time_left = min(time_left, other_time_left)\n            else:\n                time_left = other_time_left\n        if time_left is not None:\n            m['time_left'] = math.floor(time_left)\n        if num_epochs > 0:\n            if (total_exs > 0 and exs_per_epoch > 0):\n                display_epochs = int(total_exs / exs_per_epoch)\n            else:\n                display_epochs = total_epochs\n            m['num_epochs'] = display_epochs\n    return m\n\n\nclass Metrics(object):\n    \"\"\"Class that maintains evaluation metrics over dialog.\"\"\"\n\n    def __init__(self, opt):\n        self.metrics = {}\n        self.metrics['cnt'] = 0\n        self.metrics_list = ['mean_rank', 'loss', 'correct', 'f1', 'ppl']\n        for k in self.metrics_list:\n            self.metrics[k] = 0.0\n            self.metrics[k + '_cnt'] = 0\n        self.eval_pr = [1, 5, 10, 100]\n        for k in self.eval_pr:\n            self.metrics['hits@' + str(k)] = 0\n        self.metrics['hits@_cnt'] = 0\n        self.flags = {'has_text_cands': False, 'print_prediction_metrics': False}\n        if opt.get('numthreads', 1) > 1:\n            self.metrics = SharedTable(self.metrics)\n            self.flags = SharedTable(self.flags)\n\n    def __str__(self):\n        return str(self.metrics)\n\n    def __repr__(self):\n        representation = super().__repr__()\n        return representation.replace('>', ': {}>'.format(repr(self.metrics)))\n\n    def _lock(self):\n        if hasattr(self.metrics, 'get_lock'):\n            # use the shared_table's lock\n            return self.metrics.get_lock()\n        else:\n            # otherwise do nothing\n            return no_lock()\n\n    def update_ranking_metrics(self, observation, labels):\n        text_cands = observation.get('text_candidates', None)\n        if text_cands is None:\n            return\n        else:\n            text = observation.get('text', None)\n\n            # Now loop through text candidates, assuming they are sorted.\n            # If any of them is a label then score a point.\n            # maintain hits@1, 5, 10, 50, 100,  etc.\n            label_set = set(normalize_answer(l) for l in labels)\n            cnts = {k: 0 for k in self.eval_pr}\n            cnt = 0\n            for c in text_cands:\n                cnt += 1\n                if normalize_answer(c) in label_set:\n                    for k in self.eval_pr:\n                        if cnt <= k:\n                            cnts[k] += 1\n            # hits metric is 1 if cnts[k] > 0.\n            # (other metrics such as p@k and r@k take\n            # the value of cnt into account.)\n            with self._lock():\n                self.flags['has_text_cands'] = True\n                for k in self.eval_pr:\n                    if cnts[k] > 0:\n                        self.metrics['hits@' + str(k)] += 1\n                self.metrics['hits@_cnt'] += 1\n\n    def update(self, observation, labels):\n        with self._lock():\n            self.metrics['cnt'] += 1\n\n        # Exact match metric.\n        correct = 0\n        prediction = observation.get('text', None)\n        if prediction is not None:\n            if _exact_match(prediction, labels):\n                correct = 1\n            with self._lock():\n                self.flags['print_prediction_metrics'] = True\n                self.metrics['correct'] += correct\n                self.metrics['correct_cnt'] += 1\n\n            # F1 metric.\n            f1 = _f1_score(prediction, labels)\n            with self._lock():\n                self.metrics['f1'] += f1\n                self.metrics['f1_cnt'] += 1\n\n        # Ranking metrics.\n        self.update_ranking_metrics(observation, labels)\n\n        # User-reported metrics\n        if 'metrics' in observation:\n            for k, v in observation['metrics'].items():\n                if k not in ['correct', 'f1', 'hits@k']:\n                    if k in self.metrics_list:\n                        with self._lock():\n                            self.metrics[k] += v\n                            self.metrics[k + '_cnt'] += 1\n                    else:\n                        if type(self.metrics) is SharedTable:\n                            # can't share custom metrics during hogwild\n                            pass\n                        else:\n                            # no need to lock because not SharedTable\n                            if k not in self.metrics:\n                                self.metrics[k] = v\n                                self.metrics_list.append(k)\n                                self.metrics[k + '_cnt'] = 1.0\n                            else:\n                                self.metrics[k] += v\n\n        # Return a dict containing the metrics for this specific example.\n        # Metrics across all data is stored internally in the class, and\n        # can be accessed with the report method.\n        loss = {}\n        loss['correct'] = correct\n        return loss\n\n    def report(self):\n        # Report the metrics over all data seen so far.\n        m = {}\n        total = self.metrics['cnt']\n        m['exs'] = total\n        if total > 0:\n            if self.flags['print_prediction_metrics']:\n                m['accuracy'] = round_sigfigs(self.metrics['correct'] / max(1, self.metrics['correct_cnt']), 4)\n                m['f1'] = round_sigfigs(self.metrics['f1'] / max(1, self.metrics['f1_cnt']), 4)\n                if self.flags['has_text_cands']:\n                    for k in self.eval_pr:\n                        m['hits@' + str(k)] = round_sigfigs(\n                            self.metrics['hits@' + str(k)] / max(1, self.metrics['hits@_cnt']), 3)\n            for k in self.metrics_list:\n                if self.metrics[k + '_cnt'] > 0 and k != 'correct' and k != 'f1':\n                    m[k] = round_sigfigs(self.metrics[k] / max(1, self.metrics[k + '_cnt']), 4)\n        return m\n\n    def clear(self):\n        with self._lock():\n            self.metrics['cnt'] = 0\n            for k in self.metrics_list:\n                v = self.metrics[k]\n                v_typ = type(v)\n                if 'Tensor' in str(v_typ):\n                    self.metrics[k].zero_()\n                else:\n                    self.metrics[k] = 0.0\n                self.metrics[k + '_cnt'] = 0\n            for k in self.eval_pr:\n                self.metrics['hits@' + str(k)] = 0\n            self.metrics['hits@_cnt'] = 0\n", "path": "parlai/core/metrics.py"}], "after_files": [{"content": "# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n# This source code is licensed under the BSD-style license found in the\n# LICENSE file in the root directory of this source tree. An additional grant\n# of patent rights can be found in the PATENTS file in the same directory.\n\"\"\"Provides standard metric evaluations for dialog.\nUses locking and shared memory when ``numthreads`` is set to >1 to share metrics\nbetween processes.\n\"\"\"\n\nfrom parlai.core.thread_utils import SharedTable\nfrom parlai.core.utils import round_sigfigs, no_lock\nfrom collections import Counter\n\nimport re\nimport math\n\ntry:\n    from nltk.translate import bleu_score as nltkbleu\nexcept ImportError:\n    # User doesn't have nltk installed, so we can't use it for bleu\n    # We'll just turn off things, but we might want to warn the user\n    nltkbleu = None\n\nre_art = re.compile(r'\\b(a|an|the)\\b')\nre_punc = re.compile(r'[!\"#$%&()*+,-./:;<=>?@\\[\\]\\\\^`{|}~_\\']')\n\n\ndef normalize_answer(s):\n    \"\"\"Lower text and remove punctuation, articles and extra whitespace.\"\"\"\n    def remove_articles(text):\n        return re_art.sub(' ', text)\n\n    def white_space_fix(text):\n        return ' '.join(text.split())\n\n    def remove_punc(text):\n        return re_punc.sub(' ', text)  # convert punctuation to spaces\n\n    def lower(text):\n        return text.lower()\n\n    return white_space_fix(remove_articles(remove_punc(lower(s))))\n\n\ndef _exact_match(guess, answers):\n    \"\"\"Check if guess is a (normalized) exact match with any answer.\"\"\"\n    if guess is None or answers is None:\n        return False\n    guess = normalize_answer(guess)\n    for a in answers:\n        if guess == normalize_answer(a):\n            return True\n    return False\n\n\ndef _f1_score(guess, answers):\n    \"\"\"Return the max F1 score between the guess and any answer.\"\"\"\n    def _score(g_tokens, a_tokens):\n        common = Counter(g_tokens) & Counter(a_tokens)\n        num_same = sum(common.values())\n        if num_same == 0:\n            return 0\n        precision = 1.0 * num_same / len(g_tokens)\n        recall = 1.0 * num_same / len(a_tokens)\n        f1 = (2 * precision * recall) / (precision + recall)\n        return f1\n\n    if guess is None or answers is None:\n        return 0\n    g_tokens = normalize_answer(guess).split()\n    scores = [_score(g_tokens, normalize_answer(a).split()) for a in answers]\n    return max(scores)\n\n\ndef _bleu(guess, answers):\n    if nltkbleu is None:\n        # bleu library not installed, just return a default value\n        return None\n    # Warning: BLEU calculation *should* include proper tokenization and\n    # punctuation etc. We're using the normalize_answer for everything though,\n    # so we're over-estimating our BLEU scores.  Also note that NLTK's bleu is\n    # going to be slower than fairseq's (which is written in C), but fairseq's\n    # requires that everything be in arrays of ints (i.e. as tensors). NLTK's\n    # works with strings, which is better suited for this module.\n    return nltkbleu.sentence_bleu(\n        [normalize_answer(a).split(\" \") for a in answers],\n        normalize_answer(guess).split(\" \")\n    )\n\n\ndef aggregate_metrics(reporters):\n    #reporters is a list of teachers or worlds\n    m = {}\n    m['tasks'] = {}\n    sums = {'accuracy': 0, 'f1': 0, 'loss': 0, 'ppl': 0}\n    if nltkbleu is not None:\n        sums['bleu'] = 0\n    num_tasks = 0\n    total = 0\n    for i in range(len(reporters)):\n        tid = reporters[i].getID()\n        mt = reporters[i].report()\n        while tid in m['tasks']:\n            # prevent name cloberring if using multiple tasks with same ID\n            tid += '_'\n        m['tasks'][tid] = mt\n        total += mt['exs']\n        found_any = False\n        for k in sums.keys():\n            if k in mt:\n                sums[k] += mt[k]\n                found_any = True\n        if found_any:\n            num_tasks += 1\n    m['exs'] = total\n    m['accuracy'] = 0\n    if num_tasks > 0:\n        for k in sums.keys():\n            m[k] = round_sigfigs(sums[k] / num_tasks, 4)\n    return m\n\n\ndef compute_time_metrics(world, max_time):\n    # Determine time_left and num_epochs\n    exs_per_epoch = world.num_examples() if world.num_examples() else 0\n    num_epochs = world.opt.get('num_epochs', 0)\n    max_exs = exs_per_epoch * num_epochs\n    total_exs = world.get_total_exs()\n\n    m = {}\n    if (max_exs > 0 and total_exs > 0) or max_time > 0:\n        m = {}\n        time_left = None\n        time = world.get_time()\n        total_epochs = world.get_total_epochs()\n\n        if (num_epochs > 0 and total_exs > 0 and max_exs > 0):\n            exs_per_sec = time / total_exs\n            time_left = (max_exs - total_exs) * exs_per_sec\n        if max_time > 0:\n            other_time_left = max_time - time\n            if time_left is not None:\n                time_left = min(time_left, other_time_left)\n            else:\n                time_left = other_time_left\n        if time_left is not None:\n            m['time_left'] = math.floor(time_left)\n        if num_epochs > 0:\n            if (total_exs > 0 and exs_per_epoch > 0):\n                display_epochs = int(total_exs / exs_per_epoch)\n            else:\n                display_epochs = total_epochs\n            m['num_epochs'] = display_epochs\n    return m\n\n\nclass Metrics(object):\n    \"\"\"Class that maintains evaluation metrics over dialog.\"\"\"\n\n    def __init__(self, opt):\n        self.metrics = {}\n        self.metrics['cnt'] = 0\n        self.metrics_list = ['mean_rank', 'loss', 'correct', 'f1', 'ppl']\n        if nltkbleu is not None:\n            # only compute bleu if we can\n            self.metrics_list.append('bleu')\n        for k in self.metrics_list:\n            self.metrics[k] = 0.0\n            self.metrics[k + '_cnt'] = 0\n        self.eval_pr = [1, 5, 10, 100]\n        for k in self.eval_pr:\n            self.metrics['hits@' + str(k)] = 0\n        self.metrics['hits@_cnt'] = 0\n        self.flags = {'has_text_cands': False, 'print_prediction_metrics': False}\n        if opt.get('numthreads', 1) > 1:\n            self.metrics = SharedTable(self.metrics)\n            self.flags = SharedTable(self.flags)\n\n    def __str__(self):\n        return str(self.metrics)\n\n    def __repr__(self):\n        representation = super().__repr__()\n        return representation.replace('>', ': {}>'.format(repr(self.metrics)))\n\n    def _lock(self):\n        if hasattr(self.metrics, 'get_lock'):\n            # use the shared_table's lock\n            return self.metrics.get_lock()\n        else:\n            # otherwise do nothing\n            return no_lock()\n\n    def update_ranking_metrics(self, observation, labels):\n        text_cands = observation.get('text_candidates', None)\n        if text_cands is None:\n            return\n        else:\n            text = observation.get('text', None)\n\n            # Now loop through text candidates, assuming they are sorted.\n            # If any of them is a label then score a point.\n            # maintain hits@1, 5, 10, 50, 100,  etc.\n            label_set = set(normalize_answer(l) for l in labels)\n            cnts = {k: 0 for k in self.eval_pr}\n            cnt = 0\n            for c in text_cands:\n                cnt += 1\n                if normalize_answer(c) in label_set:\n                    for k in self.eval_pr:\n                        if cnt <= k:\n                            cnts[k] += 1\n            # hits metric is 1 if cnts[k] > 0.\n            # (other metrics such as p@k and r@k take\n            # the value of cnt into account.)\n            with self._lock():\n                self.flags['has_text_cands'] = True\n                for k in self.eval_pr:\n                    if cnts[k] > 0:\n                        self.metrics['hits@' + str(k)] += 1\n                self.metrics['hits@_cnt'] += 1\n\n    def update(self, observation, labels):\n        with self._lock():\n            self.metrics['cnt'] += 1\n\n        # Exact match metric.\n        correct = 0\n        prediction = observation.get('text', None)\n        if prediction is not None:\n            if _exact_match(prediction, labels):\n                correct = 1\n            with self._lock():\n                self.flags['print_prediction_metrics'] = True\n                self.metrics['correct'] += correct\n                self.metrics['correct_cnt'] += 1\n\n            # F1 and BLEU metrics.\n            f1 = _f1_score(prediction, labels)\n            bleu = _bleu(prediction, labels)\n            with self._lock():\n                self.metrics['f1'] += f1\n                self.metrics['f1_cnt'] += 1\n                if bleu is not None:\n                    self.metrics['bleu'] += bleu\n                    self.metrics['bleu_cnt'] += 1\n\n        # Ranking metrics.\n        self.update_ranking_metrics(observation, labels)\n\n        # User-reported metrics\n        if 'metrics' in observation:\n            for k, v in observation['metrics'].items():\n                if k not in ['correct', 'f1', 'hits@k', 'bleu']:\n                    if k in self.metrics_list:\n                        with self._lock():\n                            self.metrics[k] += v\n                            self.metrics[k + '_cnt'] += 1\n                    else:\n                        if type(self.metrics) is SharedTable:\n                            # can't share custom metrics during hogwild\n                            pass\n                        else:\n                            # no need to lock because not SharedTable\n                            if k not in self.metrics:\n                                self.metrics[k] = v\n                                self.metrics_list.append(k)\n                                self.metrics[k + '_cnt'] = 1.0\n                            else:\n                                self.metrics[k] += v\n\n        # Return a dict containing the metrics for this specific example.\n        # Metrics across all data is stored internally in the class, and\n        # can be accessed with the report method.\n        loss = {}\n        loss['correct'] = correct\n        return loss\n\n    def report(self):\n        # Report the metrics over all data seen so far.\n        m = {}\n        total = self.metrics['cnt']\n        m['exs'] = total\n        if total > 0:\n            if self.flags['print_prediction_metrics']:\n                m['accuracy'] = round_sigfigs(self.metrics['correct'] / max(1, self.metrics['correct_cnt']), 4)\n                m['f1'] = round_sigfigs(self.metrics['f1'] / max(1, self.metrics['f1_cnt']), 4)\n                if self.flags['has_text_cands']:\n                    for k in self.eval_pr:\n                        m['hits@' + str(k)] = round_sigfigs(\n                            self.metrics['hits@' + str(k)] / max(1, self.metrics['hits@_cnt']), 3)\n            for k in self.metrics_list:\n                if self.metrics[k + '_cnt'] > 0 and k != 'correct' and k != 'f1':\n                    m[k] = round_sigfigs(self.metrics[k] / max(1, self.metrics[k + '_cnt']), 4)\n        return m\n\n    def clear(self):\n        with self._lock():\n            self.metrics['cnt'] = 0\n            for k in self.metrics_list:\n                v = self.metrics[k]\n                v_typ = type(v)\n                if 'Tensor' in str(v_typ):\n                    self.metrics[k].zero_()\n                else:\n                    self.metrics[k] = 0.0\n                self.metrics[k + '_cnt'] = 0\n            for k in self.eval_pr:\n                self.metrics['hits@' + str(k)] = 0\n            self.metrics['hits@_cnt'] = 0\n", "path": "parlai/core/metrics.py"}]}