prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  PRF evaluator: list index out of range
  Hi! 
  I'm getting `list index out of range` when prf evaluator is used.
  
  **Config:**
  Model: TransformerRelative
  params:
    beam_width: 1
  
  train:
    maximum_features_length: 50
    maximum_labels_length: 50
    save_summary_steps: 100
    sample_buffer_size: 1000000
    keep_checkpoint_max: 20
    save_checkpoints_steps: 5000
    max_step: 2000000
  
  eval:
    batch_size: 32
    steps: 5000
    export_on_best: bleu
    external_evaluators: [ "bleu", "prf", "wer" ]
  
  infer:
    batch_size: 1024
  
  **Full stack:**
  W tensorflow/core/kernels/data/generator_dataset_op.cc:103] Error occurred when finalizing GeneratorDataset iterator: Cancelled: Operation was cancelled
  Traceback (most recent call last):
    File "/home/dima/anaconda3/envs/tf/bin/onmt-main", line 8, in <module>
      sys.exit(main())
    File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/bin/main.py", line 224, in main
      hvd=hvd)
    File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/runner.py", line 217, in train
      moving_average_decay=train_config.get("moving_average_decay"))
    File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py", line 118, in __call__
      early_stop = self._evaluate(evaluator, step, moving_average=moving_average)
    File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py", line 140, in _evaluate
      evaluator(step)
    File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/evaluation.py", line 299, in __call__
      score = scorer(self._labels_file, output_path)
    File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/scorers.py", line 132, in __call__
      precision_score, recall_score, fmeasure_score = fmeasure(ref_path, hyp_path)
    File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/fmeasure.py", line 49, in fmeasure
      if tag == classref[linecpt][tagcpt]:
  IndexError: list index out of range
  
  Can I help you with  the issue? I'm not familiar with the code base, but I can try to reproduce it locally and extract the context if necessary.
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `opennmt/utils/fmeasure.py`
  Content:
  ```
  1 """Hypotheses file scoring for Precision Recall and F-Measure."""
  2 
  3 def fmeasure(ref_path,
  4              hyp_path,
  5              return_precision_only=False,
  6              return_recall_only=False,
  7              return_fmeasure_only=False):
  8   """Compute Precision Recall and F-Measure between two files"""
  9   with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:
  10     list_null_tags = ["X", "null", "NULL", "Null", "O"]
  11     listtags = []
  12     linecpt = 0
  13     classref = []
  14     classrandom = []
  15     classhyp = []
  16     nbrtagref = {}
  17     nbrtaghyp = {}
  18     nbrtagok = {}
  19     for tag in listtags:
  20       nbrtagref[tag] = 0
  21       nbrtaghyp[tag] = 0
  22       nbrtagok[tag] = 0
  23     for line in ref_fp:
  24       line = line.strip()
  25       tabline = line.split(' ')
  26       tagcpt = 0
  27       lineref = []
  28       for tag in tabline:
  29         lineref.append(tag)
  30         if tag in nbrtagref.keys() and tag not in list_null_tags:
  31           nbrtagref[tag] = nbrtagref[tag]+1
  32         else:
  33           nbrtagref[tag] = 1
  34         tagcpt = tagcpt+1
  35       classref.append(lineref)
  36       linecpt = linecpt+1
  37     linecpt = 0
  38     for line in hyp_fp:
  39       line = line.strip()
  40       tabline = line.split(' ')
  41       tagcpt = 0
  42       linehyp = []
  43       linerandom = []
  44       for tag in tabline:
  45         linehyp.append(tag)
  46         if tag not in listtags:
  47           listtags.append(tag)
  48         linerandom.append(tag)
  49         if tag == classref[linecpt][tagcpt]:
  50           if tag in nbrtagok.keys():
  51             nbrtagok[tag] = nbrtagok[tag]+1
  52           else:
  53             nbrtagok[tag] = 1
  54         tagcpt = tagcpt+1
  55         if tag in nbrtaghyp.keys():
  56           nbrtaghyp[tag] = nbrtaghyp[tag]+1
  57         else:
  58           nbrtaghyp[tag] = 1
  59       classhyp.append(linehyp)
  60       classrandom.append(linerandom)
  61       linecpt = linecpt+1
  62 
  63   tagcpt = 0
  64   fullprecision = 0
  65   fullrecall = 0
  66   precision = {}
  67   recall = {}
  68   fulltagok = 0.00
  69   fulltaghyp = 0.00
  70   fulltagref = 0.00
  71   for tag in listtags:
  72     if tag not in nbrtagok:
  73       nbrtagok[tag] = 0
  74     if tag not in nbrtaghyp:
  75       nbrtaghyp[tag] = 0
  76     if tag not in nbrtagref:
  77       nbrtagref[tag] = 0
  78     if nbrtaghyp[tag] != 0:
  79       precision[tag] = nbrtagok[tag]/nbrtaghyp[tag]
  80     else:
  81       precision[tag] = 0
  82     if nbrtagref[tag] != 0:
  83       recall[tag] = nbrtagok[tag]/nbrtagref[tag]
  84     else:
  85       recall[tag] = 0
  86     if tag not in list_null_tags:
  87       fulltagok = fulltagok+nbrtagok[tag]
  88       fulltaghyp = fulltaghyp+nbrtaghyp[tag]
  89       fulltagref = fulltagref+nbrtagref[tag]
  90 #      fullprecision = fullprecision+precision[tag]
  91 #      fullrecall = fullrecall+recall[tag]
  92     tagcpt = tagcpt+1
  93   fullprecision = round(100*fulltagok/fulltaghyp, 2)/100
  94   fullrecall = round(100*fulltagok/fulltagref, 2)/100
  95   fullfmeasure = (round((200*fullprecision*fullrecall)/(fullprecision+fullrecall), 2))/100
  96   if return_precision_only:
  97     return fullprecision
  98   if return_recall_only:
  99     return fullrecall
  100   if return_fmeasure_only:
  101     return fullfmeasure
  102   return fullprecision, fullrecall, fullfmeasure
  103
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/opennmt/utils/fmeasure.py b/opennmt/utils/fmeasure.py
  --- a/opennmt/utils/fmeasure.py
  +++ b/opennmt/utils/fmeasure.py
  @@ -9,21 +9,15 @@
     with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:
       list_null_tags = ["X", "null", "NULL", "Null", "O"]
       listtags = []
  -    linecpt = 0
       classref = []
       classrandom = []
       classhyp = []
       nbrtagref = {}
       nbrtaghyp = {}
       nbrtagok = {}
  -    for tag in listtags:
  -      nbrtagref[tag] = 0
  -      nbrtaghyp[tag] = 0
  -      nbrtagok[tag] = 0
       for line in ref_fp:
         line = line.strip()
         tabline = line.split(' ')
  -      tagcpt = 0
         lineref = []
         for tag in tabline:
           lineref.append(tag)
  @@ -31,36 +25,29 @@
             nbrtagref[tag] = nbrtagref[tag]+1
           else:
             nbrtagref[tag] = 1
  -        tagcpt = tagcpt+1
         classref.append(lineref)
  -      linecpt = linecpt+1
  -    linecpt = 0
  -    for line in hyp_fp:
  +    for line, lineref in zip(hyp_fp, classref):
         line = line.strip()
         tabline = line.split(' ')
  -      tagcpt = 0
         linehyp = []
         linerandom = []
  -      for tag in tabline:
  +      for tagcpt, tag in enumerate(tabline):
           linehyp.append(tag)
           if tag not in listtags:
             listtags.append(tag)
           linerandom.append(tag)
  -        if tag == classref[linecpt][tagcpt]:
  +        if tagcpt < len(lineref) and tag == lineref[tagcpt]:
             if tag in nbrtagok.keys():
               nbrtagok[tag] = nbrtagok[tag]+1
             else:
               nbrtagok[tag] = 1
  -        tagcpt = tagcpt+1
           if tag in nbrtaghyp.keys():
             nbrtaghyp[tag] = nbrtaghyp[tag]+1
           else:
             nbrtaghyp[tag] = 1
         classhyp.append(linehyp)
         classrandom.append(linerandom)
  -      linecpt = linecpt+1
   
  -  tagcpt = 0
     fullprecision = 0
     fullrecall = 0
     precision = {}
  @@ -87,12 +74,11 @@
         fulltagok = fulltagok+nbrtagok[tag]
         fulltaghyp = fulltaghyp+nbrtaghyp[tag]
         fulltagref = fulltagref+nbrtagref[tag]
  -#      fullprecision = fullprecision+precision[tag]
  -#      fullrecall = fullrecall+recall[tag]
  -    tagcpt = tagcpt+1
  -  fullprecision = round(100*fulltagok/fulltaghyp, 2)/100
  -  fullrecall = round(100*fulltagok/fulltagref, 2)/100
  -  fullfmeasure = (round((200*fullprecision*fullrecall)/(fullprecision+fullrecall), 2))/100
  +  fullprecision = fulltagok / fulltaghyp if fulltaghyp != 0 else 0
  +  fullrecall = fulltagok / fulltagref if fulltagref != 0 else 0
  +  fullfmeasure = (
  +      (2 * fullprecision * fullrecall) / (fullprecision + fullrecall)
  +      if (fullprecision + fullrecall) != 0 else 0)
     if return_precision_only:
       return fullprecision
     if return_recall_only:
  
verification_info: {"golden_diff": "diff --git a/opennmt/utils/fmeasure.py b/opennmt/utils/fmeasure.py\n--- a/opennmt/utils/fmeasure.py\n+++ b/opennmt/utils/fmeasure.py\n@@ -9,21 +9,15 @@\n   with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:\n     list_null_tags = [\"X\", \"null\", \"NULL\", \"Null\", \"O\"]\n     listtags = []\n-    linecpt = 0\n     classref = []\n     classrandom = []\n     classhyp = []\n     nbrtagref = {}\n     nbrtaghyp = {}\n     nbrtagok = {}\n-    for tag in listtags:\n-      nbrtagref[tag] = 0\n-      nbrtaghyp[tag] = 0\n-      nbrtagok[tag] = 0\n     for line in ref_fp:\n       line = line.strip()\n       tabline = line.split(' ')\n-      tagcpt = 0\n       lineref = []\n       for tag in tabline:\n         lineref.append(tag)\n@@ -31,36 +25,29 @@\n           nbrtagref[tag] = nbrtagref[tag]+1\n         else:\n           nbrtagref[tag] = 1\n-        tagcpt = tagcpt+1\n       classref.append(lineref)\n-      linecpt = linecpt+1\n-    linecpt = 0\n-    for line in hyp_fp:\n+    for line, lineref in zip(hyp_fp, classref):\n       line = line.strip()\n       tabline = line.split(' ')\n-      tagcpt = 0\n       linehyp = []\n       linerandom = []\n-      for tag in tabline:\n+      for tagcpt, tag in enumerate(tabline):\n         linehyp.append(tag)\n         if tag not in listtags:\n           listtags.append(tag)\n         linerandom.append(tag)\n-        if tag == classref[linecpt][tagcpt]:\n+        if tagcpt < len(lineref) and tag == lineref[tagcpt]:\n           if tag in nbrtagok.keys():\n             nbrtagok[tag] = nbrtagok[tag]+1\n           else:\n             nbrtagok[tag] = 1\n-        tagcpt = tagcpt+1\n         if tag in nbrtaghyp.keys():\n           nbrtaghyp[tag] = nbrtaghyp[tag]+1\n         else:\n           nbrtaghyp[tag] = 1\n       classhyp.append(linehyp)\n       classrandom.append(linerandom)\n-      linecpt = linecpt+1\n \n-  tagcpt = 0\n   fullprecision = 0\n   fullrecall = 0\n   precision = {}\n@@ -87,12 +74,11 @@\n       fulltagok = fulltagok+nbrtagok[tag]\n       fulltaghyp = fulltaghyp+nbrtaghyp[tag]\n       fulltagref = fulltagref+nbrtagref[tag]\n-#      fullprecision = fullprecision+precision[tag]\n-#      fullrecall = fullrecall+recall[tag]\n-    tagcpt = tagcpt+1\n-  fullprecision = round(100*fulltagok/fulltaghyp, 2)/100\n-  fullrecall = round(100*fulltagok/fulltagref, 2)/100\n-  fullfmeasure = (round((200*fullprecision*fullrecall)/(fullprecision+fullrecall), 2))/100\n+  fullprecision = fulltagok / fulltaghyp if fulltaghyp != 0 else 0\n+  fullrecall = fulltagok / fulltagref if fulltagref != 0 else 0\n+  fullfmeasure = (\n+      (2 * fullprecision * fullrecall) / (fullprecision + fullrecall)\n+      if (fullprecision + fullrecall) != 0 else 0)\n   if return_precision_only:\n     return fullprecision\n   if return_recall_only:\n", "issue": "PRF evaluator: list index out of range\nHi! \r\nI'm getting `list index out of range` when prf evaluator is used.\r\n\r\n**Config:**\r\nModel: TransformerRelative\r\nparams:\r\n  beam_width: 1\r\n\r\ntrain:\r\n  maximum_features_length: 50\r\n  maximum_labels_length: 50\r\n  save_summary_steps: 100\r\n  sample_buffer_size: 1000000\r\n  keep_checkpoint_max: 20\r\n  save_checkpoints_steps: 5000\r\n  max_step: 2000000\r\n\r\neval:\r\n  batch_size: 32\r\n  steps: 5000\r\n  export_on_best: bleu\r\n  external_evaluators: [ \"bleu\", \"prf\", \"wer\" ]\r\n\r\ninfer:\r\n  batch_size: 1024\r\n\r\n**Full stack:**\r\nW tensorflow/core/kernels/data/generator_dataset_op.cc:103] Error occurred when finalizing GeneratorDataset iterator: Cancelled: Operation was cancelled\r\nTraceback (most recent call last):\r\n  File \"/home/dima/anaconda3/envs/tf/bin/onmt-main\", line 8, in <module>\r\n    sys.exit(main())\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/bin/main.py\", line 224, in main\r\n    hvd=hvd)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/runner.py\", line 217, in train\r\n    moving_average_decay=train_config.get(\"moving_average_decay\"))\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py\", line 118, in __call__\r\n    early_stop = self._evaluate(evaluator, step, moving_average=moving_average)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py\", line 140, in _evaluate\r\n    evaluator(step)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/evaluation.py\", line 299, in __call__\r\n    score = scorer(self._labels_file, output_path)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/scorers.py\", line 132, in __call__\r\n    precision_score, recall_score, fmeasure_score = fmeasure(ref_path, hyp_path)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/fmeasure.py\", line 49, in fmeasure\r\n    if tag == classref[linecpt][tagcpt]:\r\nIndexError: list index out of range\r\n\r\nCan I help you with  the issue? I'm not familiar with the code base, but I can try to reproduce it locally and extract the context if necessary.\r\n\n", "before_files": [{"content": "\"\"\"Hypotheses file scoring for Precision Recall and F-Measure.\"\"\"\n\ndef fmeasure(ref_path,\n             hyp_path,\n             return_precision_only=False,\n             return_recall_only=False,\n             return_fmeasure_only=False):\n  \"\"\"Compute Precision Recall and F-Measure between two files\"\"\"\n  with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:\n    list_null_tags = [\"X\", \"null\", \"NULL\", \"Null\", \"O\"]\n    listtags = []\n    linecpt = 0\n    classref = []\n    classrandom = []\n    classhyp = []\n    nbrtagref = {}\n    nbrtaghyp = {}\n    nbrtagok = {}\n    for tag in listtags:\n      nbrtagref[tag] = 0\n      nbrtaghyp[tag] = 0\n      nbrtagok[tag] = 0\n    for line in ref_fp:\n      line = line.strip()\n      tabline = line.split(' ')\n      tagcpt = 0\n      lineref = []\n      for tag in tabline:\n        lineref.append(tag)\n        if tag in nbrtagref.keys() and tag not in list_null_tags:\n          nbrtagref[tag] = nbrtagref[tag]+1\n        else:\n          nbrtagref[tag] = 1\n        tagcpt = tagcpt+1\n      classref.append(lineref)\n      linecpt = linecpt+1\n    linecpt = 0\n    for line in hyp_fp:\n      line = line.strip()\n      tabline = line.split(' ')\n      tagcpt = 0\n      linehyp = []\n      linerandom = []\n      for tag in tabline:\n        linehyp.append(tag)\n        if tag not in listtags:\n          listtags.append(tag)\n        linerandom.append(tag)\n        if tag == classref[linecpt][tagcpt]:\n          if tag in nbrtagok.keys():\n            nbrtagok[tag] = nbrtagok[tag]+1\n          else:\n            nbrtagok[tag] = 1\n        tagcpt = tagcpt+1\n        if tag in nbrtaghyp.keys():\n          nbrtaghyp[tag] = nbrtaghyp[tag]+1\n        else:\n          nbrtaghyp[tag] = 1\n      classhyp.append(linehyp)\n      classrandom.append(linerandom)\n      linecpt = linecpt+1\n\n  tagcpt = 0\n  fullprecision = 0\n  fullrecall = 0\n  precision = {}\n  recall = {}\n  fulltagok = 0.00\n  fulltaghyp = 0.00\n  fulltagref = 0.00\n  for tag in listtags:\n    if tag not in nbrtagok:\n      nbrtagok[tag] = 0\n    if tag not in nbrtaghyp:\n      nbrtaghyp[tag] = 0\n    if tag not in nbrtagref:\n      nbrtagref[tag] = 0\n    if nbrtaghyp[tag] != 0:\n      precision[tag] = nbrtagok[tag]/nbrtaghyp[tag]\n    else:\n      precision[tag] = 0\n    if nbrtagref[tag] != 0:\n      recall[tag] = nbrtagok[tag]/nbrtagref[tag]\n    else:\n      recall[tag] = 0\n    if tag not in list_null_tags:\n      fulltagok = fulltagok+nbrtagok[tag]\n      fulltaghyp = fulltaghyp+nbrtaghyp[tag]\n      fulltagref = fulltagref+nbrtagref[tag]\n#      fullprecision = fullprecision+precision[tag]\n#      fullrecall = fullrecall+recall[tag]\n    tagcpt = tagcpt+1\n  fullprecision = round(100*fulltagok/fulltaghyp, 2)/100\n  fullrecall = round(100*fulltagok/fulltagref, 2)/100\n  fullfmeasure = (round((200*fullprecision*fullrecall)/(fullprecision+fullrecall), 2))/100\n  if return_precision_only:\n    return fullprecision\n  if return_recall_only:\n    return fullrecall\n  if return_fmeasure_only:\n    return fullfmeasure\n  return fullprecision, fullrecall, fullfmeasure\n", "path": "opennmt/utils/fmeasure.py"}], "after_files": [{"content": "\"\"\"Hypotheses file scoring for Precision Recall and F-Measure.\"\"\"\n\ndef fmeasure(ref_path,\n             hyp_path,\n             return_precision_only=False,\n             return_recall_only=False,\n             return_fmeasure_only=False):\n  \"\"\"Compute Precision Recall and F-Measure between two files\"\"\"\n  with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:\n    list_null_tags = [\"X\", \"null\", \"NULL\", \"Null\", \"O\"]\n    listtags = []\n    classref = []\n    classrandom = []\n    classhyp = []\n    nbrtagref = {}\n    nbrtaghyp = {}\n    nbrtagok = {}\n    for line in ref_fp:\n      line = line.strip()\n      tabline = line.split(' ')\n      lineref = []\n      for tag in tabline:\n        lineref.append(tag)\n        if tag in nbrtagref.keys() and tag not in list_null_tags:\n          nbrtagref[tag] = nbrtagref[tag]+1\n        else:\n          nbrtagref[tag] = 1\n      classref.append(lineref)\n    for line, lineref in zip(hyp_fp, classref):\n      line = line.strip()\n      tabline = line.split(' ')\n      linehyp = []\n      linerandom = []\n      for tagcpt, tag in enumerate(tabline):\n        linehyp.append(tag)\n        if tag not in listtags:\n          listtags.append(tag)\n        linerandom.append(tag)\n        if tagcpt < len(lineref) and tag == lineref[tagcpt]:\n          if tag in nbrtagok.keys():\n            nbrtagok[tag] = nbrtagok[tag]+1\n          else:\n            nbrtagok[tag] = 1\n        if tag in nbrtaghyp.keys():\n          nbrtaghyp[tag] = nbrtaghyp[tag]+1\n        else:\n          nbrtaghyp[tag] = 1\n      classhyp.append(linehyp)\n      classrandom.append(linerandom)\n\n  fullprecision = 0\n  fullrecall = 0\n  precision = {}\n  recall = {}\n  fulltagok = 0.00\n  fulltaghyp = 0.00\n  fulltagref = 0.00\n  for tag in listtags:\n    if tag not in nbrtagok:\n      nbrtagok[tag] = 0\n    if tag not in nbrtaghyp:\n      nbrtaghyp[tag] = 0\n    if tag not in nbrtagref:\n      nbrtagref[tag] = 0\n    if nbrtaghyp[tag] != 0:\n      precision[tag] = nbrtagok[tag]/nbrtaghyp[tag]\n    else:\n      precision[tag] = 0\n    if nbrtagref[tag] != 0:\n      recall[tag] = nbrtagok[tag]/nbrtagref[tag]\n    else:\n      recall[tag] = 0\n    if tag not in list_null_tags:\n      fulltagok = fulltagok+nbrtagok[tag]\n      fulltaghyp = fulltaghyp+nbrtaghyp[tag]\n      fulltagref = fulltagref+nbrtagref[tag]\n  fullprecision = fulltagok / fulltaghyp if fulltaghyp != 0 else 0\n  fullrecall = fulltagok / fulltagref if fulltagref != 0 else 0\n  fullfmeasure = (\n      (2 * fullprecision * fullrecall) / (fullprecision + fullrecall)\n      if (fullprecision + fullrecall) != 0 else 0)\n  if return_precision_only:\n    return fullprecision\n  if return_recall_only:\n    return fullrecall\n  if return_fmeasure_only:\n    return fullfmeasure\n  return fullprecision, fullrecall, fullfmeasure\n", "path": "opennmt/utils/fmeasure.py"}]}