prompt: You will be provided with a partial code base and an issue statement explaining a problem to resolve.
  <issue>
  The scoring script is incorrect if any pruning is done on the dictionary or in the case of word models
  Hi @sw005320 and other developers,
  
  The command `json2trn.py ${dir}/data.json ${dic} --num-spkrs ${num_spkrs} --refs ${dir}/ref.trn --hyps ${dir}/hyp.trn` in the `score_sclite.sh` replaces all units in reference that are not in the dictionary with unk and then scores it.
  
  This is wrong because you are modifying the reference transcription to match your dictionary. Let's say I train a model with dictionary with only 1 term "Mr" and the hypothesis is 
  "Mr unk unk" and reference is "Mr John Doe". This would give a WER of 0% because the reference would be modified to "Mr unk unk".  
  
  </issue>
  <code>
  [start of utils/json2trn.py]
  1 #!/usr/bin/env python
  2 # encoding: utf-8
  3 
  4 # Copyright 2017 Johns Hopkins University (Shinji Watanabe)
  5 #           2018 Xuankai Chang (Shanghai Jiao Tong University)
  6 #  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
  7 
  8 import argparse
  9 import codecs
  10 import json
  11 import logging
  12 import sys
  13 
  14 from espnet.utils.cli_utils import get_commandline_args
  15 
  16 
  17 def get_parser():
  18     parser = argparse.ArgumentParser(
  19         description='convert a json to a transcription file with a token dictionary',
  20         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  21     parser.add_argument('json', type=str, help='json files')
  22     parser.add_argument('dict', type=str, help='dict')
  23     parser.add_argument('--num-spkrs', type=int, default=1, help='number of speakers')
  24     parser.add_argument('--refs', type=str, nargs='+', help='ref for all speakers')
  25     parser.add_argument('--hyps', type=str, nargs='+', help='hyp for all outputs')
  26     return parser
  27 
  28 
  29 def main(args):
  30     args = get_parser().parse_args(args)
  31     convert(args.json, args.dict, args.refs, args.hyps, args.num_spkrs)
  32 
  33 
  34 def convert(jsonf, dic, refs, hyps, num_spkrs=1):
  35     n_ref = len(refs)
  36     n_hyp = len(hyps)
  37     assert n_ref == n_hyp
  38     assert n_ref == num_spkrs
  39 
  40     # logging info
  41     logfmt = '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s'
  42     logging.basicConfig(level=logging.INFO, format=logfmt)
  43     logging.info(get_commandline_args())
  44 
  45     logging.info("reading %s", jsonf)
  46     with codecs.open(jsonf, 'r', encoding="utf-8") as f:
  47         j = json.load(f)
  48 
  49     logging.info("reading %s", dic)
  50     with codecs.open(dic, 'r', encoding="utf-8") as f:
  51         dictionary = f.readlines()
  52     char_list = [entry.split(' ')[0] for entry in dictionary]
  53     char_list.insert(0, '<blank>')
  54     char_list.append('<eos>')
  55 
  56     for ns in range(num_spkrs):
  57         hyp_file = codecs.open(hyps[ns], 'w', encoding="utf-8")
  58         ref_file = codecs.open(refs[ns], 'w', encoding="utf-8")
  59 
  60         for x in j['utts']:
  61             # hyps
  62             if num_spkrs == 1:
  63                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['rec_tokenid'].split()]
  64             else:
  65                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['rec_tokenid'].split()]
  66             hyp_file.write(" ".join(seq).replace('<eos>', '')),
  67             hyp_file.write(" (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
  68 
  69             # ref
  70             if num_spkrs == 1:
  71                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['tokenid'].split()]
  72             else:
  73                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['tokenid'].split()]
  74             ref_file.write(" ".join(seq).replace('<eos>', '')),
  75             ref_file.write(" (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
  76 
  77         hyp_file.close()
  78         ref_file.close()
  79 
  80 
  81 if __name__ == '__main__':
  82     main(sys.argv[1:])
  83
  [end of utils/json2trn.py]
  </code>
  
  I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format.
  <patch>
  diff --git a/file.py b/file.py
  --- a/file.py
  +++ b/file.py
  @@ -1,27 +1,35 @@
   def euclidean(a, b):
  -    while b:
  -        a, b = b, a % b
  -    return a
  +    if b == 0:
  +        return a
  +    return euclidean(b, a % b)
   
   
   def bresenham(x0, y0, x1, y1):
       points = []
       dx = abs(x1 - x0)
       dy = abs(y1 - y0)
  -    sx = 1 if x0 < x1 else -1
  -    sy = 1 if y0 < y1 else -1
  -    err = dx - dy
  +    x, y = x0, y0
  +    sx = -1 if x0 > x1 else 1
  +    sy = -1 if y0 > y1 else 1
   
  -    while True:
  -        points.append((x0, y0))
  -        if x0 == x1 and y0 == y1:
  -            break
  -        e2 = 2 * err
  -        if e2 > -dy:
  -            err -= dy
  -            x0 += sx
  -        if e2 < dx:
  -            err += dx
  -            y0 += sy
  +    if dx > dy:
  +        err = dx / 2.0
  +        while x != x1:
  +            points.append((x, y))
  +            err -= dy
  +            if err < 0:
  +                y += sy
  +                err += dx
  +            x += sx
  +    else:
  +        err = dy / 2.0
  +        while y != y1:
  +            points.append((x, y))
  +            err -= dx
  +            if err < 0:
  +                x += sx
  +                err += dy
  +            y += sy
  +
  +    points.append((x, y))
       return points
  
  </patch>
golden_diff: diff --git a/utils/json2trn.py b/utils/json2trn.py
  --- a/utils/json2trn.py
  +++ b/utils/json2trn.py
  @@ -58,21 +58,25 @@
           ref_file = codecs.open(refs[ns], 'w', encoding="utf-8")
   
           for x in j['utts']:
  -            # hyps
  +            # recognition hypothesis
               if num_spkrs == 1:
                   seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['rec_tokenid'].split()]
               else:
                   seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['rec_tokenid'].split()]
  +            # In the recognition hypothesis, the <eos> symbol is usually attached in the last part of the sentence
  +            # and it is removed below.
               hyp_file.write(" ".join(seq).replace('<eos>', '')),
               hyp_file.write(" (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
   
  -            # ref
  +            # reference
               if num_spkrs == 1:
  -                seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['tokenid'].split()]
  +                seq = j['utts'][x]['output'][0]['token']
               else:
  -                seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['tokenid'].split()]
  -            ref_file.write(" ".join(seq).replace('<eos>', '')),
  -            ref_file.write(" (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
  +                seq = j['utts'][x]['output'][ns][0]['token']
  +            # Unlike the recognition hypothesis, the reference is directly generated from a token without dictionary
  +            # to avoid to include <unk> symbols in the reference to make scoring normal.
  +            # The detailed discussion can be found at https://github.com/espnet/espnet/issues/993
  +            ref_file.write(seq + " (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
   
           hyp_file.close()
           ref_file.close()
  
verification_info: {"golden_diff": "diff --git a/utils/json2trn.py b/utils/json2trn.py\n--- a/utils/json2trn.py\n+++ b/utils/json2trn.py\n@@ -58,21 +58,25 @@\n         ref_file = codecs.open(refs[ns], 'w', encoding=\"utf-8\")\n \n         for x in j['utts']:\n-            # hyps\n+            # recognition hypothesis\n             if num_spkrs == 1:\n                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['rec_tokenid'].split()]\n             else:\n                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['rec_tokenid'].split()]\n+            # In the recognition hypothesis, the <eos> symbol is usually attached in the last part of the sentence\n+            # and it is removed below.\n             hyp_file.write(\" \".join(seq).replace('<eos>', '')),\n             hyp_file.write(\" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n \n-            # ref\n+            # reference\n             if num_spkrs == 1:\n-                seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['tokenid'].split()]\n+                seq = j['utts'][x]['output'][0]['token']\n             else:\n-                seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['tokenid'].split()]\n-            ref_file.write(\" \".join(seq).replace('<eos>', '')),\n-            ref_file.write(\" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n+                seq = j['utts'][x]['output'][ns][0]['token']\n+            # Unlike the recognition hypothesis, the reference is directly generated from a token without dictionary\n+            # to avoid to include <unk> symbols in the reference to make scoring normal.\n+            # The detailed discussion can be found at https://github.com/espnet/espnet/issues/993\n+            ref_file.write(seq + \" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n \n         hyp_file.close()\n         ref_file.close()\n", "issue": "The scoring script is incorrect if any pruning is done on the dictionary or in the case of word models\nHi @sw005320 and other developers,\r\n\r\nThe command `json2trn.py ${dir}/data.json ${dic} --num-spkrs ${num_spkrs} --refs ${dir}/ref.trn --hyps ${dir}/hyp.trn` in the `score_sclite.sh` replaces all units in reference that are not in the dictionary with unk and then scores it.\r\n\r\nThis is wrong because you are modifying the reference transcription to match your dictionary. Let's say I train a model with dictionary with only 1 term \"Mr\" and the hypothesis is \r\n\"Mr unk unk\" and reference is \"Mr John Doe\". This would give a WER of 0% because the reference would be modified to \"Mr unk unk\".  \n", "before_files": [{"content": "#!/usr/bin/env python\n# encoding: utf-8\n\n# Copyright 2017 Johns Hopkins University (Shinji Watanabe)\n#           2018 Xuankai Chang (Shanghai Jiao Tong University)\n#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)\n\nimport argparse\nimport codecs\nimport json\nimport logging\nimport sys\n\nfrom espnet.utils.cli_utils import get_commandline_args\n\n\ndef get_parser():\n    parser = argparse.ArgumentParser(\n        description='convert a json to a transcription file with a token dictionary',\n        formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n    parser.add_argument('json', type=str, help='json files')\n    parser.add_argument('dict', type=str, help='dict')\n    parser.add_argument('--num-spkrs', type=int, default=1, help='number of speakers')\n    parser.add_argument('--refs', type=str, nargs='+', help='ref for all speakers')\n    parser.add_argument('--hyps', type=str, nargs='+', help='hyp for all outputs')\n    return parser\n\n\ndef main(args):\n    args = get_parser().parse_args(args)\n    convert(args.json, args.dict, args.refs, args.hyps, args.num_spkrs)\n\n\ndef convert(jsonf, dic, refs, hyps, num_spkrs=1):\n    n_ref = len(refs)\n    n_hyp = len(hyps)\n    assert n_ref == n_hyp\n    assert n_ref == num_spkrs\n\n    # logging info\n    logfmt = '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s'\n    logging.basicConfig(level=logging.INFO, format=logfmt)\n    logging.info(get_commandline_args())\n\n    logging.info(\"reading %s\", jsonf)\n    with codecs.open(jsonf, 'r', encoding=\"utf-8\") as f:\n        j = json.load(f)\n\n    logging.info(\"reading %s\", dic)\n    with codecs.open(dic, 'r', encoding=\"utf-8\") as f:\n        dictionary = f.readlines()\n    char_list = [entry.split(' ')[0] for entry in dictionary]\n    char_list.insert(0, '<blank>')\n    char_list.append('<eos>')\n\n    for ns in range(num_spkrs):\n        hyp_file = codecs.open(hyps[ns], 'w', encoding=\"utf-8\")\n        ref_file = codecs.open(refs[ns], 'w', encoding=\"utf-8\")\n\n        for x in j['utts']:\n            # hyps\n            if num_spkrs == 1:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['rec_tokenid'].split()]\n            else:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['rec_tokenid'].split()]\n            hyp_file.write(\" \".join(seq).replace('<eos>', '')),\n            hyp_file.write(\" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n\n            # ref\n            if num_spkrs == 1:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['tokenid'].split()]\n            else:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['tokenid'].split()]\n            ref_file.write(\" \".join(seq).replace('<eos>', '')),\n            ref_file.write(\" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n\n        hyp_file.close()\n        ref_file.close()\n\n\nif __name__ == '__main__':\n    main(sys.argv[1:])\n", "path": "utils/json2trn.py"}]}