prompt: You will be provided with a partial code base and an issue statement explaining a problem to resolve.
  <issue>
  tools/embeddings_to_torch.py fails when some word features are included in the preprocessing step
  When there are some word features appended to each token in the source text, it seems that the `tools/embeddings_to_torch.py` script cannot extract correct vocabulary from the dataset.
  ```
  $ python tools/embeddings_to_torch.py -emb_file /path/to/word.vectors.txt -dict_file dataset.vocab.pt -output dataset.emb
  Traceback (most recent call last):
    File "tools/embeddings_to_torch.py", line 94, in <module>
      main()
    File "tools/embeddings_to_torch.py", line 62, in main
      enc_vocab, dec_vocab = get_vocabs(opt.dict_file)
    File "tools/embeddings_to_torch.py", line 24, in get_vocabs
      enc_vocab, dec_vocab = [vocab[1] for vocab in vocabs]
  ValueError: too many values to unpack (expected 2)
  ```
  
  </issue>
  <code>
  [start of tools/embeddings_to_torch.py]
  1 #!/usr/bin/env python
  2 # -*- coding: utf-8 -*-
  3 from __future__ import print_function
  4 from __future__ import division
  5 import six
  6 import sys
  7 import numpy as np
  8 import argparse
  9 import torch
  10 
  11 parser = argparse.ArgumentParser(description='embeddings_to_torch.py')
  12 parser.add_argument('-emb_file', required=True,
  13                     help="Embeddings from this file")
  14 parser.add_argument('-output_file', required=True,
  15                     help="Output file for the prepared data")
  16 parser.add_argument('-dict_file', required=True,
  17                     help="Dictionary file")
  18 parser.add_argument('-verbose', action="store_true", default=False)
  19 opt = parser.parse_args()
  20 
  21 
  22 def get_vocabs(dict_file):
  23     vocabs = torch.load(dict_file)
  24     enc_vocab, dec_vocab = [vocab[1] for vocab in vocabs]
  25 
  26     print("From: %s" % dict_file)
  27     print("\t* source vocab: %d words" % len(enc_vocab))
  28     print("\t* target vocab: %d words" % len(dec_vocab))
  29 
  30     return enc_vocab, dec_vocab
  31 
  32 
  33 def get_embeddings(file):
  34     embs = dict()
  35     for l in open(file, 'rb').readlines():
  36         l_split = l.decode('utf8').strip().split()
  37         if len(l_split) == 2:
  38             continue
  39         embs[l_split[0]] = [float(em) for em in l_split[1:]]
  40     print("Got {} embeddings from {}".format(len(embs), file))
  41 
  42     return embs
  43 
  44 
  45 def match_embeddings(vocab, emb):
  46     dim = len(six.next(six.itervalues(emb)))
  47     filtered_embeddings = np.zeros((len(vocab), dim))
  48     count = {"match": 0, "miss": 0}
  49     for w, w_id in vocab.stoi.items():
  50         if w in emb:
  51             filtered_embeddings[w_id] = emb[w]
  52             count['match'] += 1
  53         else:
  54             if opt.verbose:
  55                 print(u"not found:\t{}".format(w), file=sys.stderr)
  56             count['miss'] += 1
  57 
  58     return torch.Tensor(filtered_embeddings), count
  59 
  60 
  61 def main():
  62     enc_vocab, dec_vocab = get_vocabs(opt.dict_file)
  63     embeddings = get_embeddings(opt.emb_file)
  64 
  65     filtered_enc_embeddings, enc_count = match_embeddings(enc_vocab,
  66                                                           embeddings)
  67     filtered_dec_embeddings, dec_count = match_embeddings(dec_vocab,
  68                                                           embeddings)
  69 
  70     print("\nMatching: ")
  71     match_percent = [_['match'] / (_['match'] + _['miss']) * 100
  72                      for _ in [enc_count, dec_count]]
  73     print("\t* enc: %d match, %d missing, (%.2f%%)" % (enc_count['match'],
  74                                                        enc_count['miss'],
  75                                                        match_percent[0]))
  76     print("\t* dec: %d match, %d missing, (%.2f%%)" % (dec_count['match'],
  77                                                        dec_count['miss'],
  78                                                        match_percent[1]))
  79 
  80     print("\nFiltered embeddings:")
  81     print("\t* enc: ", filtered_enc_embeddings.size())
  82     print("\t* dec: ", filtered_dec_embeddings.size())
  83 
  84     enc_output_file = opt.output_file + ".enc.pt"
  85     dec_output_file = opt.output_file + ".dec.pt"
  86     print("\nSaving embedding as:\n\t* enc: %s\n\t* dec: %s"
  87           % (enc_output_file, dec_output_file))
  88     torch.save(filtered_enc_embeddings, enc_output_file)
  89     torch.save(filtered_dec_embeddings, dec_output_file)
  90     print("\nDone.")
  91 
  92 
  93 if __name__ == "__main__":
  94     main()
  95
  [end of tools/embeddings_to_torch.py]
  </code>
  
  I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format.
  <patch>
  diff --git a/file.py b/file.py
  --- a/file.py
  +++ b/file.py
  @@ -1,27 +1,35 @@
   def euclidean(a, b):
  -    while b:
  -        a, b = b, a % b
  -    return a
  +    if b == 0:
  +        return a
  +    return euclidean(b, a % b)
   
   
   def bresenham(x0, y0, x1, y1):
       points = []
       dx = abs(x1 - x0)
       dy = abs(y1 - y0)
  -    sx = 1 if x0 < x1 else -1
  -    sy = 1 if y0 < y1 else -1
  -    err = dx - dy
  +    x, y = x0, y0
  +    sx = -1 if x0 > x1 else 1
  +    sy = -1 if y0 > y1 else 1
   
  -    while True:
  -        points.append((x0, y0))
  -        if x0 == x1 and y0 == y1:
  -            break
  -        e2 = 2 * err
  -        if e2 > -dy:
  -            err -= dy
  -            x0 += sx
  -        if e2 < dx:
  -            err += dx
  -            y0 += sy
  +    if dx > dy:
  +        err = dx / 2.0
  +        while x != x1:
  +            points.append((x, y))
  +            err -= dy
  +            if err < 0:
  +                y += sy
  +                err += dx
  +            x += sx
  +    else:
  +        err = dy / 2.0
  +        while y != y1:
  +            points.append((x, y))
  +            err -= dx
  +            if err < 0:
  +                x += sx
  +                err += dy
  +            y += sy
  +
  +    points.append((x, y))
       return points
  
  </patch>
golden_diff: diff --git a/tools/embeddings_to_torch.py b/tools/embeddings_to_torch.py
  --- a/tools/embeddings_to_torch.py
  +++ b/tools/embeddings_to_torch.py
  @@ -21,7 +21,7 @@
   
   def get_vocabs(dict_file):
       vocabs = torch.load(dict_file)
  -    enc_vocab, dec_vocab = [vocab[1] for vocab in vocabs]
  +    enc_vocab, dec_vocab = vocabs[0][1], vocabs[-1][1]
   
       print("From: %s" % dict_file)
       print("\t* source vocab: %d words" % len(enc_vocab))
  
verification_info: {"golden_diff": "diff --git a/tools/embeddings_to_torch.py b/tools/embeddings_to_torch.py\n--- a/tools/embeddings_to_torch.py\n+++ b/tools/embeddings_to_torch.py\n@@ -21,7 +21,7 @@\n \n def get_vocabs(dict_file):\n     vocabs = torch.load(dict_file)\n-    enc_vocab, dec_vocab = [vocab[1] for vocab in vocabs]\n+    enc_vocab, dec_vocab = vocabs[0][1], vocabs[-1][1]\n \n     print(\"From: %s\" % dict_file)\n     print(\"\\t* source vocab: %d words\" % len(enc_vocab))\n", "issue": "tools/embeddings_to_torch.py fails when some word features are included in the preprocessing step\nWhen there are some word features appended to each token in the source text, it seems that the `tools/embeddings_to_torch.py` script cannot extract correct vocabulary from the dataset.\r\n```\r\n$ python tools/embeddings_to_torch.py -emb_file /path/to/word.vectors.txt -dict_file dataset.vocab.pt -output dataset.emb\r\nTraceback (most recent call last):\r\n  File \"tools/embeddings_to_torch.py\", line 94, in <module>\r\n    main()\r\n  File \"tools/embeddings_to_torch.py\", line 62, in main\r\n    enc_vocab, dec_vocab = get_vocabs(opt.dict_file)\r\n  File \"tools/embeddings_to_torch.py\", line 24, in get_vocabs\r\n    enc_vocab, dec_vocab = [vocab[1] for vocab in vocabs]\r\nValueError: too many values to unpack (expected 2)\r\n```\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nfrom __future__ import print_function\nfrom __future__ import division\nimport six\nimport sys\nimport numpy as np\nimport argparse\nimport torch\n\nparser = argparse.ArgumentParser(description='embeddings_to_torch.py')\nparser.add_argument('-emb_file', required=True,\n                    help=\"Embeddings from this file\")\nparser.add_argument('-output_file', required=True,\n                    help=\"Output file for the prepared data\")\nparser.add_argument('-dict_file', required=True,\n                    help=\"Dictionary file\")\nparser.add_argument('-verbose', action=\"store_true\", default=False)\nopt = parser.parse_args()\n\n\ndef get_vocabs(dict_file):\n    vocabs = torch.load(dict_file)\n    enc_vocab, dec_vocab = [vocab[1] for vocab in vocabs]\n\n    print(\"From: %s\" % dict_file)\n    print(\"\\t* source vocab: %d words\" % len(enc_vocab))\n    print(\"\\t* target vocab: %d words\" % len(dec_vocab))\n\n    return enc_vocab, dec_vocab\n\n\ndef get_embeddings(file):\n    embs = dict()\n    for l in open(file, 'rb').readlines():\n        l_split = l.decode('utf8').strip().split()\n        if len(l_split) == 2:\n            continue\n        embs[l_split[0]] = [float(em) for em in l_split[1:]]\n    print(\"Got {} embeddings from {}\".format(len(embs), file))\n\n    return embs\n\n\ndef match_embeddings(vocab, emb):\n    dim = len(six.next(six.itervalues(emb)))\n    filtered_embeddings = np.zeros((len(vocab), dim))\n    count = {\"match\": 0, \"miss\": 0}\n    for w, w_id in vocab.stoi.items():\n        if w in emb:\n            filtered_embeddings[w_id] = emb[w]\n            count['match'] += 1\n        else:\n            if opt.verbose:\n                print(u\"not found:\\t{}\".format(w), file=sys.stderr)\n            count['miss'] += 1\n\n    return torch.Tensor(filtered_embeddings), count\n\n\ndef main():\n    enc_vocab, dec_vocab = get_vocabs(opt.dict_file)\n    embeddings = get_embeddings(opt.emb_file)\n\n    filtered_enc_embeddings, enc_count = match_embeddings(enc_vocab,\n                                                          embeddings)\n    filtered_dec_embeddings, dec_count = match_embeddings(dec_vocab,\n                                                          embeddings)\n\n    print(\"\\nMatching: \")\n    match_percent = [_['match'] / (_['match'] + _['miss']) * 100\n                     for _ in [enc_count, dec_count]]\n    print(\"\\t* enc: %d match, %d missing, (%.2f%%)\" % (enc_count['match'],\n                                                       enc_count['miss'],\n                                                       match_percent[0]))\n    print(\"\\t* dec: %d match, %d missing, (%.2f%%)\" % (dec_count['match'],\n                                                       dec_count['miss'],\n                                                       match_percent[1]))\n\n    print(\"\\nFiltered embeddings:\")\n    print(\"\\t* enc: \", filtered_enc_embeddings.size())\n    print(\"\\t* dec: \", filtered_dec_embeddings.size())\n\n    enc_output_file = opt.output_file + \".enc.pt\"\n    dec_output_file = opt.output_file + \".dec.pt\"\n    print(\"\\nSaving embedding as:\\n\\t* enc: %s\\n\\t* dec: %s\"\n          % (enc_output_file, dec_output_file))\n    torch.save(filtered_enc_embeddings, enc_output_file)\n    torch.save(filtered_dec_embeddings, dec_output_file)\n    print(\"\\nDone.\")\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "tools/embeddings_to_torch.py"}]}