prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Bug in "onmt-ark-to-records" code
  I have found a small bug in the code line referenced below. It causes the script to terminate with a `TypeError: data type not understood`. Just for the sake of completeness, this is caused by the fact that numpy doesn't understand the object `tf.float32`. I changed that to `float` and it worked as it was supposed to. I can create a PR for this, but I suppose it is too trivial to do so and claim a contribution, unless you want me to.
  
  https://github.com/OpenNMT/OpenNMT-tf/blob/5809c293d7bc65d923274cfd56b3339fc4107af6/opennmt/bin/ark_to_records.py#L46
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `opennmt/bin/ark_to_records.py`
  Content:
  ```
  1 """ARK data file to TFRecords converter.
  2 
  3 The scripts takes the ARK data file and optionally the indexed target text
  4 to write aligned source and target data.
  5 """
  6 
  7 import argparse
  8 import numpy as np
  9 import tensorflow as tf
  10 
  11 from opennmt.inputters.record_inputter import write_sequence_record
  12 
  13 
  14 def consume_next_vector(ark_file):
  15   """Consumes the next vector.
  16 
  17   Args:
  18     ark_file: The ARK data file.
  19 
  20   Returns:
  21     The next vector as a 2D Numpy array.
  22   """
  23   idx = None
  24   vector = []
  25 
  26   for line in ark_file:
  27     line = line.strip()
  28     fields = line.split()
  29 
  30     if not idx:
  31       idx = fields[0]
  32       fields.pop(0)
  33       fields.pop(0)
  34 
  35     end = fields and fields[-1] == "]"
  36 
  37     if end:
  38       fields.pop()
  39 
  40     if fields:
  41       vector.append(fields)
  42 
  43     if end:
  44       break
  45 
  46   return idx, np.asarray(vector, dtype=tf.float32)
  47 
  48 def consume_next_text(text_file):
  49   """Consumes the next text line from `text_file`."""
  50   idx = None
  51   text = text_file.readline()
  52 
  53   if text:
  54     tokens = text.strip().split()
  55     idx = tokens[0]
  56     tokens.pop(0)
  57     text = " ".join(tokens)
  58 
  59   return idx, text
  60 
  61 def write_text(text, writer):
  62   """Serializes a line of text."""
  63   writer.write(text)
  64   writer.write("\n")
  65 
  66 def ark_to_records_aligned(ark_filename, text_filename, out_prefix, compression_type=None):
  67   """Converts ARK and text datasets to aligned TFRecords and text datasets."""
  68   record_filename = "%s.records" % out_prefix
  69   if compression_type == "GZIP":
  70     record_filename = "%s.gz" % record_filename
  71   record_writer = tf.io.TFRecordWriter(record_filename, options=compression_type)
  72   text_writer = open(out_prefix + ".txt", encoding="utf-8", mode="w")
  73 
  74   ark_buffer = {}
  75   text_buffer = {}
  76   count = 0
  77 
  78   def _write_example(vector, text):
  79     write_sequence_record(vector, record_writer)
  80     write_text(text, text_writer)
  81 
  82   def _search_aligned():
  83     for idx in ark_buffer:
  84       if idx in text_buffer:
  85         vector = ark_buffer[idx]
  86         text = text_buffer[idx]
  87 
  88         del ark_buffer[idx]
  89         del text_buffer[idx]
  90 
  91         return vector, text
  92 
  93     return None, None
  94 
  95   with open(ark_filename, encoding="utf-8") as ark_file, open(text_filename, encoding="utf-8") as text_file: #pylint: disable=line-too-long
  96     while True:
  97       ark_idx, vector = consume_next_vector(ark_file)
  98       text_idx, text = consume_next_text(text_file)
  99 
  100       if not ark_idx and not text_idx:
  101         # Both files are empty.
  102         break
  103 
  104       if ark_idx == text_idx:
  105         # If the indices match, write the example.
  106         _write_example(vector, text)
  107         count += 1
  108       else:
  109         # Otherwise store the entries.
  110         if ark_idx:
  111           ark_buffer[ark_idx] = vector
  112         if text_idx:
  113           text_buffer[text_idx] = text
  114 
  115         # Look if we can now find aligned entries.
  116         vector, text = _search_aligned()
  117 
  118         if vector is not None:
  119           _write_example(vector, text)
  120           count += 1
  121 
  122   # Search alignments in stored entries.
  123   while True:
  124     vector, text = _search_aligned()
  125     if vector is None:
  126       break
  127     _write_example(vector, text)
  128     count += 1
  129 
  130   record_writer.close()
  131   text_writer.close()
  132 
  133   print("Saved {} aligned records.".format(count))
  134 
  135 def ark_to_records(ark_filename, out_prefix, compression_type=None):
  136   """Converts ARK dataset to TFRecords."""
  137   record_writer = tf.io.TFRecordWriter(out_prefix + ".records", options=compression_type)
  138   count = 0
  139 
  140   with open(ark_filename, encoding="utf-8") as ark_file:
  141     while True:
  142       ark_idx, vector = consume_next_vector(ark_file)
  143       if not ark_idx:
  144         break
  145       write_sequence_record(vector, record_writer)
  146       count += 1
  147 
  148   record_writer.close()
  149   print("Saved {} records.".format(count))
  150 
  151 
  152 def main():
  153   parser = argparse.ArgumentParser()
  154   parser.add_argument("--ark", required=True,
  155                       help="Indexed ARK data file.")
  156   parser.add_argument("--txt",
  157                       help=("Indexed target text data file "
  158                             "(must set it to align source and target files)."))
  159   parser.add_argument("--out", required=True,
  160                       help="Output files prefix (will be suffixed by .records and .txt).")
  161   parser.add_argument("--compression_type", default=None, choices=["GZIP"],
  162                       help="Optional compression type.")
  163   args = parser.parse_args()
  164 
  165   if args.txt:
  166     ark_to_records_aligned(args.ark, args.txt, args.out, compression_type=args.compression_type)
  167   else:
  168     ark_to_records(args.ark, args.out, compression_type=args.compression_type)
  169 
  170 if __name__ == "__main__":
  171   main()
  172
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/opennmt/bin/ark_to_records.py b/opennmt/bin/ark_to_records.py
  --- a/opennmt/bin/ark_to_records.py
  +++ b/opennmt/bin/ark_to_records.py
  @@ -43,7 +43,7 @@
       if end:
         break
   
  -  return idx, np.asarray(vector, dtype=tf.float32)
  +  return idx, np.asarray(vector, dtype=np.float32)
   
   def consume_next_text(text_file):
     """Consumes the next text line from `text_file`."""
  
verification_info: {"golden_diff": "diff --git a/opennmt/bin/ark_to_records.py b/opennmt/bin/ark_to_records.py\n--- a/opennmt/bin/ark_to_records.py\n+++ b/opennmt/bin/ark_to_records.py\n@@ -43,7 +43,7 @@\n     if end:\n       break\n \n-  return idx, np.asarray(vector, dtype=tf.float32)\n+  return idx, np.asarray(vector, dtype=np.float32)\n \n def consume_next_text(text_file):\n   \"\"\"Consumes the next text line from `text_file`.\"\"\"\n", "issue": "Bug in \"onmt-ark-to-records\" code\nI have found a small bug in the code line referenced below. It causes the script to terminate with a `TypeError: data type not understood`. Just for the sake of completeness, this is caused by the fact that numpy doesn't understand the object `tf.float32`. I changed that to `float` and it worked as it was supposed to. I can create a PR for this, but I suppose it is too trivial to do so and claim a contribution, unless you want me to.\r\n\r\nhttps://github.com/OpenNMT/OpenNMT-tf/blob/5809c293d7bc65d923274cfd56b3339fc4107af6/opennmt/bin/ark_to_records.py#L46\n", "before_files": [{"content": "\"\"\"ARK data file to TFRecords converter.\n\nThe scripts takes the ARK data file and optionally the indexed target text\nto write aligned source and target data.\n\"\"\"\n\nimport argparse\nimport numpy as np\nimport tensorflow as tf\n\nfrom opennmt.inputters.record_inputter import write_sequence_record\n\n\ndef consume_next_vector(ark_file):\n  \"\"\"Consumes the next vector.\n\n  Args:\n    ark_file: The ARK data file.\n\n  Returns:\n    The next vector as a 2D Numpy array.\n  \"\"\"\n  idx = None\n  vector = []\n\n  for line in ark_file:\n    line = line.strip()\n    fields = line.split()\n\n    if not idx:\n      idx = fields[0]\n      fields.pop(0)\n      fields.pop(0)\n\n    end = fields and fields[-1] == \"]\"\n\n    if end:\n      fields.pop()\n\n    if fields:\n      vector.append(fields)\n\n    if end:\n      break\n\n  return idx, np.asarray(vector, dtype=tf.float32)\n\ndef consume_next_text(text_file):\n  \"\"\"Consumes the next text line from `text_file`.\"\"\"\n  idx = None\n  text = text_file.readline()\n\n  if text:\n    tokens = text.strip().split()\n    idx = tokens[0]\n    tokens.pop(0)\n    text = \" \".join(tokens)\n\n  return idx, text\n\ndef write_text(text, writer):\n  \"\"\"Serializes a line of text.\"\"\"\n  writer.write(text)\n  writer.write(\"\\n\")\n\ndef ark_to_records_aligned(ark_filename, text_filename, out_prefix, compression_type=None):\n  \"\"\"Converts ARK and text datasets to aligned TFRecords and text datasets.\"\"\"\n  record_filename = \"%s.records\" % out_prefix\n  if compression_type == \"GZIP\":\n    record_filename = \"%s.gz\" % record_filename\n  record_writer = tf.io.TFRecordWriter(record_filename, options=compression_type)\n  text_writer = open(out_prefix + \".txt\", encoding=\"utf-8\", mode=\"w\")\n\n  ark_buffer = {}\n  text_buffer = {}\n  count = 0\n\n  def _write_example(vector, text):\n    write_sequence_record(vector, record_writer)\n    write_text(text, text_writer)\n\n  def _search_aligned():\n    for idx in ark_buffer:\n      if idx in text_buffer:\n        vector = ark_buffer[idx]\n        text = text_buffer[idx]\n\n        del ark_buffer[idx]\n        del text_buffer[idx]\n\n        return vector, text\n\n    return None, None\n\n  with open(ark_filename, encoding=\"utf-8\") as ark_file, open(text_filename, encoding=\"utf-8\") as text_file: #pylint: disable=line-too-long\n    while True:\n      ark_idx, vector = consume_next_vector(ark_file)\n      text_idx, text = consume_next_text(text_file)\n\n      if not ark_idx and not text_idx:\n        # Both files are empty.\n        break\n\n      if ark_idx == text_idx:\n        # If the indices match, write the example.\n        _write_example(vector, text)\n        count += 1\n      else:\n        # Otherwise store the entries.\n        if ark_idx:\n          ark_buffer[ark_idx] = vector\n        if text_idx:\n          text_buffer[text_idx] = text\n\n        # Look if we can now find aligned entries.\n        vector, text = _search_aligned()\n\n        if vector is not None:\n          _write_example(vector, text)\n          count += 1\n\n  # Search alignments in stored entries.\n  while True:\n    vector, text = _search_aligned()\n    if vector is None:\n      break\n    _write_example(vector, text)\n    count += 1\n\n  record_writer.close()\n  text_writer.close()\n\n  print(\"Saved {} aligned records.\".format(count))\n\ndef ark_to_records(ark_filename, out_prefix, compression_type=None):\n  \"\"\"Converts ARK dataset to TFRecords.\"\"\"\n  record_writer = tf.io.TFRecordWriter(out_prefix + \".records\", options=compression_type)\n  count = 0\n\n  with open(ark_filename, encoding=\"utf-8\") as ark_file:\n    while True:\n      ark_idx, vector = consume_next_vector(ark_file)\n      if not ark_idx:\n        break\n      write_sequence_record(vector, record_writer)\n      count += 1\n\n  record_writer.close()\n  print(\"Saved {} records.\".format(count))\n\n\ndef main():\n  parser = argparse.ArgumentParser()\n  parser.add_argument(\"--ark\", required=True,\n                      help=\"Indexed ARK data file.\")\n  parser.add_argument(\"--txt\",\n                      help=(\"Indexed target text data file \"\n                            \"(must set it to align source and target files).\"))\n  parser.add_argument(\"--out\", required=True,\n                      help=\"Output files prefix (will be suffixed by .records and .txt).\")\n  parser.add_argument(\"--compression_type\", default=None, choices=[\"GZIP\"],\n                      help=\"Optional compression type.\")\n  args = parser.parse_args()\n\n  if args.txt:\n    ark_to_records_aligned(args.ark, args.txt, args.out, compression_type=args.compression_type)\n  else:\n    ark_to_records(args.ark, args.out, compression_type=args.compression_type)\n\nif __name__ == \"__main__\":\n  main()\n", "path": "opennmt/bin/ark_to_records.py"}], "after_files": [{"content": "\"\"\"ARK data file to TFRecords converter.\n\nThe scripts takes the ARK data file and optionally the indexed target text\nto write aligned source and target data.\n\"\"\"\n\nimport argparse\nimport numpy as np\nimport tensorflow as tf\n\nfrom opennmt.inputters.record_inputter import write_sequence_record\n\n\ndef consume_next_vector(ark_file):\n  \"\"\"Consumes the next vector.\n\n  Args:\n    ark_file: The ARK data file.\n\n  Returns:\n    The next vector as a 2D Numpy array.\n  \"\"\"\n  idx = None\n  vector = []\n\n  for line in ark_file:\n    line = line.strip()\n    fields = line.split()\n\n    if not idx:\n      idx = fields[0]\n      fields.pop(0)\n      fields.pop(0)\n\n    end = fields and fields[-1] == \"]\"\n\n    if end:\n      fields.pop()\n\n    if fields:\n      vector.append(fields)\n\n    if end:\n      break\n\n  return idx, np.asarray(vector, dtype=np.float32)\n\ndef consume_next_text(text_file):\n  \"\"\"Consumes the next text line from `text_file`.\"\"\"\n  idx = None\n  text = text_file.readline()\n\n  if text:\n    tokens = text.strip().split()\n    idx = tokens[0]\n    tokens.pop(0)\n    text = \" \".join(tokens)\n\n  return idx, text\n\ndef write_text(text, writer):\n  \"\"\"Serializes a line of text.\"\"\"\n  writer.write(text)\n  writer.write(\"\\n\")\n\ndef ark_to_records_aligned(ark_filename, text_filename, out_prefix, compression_type=None):\n  \"\"\"Converts ARK and text datasets to aligned TFRecords and text datasets.\"\"\"\n  record_filename = \"%s.records\" % out_prefix\n  if compression_type == \"GZIP\":\n    record_filename = \"%s.gz\" % record_filename\n  record_writer = tf.io.TFRecordWriter(record_filename, options=compression_type)\n  text_writer = open(out_prefix + \".txt\", encoding=\"utf-8\", mode=\"w\")\n\n  ark_buffer = {}\n  text_buffer = {}\n  count = 0\n\n  def _write_example(vector, text):\n    write_sequence_record(vector, record_writer)\n    write_text(text, text_writer)\n\n  def _search_aligned():\n    for idx in ark_buffer:\n      if idx in text_buffer:\n        vector = ark_buffer[idx]\n        text = text_buffer[idx]\n\n        del ark_buffer[idx]\n        del text_buffer[idx]\n\n        return vector, text\n\n    return None, None\n\n  with open(ark_filename, encoding=\"utf-8\") as ark_file, open(text_filename, encoding=\"utf-8\") as text_file: #pylint: disable=line-too-long\n    while True:\n      ark_idx, vector = consume_next_vector(ark_file)\n      text_idx, text = consume_next_text(text_file)\n\n      if not ark_idx and not text_idx:\n        # Both files are empty.\n        break\n\n      if ark_idx == text_idx:\n        # If the indices match, write the example.\n        _write_example(vector, text)\n        count += 1\n      else:\n        # Otherwise store the entries.\n        if ark_idx:\n          ark_buffer[ark_idx] = vector\n        if text_idx:\n          text_buffer[text_idx] = text\n\n        # Look if we can now find aligned entries.\n        vector, text = _search_aligned()\n\n        if vector is not None:\n          _write_example(vector, text)\n          count += 1\n\n  # Search alignments in stored entries.\n  while True:\n    vector, text = _search_aligned()\n    if vector is None:\n      break\n    _write_example(vector, text)\n    count += 1\n\n  record_writer.close()\n  text_writer.close()\n\n  print(\"Saved {} aligned records.\".format(count))\n\ndef ark_to_records(ark_filename, out_prefix, compression_type=None):\n  \"\"\"Converts ARK dataset to TFRecords.\"\"\"\n  record_writer = tf.io.TFRecordWriter(out_prefix + \".records\", options=compression_type)\n  count = 0\n\n  with open(ark_filename, encoding=\"utf-8\") as ark_file:\n    while True:\n      ark_idx, vector = consume_next_vector(ark_file)\n      if not ark_idx:\n        break\n      write_sequence_record(vector, record_writer)\n      count += 1\n\n  record_writer.close()\n  print(\"Saved {} records.\".format(count))\n\n\ndef main():\n  parser = argparse.ArgumentParser()\n  parser.add_argument(\"--ark\", required=True,\n                      help=\"Indexed ARK data file.\")\n  parser.add_argument(\"--txt\",\n                      help=(\"Indexed target text data file \"\n                            \"(must set it to align source and target files).\"))\n  parser.add_argument(\"--out\", required=True,\n                      help=\"Output files prefix (will be suffixed by .records and .txt).\")\n  parser.add_argument(\"--compression_type\", default=None, choices=[\"GZIP\"],\n                      help=\"Optional compression type.\")\n  args = parser.parse_args()\n\n  if args.txt:\n    ark_to_records_aligned(args.ark, args.txt, args.out, compression_type=args.compression_type)\n  else:\n    ark_to_records(args.ark, args.out, compression_type=args.compression_type)\n\nif __name__ == \"__main__\":\n  main()\n", "path": "opennmt/bin/ark_to_records.py"}]}