simplify logic for gemini annototate
re
MODIFY
gemini/gemini_annotate.py
gemini/gemini_annotate.py
@@ -89,16 +89,12 @@ def _annotate_variants(args, conn, metadata, get_val_fn, col_names=None, col_typ
conn, metadata = database.get_session_metadata(str(conn.bind.url))
cursor = conn.bind.connect()
- last_id = 0
- current_id = 0
total = 0
- CHUNK_SIZE = 100000
+ update_size = 5000
to_update = []
select_res = cursor.execution_options(stream_results=True).execute('''SELECT chrom, start, end, ref, alt, variant_id FROM variants''')
- while True:
-
- for row in select_res.fetchmany(CHUNK_SIZE):
+ for row in select_res:
# update_data starts out as a list of the values that should
# be used to populate the new columns for the current row.
@@ -116,17 +112,15 @@ def _annotate_variants(args, conn, metadata, get_val_fn, col_names=None, col_typ
update_data.append(str(row["variant_id"]))
to_update.append(tuple(update_data))
- current_id = row["variant_id"]
-
- if current_id <= last_id:
- break
- elif len(to_update) > 0:
+ if len(to_update) > update_size:
_update_variants(metadata, to_update, col_names, cursor)
-
total += len(to_update)
- print("updated", total, "variants")
- last_id = current_id
to_update = []
+ print("updated", total, "variants")
+ if len(to_update):
+ _update_variants(metadata, to_update, col_names, cursor)
+ total += len(to_update)
+
print("finished updating", total, "variants")
def _update_variants(metadata, to_update, col_names, cursor):

simplify logic for gemini annototate

re #803