fix unicode errors on load and on gemini annotate
MODIFY
docs/content/history.rst
docs/content/history.rst
@@ -6,6 +6,10 @@ Release History
===============
#. Use vcfanno for faster, more generalized variant annotation and database creation.
+0.20.2
+======
+#. fix unicode errors in gemini annotate and load
+
0.20.1
======
#. Allow `--gt-filter` to specify the gt_type, e.g. `(gt_quals).(=HET).(<20).(all)` (thanks Jessica for suggesting)

MODIFY
gemini/database.py
gemini/database.py
@@ -437,6 +437,19 @@ def insert_variation(session, metadata, buffer):
left = set(buffer[0].keys()) - set(cols)
assert len(left) == 0, left
+ try:
+ session.execute(tbl.insert(), buffer)
+ session.commit()
+ except:
+ try:
+ session.rollback()
+ except sql.exc.OperationalError:
+ pass
+
+ for b in buffer:
+ for k, v in b.items():
+ if isinstance(v, str):
+ b[k] = v.decode('utf8', 'ignore')
try:
session.execute(tbl.insert(), buffer)
session.commit()
@@ -446,11 +459,14 @@ def insert_variation(session, metadata, buffer):
session.rollback()
except sql.exc.OperationalError:
pass
+
+
stmt = tbl.insert()
+
with session.bind.begin() as trans:
for b in buffer:
trans.execute(stmt, b)
- raise
+ #raise
def insert_variation_impacts(session, metadata, buffer):

MODIFY
gemini/gemini_annotate.py
gemini/gemini_annotate.py
@@ -112,7 +112,7 @@ def _annotate_variants(args, conn, metadata, get_val_fn, col_names=None, col_typ
update_data.append(str(row["variant_id"]))
to_update.append(tuple(update_data))
- if len(to_update) > update_size:
+ if len(to_update) >= update_size:
_update_variants(metadata, to_update, col_names, cursor)
total += len(to_update)
to_update = []
@@ -182,10 +182,10 @@ def gemops_sum(li, col_type):
return np.sum(_map_list_types(li, col_type))
def gemops_list(li, col_type):
- return ",".join(li)
+ return ",".join(x.encode('utf8', 'ignore') for x in li)
def gemops_uniq_list(li, col_type):
- return ",".join(set(li))
+ return ",".join(x.encode('utf8', 'ignore') for x in set(li))
def gemops_median(li, col_type):
return np.median(_map_list_types(li, col_type))

MODIFY
test/data-setup.sh
test/data-setup.sh
set -eo pipefail
gemini load -v test.auto_rec.vcf tt.db --no-genotypes && rm tt.db
+gemini load -t BCFT --skip-gene-tables -v unicode.vcf.gz -p unicode.ped unicode.test.db &
gemini load --cores 2 --skip-gene-tables --test-mode --skip-pls -v test.x-linked.vcf --skip-gerp-bp --skip-cadd -t VEP -p test.x-linked.ped test.x_linked.db
gemini load --skip-gene-tables --test-mode -v test.x-linked.vcf --skip-gerp-bp --skip-cadd -t VEP -p test.x-linked.ped test.x_linked.db
gemini load --skip-gene-tables --test-mode -t VEP -v test-vep-extra.vcf test.vep.extra.db &

MODIFY
test/test-annotate-tool.sh
test/test-annotate-tool.sh
@@ -526,7 +526,8 @@ tabix -p bed anno.bed.gz
echo $'ValueError: Column operation [model] not supported.\n' > exp
gemini annotate -f anno.bed.gz -a extract -c anno27,anno28 -e 4,5 -t text,float -o last,model test.snpeff.vcf.db 2> obs
+gemini annotate -a extract -t text,text -o list,list -e CLNDBN,CLNSIG unicode.test.db -f unicode.vcf.gz
check <(tail -n2 obs | grep -v ^$) <(tail -n2 exp | grep -v ^$) annotate-tool.t17
rm obs exp
-rm *.gz*
+

ADD
test/unicode.ped
+fam1 sample -9 -9 1 1

ADD
test/unicode.vcf.gz
test/unicode.vcf.gz
Binary files /dev/null and b/test/unicode.vcf.gz differ

ADD
test/unicode.vcf.gz.tbi
test/unicode.vcf.gz.tbi
Binary files /dev/null and b/test/unicode.vcf.gz.tbi differ

fix unicode errors on load and on gemini annotate