# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + papermill={"duration": 0.111927, "end_time": "2020-01-06T17:19:49.747744", "exception": false, "start_time": "2020-01-06T17:19:49.635817", "status": "completed"} tags=[]
# %load_ext autoreload
# %autoreload 2

# + papermill={"duration": 1.517996, "end_time": "2020-01-06T17:19:51.363093", "exception": false, "start_time": "2020-01-06T17:19:49.845097", "status": "completed"} tags=[]
import os
import pickle
from glob import glob
from concurrent.futures import ProcessPoolExecutor, as_completed

import numpy as np
import pandas as pd
from scipy import stats

import settings as conf
import metadata
from utils import is_number, chunker
from results.multixcan import MXPhenoInfo, MXPhenoResults

# + papermill={"duration": 0.11345, "end_time": "2020-01-06T17:19:51.868718", "exception": false, "start_time": "2020-01-06T17:19:51.755268", "status": "completed"} tags=[]
os.makedirs(conf.WEBAPP_DIR, exist_ok=True)

# + [markdown] papermill={"duration": 0.162855, "end_time": "2020-01-06T17:19:52.318438", "exception": false, "start_time": "2020-01-06T17:19:52.155583", "status": "completed"} tags=[]
# # Gene mappings

# + papermill={"duration": 0.82939, "end_time": "2020-01-06T17:19:53.226201", "exception": false, "start_time": "2020-01-06T17:19:52.396811", "status": "completed"} tags=[]
metadata.GENES_MAPPINGS.head()

# + papermill={"duration": 0.411866, "end_time": "2020-01-06T17:19:53.742814", "exception": false, "start_time": "2020-01-06T17:19:53.330948", "status": "completed"} tags=[]
gene_name_to_id_long = metadata.GENES_MAPPINGS[['gene_name', 'gene']].set_index('gene_name').to_dict()['gene']

# + papermill={"duration": 0.119402, "end_time": "2020-01-06T17:19:54.086019", "exception": false, "start_time": "2020-01-06T17:19:53.966617", "status": "completed"} tags=[]
gene_name_to_id_long['A2M']

# + papermill={"duration": 0.12816, "end_time": "2020-01-06T17:19:54.578628", "exception": false, "start_time": "2020-01-06T17:19:54.450468", "status": "completed"} tags=[]
gene_id_long_to_name = metadata.GENES_MAPPINGS[['gene_name', 'gene']].set_index('gene').to_dict()['gene_name']

# + papermill={"duration": 0.239462, "end_time": "2020-01-06T17:19:55.001615", "exception": false, "start_time": "2020-01-06T17:19:54.762153", "status": "completed"} tags=[]
gene_id_long_to_name['ENSG00000175899.14']

# + papermill={"duration": 0.181006, "end_time": "2020-01-06T17:19:55.449144", "exception": false, "start_time": "2020-01-06T17:19:55.268138", "status": "completed"} tags=[]
gene_id_long_to_band = metadata.GENES_MAPPINGS[['band', 'gene']].set_index('gene').to_dict()['band']

# + papermill={"duration": 0.216842, "end_time": "2020-01-06T17:19:55.810965", "exception": false, "start_time": "2020-01-06T17:19:55.594123", "status": "completed"} tags=[]
gene_id_long_to_band['ENSG00000175899.14']

# + [markdown] papermill={"duration": 0.187767, "end_time": "2020-01-06T17:19:56.197852", "exception": false, "start_time": "2020-01-06T17:19:56.010085", "status": "completed"} tags=[]
# # Load UKB x ClinVar z2

# + papermill={"duration": 0.410985, "end_time": "2020-01-06T17:19:56.831610", "exception": false, "start_time": "2020-01-06T17:19:56.420625", "status": "completed"} tags=[]
ukb_clinvar_z2_filename = os.path.join(conf.GENE_ASSOC_DIR, 'smultixcan_and_clinvar-z2.pkl.xz')
display(ukb_clinvar_z2_filename)

# + papermill={"duration": 15.003, "end_time": "2020-01-06T17:20:11.942393", "exception": false, "start_time": "2020-01-06T17:19:56.939393", "status": "completed"} tags=[]
ukb_clinvar_z2 = pd.read_pickle(ukb_clinvar_z2_filename)

# + papermill={"duration": 0.225181, "end_time": "2020-01-06T17:20:12.253106", "exception": false, "start_time": "2020-01-06T17:20:12.027925", "status": "completed"} tags=[]
ukb_clinvar_z2.shape

# + papermill={"duration": 0.162729, "end_time": "2020-01-06T17:20:12.500770", "exception": false, "start_time": "2020-01-06T17:20:12.338041", "status": "completed"} tags=[]
ukb_clinvar_z2.head()

# + [markdown] papermill={"duration": 0.085773, "end_time": "2020-01-06T17:20:12.670686", "exception": false, "start_time": "2020-01-06T17:20:12.584913", "status": "completed"} tags=[]
# # Load S-MultiXcan gene associations

# + papermill={"duration": 0.118479, "end_time": "2020-01-06T17:20:12.876682", "exception": false, "start_time": "2020-01-06T17:20:12.758203", "status": "completed"} tags=[]
smultixcan_pvalues_file = os.path.join(conf.GENE_ASSOC_DIR, f'smultixcan-mashr-pvalues.pkl.xz')
display(smultixcan_pvalues_file)

# + papermill={"duration": 92.597308, "end_time": "2020-01-06T17:21:45.555240", "exception": false, "start_time": "2020-01-06T17:20:12.957932", "status": "completed"} tags=[]
smultixcan_gene_associations = pd.read_pickle(smultixcan_pvalues_file)

# + papermill={"duration": 0.759714, "end_time": "2020-01-06T17:21:46.416678", "exception": false, "start_time": "2020-01-06T17:21:45.656964", "status": "completed"} tags=[]
smultixcan_gene_associations.shape

# + papermill={"duration": 0.228747, "end_time": "2020-01-06T17:21:46.716146", "exception": false, "start_time": "2020-01-06T17:21:46.487399", "status": "completed"} tags=[]
smultixcan_gene_associations.head(5)

# + [markdown] papermill={"duration": 0.076583, "end_time": "2020-01-06T17:21:46.872683", "exception": false, "start_time": "2020-01-06T17:21:46.796100", "status": "completed"} tags=[]
# # Load ClinVar gene associations

# + papermill={"duration": 0.104345, "end_time": "2020-01-06T17:21:47.049836", "exception": false, "start_time": "2020-01-06T17:21:46.945491", "status": "completed"} tags=[]
clinvar_genes_associations_filename = os.path.join(conf.GENE_ASSOC_DIR, 'clinvar-gene_associations.pkl.xz')
display(clinvar_genes_associations_filename)

# + papermill={"duration": 0.213113, "end_time": "2020-01-06T17:21:47.336388", "exception": false, "start_time": "2020-01-06T17:21:47.123275", "status": "completed"} tags=[]
clinvar_genes_associations = pd.read_pickle(clinvar_genes_associations_filename)

# + papermill={"duration": 0.127542, "end_time": "2020-01-06T17:21:47.536564", "exception": false, "start_time": "2020-01-06T17:21:47.409022", "status": "completed"} tags=[]
clinvar_genes_associations.shape

# + papermill={"duration": 0.136695, "end_time": "2020-01-06T17:21:47.756508", "exception": false, "start_time": "2020-01-06T17:21:47.619813", "status": "completed"} tags=[]
clinvar_genes_associations.head(5)

# + [markdown] papermill={"duration": 0.086313, "end_time": "2020-01-06T17:21:48.023080", "exception": false, "start_time": "2020-01-06T17:21:47.936767", "status": "completed"} tags=[]
# # Compute unique list of genes and non-empty diseases

# + papermill={"duration": 0.126726, "end_time": "2020-01-06T17:21:48.222104", "exception": false, "start_time": "2020-01-06T17:21:48.095378", "status": "completed"} tags=[]
clinvar_genes_to_id = pd.Index([metadata.GENE_NAME_TO_ID_MAP[x] for x in clinvar_genes_associations.index if x in metadata.GENE_NAME_TO_ID_MAP])
display(len(clinvar_genes_to_id))

# + papermill={"duration": 0.12208, "end_time": "2020-01-06T17:21:48.421358", "exception": false, "start_time": "2020-01-06T17:21:48.299278", "status": "completed"} tags=[]
shared_gene_ids = clinvar_genes_to_id.intersection(smultixcan_gene_associations.index)
display(len(shared_gene_ids))

# + papermill={"duration": 0.124893, "end_time": "2020-01-06T17:21:48.623563", "exception": false, "start_time": "2020-01-06T17:21:48.498670", "status": "completed"} tags=[]
shared_gene_names = pd.Index([metadata.GENE_ID_TO_NAME_MAP[x] for x in shared_gene_ids])
display(len(shared_gene_names))

# + papermill={"duration": 0.122702, "end_time": "2020-01-06T17:21:48.927677", "exception": false, "start_time": "2020-01-06T17:21:48.804975", "status": "completed"} tags=[]
assert len(shared_gene_ids) == len(shared_gene_names)

# + papermill={"duration": 0.531795, "end_time": "2020-01-06T17:21:49.534468", "exception": false, "start_time": "2020-01-06T17:21:49.002673", "status": "completed"} tags=[]
smultixcan_gene_associations = smultixcan_gene_associations.loc[shared_gene_ids]
clinvar_genes_associations = clinvar_genes_associations.loc[shared_gene_names]

# + papermill={"duration": 1.107486, "end_time": "2020-01-06T17:21:50.790747", "exception": false, "start_time": "2020-01-06T17:21:49.683261", "status": "completed"} tags=[]
_tmp = clinvar_genes_associations.sum()
_clinvar_diseases_to_remove = _tmp[_tmp == 0].index
display(_clinvar_diseases_to_remove.shape)

# + papermill={"duration": 0.190849, "end_time": "2020-01-06T17:21:51.078711", "exception": false, "start_time": "2020-01-06T17:21:50.887862", "status": "completed"} tags=[]
# remove diseases that have no-genes associated (after selecting just shared ones above)
clinvar_genes_associations = clinvar_genes_associations.drop(columns=_clinvar_diseases_to_remove)

# + papermill={"duration": 0.630781, "end_time": "2020-01-06T17:21:51.783568", "exception": false, "start_time": "2020-01-06T17:21:51.152787", "status": "completed"} tags=[]
_tmp = clinvar_genes_associations.sum()
assert _tmp[_tmp == 0].shape[0] == 0

# + papermill={"duration": 0.119569, "end_time": "2020-01-06T17:21:51.988573", "exception": false, "start_time": "2020-01-06T17:21:51.869004", "status": "completed"} tags=[]
display(smultixcan_gene_associations.shape)
display(clinvar_genes_associations.shape)

# + [markdown] papermill={"duration": 0.077235, "end_time": "2020-01-06T17:21:52.143857", "exception": false, "start_time": "2020-01-06T17:21:52.066622", "status": "completed"} tags=[]
# # ClinVar: rename genes

# + papermill={"duration": 0.145007, "end_time": "2020-01-06T17:21:52.365067", "exception": false, "start_time": "2020-01-06T17:21:52.220060", "status": "completed"} tags=[]
clinvar_genes_associations = clinvar_genes_associations.rename(index=gene_name_to_id_long)

# + papermill={"duration": 0.247098, "end_time": "2020-01-06T17:21:52.827026", "exception": false, "start_time": "2020-01-06T17:21:52.579928", "status": "completed"} tags=[]
clinvar_genes_associations.shape

# + papermill={"duration": 0.133603, "end_time": "2020-01-06T17:21:53.046726", "exception": false, "start_time": "2020-01-06T17:21:52.913123", "status": "completed"} tags=[]
clinvar_genes_associations.head()

# + papermill={"duration": 0.111079, "end_time": "2020-01-06T17:21:53.237005", "exception": false, "start_time": "2020-01-06T17:21:53.125926", "status": "completed"} tags=[]
assert clinvar_genes_associations.index.is_unique
assert clinvar_genes_associations.columns.is_unique

# + [markdown] papermill={"duration": 0.080534, "end_time": "2020-01-06T17:21:53.394939", "exception": false, "start_time": "2020-01-06T17:21:53.314405", "status": "completed"} tags=[]
# # ClinVar genes per trait

# + papermill={"duration": 1.844528, "end_time": "2020-01-06T17:21:55.317655", "exception": false, "start_time": "2020-01-06T17:21:53.473127", "status": "completed"} tags=[]
clinvar_genes = clinvar_genes_associations.unstack().to_frame('associated')

# + papermill={"duration": 0.110868, "end_time": "2020-01-06T17:21:55.615918", "exception": false, "start_time": "2020-01-06T17:21:55.505050", "status": "completed"} tags=[]
clinvar_genes.shape

# + papermill={"duration": 0.124994, "end_time": "2020-01-06T17:21:55.816696", "exception": false, "start_time": "2020-01-06T17:21:55.691702", "status": "completed"} tags=[]
assert clinvar_genes.shape[0] == clinvar_genes_associations.size

# + papermill={"duration": 0.10873, "end_time": "2020-01-06T17:21:56.009297", "exception": false, "start_time": "2020-01-06T17:21:55.900567", "status": "completed"} tags=[]
clinvar_genes.head()

# + papermill={"duration": 0.185828, "end_time": "2020-01-06T17:21:56.275402", "exception": false, "start_time": "2020-01-06T17:21:56.089574", "status": "completed"} tags=[]
clinvar_genes = clinvar_genes[clinvar_genes['associated'] == 1].reset_index().rename(columns={'level_0': 'clinvar_trait', 'level_1': 'gene'}).drop(columns=['associated'])

# + papermill={"duration": 0.198273, "end_time": "2020-01-06T17:21:56.641011", "exception": false, "start_time": "2020-01-06T17:21:56.442738", "status": "completed"} tags=[]
clinvar_genes.shape

# + papermill={"duration": 0.127424, "end_time": "2020-01-06T17:21:56.931538", "exception": false, "start_time": "2020-01-06T17:21:56.804114", "status": "completed"} tags=[]
clinvar_genes.head()

# + papermill={"duration": 0.106514, "end_time": "2020-01-06T17:21:57.115504", "exception": false, "start_time": "2020-01-06T17:21:57.008990", "status": "completed"} tags=[]
clinvar_genes = clinvar_genes.assign(gene_name=clinvar_genes['gene'].apply(lambda x: gene_id_long_to_name[x]))

# + papermill={"duration": 0.136921, "end_time": "2020-01-06T17:21:57.331325", "exception": false, "start_time": "2020-01-06T17:21:57.194404", "status": "completed"} tags=[]
clinvar_genes = clinvar_genes.assign(gene_band=clinvar_genes['gene'].apply(lambda x: gene_id_long_to_band[x]))

# + papermill={"duration": 0.11353, "end_time": "2020-01-06T17:21:57.524194", "exception": false, "start_time": "2020-01-06T17:21:57.410664", "status": "completed"} tags=[]
clinvar_genes.head()

# + papermill={"duration": 0.11323, "end_time": "2020-01-06T17:21:57.721379", "exception": false, "start_time": "2020-01-06T17:21:57.608149", "status": "completed"} tags=[]
assert clinvar_genes.drop_duplicates().shape == clinvar_genes.shape


# + papermill={"duration": 0.109722, "end_time": "2020-01-06T17:21:57.925172", "exception": false, "start_time": "2020-01-06T17:21:57.815450", "status": "completed"} tags=[]
def _format_gene(x):
    return pd.Series([
        ', '.join(x.gene.values),
        ', '.join([f'{g} ({b})' for g, b in zip(x.gene_name.values, x.gene_band.values)]),
    ], index=['gene_ids', 'gene_names'])


# + papermill={"duration": 4.199689, "end_time": "2020-01-06T17:22:02.203600", "exception": false, "start_time": "2020-01-06T17:21:58.003911", "status": "completed"} tags=[]
clinvar_genes_grp = clinvar_genes.groupby('clinvar_trait').apply(_format_gene)

# + papermill={"duration": 0.106296, "end_time": "2020-01-06T17:22:02.395746", "exception": false, "start_time": "2020-01-06T17:22:02.289450", "status": "completed"} tags=[]
clinvar_genes_grp.shape

# + papermill={"duration": 0.131711, "end_time": "2020-01-06T17:22:02.606355", "exception": false, "start_time": "2020-01-06T17:22:02.474644", "status": "completed"} tags=[]
clinvar_genes_grp.head()

# + papermill={"duration": 0.104521, "end_time": "2020-01-06T17:22:02.798724", "exception": false, "start_time": "2020-01-06T17:22:02.694203", "status": "completed"} tags=[]
assert clinvar_genes_grp.index.is_unique

# + [markdown] papermill={"duration": 0.07952, "end_time": "2020-01-06T17:22:02.959390", "exception": false, "start_time": "2020-01-06T17:22:02.879870", "status": "completed"} tags=[]
# ### Save

# + papermill={"duration": 0.108618, "end_time": "2020-01-06T17:22:03.145743", "exception": false, "start_time": "2020-01-06T17:22:03.037125", "status": "completed"} tags=[]
clinvar_traits = clinvar_genes_grp[['gene_names']].reset_index()
clinvar_traits.index.rename('id', inplace=True)

# + papermill={"duration": 0.108161, "end_time": "2020-01-06T17:22:03.332834", "exception": false, "start_time": "2020-01-06T17:22:03.224673", "status": "completed"} tags=[]
clinvar_traits = clinvar_traits.rename(columns={
    'clinvar_traits': 'description',
    'gene_names': 'genes_info',
})

# + papermill={"duration": 0.114459, "end_time": "2020-01-06T17:22:03.530795", "exception": false, "start_time": "2020-01-06T17:22:03.416336", "status": "completed"} tags=[]
display(clinvar_traits.shape)
display(clinvar_traits.head())

# + papermill={"duration": 0.109257, "end_time": "2020-01-06T17:22:03.724027", "exception": false, "start_time": "2020-01-06T17:22:03.614770", "status": "completed"} tags=[]
output_file = os.path.join(conf.WEBAPP_DIR, 'ukb_clinvar-clinvar_traits.tsv')
display(output_file)

# + papermill={"duration": 0.130948, "end_time": "2020-01-06T17:22:03.942709", "exception": false, "start_time": "2020-01-06T17:22:03.811761", "status": "completed"} tags=[]
clinvar_traits.to_csv(output_file, sep='\t')

# + papermill={"duration": 0.11173, "end_time": "2020-01-06T17:22:04.135929", "exception": false, "start_time": "2020-01-06T17:22:04.024199", "status": "completed"} tags=[]
clinvar_traits_to_id = clinvar_traits.reset_index()[['id', 'clinvar_trait']].set_index('clinvar_trait').to_dict()['id']

# + papermill={"duration": 0.106934, "end_time": "2020-01-06T17:22:04.326594", "exception": false, "start_time": "2020-01-06T17:22:04.219660", "status": "completed"} tags=[]
clinvar_traits_to_id['2-aminoadipic 2-oxoadipic aciduria']

# + [markdown] papermill={"duration": 0.081127, "end_time": "2020-01-06T17:22:04.488601", "exception": false, "start_time": "2020-01-06T17:22:04.407474", "status": "completed"} tags=[]
# # Load phenotype information

# + papermill={"duration": 0.110435, "end_time": "2020-01-06T17:22:04.679992", "exception": false, "start_time": "2020-01-06T17:22:04.569557", "status": "completed"} tags=[]
os.path.join(conf.DELIVERABLES_DIR, 'phenotypes_info.tsv.gz')

# + papermill={"duration": 0.146692, "end_time": "2020-01-06T17:22:04.908903", "exception": false, "start_time": "2020-01-06T17:22:04.762211", "status": "completed"} tags=[]
pheno_info = pd.read_csv(os.path.join(conf.DELIVERABLES_DIR, 'phenotypes_info.tsv.gz'), sep='\t')

# + papermill={"duration": 0.129881, "end_time": "2020-01-06T17:22:05.146253", "exception": false, "start_time": "2020-01-06T17:22:05.016372", "status": "completed"} tags=[]
pheno_info.shape

# + papermill={"duration": 0.125074, "end_time": "2020-01-06T17:22:05.353150", "exception": false, "start_time": "2020-01-06T17:22:05.228076", "status": "completed"} tags=[]
pheno_info.head()

# + papermill={"duration": 0.11197, "end_time": "2020-01-06T17:22:05.546643", "exception": false, "start_time": "2020-01-06T17:22:05.434673", "status": "completed"} tags=[]
pheno_desc_to_id = pheno_info[['unique_description', 'pheno_id']].set_index('unique_description').to_dict()['pheno_id']

# + papermill={"duration": 0.111317, "end_time": "2020-01-06T17:22:05.744858", "exception": false, "start_time": "2020-01-06T17:22:05.633541", "status": "completed"} tags=[]
pheno_desc_to_id['Fat']

# + papermill={"duration": 0.112174, "end_time": "2020-01-06T17:22:05.946630", "exception": false, "start_time": "2020-01-06T17:22:05.834456", "status": "completed"} tags=[]
pheno_full_code_to_id = pheno_info[['full_code', 'pheno_id']].set_index('full_code').to_dict()['pheno_id']

# + papermill={"duration": 0.108754, "end_time": "2020-01-06T17:22:06.144109", "exception": false, "start_time": "2020-01-06T17:22:06.035355", "status": "completed"} tags=[]
pheno_full_code_to_id['100001_raw-Food_weight']

# + [markdown] papermill={"duration": 0.08514, "end_time": "2020-01-06T17:22:06.316395", "exception": false, "start_time": "2020-01-06T17:22:06.231255", "status": "completed"} tags=[]
# ### Obtain z2

# + papermill={"duration": 13.955032, "end_time": "2020-01-06T17:22:20.381907", "exception": false, "start_time": "2020-01-06T17:22:06.426875", "status": "completed"} tags=[]
clinvar_unstacked = ukb_clinvar_z2.rename(
    index=pheno_full_code_to_id,
    columns=clinvar_traits_to_id,
).T.unstack().to_frame('z2_avg')

# + papermill={"duration": 19.655958, "end_time": "2020-01-06T17:22:40.214117", "exception": false, "start_time": "2020-01-06T17:22:20.558159", "status": "completed"} tags=[]
assert clinvar_unstacked.index.is_unique

# + papermill={"duration": 0.12807, "end_time": "2020-01-06T17:22:40.439875", "exception": false, "start_time": "2020-01-06T17:22:40.311805", "status": "completed"} tags=[]
assert ukb_clinvar_z2.size == clinvar_unstacked.shape[0]

# + papermill={"duration": 0.109441, "end_time": "2020-01-06T17:22:40.636064", "exception": false, "start_time": "2020-01-06T17:22:40.526623", "status": "completed"} tags=[]
clinvar_unstacked.index.rename(['ukb_pheno_id', 'clinvar_pheno_id'], inplace=True)

# + papermill={"duration": 77.407115, "end_time": "2020-01-06T17:23:58.130332", "exception": false, "start_time": "2020-01-06T17:22:40.723217", "status": "completed"} tags=[]
clinvar_unstacked = clinvar_unstacked.sort_index()

# + papermill={"duration": 0.688648, "end_time": "2020-01-06T17:23:58.907678", "exception": false, "start_time": "2020-01-06T17:23:58.219030", "status": "completed"} tags=[]
clinvar_unstacked = clinvar_unstacked.assign(sqrt_z2_avg=np.sqrt(clinvar_unstacked['z2_avg']))

# + papermill={"duration": 18.114298, "end_time": "2020-01-06T17:24:17.168054", "exception": false, "start_time": "2020-01-06T17:23:59.053756", "status": "completed"} tags=[]
assert clinvar_unstacked.index.is_unique

# + papermill={"duration": 0.162857, "end_time": "2020-01-06T17:24:17.415413", "exception": false, "start_time": "2020-01-06T17:24:17.252556", "status": "completed"} tags=[]
clinvar_unstacked.shape

# + papermill={"duration": 0.121392, "end_time": "2020-01-06T17:24:17.622624", "exception": false, "start_time": "2020-01-06T17:24:17.501232", "status": "completed"} tags=[]
clinvar_unstacked.head()

# + [markdown] papermill={"duration": 0.082849, "end_time": "2020-01-06T17:24:17.789974", "exception": false, "start_time": "2020-01-06T17:24:17.707125", "status": "completed"} tags=[]
# # Some testing

# + papermill={"duration": 0.107394, "end_time": "2020-01-06T17:24:17.980866", "exception": false, "start_time": "2020-01-06T17:24:17.873472", "status": "completed"} tags=[]
pheno_desc_to_id['Depression possibly related to childbirth']

# + papermill={"duration": 0.111209, "end_time": "2020-01-06T17:24:18.174985", "exception": false, "start_time": "2020-01-06T17:24:18.063776", "status": "completed"} tags=[]
clinvar_traits_to_id['Keratosis palmoplantaris papulosa']

# + papermill={"duration": 0.782663, "end_time": "2020-01-06T17:24:19.035306", "exception": false, "start_time": "2020-01-06T17:24:18.252643", "status": "completed"} tags=[]
clinvar_unstacked.loc[1341, 2707]

# + papermill={"duration": 0.112325, "end_time": "2020-01-06T17:24:19.232275", "exception": false, "start_time": "2020-01-06T17:24:19.119950", "status": "completed"} tags=[]
assert clinvar_unstacked.loc[pheno_desc_to_id['Other epidermal thickening'], clinvar_traits_to_id["Alzheimer's disease"]]['sqrt_z2_avg'] == np.sqrt(1.3472206237157518)

# + papermill={"duration": 0.136427, "end_time": "2020-01-06T17:24:19.456928", "exception": false, "start_time": "2020-01-06T17:24:19.320501", "status": "completed"} tags=[]
assert clinvar_unstacked.loc[pheno_desc_to_id['Depression possibly related to childbirth'], clinvar_traits_to_id["Keratosis palmoplantaris papulosa"]]['sqrt_z2_avg'] == np.sqrt(0.08090372843310314)

# + papermill={"duration": 0.167972, "end_time": "2020-01-06T17:24:19.716187", "exception": false, "start_time": "2020-01-06T17:24:19.548215", "status": "completed"} tags=[]
clinvar_unstacked[clinvar_unstacked['sqrt_z2_avg'] > 6].head()

# + [markdown] papermill={"duration": 0.087482, "end_time": "2020-01-06T17:24:19.889487", "exception": false, "start_time": "2020-01-06T17:24:19.802005", "status": "completed"} tags=[]
# #### Save all results

# + papermill={"duration": 1.335723, "end_time": "2020-01-06T17:24:21.310307", "exception": false, "start_time": "2020-01-06T17:24:19.974584", "status": "completed"} tags=[]
clinvar_unstacked = clinvar_unstacked[['sqrt_z2_avg']]

# + papermill={"duration": 0.131563, "end_time": "2020-01-06T17:24:21.532405", "exception": false, "start_time": "2020-01-06T17:24:21.400842", "status": "completed"} tags=[]
display(clinvar_unstacked.shape)
display(clinvar_unstacked.head())

# + papermill={"duration": 0.113469, "end_time": "2020-01-06T17:24:21.732443", "exception": false, "start_time": "2020-01-06T17:24:21.618974", "status": "completed"} tags=[]
clinvar_unstacked_filename = os.path.join(conf.GENE_ASSOC_DIR, 'ukb_clinvar.tsv')
display(clinvar_unstacked_filename)

# + papermill={"duration": 68.926298, "end_time": "2020-01-06T17:25:30.747548", "exception": false, "start_time": "2020-01-06T17:24:21.821250", "status": "completed"} tags=[]
clinvar_unstacked.to_csv(clinvar_unstacked_filename, sep='\t')

# + papermill={"duration": 0.089877, "end_time": "2020-01-06T17:25:30.948719", "exception": false, "start_time": "2020-01-06T17:25:30.858842", "status": "completed"} tags=[]


StarcoderdataJupyterScriptsDedupFiltered