Add validate

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 media/pfp/*.png filter=lfs diff=lfs merge=lfs -text
+data/*.csv filter=lfs diff=lfs merge=lfs -text

data/kg_edge_types.csv

@@ -1,75 +1,3 @@
-x_type,relation,display_relation,y_type,direction,N
-anatomy,anatomy_protein_present,expression present,gene/protein,forward,3831782
-gene/protein,rev_anatomy_protein_present,expression present,anatomy,reverse,3831782
-drug,drug_drug,synergistic interaction,drug,forward,1433261
-drug,rev_drug_drug,synergistic interaction,drug,reverse,1433261
-anatomy,anatomy_protein_absent,expression absent,gene/protein,forward,324186
-gene/protein,rev_anatomy_protein_absent,expression absent,anatomy,reverse,324186
-gene/protein,protein_protein,ppi,gene/protein,forward,321090
-gene/protein,rev_protein_protein,ppi,gene/protein,reverse,321090
-disease,disease_phenotype_positive,phenotype present,effect/phenotype,forward,200354
-effect/phenotype,rev_disease_phenotype_positive,phenotype present,disease,reverse,200354
-disease,disease_protein,associated with,gene/protein,forward,147984
-gene/protein,rev_disease_protein,associated with,disease,reverse,147984
-biological_process,bioprocess_protein,interacts with,gene/protein,forward,138297
-gene/protein,rev_bioprocess_protein,interacts with,biological_process,reverse,138297
-cellular_component,cellcomp_protein,interacts with,gene/protein,forward,83089
-gene/protein,rev_cellcomp_protein,interacts with,cellular_component,reverse,83089
-disease,disease_protein_negative,expression downregulated,gene/protein,forward,71135
-gene/protein,rev_disease_protein_negative,expression downregulated,disease,reverse,71135
-gene/protein,molfunc_protein,interacts with,molecular_function,forward,70291
-molecular_function,rev_molfunc_protein,interacts with,gene/protein,reverse,70291
-disease,disease_protein_positive,expression upregulated,gene/protein,forward,69488
-gene/protein,rev_disease_protein_positive,expression upregulated,disease,reverse,69488
-drug,drug_effect,side effect,effect/phenotype,forward,64249
-effect/phenotype,rev_drug_effect,side effect,drug,reverse,64249
-biological_process,bioprocess_bioprocess,parent-child,biological_process,forward,50232
-biological_process,rev_bioprocess_bioprocess,parent-child,biological_process,reverse,50232
-gene/protein,pathway_protein,interacts with,pathway,forward,44116
-pathway,rev_pathway_protein,interacts with,gene/protein,reverse,44116
-disease,disease_disease,parent-child,disease,forward,37808
-disease,rev_disease_disease,parent-child,disease,reverse,37808
-disease,contraindication,contraindication,drug,forward,26899
-drug,rev_contraindication,contraindication,disease,reverse,26899
-effect/phenotype,phenotype_phenotype,parent-child,effect/phenotype,forward,20183
-effect/phenotype,rev_phenotype_phenotype,parent-child,effect/phenotype,reverse,20183
-drug,drug_protein,target,gene/protein,forward,18513
-gene/protein,rev_drug_protein,target,drug,reverse,18513
-disease,weak_clinical_evidence,clinical candidate,drug,forward,16111
-drug,rev_weak_clinical_evidence,clinical candidate,disease,reverse,16111
-anatomy,anatomy_anatomy,parent-child,anatomy,forward,14383
-anatomy,rev_anatomy_anatomy,parent-child,anatomy,reverse,14383
-molecular_function,molfunc_molfunc,parent-child,molecular_function,forward,13735
-molecular_function,rev_molfunc_molfunc,parent-child,molecular_function,reverse,13735
-disease,indication,indication,drug,forward,12608
-drug,rev_indication,indication,disease,reverse,12608
-drug,drug_protein,enzyme,gene/protein,forward,5919
-gene/protein,rev_drug_protein,enzyme,drug,reverse,5919
-disease,strong_clinical_evidence,clinical candidate,drug,forward,5352
-drug,rev_strong_clinical_evidence,clinical candidate,disease,reverse,5352
-cellular_component,cellcomp_cellcomp,parent-child,cellular_component,forward,4683
-cellular_component,rev_cellcomp_cellcomp,parent-child,cellular_component,reverse,4683
-effect/phenotype,phenotype_protein,associated with,gene/protein,forward,4437
-gene/protein,rev_phenotype_protein,associated with,effect/phenotype,reverse,4437
-drug,drug_protein,transporter,gene/protein,forward,3349
-gene/protein,rev_drug_protein,transporter,drug,reverse,3349
-pathway,pathway_pathway,parent-child,pathway,forward,2647
-pathway,rev_pathway_pathway,parent-child,pathway,reverse,2647
-disease,exposure_disease,linked to,exposure,forward,2421
-exposure,rev_exposure_disease,linked to,disease,reverse,2421
-disease,off_label_use,off-label use,drug,forward,2370
-drug,rev_off_label_use,off-label use,disease,reverse,2370
-exposure,exposure_exposure,parent-child,exposure,forward,2263
-exposure,rev_exposure_exposure,parent-child,exposure,reverse,2263
-exposure,exposure_protein,interacts with,gene/protein,forward,2012
-gene/protein,rev_exposure_protein,interacts with,exposure,reverse,2012
-biological_process,exposure_bioprocess,interacts with,exposure,forward,1990
-exposure,rev_exposure_bioprocess,interacts with,biological_process,reverse,1990
-drug,drug_protein,carrier,gene/protein,forward,993
-gene/protein,rev_drug_protein,carrier,drug,reverse,993
-disease,disease_phenotype_negative,phenotype absent,effect/phenotype,forward,508
-effect/phenotype,rev_disease_phenotype_negative,phenotype absent,disease,reverse,508
-exposure,exposure_molfunc,interacts with,molecular_function,forward,45
-molecular_function,rev_exposure_molfunc,interacts with,exposure,reverse,45
-cellular_component,exposure_cellcomp,interacts with,exposure,forward,12
-exposure,rev_exposure_cellcomp,interacts with,cellular_component,reverse,12
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ee79b2f5021304a4dd82581568e8a8c940f94b29cd1206f7730bdff6b82cab4
+size 5288

data/kg_edges.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab7d0e23c56381abf8e214cc5d4fae4e6a8b98957c8f2e5272b4f800953b1461
+size 2765378133

data/kg_node_types.csv

@@ -1,11 +1,3 @@
-node_type,N
-gene/protein,35198
-biological_process,27668
-disease,22201
-effect/phenotype,16711
-anatomy,14384
-molecular_function,11228
-drug,8160
-cellular_component,4054
-pathway,2629
-exposure,860
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1a0afff52deec5f48689a22a479d14cd49333759e054624366687ec4ef306c8
+size 192

pages/predict.py

@@ -12,6 +12,10 @@ import torch.nn.functional as F
 from pathlib import Path
 from huggingface_hub import hf_hub_download
 
+# Plotting
+import matplotlib.pyplot as plt
+plt.rcParams['font.sans-serif'] = 'Arial'
+
 # Custom and other imports
 import project_config
 from utils import capitalize_after_slash, load_kg
@@ -134,6 +138,9 @@ with st.spinner('Computing predictions...'):
     # Add URLs to database column
     display_data['Database'] = display_data.apply(lambda x: map_dbs[target_node_type](x['ID']), axis = 1)
 
+
+    # NODE SEARCH
+
     # Use multiselect to search for specific nodes
     selected_nodes = st.multiselect(f"Search for specific {st.session_state.query['source_node_type'].replace('_', ' ')} nodes to determine their ranking.",
                                     display_data.Name, placeholder = "Type to search...")
@@ -151,6 +158,25 @@ with st.spinner('Computing predictions...'):
         else:
             st.dataframe(selected_display_data, use_container_width = True)
 
+        # Plot rank vs. score using matplotlib
+        st.markdown("**Rank vs. Score**")
+        fig, ax = plt.subplots(figsize = (10, 6))
+        ax.plot(display_data['Rank'], display_data['Score'])
+        ax.set_xlabel('Rank', fontsize = 12)
+        ax.set_ylabel('Score', fontsize = 12)
+        ax.set_xlim(1, display_data['Rank'].max())
+
+        # Add vertical line for selected nodes
+        for i, node in selected_display_data.iterrows():
+            ax.axvline(node['Rank'], color = 'red', linestyle = '--', label = node['Name'])
+            ax.text(node['Rank'] + 100, node['Score'], node['Name'], fontsize = 10, color = 'red')
+
+        # Show plot
+        st.pyplot(fig)
+    
+    
+    # FULL RESULTS
+
     # Show top ranked nodes
     st.subheader("Model Predictions", divider = "blue")
     top_k = st.slider('Select number of top ranked nodes to show.', 1, target_nodes.shape[0], min(500, target_nodes.shape[0])) 
@@ -165,3 +191,4 @@ with st.spinner('Computing predictions...'):
 
     # Save to session state
     st.session_state.predictions = display_data
+    st.session_state.display_database = display_database

pages/validate.py

@@ -1,11 +1,21 @@
 import streamlit as st
 from menu import menu_with_redirect
 
+# Standard imports
+import numpy as np
+import pandas as pd
+
 # Path manipulation
 from pathlib import Path
 
+# Plotting
+import matplotlib.pyplot as plt
+plt.rcParams['font.sans-serif'] = 'Arial'
+import matplotlib.colors as mcolors
+
 # Custom and other imports
 import project_config
+from utils import load_kg, load_kg_edges
 
 # Redirect to app.py if not logged in, otherwise show the navigation menu
 menu_with_redirect()
@@ -21,6 +31,76 @@ st.subheader("Validate Predictions", divider = "green")
 # Print current query
 st.markdown(f"**Query:** {st.session_state.query['source_node'].replace('_', ' ')} ➡️ {st.session_state.query['relation'].replace('_', '-')} ➡️ {st.session_state.query['target_node_type'].replace('_', ' ')}")
 
-
 # Coming soon
-st.write("Coming soon...")
+# st.write("Coming soon...")
+
+source_node_type = st.session_state.query['source_node_type']
+source_node = st.session_state.query['source_node']
+relation = st.session_state.query['relation']
+target_node_type = st.session_state.query['target_node_type']
+predictions = st.session_state.predictions
+
+kg_nodes = load_kg()
+kg_edges = load_kg_edges()
+
+
+with st.spinner('Searching known relationships...'):
+
+    # Subset existing edges
+    edge_subset = kg_edges[(kg_edges.x_type == source_node_type) & (kg_edges.x_name == source_node)]
+    edge_subset = edge_subset[edge_subset.y_type == target_node_type]
+
+    # Merge edge subset with predictions
+    edges_in_kg = pd.merge(predictions, edge_subset[['relation', 'y_id']], left_on = 'ID', right_on = 'y_id', how = 'right')
+    edges_in_kg = edges_in_kg.sort_values(by = 'Score', ascending = False)
+    edges_in_kg = edges_in_kg.drop(columns = 'y_id')
+
+    # Rename relation to ground-truth
+    edges_in_kg = edges_in_kg[['relation'] + [col for col in edges_in_kg.columns if col != 'relation']]
+    edges_in_kg = edges_in_kg.rename(columns = {'relation': 'Known Relation'})
+
+
+with st.spinner('Plotting known relationships...'):
+
+    # Define a color map for different relations
+    color_map = plt.cm.get_cmap('tab10')
+
+    # Group by relation and create separate plots
+    relations = edges_in_kg['Known Relation'].unique()
+    for idx, relation in enumerate(relations):
+        fig, ax = plt.subplots(figsize=(10, 3))
+        ax.plot(predictions['Rank'], predictions['Score'])
+        ax.set_xlabel('Rank', fontsize=12)
+        ax.set_ylabel('Score', fontsize=12)
+        ax.set_xlim(1, predictions['Rank'].max())
+
+        relation_data = edges_in_kg[edges_in_kg['Known Relation'] == relation]
+        color = color_map(idx % color_map.N)  # Get a color from the color map
+        
+        for i, node in relation_data.iterrows():
+            ax.axvline(node['Rank'], color=color, linestyle='--', label=node['Name'])
+            # ax.text(node['Rank'] + 100, node['Score'], node['Name'], fontsize=10, color=color)
+        
+        # ax.set_title(f'{relation.replace("_", "-")}')
+        # ax.legend()
+
+        # Convert tuple to hex
+        def rgba_to_hex(rgba):
+            return mcolors.to_hex(rgba[:3])
+        color_hex = rgba_to_hex(color)
+
+        # Write header in color of relation
+        st.markdown(f"<h3 style='color:{color_hex}'>{relation.replace('_', ' ').title()}</h2>", unsafe_allow_html=True)
+
+        # Show plot
+        st.pyplot(fig)
+
+        # Drop known relation column
+        relation_data = relation_data.drop(columns = 'Known Relation')
+        if target_node_type not in ['disease', 'anatomy']:
+            st.dataframe(relation_data, use_container_width=True,
+                         column_config={"Database": st.column_config.LinkColumn(width = "small",
+                                                                                help = "Click to visit external database.",
+                                                                                display_text = st.session_state.display_database)})
+        else:
+            st.dataframe(relation_data, use_container_width=True)

utils.py

@@ -3,12 +3,21 @@ import pandas as pd
 import project_config
 import base64
 
-@st.cache_data(show_spinner = 'Loading knowledge graph...')
+
+@st.cache_data(show_spinner = 'Loading knowledge graph nodes...')
 def load_kg():
     # with st.spinner('Loading knowledge graph...'):
     kg_nodes = pd.read_csv(project_config.DATA_DIR / 'kg_nodes.csv', dtype = {'node_index': int}, low_memory = False)
     return kg_nodes
 
+
+@st.cache_data(show_spinner = 'Loading knowledge graph edges...')
+def load_kg_edges():
+    # with st.spinner('Loading knowledge graph...'):
+    kg_edges = pd.read_csv(project_config.DATA_DIR / 'kg_edges.csv', dtype = {'edge_index': int, 'x_index': int, 'y_index': int}, low_memory = False)
+    return kg_edges
+
+
 def capitalize_after_slash(s):
     # Split the string by slashes first
     parts = s.split('/')
@@ -18,6 +27,7 @@ def capitalize_after_slash(s):
     capitalized_string = '/'.join(capitalized_parts).replace('_', ' ')
     return capitalized_string
 
+
 # From https://stackoverflow.com/questions/73251012/put-logo-and-title-above-on-top-of-page-navigation-in-sidebar-of-streamlit-multi
 # See also https://arnaudmiribel.github.io/streamlit-extras/extras/app_logo/ 
 @st.cache_data()
@@ -26,6 +36,7 @@ def get_base64_of_bin_file(png_file):
         data = f.read()
     return base64.b64encode(data).decode()
 
+
 def build_markup_for_logo(
     png_file,
     background_position="50% 10%",
@@ -55,6 +66,7 @@ def build_markup_for_logo(
         image_height,
     )
 
+
 def add_logo(png_file):
     logo_markup = build_markup_for_logo(png_file)
     st.markdown(