Duplicate from carisackc/Clinical

Co-authored-by: Carisa Choy <[email protected]>

.gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+shpi_w_rouge21Nov.csv filter=lfs diff=lfs merge=lfs -text
+shpi25nov.csv filter=lfs diff=lfs merge=lfs -text
+24hourevents.csv filter=lfs diff=lfs merge=lfs -text
+24houreventsFulltextwdifference.csv filter=lfs diff=lfs merge=lfs -text

24hourevents.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4800fd232eac42eb8cb61ec0161e40dd87ec580db92920e35980bb6321be3f42
+size 78926133

24houreventsFulltextwdifference.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fae31ed44681dce63ecaf00a885ebcc947673c7021aea7af4206625925a984d
+size 59516709

README.md

@@ -0,0 +1,14 @@
+---
+title: Clinical
+emoji: 💩
+colorFrom: blue
+colorTo: red
+sdk: streamlit
+sdk_version: 1.10.0
+app_file: app.py
+pinned: false
+license: other
+duplicated_from: carisackc/Clinical
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,409 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+from math import ceil
+from collections import Counter
+from string import punctuation
+import spacy
+from negspacy.negation import Negex
+from spacy import displacy
+from spacy.lang.en import English
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Span
+#import en_ner_bc5cdr_md
+import re
+
+
+from streamlit.components.v1 import html
+
+# Store the initial value of widgets in session state
+if "visibility" not in st.session_state:
+    st.session_state.visibility = "visible"
+    st.session_state.disabled = False
+
+#nlp = en_core_web_lg.load()
+nlp = spacy.load("en_ner_bc5cdr_md")
+
+st.set_page_config(page_title ='Patient Inpatient Progression Dashboard', 
+                   #page_icon= "Notes",
+                   layout='wide')
+st.title('Patient Inpatient Progression Dashboard')
+st.markdown(
+    """
+    <style>
+    [data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
+        width: 400px;
+    }
+    [data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
+        width: 400px;
+        margin-left: -230px;
+    }
+    </style>
+    """,
+    unsafe_allow_html=True,
+)
+st.sidebar.markdown('Using transformer model')
+
+## ======== Loading dataset ========
+## Loading in Admission Dataset
+df = pd.read_csv('shpi25nov.csv')
+df.sort_values(by='SUBJECT_ID',ascending = True, inplace=True)
+
+# Loading in Admission chief Complaint and diagnosis
+df2 = pd.read_csv('cohort_cc_adm_diag.csv')
+
+# Loading in Dischare History
+df3 = pd.read_csv('cohort_past_history_12072022.csv')
+df3.sort_values(by='CHARTDATE',ascending = False, inplace=True)
+
+# Loading in Daily Narrative 
+df4 = pd.read_csv('24houreventsFulltextwdifference.csv')
+df4.sort_values(by=['SUBJECT_ID','HADM_ID','STORETIME'],ascending = True, inplace=True)
+
+
+# combining both data into one 
+df = pd.merge(df, df2, on=['HADM_ID','SUBJECT_ID'])
+
+# Deleting admission chief complaint and diagnosis after combining
+del df2
+
+# Remove decimal point from Admission ID
+df['HADM_ID'] = df['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
+df3['HADM_ID'] = df3['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
+df4['HADM_ID'] = df4['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
+df3['INDEX_HADM_ID'] = df3['INDEX_HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
+df3["CHARTDATE_HADM_ID"] = df3["CHARTDATE"].astype(str) +' ('+ df3["HADM_ID"] +')'
+df3["DIAGNOSIS"] = df3["DIAGNOSIS"].str.capitalize()
+df3["DISCHARGE_LOCATION"] = df3["DISCHARGE_LOCATION"].str.capitalize()
+
+df3["TEXT"] =df3["TEXT"].replace(r'\n','  \n ', regex=True)
+df3["TEXT"] =df3["TEXT"].replace(r'#',' ', regex=True) 
+df3["BertSummarizer"] =df3["BertSummarizer"].replace(r'#',' ', regex=True) 
+
+
+#Renaming column
+df.rename(columns={'SUBJECT_ID':'Patient_ID',
+                  'HADM_ID':'Admission_ID',
+                  'hpi_input_text':'Original_Text',
+                  'hpi_reference_summary':'Reference_text'}, inplace = True)
+df3.rename(columns={'SUBJECT_ID':'Patient_ID',
+                   'HADM_ID':'PAST_Admission_ID',
+                   'INDEX_HADM_ID':'Admission_ID'}, inplace = True) 
+
+df4.rename(columns={'SUBJECT_ID':'Patient_ID',
+                   'HADM_ID':'Admission_ID',
+                   'Full_24_Hour_Events':'Full Text'}, inplace = True) 
+
+
+#Filter selection 
+st.sidebar.header("Search for Patient:")
+
+# ===== Initial filter for patient and admission id =====
+patientid = df['Patient_ID'].unique()
+patient = st.sidebar.selectbox('Select Patient ID:', patientid)    #Filter Patient
+admissionid = df['Admission_ID'].loc[df['Patient_ID'] == patient]  #Filter available Admission id for patient
+HospitalAdmission = st.sidebar.selectbox(' ', admissionid)   
+pastHistoryEpDate = df3['CHARTDATE_HADM_ID'].loc[(df3['Patient_ID'] == patient) & (df3['Admission_ID']== HospitalAdmission)] 
+countOfAdmission = len(pastHistoryEpDate)
+
+
+# List of Model available
+model = st.sidebar.selectbox('Select Model', ('BertSummarizer','BertGPT2','t5seq2eq','t5','gensim','pysummarizer'))
+
+# ===== to display selected patient and admission id on main page
+col3,col4 = st.columns(2) 
+patientid = col3.write(f"Patient ID:  {patient} ")
+admissionid =col4.write(f"Admission ID:  {HospitalAdmission} ")
+
+runtext = ''
+inputNote  ='Input note here:'
+# Query out relevant Clinical notes
+original_text =  df.query(
+    "Patient_ID  == @patient & Admission_ID == @HospitalAdmission"
+)
+
+original_text2 = original_text['Original_Text'].values
+AdmissionChiefCom = original_text['Admission_Chief_Complaint'].values
+diagnosis =original_text['DIAGNOSIS'].values
+reference_text = original_text['Reference_text'].values
+
+#dailyNoteChange = df4['_24_Hour_Events'].loc[(df4['Patient_ID'] == patient) & (df3['Admission_ID']== HospitalAdmission)] 
+dailyNoteChange =df4[['STORETIME','_24_Hour_Events']].loc[(df4['Patient_ID'] == patient) & (df4['Admission_ID']==HospitalAdmission) & df4['_24_Hour_Events'].notnull()]
+                        
+dailyNoteChange.rename(columns={'STORETIME':'Time of Record',
+                   '_24_Hour_Events':'Note Changes'}, inplace = True) 
+dailyNote = df4['Full Text'].loc[(df4['Patient_ID'] == patient) & (df4['Admission_ID']==HospitalAdmission)]
+dailyNote = dailyNote.unique()
+
+##========= Buttons to the 5 tabs ======== Temp disabled Discharge Plan and Social Notes 
+##col1, col2, col3, col4, col5 = st.columns([1,1,1,1,1]) -- to uncomment and comment below line to include discharge plan and social notes 
+col1, col2, col5 = st.columns([1,1,1])
+col6, col7 =st.columns([2,2])
+with st.container():
+    with col1:
+        btnAdmission = st.button("🏥 Admission")
+        if btnAdmission:
+            #nav_page('Admission')
+            inputNote = "Input Admission Note"
+    with col2:
+        btnDailyNarrative = st.button('📆Daily Narrative')
+        if btnDailyNarrative:
+            inputNote = "Input Daily Narrative Note"
+#    with col3:
+#        btnDischargePlan = st.button('🗒️Discharge Plan')
+#        if btnDischargePlan:
+#            inputNote = "Input Discharge Plan"
+#    with col4:
+#        btnSocialNotes = st.button('📝Social Notes')
+#        if btnSocialNotes: 
+#            inputNote = "Input Social Note"
+    with col5:
+        btnPastHistory = st.button('📇Past History (6 Mths)')
+           
+           
+            
+##========= on Past History Tab  =========
+
+with st.container(): 
+    if btnPastHistory:
+        with col6: 
+            st.markdown('**No. of admission past 6 months:**')
+            st.markdown(countOfAdmission)
+       
+        with col7:
+            #st.date_input('Select Admission Date') # To replace with a dropdown filter instead 
+            #st.selectbox('Past Episodes',pastHistoryEp)
+            pastHistory = st.selectbox('Select Past History Admission', pastHistoryEpDate, format_func=lambda x: 'Select an option' if x == '' else x)
+            
+if btnPastHistory:
+    
+    #st.write('Past History')
+    historyAdmission =  df3.query(
+                "Patient_ID  == @patient & CHARTDATE_HADM_ID == @pastHistory"
+                )
+    runtext = historyAdmission['hospital_course_processed'].values[0]
+
+if not(btnPastHistory) and not(btnDailyNarrative):
+    runtext =st.text_area(inputNote, str(original_text2)[1:-1], height=300)
+    
+    
+
+    
+##========= END on Past History Tab  =========
+
+## ===== Commented out as no longer in use ===== 
+# Extract words associated with each entity
+#def genEntities(ann, entity):
+#              # entity colour dict
+#              #ent_col = {'DISEASE':'#B42D1B', 'CHEMICAL':'#F06292'}
+#              ent_col = {'DISEASE':'pink', 'CHEMICAL':'orange'}
+#              # separate into the different entities
+#              entities = trans_df['Class'].unique()
+#
+#              if entity in entities:
+#                             ent = list(trans_df[trans_df['Class']==entity]['Entity'].unique())
+#                             entlist = ",".join(ent)
+#                             st.markdown(f'<p style="background-color:{ent_col[entity]};color:#080808;font-size:16px;">{entlist}</p>', #unsafe_allow_html=True)
+
+
+##======================== Start of NER Tagging ========================
+# ====== Old NER ======
+# doc = nlp(str(original_text2))
+# colors = { "DISEASE": "pink","CHEMICAL": "orange"}
+# options = {"ents": [ "DISEASE", "CHEMICAL"],"colors": colors}
+# ent_html = displacy.render(doc, style="ent", options=options)
+# ====== End of Old NER ======
+                  
+#lemmatizing the notes to capture all forms of negation(e.g., deny: denies, denying)
+def lemmatize(note, nlp):
+    doc = nlp(note)
+    lemNote = [wd.lemma_ for wd in doc]
+    return " ".join(lemNote)
+
+#function to modify options for displacy NER visualization
+def get_entity_options():
+    entities = ["DISEASE", "CHEMICAL", "NEG_ENTITY"]
+    colors = {'DISEASE': 'pink', 'CHEMICAL': 'orange', "NEG_ENTITY":'white'}
+    options = {"ents": entities, "colors": colors}    
+    return options
+
+#adding a new pipeline component to identify negation
+def neg_model():
+    nlp.add_pipe('sentencizer')
+    nlp.add_pipe(
+    "negex",
+    config={
+        "chunk_prefix": ["no"],
+    },
+    last=True)
+    return nlp
+
+def negation_handling(note, neg_model):
+    results = []
+    nlp = neg_model() 
+    note = note.split(".") #sentence tokenizing based on delimeter 
+    note = [n.strip() for n in note] #removing extra spaces at the begining and end of sentence
+    for t in note:
+        doc = nlp(t)
+        for e in doc.ents:
+            rs = str(e._.negex)
+            if rs == "True": 
+                results.append(e.text)
+    return results
+
+#function to identify span objects of matched negative phrases from text
+def match(nlp,terms,label):
+    patterns = [nlp.make_doc(text) for text in terms]
+    matcher = PhraseMatcher(nlp.vocab)
+    matcher.add(label, None, *patterns)
+    return matcher
+
+#replacing the labels for identified negative entities
+def overwrite_ent_lbl(matcher, doc):
+    matches = matcher(doc)
+    seen_tokens = set()
+    new_entities = []
+    entities = doc.ents
+    for match_id, start, end in matches:
+        if start not in seen_tokens and end - 1 not in seen_tokens:
+            new_entities.append(Span(doc, start, end, label=match_id))
+            entities = [e for e in entities if not (e.start < end and e.end > start)]
+            seen_tokens.update(range(start, end))
+    doc.ents = tuple(entities) + tuple(new_entities)
+    return doc
+
+#deduplicate repeated entities 
+def dedupe(items):
+    seen = set()
+    for item in items:
+        item = str(item).strip()
+        if item not in seen:
+            yield item
+            seen.add(item)
+
+lem_clinical_note= lemmatize(runtext, nlp)
+#creating a doc object using BC5CDR model
+doc = nlp(lem_clinical_note)
+options = get_entity_options()
+
+#list of negative concepts from clinical note identified by negspacy
+results0 = negation_handling(lem_clinical_note, neg_model)
+
+matcher = match(nlp, results0,"NEG_ENTITY")
+
+#doc0: new doc object with added "NEG_ENTITY label"
+doc0 = overwrite_ent_lbl(matcher,doc)
+
+#visualizing identified Named Entities in clinical input text 
+ent_html = displacy.render(doc0, style='ent', options=options)
+
+##======================== End of NER Tagging ========================
+
+def run_model(input_text):    
+    if model == "BertSummarizer":
+        output = original_text['BertSummarizer2s'].values
+        st.write('Summary')
+
+    elif model == "BertGPT2":
+        output = original_text['BertGPT2'].values
+        st.write('Summary')
+        
+      
+    elif model == "t5seq2eq": 
+        output = original_text['t5seq2eq'].values
+        st.write('Summary')
+         
+    elif model == "t5":
+        output = original_text['t5'].values
+        st.write('Summary')
+        
+    elif model == "gensim": 
+        output = original_text['gensim'].values
+        st.write('Summary')
+        
+    elif model == "pysummarizer": 
+        output = original_text['pysummarizer'].values
+        st.write('Summary')
+
+
+    
+    st.success(output)
+
+
+col1, col2 = st.columns([1,1])
+
+#to not show summary and references text for Past History and Daily Narrative
+if not(btnPastHistory) and not(btnDailyNarrative):
+        with col1:
+            st.button('Summarize')
+            run_model(runtext)
+            #sentences=runtext.split('.')
+            st.text_area('Reference text', str(reference_text), height=150)
+        with col2:
+            st.button('NER')
+            # ===== Adding the Disease/Chemical into a list =====    
+            problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
+            medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
+            st.markdown('**CHIEF COMPLAINT:**')
+            st.write(str(AdmissionChiefCom)[1:-1])
+            st.markdown('**ADMISSION DIAGNOSIS:**')
+            st.markdown(str(diagnosis)[1:-1].capitalize())
+            st.markdown('**PROBLEM/ISSUE**')
+            #st.markdown(problem_entities)
+            st.markdown(f'<p style="background-color:PINK;color:#080808;font-size:16px;">{str(problem_entities)[1:-1]}</p>', unsafe_allow_html=True)
+            #genEntities(trans_df, 'DISEASE')
+            st.markdown('**MEDICATION**')
+            st.markdown(f'<p style="background-color:orange;color:#080808;font-size:16px;">{str(medication_entities)[1:-1]}</p>', unsafe_allow_html=True)
+            #genEntities(trans_df, 'CHEMICAL')
+            #st.table(trans_df)   
+            st.markdown('**NER**')
+            with st.expander("See NER Details"):
+                st.markdown(ent_html, unsafe_allow_html=True)
+if btnDailyNarrative: 
+    with st.container():
+        st.markdown('Daily Progress Note (24 hour event only):') 
+        st.markdown(str(dailyNote)[1:-1])
+ 
+    
+    with st.container():
+#        hide_table_row_index = """
+#                    <style>
+#                    thead tr th:first-child {display:none}
+#                    tbody th {display:none}
+#                    </style>
+#                    """
+#
+#        # Inject CSS with Markdown
+#        st.markdown(hide_table_row_index, unsafe_allow_html=True)
+#        st.table(dailyNoteChange)
+        styler = dailyNoteChange.style.hide_index()
+        st.write(styler.to_html(), unsafe_allow_html=True)
+    st.markdown(f'<p style="color:#828080;font-size:12px;">*Current prototype displays only a single section within the daily progress note, could also potentially include all sections within each progress note and allow user to select the section changes the user wants to look at</p>', unsafe_allow_html=True)
+
+#else:
+if btnPastHistory:
+# ===== Adding the Disease/Chemical into a list =====    
+    problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
+    medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
+    st.markdown('Admission Date: ' + historyAdmission['ADMITTIME'].values[0])
+    st.markdown('Date of Discharge: ' + historyAdmission['DISCHTIME'].values[0])
+    st.markdown('Days from current admission: ' + str(historyAdmission['days_from_index'].values[0]) +' days')
+    #st.markdown('Summary: ')
+    st.markdown(f'<p style="color:#080808;font-size:16px;"><b>Summary: </b></p>', unsafe_allow_html=True)
+    
+              
+    if model == "BertSummarizer":
+        st.markdown(str(historyAdmission['BertSummarizer'].values[0]))
+    elif model == "t5seq2eq":
+        st.markdown(str(historyAdmission['t5seq2eq'].values[0]))
+    st.markdown(f'<p style="color:#080808;font-size:16px;"><b>Diagnosis: </b></p>', unsafe_allow_html=True)
+    st.markdown(str(historyAdmission['Diagnosis_Description'].values[0]))
+    st.markdown('**PROBLEM/ISSUE**')
+    st.markdown(f'<p style="background-color:PINK;color:#080808;font-size:16px;">{str(problem_entities)[1:-1]}</p>', unsafe_allow_html=True)
+    st.markdown('**MEDICATION**')
+    st.markdown(f'<p style="background-color:orange;color:#080808;font-size:16px;">{str(medication_entities)[1:-1]}</p>', unsafe_allow_html=True)        
+    st.markdown('Discharge Disposition: ' + str(historyAdmission['DISCHARGE_LOCATION'].values[0]))
+    with st.expander('Full Discharge Summary'):
+            #st.write("line 1  \n  line 2  \n line 3")
+            fulldischargesummary = historyAdmission['TEXT'].values[0]
+            st.write(fulldischargesummary)

requirements.txt

@@ -0,0 +1,10 @@
+streamlit==1.13.0
+pandas==1.3.5
+numpy==1.20.0
+regex==2022.9.13
+spacy==3.2.3
+scispacy==0.5.0
+click==7.1.2
+typer==0.4.0
+https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_ner_bc5cdr_md-0.5.0.tar.gz
+negspacy==1.0.3

shpi25nov.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56043b5fd95df367dfe2ffc1daf046271355884ab8387fb5f0696ef903cbfb6f
+size 15879957

shpi_w_rouge21Nov.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2278ed74dd58349449d2ec031faaf2cf23bd8a37d32f80543ed410e947062fa3
+size 14876249