Initial

app.py

@@ -18,23 +18,24 @@ SPECIFIC_TAG=":__entity__"
 
 
 
+
 @st.cache(suppress_st_warning=True, allow_output_mutation=True)
 def POS_get_model(model_name):
   val = SequenceTagger.load(model_name) # Load the model
   return val
-  
+
 def getPos(s: Sentence):
   texts = []
   labels = []
   for t in s.tokens:
     for label in t.annotation_layers.keys():
       texts.append(t.text)
-      labels.append(t.get_labels(label)[0].value)          
+      labels.append(t.get_labels(label)[0].value)
   return texts, labels
-  
+
 def getDictFromPOS(texts, labels):
   return [["dummy",t,l,"dummy","dummy" ] for t, l in zip(texts, labels)]
-  
+
 def decode(tokenizer, pred_idx, top_clean):
   ignore_tokens = string.punctuation + '[PAD]'
   tokens = []
@@ -74,9 +75,9 @@ def get_bert_prediction(input_text,top_k):
 def load_pos_model():
   checkpoint = "flair/pos-english"
   return  POS_get_model(checkpoint)
-  
 
-  
+
+
 
 def init_session_states():
   if 'top_k' not in st.session_state:
@@ -94,8 +95,8 @@ def init_session_states():
   if 'aggr' not in st.session_state:
     st.session_state['aggr'] = None
 
-    
-    
+
+
 def get_pos_arr(input_text,display_area):
    if (st.session_state['pos_model'] is None):
      display_area.text("Loading model 3 of 3.Loading POS model...")
@@ -105,37 +106,37 @@ def get_pos_arr(input_text,display_area):
    texts, labels = getPos(s)
    pos_results = getDictFromPOS(texts, labels)
    return pos_results
- 
+
 def perform_inference(text,display_area):
-  
+
   if (st.session_state['bio_model'] is None):
     display_area.text("Loading model 1 of 3. Bio model...")
     st.session_state['bio_model'] = bd.BatchInference("bio/desc_a100_config.json",'ajitrajasekharan/biomedical',False,False,DEFAULT_TOP_K,True,True,       "bio/","bio/a100_labels.txt",False)
-    
+
   if (st.session_state['phi_model'] is None):
     display_area.text("Loading model 2 of 3. PHI model...")
     st.session_state['phi_model'] = bd.BatchInference("bbc/desc_bbc_config.json",'bert-base-cased',False,False,DEFAULT_TOP_K,True,True,       "bbc/","bbc/bbc_labels.txt",False)
-    
+
   #Load POS model if needed and gets POS tags
   if (SPECIFIC_TAG not in text):
     pos_arr = get_pos_arr(text,display_area)
   else:
     pos_arr = None
-    
+
   if (st.session_state['ner_bio'] is None):
     display_area.text("Initializing BIO module...")
     st.session_state['ner_bio'] = ner.UnsupNER("bio/ner_a100_config.json")
-    
+
   if (st.session_state['ner_phi'] is None):
     display_area.text("Initializing PHI module...")
     st.session_state['ner_phi'] = ner.UnsupNER("bbc/ner_bbc_config.json")
-    
+
   if (st.session_state['aggr'] is None):
     display_area.text("Initializing Aggregation modeule...")
     st.session_state['aggr'] = aggr.AggregateNER("./ensemble_config.json")
-  
-  
-  
+
+
+
   display_area.text("Getting results from BIO model...")
   bio_descs = st.session_state['bio_model'].get_descriptors(text,pos_arr)
   display_area.text("Getting results from PHI model...")
@@ -143,65 +144,65 @@ def perform_inference(text,display_area):
   display_area.text("Aggregating BIO & PHI results...")
   bio_ner = st.session_state['ner_bio'].tag_sentence_service(text,bio_descs)
   phi_ner = st.session_state['ner_phi'].tag_sentence_service(text,phi_results)
-  
+
   combined_arr = [json.loads(bio_ner),json.loads(phi_ner)]
 
   aggregate_results = st.session_state['aggr'].fetch_all(text,combined_arr)
   return aggregate_results
-  
+
 
 sent_arr = [
-"Lou Gehrig who works for XCorp and lives in New York suffers from Parkinson's ",  
-"Parkinson who works for XCorp and lives in New York suffers from Lou Gehrig's", 
-"lou gehrig was diagnosed with Parkinson's ",  
-"A eGFR below 60 indicates chronic kidney disease", 
-"Overexpression of EGFR occurs across a wide range of different cancers", 
-"Stanford called", 
-"He was diagnosed with non small cell lung cancer", 
-"I met my girl friends at the pub ",  
-"I met my New York friends at the pub", 
-"I met my XCorp friends at the pub", 
-"I met my two friends at the pub", 
-"Bio-Techne's genomic tools include advanced tissue-based in-situ hybridization assays sold under the ACD brand as well as a portfolio of     assays for prostate cancer diagnosis ",  
-"There are no treatment options specifically indicated for ACD and physicians must utilize agents approved for other dermatology conditions", "As ACD has been implicated in apoptosis-resistant glioblastoma (GBM), there is a high medical need for identifying novel ACD-inducing drugs  ", 
-"Located in the heart of Dublin , in the family home of acclaimed writer Oscar Wilde , ACD provides the perfect backdrop to inspire Irish     (and Irish-at-heart) students to excel in business and the arts", 
-"Patients treated with anticancer chemotherapy drugs ( ACD ) are vulnerable to infectious diseases due to immunosuppression and to the direct impact of ACD on their intestinal microbiota ", 
-"In the LASOR trial , increasing daily imatinib dose from 400 to 600mg induced MMR at 12 and 24 months in 25% and 36% of the patients,        respectively, who had suboptimal cytogenetic responses ", 
-"The sky turned dark in advance of the storm that was coming from the east ", 
-"She loves to watch Sunday afternoon football with her family ", 
-"Paul Erdos died at 83 " 
+"Lou Gehrig who works for XCorp and lives in New York suffers from Parkinson's ",
+"Parkinson who works for XCorp and lives in New York suffers from Lou Gehrig's",
+"lou gehrig was diagnosed with Parkinson's ",
+"A eGFR below 60 indicates chronic kidney disease",
+"Overexpression of EGFR occurs across a wide range of different cancers",
+"Stanford called",
+"He was diagnosed with non small cell lung cancer",
+"I met my girl friends at the pub ",
+"I met my New York friends at the pub",
+"I met my XCorp friends at the pub",
+"I met my two friends at the pub",
+"Bio-Techne's genomic tools include advanced tissue-based in-situ hybridization assays sold under the ACD brand as well as a portfolio of     assays for prostate cancer diagnosis ",
+"There are no treatment options specifically indicated for ACD and physicians must utilize agents approved for other dermatology conditions", "As ACD has been implicated in apoptosis-resistant glioblastoma (GBM), there is a high medical need for identifying novel ACD-inducing drugs  ",
+"Located in the heart of Dublin , in the family home of acclaimed writer Oscar Wilde , ACD provides the perfect backdrop to inspire Irish     (and Irish-at-heart) students to excel in business and the arts",
+"Patients treated with anticancer chemotherapy drugs ( ACD ) are vulnerable to infectious diseases due to immunosuppression and to the direct impact of ACD on their intestinal microbiota ",
+"In the LASOR trial , increasing daily imatinib dose from 400 to 600mg induced MMR at 12 and 24 months in 25% and 36% of the patients,        respectively, who had suboptimal cytogenetic responses ",
+"The sky turned dark in advance of the storm that was coming from the east ",
+"She loves to watch Sunday afternoon football with her family ",
+"Paul Erdos died at 83 "
 ]
 
 
 sent_arr_masked = [
-"Lou Gehrig:__entity__ who works for XCorp:__entity__ and lives in New:__entity__ York:__entity__ suffers from Parkinson's:__entity__ ",  
-"Parkinson:__entity__ who works for XCorp:__entity__ and lives in New:__entity__ York:__entity__ suffers from Lou Gehrig's:__entity__", 
-"lou:__entity__ gehrig:__entity__ was diagnosed with Parkinson's:__entity__ ",  
-"A eGFR:__entity__ below 60 indicates chronic kidney disease", 
-"Overexpression of EGFR:__entity__ occurs across a wide range of different cancers", 
-"Stanford:__entity__ called", 
-"He was diagnosed with non:__entity__ small:__entity__ cell:__entity__ lung:__entity__ cancer:__entity__", 
-"I met my girl:__entity__ friends at the pub ",  
-"I met my New:__entity__ York:__entity__ friends at the pub", 
-"I met my XCorp:__entity__ friends at the pub", 
-"I met my two:__entity__ friends at the pub", 
-"Bio-Techne's genomic tools include advanced tissue-based in-situ hybridization assays sold under the ACD:__entity__ brand as well as a portfolio of     assays for prostate cancer diagnosis ",  
-"There are no treatment options specifically indicated for ACD:__entity__ and physicians must utilize agents approved for other dermatology conditions", 
-"As ACD:__entity__ has been implicated in apoptosis-resistant glioblastoma (GBM), there is a high medical need for identifying novel ACD-inducing drugs  ", 
-"Located in the heart of Dublin , in the family home of acclaimed writer Oscar Wilde , ACD:__entity__ provides the perfect backdrop to inspire Irish     (and Irish-at-heart) students to excel in business and the arts", 
-"Patients treated with anticancer chemotherapy drugs ( ACD:__entity__ ) are vulnerable to infectious diseases due to immunosuppression and to the direct impact of ACD on their intestinal microbiota ", 
-"In the LASOR:__entity__ trial:__entity__ , increasing daily imatinib dose from 400 to 600mg induced MMR at 12 and 24 months in 25% and 36% of the patients,        respectively, who had suboptimal cytogenetic responses ", 
-"The sky turned dark:__entity__ in advance of the storm that was coming from the east ", 
-"She loves to watch Sunday afternoon football:__entity__ with her family ", 
-"Paul:__entity__ Erdos:__entity__ died at 83:__entity__ " 
+"Lou Gehrig:__entity__ who works for XCorp:__entity__ and lives in New:__entity__ York:__entity__ suffers from Parkinson's:__entity__ ",
+"Parkinson:__entity__ who works for XCorp:__entity__ and lives in New:__entity__ York:__entity__ suffers from Lou Gehrig's:__entity__",
+"lou:__entity__ gehrig:__entity__ was diagnosed with Parkinson's:__entity__ ",
+"A eGFR:__entity__ below 60 indicates chronic kidney disease",
+"Overexpression of EGFR:__entity__ occurs across a wide range of different cancers",
+"Stanford:__entity__ called",
+"He was diagnosed with non:__entity__ small:__entity__ cell:__entity__ lung:__entity__ cancer:__entity__",
+"I met my girl:__entity__ friends at the pub ",
+"I met my New:__entity__ York:__entity__ friends at the pub",
+"I met my XCorp:__entity__ friends at the pub",
+"I met my two:__entity__ friends at the pub",
+"Bio-Techne's genomic tools include advanced tissue-based in-situ hybridization assays sold under the ACD:__entity__ brand as well as a portfolio of     assays for prostate cancer diagnosis ",
+"There are no treatment options specifically indicated for ACD:__entity__ and physicians must utilize agents approved for other dermatology conditions",
+"As ACD:__entity__ has been implicated in apoptosis-resistant glioblastoma (GBM), there is a high medical need for identifying novel ACD-inducing drugs  ",
+"Located in the heart of Dublin , in the family home of acclaimed writer Oscar Wilde , ACD:__entity__ provides the perfect backdrop to inspire Irish     (and Irish-at-heart) students to excel in business and the arts",
+"Patients treated with anticancer chemotherapy drugs ( ACD:__entity__ ) are vulnerable to infectious diseases due to immunosuppression and to the direct impact of ACD on their intestinal microbiota ",
+"In the LASOR:__entity__ trial:__entity__ , increasing daily imatinib dose from 400 to 600mg induced MMR at 12 and 24 months in 25% and 36% of the patients,        respectively, who had suboptimal cytogenetic responses ",
+"The sky turned dark:__entity__ in advance of the storm that was coming from the east ",
+"She loves to watch Sunday afternoon football:__entity__ with her family ",
+"Paul:__entity__ Erdos:__entity__ died at 83:__entity__ "
 ]
 
 def init_selectbox():
   return st.selectbox(
      'Choose any of the sentences in pull-down below',
      sent_arr,key='my_choice')
-  
-   
+
+
 def on_text_change():
   text = st.session_state.my_text
   print("in callback: " + text)
@@ -211,22 +212,22 @@ def main():
   try:
 
     init_session_states()
-    
+
     st.markdown("<h3 style='text-align: center;'>NER using pretrained models with <a href='https://ajitrajasekharan.github.io/2021/01/02/my-first-post.html'>no fine tuning</a></h3>", unsafe_allow_html=True)
     #st.markdown("""
     #<h3 style="font-size:16px; color: #ff0000; text-align: center"><b>App under construction... (not in working condition yet)</b></h3>
   #""", unsafe_allow_html=True)
-   
-    
+
+
     st.markdown("""
     <p style="text-align:center;"><img src="https://ajitrajasekharan.github.io/images/1.png" width="700"></p>
     <br/>
     <br/>
   """, unsafe_allow_html=True)
-  
+
     st.write("This app uses 3 models.  Two Pretrained Bert models (**no fine tuning**) and a POS tagger")
-    
-    
+
+
     with st.form('my_form'):
       selected_sentence = init_selectbox()
       text_input = st.text_area(label='Type any sentence below',value="")
@@ -243,16 +244,16 @@ def main():
             with display_area.container():
               st.text(f"prediction took {time.time() - start:.2f}s")
               st.json(results)
-              
-              
-              
- 
+
+
+
+
 
     #input_text = st.text_area(
     #  label="Type any sentence",
    #   on_change=on_text_change,key='my_text'
    # )
-   
+
     st.markdown("""
     <small style="font-size:16px; color: #7f7f7f; text-align: left"><br/><br/>Models used: <br/>(1) <a href='https://huggingface.co/ajitrajasekharan/biomedical' target='_blank'>Biomedical model</a> pretrained on Pubmed,Clinical trials and BookCorpus subset.<br/>(2) Bert-base-cased (for PHI entities - Person/location/organization etc.)<br/>(3) Flair POS tagger</small>
   #""", unsafe_allow_html=True)
@@ -264,8 +265,8 @@ def main():
   """, unsafe_allow_html=True)
 
   except Exception as e:
-    print("Some error occurred in main") 
-    st.exception(e) 
-	
+    print("Some error occurred in main")
+    st.exception(e)
+
 if __name__ == "__main__":
    main()