Update app.py

app.py

@@ -54,31 +54,27 @@ sp = en_core_sci_lg.load()
 all_stopwords = sp.Defaults.stop_words
 
 
+def remove_stopwords(sen):
+    sen_new = " ".join([i for i in sen if i not in stop_words])
+    return sen_new
 
 
 
-
-word_embedding_model = models.Transformer('cambridgeltl/SapBERT-from-PubMedBERT-fulltext')
-pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),
+def keyphrase_generator(article_link, model_1, model_2, max_num_keywords, model_3):
+  word_embedding_model = models.Transformer(model_3)
+  pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),
                                pooling_mode_mean_tokens=True,
                                pooling_mode_cls_token=False,
                                pooling_mode_max_tokens=False)
 
-embedder = SentenceTransformer(modules=[word_embedding_model, pooling_model])
-
-
-def remove_stopwords(sen):
-    sen_new = " ".join([i for i in sen if i not in stop_words])
-    return sen_new
-
-
-
-def keyphrase_generator(article_link, model_1, model_2, max_num_keywords):
+  embedder = SentenceTransformer(modules=[word_embedding_model, pooling_model])
   element=[]
   cluster_list_final=[]
   comb_list=[] 
   comb=[]
   title_list=[]
+  titles_list=[]
+  abstracts_list=[]
   silhouette_score_list=[]
   final_textrank_list=[]
   document=[]
@@ -89,8 +85,6 @@ def keyphrase_generator(article_link, model_1, model_2, max_num_keywords):
   model_1 = SentenceTransformer(model_1)
   model_2 = SentenceTransformer(model_2)
   url = article_link
-  if (url == False):
-    print("error")
   html = requests.get(url).text
   article = fulltext(html)
   corpus=sent_tokenize(article)
@@ -203,33 +197,42 @@ def keyphrase_generator(article_link, model_1, model_2, max_num_keywords):
   last_url='esearch.fcgi?db=pubmed'+'&term='+f_1
   search_rettype = '&rettype=json'
   overall_url=ncbi_url+last_url+search_rettype+'&sort=relevance'
-  r = requests.get(overall_url)
+  pubmed_search_request = requests.get(overall_url)
   
-  root = ET.fromstring(r.text)
+  root = ET.fromstring(pubmed_search_request.text)
   levels = root.findall('.//Id')
-  name_list=[]
+  search_id_list=[]
   for level in levels:  
     name = level.text
-    name_list.append(name)
-  name_1 = ','.join(name_list)
+    search_id_list.append(name)
+  all_search_ids = ','.join(search_id_list)
   fetch_url='efetch.fcgi?db=pubmed'
-  search_id='&id='+name_1
+  search_id='&id='+all_search_ids
   ret_type='&rettype=text'
   ret_mode='&retmode=xml'
-  ret_max='&retmax=10'
+  ret_max='&retmax=20'
   ret_sort='&sort=relevance'
   return_url=ncbi_url+fetch_url+search_id+ret_type+ret_mode+ret_max+ret_sort
-  r_1 = requests.get(return_url)
-  root_1 = ET.fromstring(r_1.text)
-  levels_1 = root_1.findall('.//ArticleTitle')
-  for level in levels_1:  
-    name = level.text
-    title_list.append(name)
+  pubmed_abstract_request = requests.get(return_url)
+  root_1 = ET.fromstring(pubmed_abstract_request.text)
+  article_title = root_1.findall('.//ArticleTitle')
+  for a in article_title:  
+    article_title_name = a.text
+    titles_list.append(article_title_name)
+  article_abstract = root_1.findall('.//AbstractText')
+  for b in article_abstract:
+    article_abstract_name = b.text
+    abstracts_list.append(article_abstract_name)
+  mydict = {'Title': titles_list, 'Abstract':abstracts_list}
+    
+
+  df_new = pd.DataFrame(dict([ (k,pd.Series(v)) for k,v in mydict.items() ]))
 
   
-  return title_list
+  return df_new
   
-gr.Interface(keyphrase_generator, 
+
+igen_pubmed = gr.Interface(keyphrase_generator, 
              inputs=[gr.inputs.Textbox(lines=1, placeholder="Provide article web link here",default="", label="Article web link"),
                      gr.inputs.Dropdown(choices=['sentence-transformers/all-mpnet-base-v2',
                                                  'sentence-transformers/all-mpnet-base-v1',                       
@@ -261,13 +264,21 @@ gr.Interface(keyphrase_generator,
                                                  'sentence-transformers/all-MiniLM-L6-v2'], 
                                         type="value", 
                                         default='sentence-transformers/all-mpnet-base-v1', 
-                                        label="Select any SBERT model for keyphrases from the list below"),                     
-                     gr.inputs.Slider(minimum=5, maximum=30, step=1, default=10, label="Max Keywords")], 
-             outputs=gr.outputs.Textbox(type="auto", label="Stuff"), 
+                                        label="Select any SBERT model for keyphrases from the list below"),                                                     
+                     gr.inputs.Slider(minimum=5, maximum=30, step=1, default=10, label="Max Keywords")
+                     gr.inputs.Dropdown(choices=['cambridgeltl/SapBERT-from-PubMedBERT-fulltext',
+                                                 'cambridgeltl/SapBERT-from-PubMedBERT-fulltext-mean-token'], 
+                                        type="value", 
+                                        default='cambridgeltl/SapBERT-from-PubMedBERT-fulltext', 
+                                        label="Select any SapBERT model for clustering from the list below"),], 
+             outputs=gr.outputs.Dataframe(type="auto", label="dataframe",max_rows=10, max_cols=None, overflow_row_behaviour="paginate"), 
              theme="peach", 
              title="Scientific Article Keyphrase Generator", description="Generates the keyphrases from an article which best describes the article.",
-             article= "The work is based on a part of the paper <a href=https://dl.acm.org/doi/10.1145/3487664.3487701>provided here</a>." 
+             article= "The work is based the paper <a href=https://dl.acm.org/doi/10.1145/3487664.3487701>provided here</a>." 
              "\t It uses the TextRank algorithm with SBERT to first find the top sentences and then extracts the keyphrases from those sentences using scispaCy and SBERT."
+             "\t The application then uses a <a href=https://arxiv.org/abs/2010.11784>UMLS based Bert model</a> to cluster the keyphrases using K-means clustering method and finally create a boolean query. After that the top 20 titles and abstracts are retrieved from PubMed database and displayed according to relevancy. The UMLS Bert models can be chosen from the list provided. "
              "\t The list of SBERT models required in the textboxes can be found in <a href=www.sbert.net/docs/pretrained_models.html>SBERT Pre-trained models hub</a>."
              "\t The default model names are provided which can be changed from the list of pretrained models. "
-             "\t The value of output keyphrases can be changed. The default value is 10, minimum is 5 and a maximum value of 30.").launch(share=True,server_name='0.0.0.0',show_error=True)
+             "\t The value of keyphrases can be changed. The default value is 10, minimum is 5 and a maximum value of 30.")
+
+igen_pubmed.launch(share=True,server_name='0.0.0.0',show_error=True)