cleaned repo, added utils

app.py

@@ -1,13 +1,7 @@
 import streamlit as st  #Web App
-import urllib
-from lxml import html
-import requests
-import re
 import os
-from stqdm import stqdm
-import time
-import shutil
 from PIL import Image
+from unitls import *
 
 import pickle
 docs = None
@@ -36,118 +30,14 @@ if len(api_key) != 51:
     st.warning('Please enter a valid OpenAI API key.', icon="⚠️")
 
 
-
-def call_arXiv_API(search_query, search_by='all', sort_by='relevance', max_results='10', folder_name='arxiv-dl'):
-    '''
-      Scraps the arXiv's html to get data from each entry in a search. Entries has the following formatting:
-      <entry>\n    
-      <id>http://arxiv.org/abs/2008.04584v2</id>\n    
-      <updated>2021-05-11T12:00:24Z</updated>\n    
-      <published>2020-08-11T08:47:06Z</published>\n    
-      <title>Bayesian Selective Inference: Non-informative Priors</title>\n    
-      <summary>  We discuss Bayesian inference for parameters selected using the data. First,\nwe provide a critical analysis of the existing positions in the literature\nregarding the correct Bayesian approach under selection. Second, we propose two\ntypes of non-informative priors for selection models. These priors may be\nemployed to produce a posterior distribution in the absence of prior\ninformation as well as to provide well-calibrated frequentist inference for the\nselected parameter. We test the proposed priors empirically in several\nscenarios.\n</summary>\n    
-      <author>\n      <name>Daniel G. Rasines</name>\n    </author>\n    <author>\n      <name>G. Alastair Young</name>\n    </author>\n    
-      <arxiv:comment xmlns:arxiv="http://arxiv.org/schemas/atom">24 pages, 7 figures</arxiv:comment>\n    
-      <link href="http://arxiv.org/abs/2008.04584v2" rel="alternate" type="text/html"/>\n    
-      <link title="pdf" href="http://arxiv.org/pdf/2008.04584v2" rel="related" type="application/pdf"/>\n    
-      <arxiv:primary_category xmlns:arxiv="http://arxiv.org/schemas/atom" term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
-      <category term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
-      <category term="stat.TH" scheme="http://arxiv.org/schemas/atom"/>\n  
-      </entry>\n  
-    '''
-
-    # Remove space in seach query
-    search_query=search_query.strip().replace(" ", "+")
-    # Call arXiv API
-    arXiv_url=f'http://export.arxiv.org/api/query?search_query={search_by}:{search_query}&sortBy={sort_by}&start=0&max_results={max_results}'
-    with urllib.request.urlopen(arXiv_url) as url:
-        s = url.read()
-    
-    # Parse the xml data
-    root = html.fromstring(s)
-    # Fetch relevant pdf information
-    pdf_entries = root.xpath("entry")
-
-    pdf_titles   = []
-    pdf_authors  = []
-    pdf_urls     = []
-    pdf_categories = []
-    folder_names = []
-    pdf_citation = []
-    pdf_years = []
-
-    for i, pdf in enumerate(pdf_entries):
-      # print(pdf.xpath('updated/text()')[0][:4])
-      # xpath return a list with every ocurrence of the html path. Since we're getting each entry individually, we'll take the first element to avoid an unecessary list
-      pdf_titles.append(re.sub('[^a-zA-Z0-9]', ' ', pdf.xpath("title/text()")[0]))
-      pdf_authors.append(pdf.xpath("author/name/text()"))
-      pdf_urls.append(pdf.xpath("link[@title='pdf']/@href")[0])
-      pdf_categories.append(pdf.xpath("category/@term"))
-      folder_names.append(folder_name)
-      pdf_years.append(pdf.xpath('updated/text()')[0][:4])
-      pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. arXiv [{pdf_categories[i][0]}] ({pdf_years[i]}), (available at {pdf_urls[i]}).")
-
-      
-
-    pdf_info=list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
-    
-    # Check number of available files
-    # print('Requesting {max_results} files'.format(max_results=max_results))
-    if len(pdf_urls)<int(max_results):
-        matching_pdf_num=len(pdf_urls)
-        # print('Only {matching_pdf_num} files available'.format(matching_pdf_num=matching_pdf_num))
-    return pdf_info, pdf_citation
-
-
-def download_pdf(pdf_info):
-   
-    # if len(os.listdir(f'./{folder_name}') ) != 0:
-            # check folder is empty to avoid using papers from old runs:
-            # os.remove(f'./{folder_name}/*')
-    all_reference_text = []
-    for i,p in enumerate(stqdm(pdf_info, desc='Searching and downloading papers')):
-
-        pdf_title=p[0]
-        pdf_url=p[1]
-        pdf_author=p[2]
-        pdf_category=p[3]
-        folder_name=p[4]
-        pdf_citation=p[5]
-        r = requests.get(pdf_url, allow_redirects=True)
-        if  i == 0:
-            if not os.path.exists(f'{folder_name}'):
-                os.makedirs(f"{folder_name}")
-            else:
-                shutil.rmtree(f'{folder_name}') 
-                os.makedirs(f"{folder_name}")
-        with open(f'{folder_name}/{pdf_title}.pdf', 'wb') as currP:
-            currP.write(r.content)
-        if i == 0:
-            st.markdown("###### Papers found:")
-        st.markdown(f"{i+1}. {pdf_citation}")
-        time.sleep(0.15)
-        all_reference_text.append(f"{i+1}. {pdf_citation}\n")
-    if 'all_reference_text' not in st.session_state:
-        st.session_state.key = 'all_reference_text'
-    st.session_state['all_reference_text'] = ' '.join(all_reference_text)
-
-    # print(all_reference_text)
-    
-
-
 max_results_current = 5
 max_results = max_results_current
-# pdf_info = ''
-# pdf_citation = ''
 def search_click_callback(search_query, max_results):
     global pdf_info, pdf_citation
     pdf_info, pdf_citation = call_arXiv_API(f'{search_query}', max_results=max_results)
     download_pdf(pdf_info)
     return pdf_info
 
-
-
-
 with st.form(key='columns_in_form', clear_on_submit = False):
     c1, c2 = st.columns([8,1])
     with c1:
@@ -158,7 +48,6 @@ with st.form(key='columns_in_form', clear_on_submit = False):
         max_results = st.text_input("Max papers", value=max_results_current)
         max_results_current = max_results_current
     searchButton = st.form_submit_button(label = 'Search')
-    #         search_click(search_query, max_results_default)
 
 if searchButton:
     global pdf_info
@@ -166,39 +55,11 @@ if searchButton:
     if 'pdf_info' not in st.session_state:
         st.session_state.key = 'pdf_info'
     st.session_state['pdf_info'] = pdf_info
-    # print(f'This is PDF info from search:{pdf_info}')
-
-
-# def tokenize_callback():
-
-    # return docs
-
-# tokenization_form = st.form(key='tokenization-form')
-# tokenization_form.markdown(f"Happy with your paper search results? ")
-# toknizeButton = tokenization_form.form_submit_button(label = "Yes! Let's tokenize.", on_click=tokenize_callback())
-# tokenization_form.markdown("If not, change keywords and search again. [This step costs!](https://openai.com/api/pricing/)")
-
-
-
-# submitButton = form.form_submit_button('Submit')
-# with st.form(key='tokenization_form', clear_on_submit = False):
-#     st.markdown(f"Happy with your paper search results? If not, change keywords and search again. [This step costs!](https://openai.com/api/pricing/)")
-#     # st.text_input("Input search query here:", placeholder='Keywords for most relevant search...'
-#     #                                )#search_query, max_results_current))
-#     toknizeButton = st.form_submit_button(label = "Yes! Let's tokenize.")
-
-# if toknizeButton:
-#     tokenize_callback()
-
-# tokenize_callback()
-
-
 
 
 def answer_callback(question_query):
     import paperqa
     global docs
-    # global pdf_info
     progress_text = "Please wait..."
     # my_bar = st.progress(0, text = progress_text)
     st.info('Please wait...', icon="🔥")
@@ -221,8 +82,6 @@ def answer_callback(question_query):
     st.success('Voila!')
     return answer.formatted_answer
 
-
-
 form = st.form(key='question_form')
 question_query = form.text_input("What do you wanna know from these papers?", placeholder='Input questions here...',
                 value='')
@@ -232,39 +91,3 @@ if submitButton:
     with st.expander("Found papers:", expanded=True):
         st.write(f"{st.session_state['all_reference_text']}")
     st.text_area("Answer:", answer_callback(question_query), height=600)
-
-# with st.form(key='question_form', clear_on_submit = False):
-#     question_query = st.text_input("What do you wanna know from these papers?", placeholder='Input questions here')
-#     # st.text_input("Input search query here:", placeholder='Keywords for most relevant search...'
-#     #                                )#search_query, max_results_current))
-#     submitButton = form.form_submit_button(label = "Submit", on_click=answer_callback(question_query))
-
-
-# Simulation-based inference bayesian model selection
-
-
-
-
-
-# test = "<ul> \
-#         <li>List item here</li> \
-#         <li>List item here</li> \
-#         <li>List item here</li> \
-#         <li>List item here</li> \
-#         </ul>"
-# test = "'''It was the best of times, it was the worst of times, it was \
-#             the age of wisdom, it was the age of foolishness, it was \
-#             the epoch of belief, it was the epoch of incredulity, it \
-#             was the season of Light, it was the season of Darkness, it\
-#             was the spring of hope, it was the winter of despair, (...)'''"
-
-# citation_text = st.text_area('Papers found:',test, height=300) # f'{pdf_citation}'
-
-
-# for i, cite in enumerate(pdf_citation):
-#     st.markdown(f'{i+1}. {cite}')
-#     time.sleep(1)
-
-
-# def make_clickable('link',text):
-#     return f'<a target="_blank" href="{link}">{text}'

unitls.py

@@ -0,0 +1,103 @@
+import urllib
+from lxml import html
+import streamlit as st 
+import requests
+import re
+from stqdm import stqdm
+import os
+import shutil
+import time
+
+def call_arXiv_API(search_query, search_by='all', sort_by='relevance', max_results='10', folder_name='arxiv-dl'):
+    '''
+      Scraps the arXiv's html to get data from each entry in a search. Entries has the following formatting:
+      <entry>\n    
+      <id>http://arxiv.org/abs/2008.04584v2</id>\n    
+      <updated>2021-05-11T12:00:24Z</updated>\n    
+      <published>2020-08-11T08:47:06Z</published>\n    
+      <title>Bayesian Selective Inference: Non-informative Priors</title>\n    
+      <summary>  We discuss Bayesian inference for parameters selected using the data. First,\nwe provide a critical analysis of the existing positions in the literature\nregarding the correct Bayesian approach under selection. Second, we propose two\ntypes of non-informative priors for selection models. These priors may be\nemployed to produce a posterior distribution in the absence of prior\ninformation as well as to provide well-calibrated frequentist inference for the\nselected parameter. We test the proposed priors empirically in several\nscenarios.\n</summary>\n    
+      <author>\n      <name>Daniel G. Rasines</name>\n    </author>\n    <author>\n      <name>G. Alastair Young</name>\n    </author>\n    
+      <arxiv:comment xmlns:arxiv="http://arxiv.org/schemas/atom">24 pages, 7 figures</arxiv:comment>\n    
+      <link href="http://arxiv.org/abs/2008.04584v2" rel="alternate" type="text/html"/>\n    
+      <link title="pdf" href="http://arxiv.org/pdf/2008.04584v2" rel="related" type="application/pdf"/>\n    
+      <arxiv:primary_category xmlns:arxiv="http://arxiv.org/schemas/atom" term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
+      <category term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
+      <category term="stat.TH" scheme="http://arxiv.org/schemas/atom"/>\n  
+      </entry>\n  
+    '''
+
+    # Remove space in seach query
+    search_query=search_query.strip().replace(" ", "+")
+    # Call arXiv API
+    arXiv_url=f'http://export.arxiv.org/api/query?search_query={search_by}:{search_query}&sortBy={sort_by}&start=0&max_results={max_results}'
+    with urllib.request.urlopen(arXiv_url) as url:
+        s = url.read()
+    
+    # Parse the xml data
+    root = html.fromstring(s)
+    # Fetch relevant pdf information
+    pdf_entries = root.xpath("entry")
+
+    pdf_titles   = []
+    pdf_authors  = []
+    pdf_urls     = []
+    pdf_categories = []
+    folder_names = []
+    pdf_citation = []
+    pdf_years = []
+
+    for i, pdf in enumerate(pdf_entries):
+      # print(pdf.xpath('updated/text()')[0][:4])
+      # xpath return a list with every ocurrence of the html path. Since we're getting each entry individually, we'll take the first element to avoid an unecessary list
+      pdf_titles.append(re.sub('[^a-zA-Z0-9]', ' ', pdf.xpath("title/text()")[0]))
+      pdf_authors.append(pdf.xpath("author/name/text()"))
+      pdf_urls.append(pdf.xpath("link[@title='pdf']/@href")[0])
+      pdf_categories.append(pdf.xpath("category/@term"))
+      folder_names.append(folder_name)
+      pdf_years.append(pdf.xpath('updated/text()')[0][:4])
+      pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. arXiv [{pdf_categories[i][0]}] ({pdf_years[i]}), (available at {pdf_urls[i]}).")
+
+      
+
+    pdf_info=list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
+    
+    # Check number of available files
+    # print('Requesting {max_results} files'.format(max_results=max_results))
+    if len(pdf_urls)<int(max_results):
+        matching_pdf_num=len(pdf_urls)
+        # print('Only {matching_pdf_num} files available'.format(matching_pdf_num=matching_pdf_num))
+    return pdf_info, pdf_citation
+
+
+def download_pdf(pdf_info):
+   
+    # if len(os.listdir(f'./{folder_name}') ) != 0:
+            # check folder is empty to avoid using papers from old runs:
+            # os.remove(f'./{folder_name}/*')
+    all_reference_text = []
+    for i,p in enumerate(stqdm(pdf_info, desc='Searching and downloading papers')):
+
+        pdf_title=p[0]
+        pdf_url=p[1]
+        pdf_author=p[2]
+        pdf_category=p[3]
+        folder_name=p[4]
+        pdf_citation=p[5]
+        r = requests.get(pdf_url, allow_redirects=True)
+        if  i == 0:
+            if not os.path.exists(f'{folder_name}'):
+                os.makedirs(f"{folder_name}")
+            else:
+                shutil.rmtree(f'{folder_name}') 
+                os.makedirs(f"{folder_name}")
+        with open(f'{folder_name}/{pdf_title}.pdf', 'wb') as currP:
+            currP.write(r.content)
+        if i == 0:
+            st.markdown("###### Papers found:")
+        st.markdown(f"{i+1}. {pdf_citation}")
+        time.sleep(0.15)
+        all_reference_text.append(f"{i+1}. {pdf_citation}\n")
+    if 'all_reference_text' not in st.session_state:
+        st.session_state.key = 'all_reference_text'
+    st.session_state['all_reference_text'] = ' '.join(all_reference_text)