app.py

import streamlit as st  #Web App
import urllib
from lxml import html
import requests
import re
import os
from stqdm import stqdm
import time
import shutil
from PIL import Image

import pickle
docs = None
api_key = ' '

st.set_page_config(layout="wide")

image = Image.open('arxiv_decode.png')
st.image(image, width=1000)


#title
st.title("Answering questions from scientific papers")
st.markdown("##### This tool will allow you to ask questions and get based on scientific papers. It uses OpenAI's GPT models, and you must have your own API key. Each query is about 10k tokens, which costs about $0.20.")
st.markdown("##### Current version searches on ArXiv papers only. 🚧Under development🚧")
st.markdown("Used libraries:\n * [PaperQA](https://github.com/whitead/paper-qa) \n* [langchain](https://github.com/hwchase17/langchain)")

api_key_url = 'https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key'

api_key = st.text_input('OpenAI API Key',
         placeholder='sk-...', 
         help=f"['What is that?']({api_key_url})",
         type="password")     

os.environ["OPENAI_API_KEY"] = f"{api_key}" # 
if len(api_key) != 51:
    st.warning('Please enter a valid OpenAI API key.', icon="⚠️")



def call_arXiv_API(search_query, search_by='all', sort_by='relevance', max_results='10', folder_name='arxiv-dl'):
    '''
      Scraps the arXiv's html to get data from each entry in a search. Entries has the following formatting:
      <entry>\n    
      <id>http://arxiv.org/abs/2008.04584v2</id>\n    
      <updated>2021-05-11T12:00:24Z</updated>\n    
      <published>2020-08-11T08:47:06Z</published>\n    
      <title>Bayesian Selective Inference: Non-informative Priors</title>\n    
      <summary>  We discuss Bayesian inference for parameters selected using the data. First,\nwe provide a critical analysis of the existing positions in the literature\nregarding the correct Bayesian approach under selection. Second, we propose two\ntypes of non-informative priors for selection models. These priors may be\nemployed to produce a posterior distribution in the absence of prior\ninformation as well as to provide well-calibrated frequentist inference for the\nselected parameter. We test the proposed priors empirically in several\nscenarios.\n</summary>\n    
      <author>\n      <name>Daniel G. Rasines</name>\n    </author>\n    <author>\n      <name>G. Alastair Young</name>\n    </author>\n    
      <arxiv:comment xmlns:arxiv="http://arxiv.org/schemas/atom">24 pages, 7 figures</arxiv:comment>\n    
      <link href="http://arxiv.org/abs/2008.04584v2" rel="alternate" type="text/html"/>\n    
      <link title="pdf" href="http://arxiv.org/pdf/2008.04584v2" rel="related" type="application/pdf"/>\n    
      <arxiv:primary_category xmlns:arxiv="http://arxiv.org/schemas/atom" term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
      <category term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
      <category term="stat.TH" scheme="http://arxiv.org/schemas/atom"/>\n  
      </entry>\n  
    '''

    # Remove space in seach query
    search_query=search_query.strip().replace(" ", "+")
    # Call arXiv API
    arXiv_url=f'http://export.arxiv.org/api/query?search_query={search_by}:{search_query}&sortBy={sort_by}&start=0&max_results={max_results}'
    with urllib.request.urlopen(arXiv_url) as url:
        s = url.read()
    
    # Parse the xml data
    root = html.fromstring(s)
    # Fetch relevant pdf information
    pdf_entries = root.xpath("entry")

    pdf_titles   = []
    pdf_authors  = []
    pdf_urls     = []
    pdf_categories = []
    folder_names = []
    pdf_citation = []
    pdf_years = []

    for i, pdf in enumerate(pdf_entries):
      # print(pdf.xpath('updated/text()')[0][:4])
      # xpath return a list with every ocurrence of the html path. Since we're getting each entry individually, we'll take the first element to avoid an unecessary list
      pdf_titles.append(re.sub('[^a-zA-Z0-9]', ' ', pdf.xpath("title/text()")[0]))
      pdf_authors.append(pdf.xpath("author/name/text()"))
      pdf_urls.append(pdf.xpath("link[@title='pdf']/@href")[0])
      pdf_categories.append(pdf.xpath("category/@term"))
      folder_names.append(folder_name)
      pdf_years.append(pdf.xpath('updated/text()')[0][:4])
      pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. arXiv [{pdf_categories[i][0]}] ({pdf_years[i]}), (available at {pdf_urls[i]}).")

      

    pdf_info=list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
    
    # Check number of available files
    # print('Requesting {max_results} files'.format(max_results=max_results))
    if len(pdf_urls)<int(max_results):
        matching_pdf_num=len(pdf_urls)
        # print('Only {matching_pdf_num} files available'.format(matching_pdf_num=matching_pdf_num))
    return pdf_info, pdf_citation


def download_pdf(pdf_info):
   
    # if len(os.listdir(f'./{folder_name}') ) != 0:
            # check folder is empty to avoid using papers from old runs:
            # os.remove(f'./{folder_name}/*')
    all_reference_text = []
    for i,p in enumerate(stqdm(pdf_info, desc='Searching and downloading papers')):

        pdf_title=p[0]
        pdf_url=p[1]
        pdf_author=p[2]
        pdf_category=p[3]
        folder_name=p[4]
        pdf_citation=p[5]
        r = requests.get(pdf_url, allow_redirects=True)
        if  i == 0:
            if not os.path.exists(f'{folder_name}'):
                os.makedirs(f"{folder_name}")
            else:
                shutil.rmtree(f'{folder_name}') 
                os.makedirs(f"{folder_name}")
        with open(f'{folder_name}/{pdf_title}.pdf', 'wb') as currP:
            currP.write(r.content)
        if i == 0:
            st.markdown("###### Papers found:")
        st.markdown(f"{i+1}. {pdf_citation}")
        time.sleep(0.15)
        all_reference_text.append(f"{i+1}. {pdf_citation}\n")
    if 'all_reference_text' not in st.session_state:
        st.session_state.key = 'all_reference_text'
    st.session_state['all_reference_text'] = ' '.join(all_reference_text)

    # print(all_reference_text)
    


max_results_current = 5
max_results = max_results_current
# pdf_info = ''
# pdf_citation = ''
def search_click_callback(search_query, max_results):
    global pdf_info, pdf_citation
    pdf_info, pdf_citation = call_arXiv_API(f'{search_query}', max_results=max_results)
    download_pdf(pdf_info)
    return pdf_info




with st.form(key='columns_in_form', clear_on_submit = False):
    c1, c2 = st.columns([8,1])
    with c1:
        search_query = st.text_input("Input search query here:", placeholder='Keywords for most relevant search...', value=''
                                       )#search_query, max_results_current))

    with c2:
        max_results = st.text_input("Max papers", value=max_results_current)
        max_results_current = max_results_current
    searchButton = st.form_submit_button(label = 'Search')
    #         search_click(search_query, max_results_default)

if searchButton:
    global pdf_info
    pdf_info =  search_click_callback(search_query, max_results)
    if 'pdf_info' not in st.session_state:
        st.session_state.key = 'pdf_info'
    st.session_state['pdf_info'] = pdf_info
    # print(f'This is PDF info from search:{pdf_info}')


# def tokenize_callback():

    # return docs

# tokenization_form = st.form(key='tokenization-form')
# tokenization_form.markdown(f"Happy with your paper search results? ")
# toknizeButton = tokenization_form.form_submit_button(label = "Yes! Let's tokenize.", on_click=tokenize_callback())
# tokenization_form.markdown("If not, change keywords and search again. [This step costs!](https://openai.com/api/pricing/)")



# submitButton = form.form_submit_button('Submit')
# with st.form(key='tokenization_form', clear_on_submit = False):
#     st.markdown(f"Happy with your paper search results? If not, change keywords and search again. [This step costs!](https://openai.com/api/pricing/)")
#     # st.text_input("Input search query here:", placeholder='Keywords for most relevant search...'
#     #                                )#search_query, max_results_current))
#     toknizeButton = st.form_submit_button(label = "Yes! Let's tokenize.")

# if toknizeButton:
#     tokenize_callback()

# tokenize_callback()




def answer_callback(question_query):
    import paperqa
    global docs
    # global pdf_info
    progress_text = "Please wait..."
    # my_bar = st.progress(0, text = progress_text)
    st.info('Please wait...', icon="🔥")
    if docs is None:
        # my_bar.progress(0.2, "Please wait...")
        pdf_info = st.session_state['pdf_info']
        # print('buliding docs')
        docs = paperqa.Docs()
        pdf_paths = [f"{p[4]}/{p[0]}.pdf" for p in pdf_info]
        pdf_citations = [p[5] for p in pdf_info]
        print(list(zip(pdf_paths, pdf_citations)))

        for d, c in zip(pdf_paths, pdf_citations):
            # print(d,c)
            docs.add(d, c)
    # docs._build_faiss_index()
    answer = docs.query(question_query)
    # print(answer.formatted_answer)
    # my_bar.progress(1.0, "Done!")
    st.success('Done!')
    return answer.formatted_answer



form = st.form(key='question_form')
question_query = form.text_input("What do you wanna know from these papers?", placeholder='Input questions here...',
                value='')
submitButton = form.form_submit_button('Submit')

if submitButton:
    with st.expander("Found papers:", expanded=True):
        st.write(f"{st.session_state['all_reference_text']}")
    st.text_area("Answer:", answer_callback(question_query), height=600)

# with st.form(key='question_form', clear_on_submit = False):
#     question_query = st.text_input("What do you wanna know from these papers?", placeholder='Input questions here')
#     # st.text_input("Input search query here:", placeholder='Keywords for most relevant search...'
#     #                                )#search_query, max_results_current))
#     submitButton = form.form_submit_button(label = "Submit", on_click=answer_callback(question_query))


# Simulation-based inference bayesian model selection





# test = "<ul> \
#         <li>List item here</li> \
#         <li>List item here</li> \
#         <li>List item here</li> \
#         <li>List item here</li> \
#         </ul>"
# test = "'''It was the best of times, it was the worst of times, it was \
#             the age of wisdom, it was the age of foolishness, it was \
#             the epoch of belief, it was the epoch of incredulity, it \
#             was the season of Light, it was the season of Darkness, it\
#             was the spring of hope, it was the winter of despair, (...)'''"

# citation_text = st.text_area('Papers found:',test, height=300) # f'{pdf_citation}'


# for i, cite in enumerate(pdf_citation):
#     st.markdown(f'{i+1}. {cite}')
#     time.sleep(1)


# def make_clickable('link',text):
#     return f'<a target="_blank" href="{link}">{text}'