app init

app.py

@@ -0,0 +1,269 @@
+import streamlit as st
+import pandas as pd
+import ast
+import openai
+from openai import OpenAI
+import PyPDF2
+from PyPDF2 import PdfReader
+import openreview
+import json
+import urllib.request
+import requests
+import os
+my_api_key = os.environ["OPENAI_API_KEY"]
+client_openai = OpenAI(api_key=my_api_key)
+
+# converts paper pdf into text
+def pdf_to_text(pdf):
+    paper_text = ''
+
+    reader = PdfReader(pdf)
+    number_of_pages = len(reader.pages)
+    for p in range(number_of_pages):
+        page = reader.pages[p].get_object()
+        text = page.extract_text()
+        paper_text += text
+
+    return paper_text
+
+# loads all 2024 ICLR submissions from OpenReview
+def load_ICLR_submissions():
+    client = openreview.api.OpenReviewClient(
+    baseurl='https://api2.openreview.net',
+    username="[email protected]",
+    password=os.environ["OPENREVIEW_PASSWORD"]
+    )
+
+    venue_id = 'ICLR.cc/2024/Conference'
+    venue_group = client.get_group(venue_id)
+    submission_name = venue_group.content['submission_name']['value']
+    submissions = client.get_all_notes(invitation=f'{venue_id}/-/{submission_name}', details='replies')
+
+    return submissions
+
+# returns review and pdf paper text given paper id and reviewer id
+def parse_openreview_id(submissions, paper_id, reviewer_id):
+    paper_url = f"https://openreview.net/pdf?id={paper_id}"
+
+    ind = next((index for index, entry in enumerate(submissions) if entry.id == paper_id), None)
+    curr_review = submissions[ind]
+    bad_review = next((i for i, entry in enumerate(curr_review.details['replies'])
+              if entry['signatures'][0].split('_')[-1] == reviewer_id), None)
+    
+    review_content = curr_review.details["replies"][bad_review]["content"]
+
+    urllib.request.urlretrieve(paper_url, f"{paper_id}.pdf")
+    pdf = f"{paper_id}.pdf"
+    pdf_text = pdf_to_text(pdf)
+
+    return review_content, pdf, pdf_text
+
+
+def create_feedback(review, pdf_text, agent_prompt):
+
+    messages = [{
+        "role": "user",
+        "content": [
+      {
+        "type": "text",
+        "text": agent_prompt
+      },
+      {
+        "type": "text",
+        "text": "Here is the ML conference review"
+      },
+      {
+        "type": "text",
+        "text": json.dumps(review) #json.dumps(review)
+      },
+      {
+        "type": "text",
+        "text": "Finally, read the paper this review was written about"
+      },
+      {
+        "type": "text",
+        "text": pdf_text
+      }
+      ]}]
+
+    headers = {
+    "Content-Type": "application/json",
+    "Authorization": f"Bearer {my_api_key}"
+    }
+    payload = {
+    "model": "gpt-4o",
+    "messages": messages,
+    "max_tokens": 1000
+    }
+
+    try:
+        response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
+        feedback = response.json()["choices"][0]["message"]["content"]
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        feedback = "an error occured"
+
+    return feedback
+
+agent_prompt = """
+You are given a peer review of a machine learning paper submitted to a top-tier ML conference on OpenReview.
+First, you will read the text of the review and then the paper it was written about.
+Give feedback to the to the reviewer so that it becomes a high-quality review.
+Here is a checklist. For each item, determine if the review passes or fails. If it fails, provide feedback accordingly. Follow the below guidelines for your feedback:
+
+1. The reviewer does not ask for anything that is OBVIOUSLY already present in the paper. You should be ABSOLUTELY certain the reviewer made an error, otherwise do NOT comment on their point.
+If you find such a case, quote the specific statement from the review and provide the exact quote from the paper that directly addresses or contradicts it. 
+Ensure that the quote from the paper specifically answers the claim or request made by the reviewer, and explain why the selected quote is relevant.
+If the reviewer has a question, you do not need to respond. If you respond, point to a specific section or quote from the paper and ask “does this provide what you were looking for?”
+2. The review does not make any vague, unjustifiable, or unsupported claims. 
+If there are any vague comments in the review, ask the reviewer to be more specific.
+3. If the reviewer claims the paper isn't novel, they should explain why and specify the paper(s) they believe this work is similar to. 
+If they do not already do this, ask the reviewer to do so.
+4. The reviewer does not make any personal attacks against the paper and/or author(s). 
+For example, they call the work "incompetent" without justifying why. 
+If they do, suggest the reviewer revise their feedback to remove those attacks.
+
+Be concise, limiting your response to 2-3 sentences per checklist item. Do not repeat anything that the reviewer already included in their review, and do not summarize your feedback at the end.
+
+Here are some examples of good responses to reviews that may fall under each of these categories.
+
+1. A question / claim that is either already answered in the paper, or contradicts with what is provided in the paper.
+
+For such cases, we would like to point the reviewer specifically to the relevant part of the paper through direct quoting. Only send a quote if it verbatim exists in the paper or review.
+
+Example 1:
+
+**Reviewer Comment:** *..In Figure 4, the efficiency experiments have no results for Transformer models, which is a key limitation of the paper.I*
+
+**Feedback to reviewer:** You may want to check Section 3, Figure 5 of the paper that has the Transformer results. See: “In Transformers, the proposed technique provides 25% relative improvement in wall-clock time (Figure 5)”.
+
+Example 2: 
+**Reviewer Comment:** *Figure 2. Are people forced to select a choice or could they select 'I don't know'? Did you monitor response times to see if the manipulated images required longer times for individuals to pass decisions? In Appendix A, you mention “the response times will also be released upon publication”, however I do not see further information about this in the paper.*
+
+As the reviewer already refers to the key part of the paper with quotes and asks a question that is not answered in the paper, we do not need to give feedback to this comment.
+
+2. Lack of clarity or justification for a criticism: The reviewer criticizes the paper without providing a concrete justification. This results in points that are not actionable or harder to respond to. 
+
+For such cases, we would like to nudge the reviewer to justify their claim.
+
+Example 1:
+**Reviewer Comment:** *A key limitation is that this paper motivates from the TTT perspective, but no TTT experiments are performed and compared, e.g., comparing with (Sun et al., 2020).*
+
+**Feedback to reviewer:** In general, a “TTT experiment” is not a broadly known terminology. It would be very helpful if you could describe what specific experiment you are referring to, and why it is important for this paper.
+
+Example 2: 
+**Reviewer Comment:** *W1: Many of the key pieces of this work have been considered before separately. I think some additional discussion of related work is needed.
+
+W2: It appears that the linear mode connectivity results may be somewhat brittle.*
+
+**Feedback to reviewer:** For W1, It would be very helpful to specify or cite what key pieces are considered in earlier works, how they relate to the current work, and specify what is missing. For W2, could you elaborate on why you see the results as brittle and what would be convincing evidence so that the feedback is more actionable?
+
+Example 3: 
+**Reviewer Comment:** *The data/model size are not large-scale, thus the paper will not be impactful.*
+
+**Feedback to reviewer:** Please consider putting this claim in context. If possible, please discuss why you consider the experiments to be not large scale using precedents, why it is important to have larger scale experiments in this context and how it would inform or improve the manuscript.
+
+3. Novelty claims: The reviewer claims lack of novelty without providing concrete justification.
+
+For this case, we would like to nudge the reviewer to provide the justification of the claim, through prompting them to provide the most relevant references, the relationships, and specifying similarities or differences.
+
+Example 1:
+**Reviewer Comment:** *.. The paper’s novelty is limited considering the ICLR standards, as there are very close works [1,2,3]. *
+
+**Feedback to reviewer:** Please consider the reasons for why the novelty is limited, and specify what ICLR standards are in this context. In particular, it would be very helpful to all parties involved if you give examples of the closest papers, its similarities, and differences with the methods or results in the current paper.
+
+Example 2: 
+**Reviewer Comment:** *.. DASHA is a mash-up between MARINA and existing distributed nonconvex optimization methods. Other than the fact that three variants of DASHA get rid of the uncompressed synchronization in MARINA, this reviewer could not pinpoint a difference between MARINA and DASHA. As such, the main novelty of this work seems to be in terms of theoretical analysis of MARINA when the uncompressed synchronization step is removed. The authors could have done a better job of clarifying where does this novelty lie in the analysis (e.g., pinpointing the key analytical approaches in the lemma that helped improve the analysis)*
+
+As the novelty claim is already well-justified and an actionable question is asked, we do not need to give feedback to this review.
+
+4. Personal attack, ad hominem: The review contains an attack on the authors. This can be about the personality, the knowledge, or the experience of the authors. 
+
+For this case, we would like to kindly warn the reviewer about their comment. 
+
+Example 1: 
+**Reviewer Comment:** *.. The authors clearly do not live in the real world and do not care about people or downstream effects of their research. *
+
+**Feedback to the reviewer:** We kindly suggest you revise this comment, as it includes remarks about the personalities or intents of the authors.
+
+Your instructions: Go through all comments in the review. Check each comment using the above guidelines. Only address issues with the review that fail according to the above guidelines.
+Your feedback will be sent directly to reviewers. Follow the below format when sending your review:
+
+- Comment: {{the comment of interest}}
+- Feedback: {{write your short feedback}}
+
+and send a list of comments and feedbacks. If you do not identify issues for a comment, do not add it to the list or send feedback. DO NOT mention that there is a checklist or guidelines.
+
+If you cannot identify any issues, respond with "Great review, thanks for your hard work!"
+"""
+
+# Set page to wide mode
+st.set_page_config(layout="wide")
+
+# Sidebar for navigation
+st.sidebar.title('Input review and PDF or OpenReview ID')
+
+option = st.sidebar.radio(
+    "Choose one input option:",
+    ("Upload PDF with Review", "OpenReview paper ID"))
+
+# # Add a text input box to the sidebar for user input
+# user_input = st.sidebar.text_input("Enter OpenReview ID (e.g. xC8xh2RSs2)")
+
+user_input = False
+
+if option == "OpenReview paper ID":
+    paper_id = st.text_input("Enter OpenReview ID (e.g. xC8xh2RSs2):")
+    reviewer_id = st.text_input("Enter reviewer ID (e.g. gNxe):")
+    if paper_id and reviewer_id:
+        upload_file = False
+        user_input = True
+        submissions = load_ICLR_submissions()
+        review, pdf, pdf_text = parse_openreview_id(submissions, paper_id, reviewer_id)
+        # st.write(f"Review text: {pdf_text}")
+
+elif option == "Upload PDF with Review":
+    user_text = st.text_area("Enter review:")
+    uploaded_file = st.file_uploader("Upload PDF", type="pdf")
+    if user_text and uploaded_file:
+        upload_file = True
+        user_input = True
+        review, pdf = user_text, uploaded_file
+        pdf_text = pdf_to_text(pdf)
+
+        # st.write(f"Review text: {pdf_text}")
+        # st.write(f"You entered review: {user_text}")
+        # st.write(f"Uploaded file: {uploaded_file.name}")
+
+if user_input:
+    # Run your pipeline to generate the dataframe based on user input
+    feedback = create_feedback(review, pdf_text, agent_prompt)
+
+    st.title(f'Review feedback')
+
+    if not upload_file:
+        # Create three columns
+        col1, col2, col3 = st.columns(3)
+
+        with col3:
+            st.subheader('PDF Link')
+            pdf_url = f"https://openreview.net/pdf?id={paper_id}"
+            st.markdown(f"[Click here to view PDF]({pdf_url})")
+    
+    else:
+        # Create two columns
+        col1, col2 = st.columns(2)
+        
+
+    # Column 1: Display review fields
+    with col1:
+        st.subheader("Review")
+        st.write(json.dumps(review))
+
+    # Column 2: Display feedback
+    with col2:
+        st.subheader('Feedback')
+        st.write(feedback)
+
+else:
+    st.title('Please enter OpenReview ID or upload PDF with review to generate feedback')