app.py

# from reviewer_feedback_agent.apps import app
# app()

import streamlit as st
import pandas as pd
import ast
import re
import PyPDF2
from PyPDF2 import PdfReader
import openreview
import json
import urllib.request
import requests
import os
import pickle
import openai
from openai import OpenAI
import anthropic
from anthropic import Anthropic

openai_api_key = os.environ["OPENAI_API_KEY"]
client_openai = OpenAI(api_key=openai_api_key)

anthropic_api_key = os.environ["ANTHROPIC_API_KEY"]
client_anthropic = Anthropic(api_key=anthropic_api_key)

def parse_quotes(input_string, pdf_text):

    header_pattern = r'\d+Under review as a conference paper at ICLR 2024'
    #r'^\d+Under review as a conference paper at ICLR 2024\s*'  # Pattern to match headers with variable page numbers
    # r'(?m)^Under review as a conference paper at ICLR 2024*$'

    pdf_text_wo_header = re.sub(header_pattern, '', pdf_text)
    # Remove new lines that are not followed by a period, exclamation mark, or question mark
    pdf_text_wo_header = re.sub(r'(?<!\.\s)(?<!\!\s)(?<!\?\s)\n+', ' ', pdf_text_wo_header)
    # Remove extra spaces that may have been introduced
    pdf_text_wo_header = re.sub(r'\s{2,}', ' ', pdf_text_wo_header).strip()
    
    # Find all matches of <quote>...</quote> and extract the content between the tags
    matches = re.findall(r'<quote>(.*?)</quote>', input_string)
    
    count = len(matches)
    extracted_texts = matches
    # with open('pdf_text.txt', 'w') as file:
    #     file.write(pdf_text_wo_header)
    # print(extracted_texts)
    match_count = sum(1 for text in extracted_texts if text in pdf_text_wo_header)
    
    return count, match_count

# load manual (human) annotations
def load_annotations():
    path = './annotations_8_26.csv'
    annotations = pd.read_csv(path)
    annotations.columns = annotations.columns.str.replace(' ', '_').str.lower()

    # 36 0, 15 1
    annotations['annotation_label'] = annotations['annotation_label'].astype(int)
    
    return annotations

# Check if the password matches the env variable
def check_password():
    def password_entered():
        if st.session_state["password"] == os.environ["APP_PASSCODE"]:
            st.session_state["password_correct"] = True
            del st.session_state["password"]  # Clear password from memory
        else:
            st.session_state["password_correct"] = False

    if "password_correct" not in st.session_state:
        st.text_input("Enter Password", type="password", on_change=password_entered, key="password")
        return False
    elif not st.session_state["password_correct"]:
        st.error("Password incorrect")
        return False
    else:
        return True

# Ask for password at the beginning
if not check_password():
    st.stop()

# converts paper pdf into text
def pdf_to_text(pdf):
    paper_text = ''

    reader = PdfReader(pdf)
    number_of_pages = len(reader.pages)
    for p in range(number_of_pages):
        page = reader.pages[p].get_object()
        text = page.extract_text()
        paper_text += text

    return paper_text

# loads all 2024 ICLR submissions from OpenReview
submissions_path = "./submissions_iclr2024.pkl"
# API V2
client = openreview.api.OpenReviewClient(
    baseurl='https://api2.openreview.net',
)

# Get ICLR venues
venues = client.get_group(id='venues').members
iclr_venues = [v for v in venues if "iclr.cc" in v.lower() and "conference" in v.lower()]

@st.cache_data
def load_ICLR_submissions():
    if os.path.exists(submissions_path):
        with open(submissions_path, "rb") as f:
            return pickle.load(f)
    else:
        # Select the second latest venue
        venue_id = iclr_venues[-2]
        venue_group = client.get_group(venue_id)
        submission_name = venue_group.content['submission_name']['value']
        submissions = client.get_all_notes(invitation=f'{venue_id}/-/{submission_name}', details='replies')
        with open(submissions_path, "wb") as f:
            pickle.dump(submissions, f)
        return submissions

# returns review and pdf paper text given paper id and reviewer id
def parse_openreview_id(submissions, paper_id, reviewer_id):
    paper_url = f"https://openreview.net/pdf?id={paper_id}"
    fields = ['summary', 'soundness', 'presentation', 'contribution', 'strengths', 'weaknesses', 'questions']

    ind = next((index for index, entry in enumerate(submissions) if entry.id == paper_id), None)
    curr_review = submissions[ind]
    bad_review = next((i for i, entry in enumerate(curr_review.details['replies'])
              if entry['signatures'][0].split('_')[-1] == reviewer_id), None)

    try:
        review_content = curr_review.details["replies"][bad_review]["content"]
    except:
        st.error("Paper ID or reviewer ID are incorrect")

    review_content = {key: review_content[key] if key in fields else None for key in review_content.keys() if key in fields}

    urllib.request.urlretrieve(paper_url, f"{paper_id}.pdf")
    pdf = f"{paper_id}.pdf"
    pdf_text = pdf_to_text(pdf)

    return review_content, pdf, pdf_text


def create_feedback(review, pdf_text, agent_prompt, model):

    messages = [{
        "role": "user",
        "content": [
      {
        "type": "text",
        "text": agent_prompt
      },
      {
        "type": "text",
        "text": "Here is the ML conference review"
      },
      {
        "type": "text",
        "text": json.dumps(review) #json.dumps(review)
      },
      {
        "type": "text",
        "text": "Finally, read the paper this review was written about"
      },
      {
        "type": "text",
        "text": pdf_text
      }
      ]}]

    if model == 'gpt':
        headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {openai_api_key}"
        }
        payload = {
        "model": "gpt-4o",
        "messages": messages,
        "max_tokens": 1000,
        "temperature": 0.2
        }

        try:
            response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
            feedback = response.json()["choices"][0]["message"]["content"]
        except Exception as e:
            print(f"An unexpected error occurred: {e}")
            feedback = "an error occured"

    else:
        message = client_anthropic.messages.create(max_tokens=1024, messages=messages, model="claude-3-5-sonnet-20240620")
        feedback = message.content[0].text

    return feedback

def aggregator(feedback_list, aggregator_prompt, agent_prompt, review, pdf_text, model):

    messages = [{
        "role": "user",
        "content": [
      {
        "type": "text",
        "text": aggregator_prompt
      },
      {
        "type": "text",
        "text": "Here is the ML conference review"
      },
      {
        "type": "text",
        "text": json.dumps(review) #json.dumps(review)
      },
      {
        "type": "text",
        "text": "Here is the list of feedback about the review, where each item is separated by the '^' character"
      },
      {
        "type": "text",
        "text": "^ ".join(feedback_list)
      },
      {
        "type": "text",
        "text": "Finally, read the paper this review was written about"
      },
      {
        "type": "text",
        "text": pdf_text
      }
      ]}]

    if model == 'gpt':
        headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {openai_api_key}"
        }
        payload = {
        "model": "gpt-4o-mini",
        "messages": messages,
        "max_tokens": 1000,
        "temperature": 0.2
        }

        try:
            response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
            best_feedback = response.json()["choices"][0]["message"]["content"]
        except Exception as e:
            print(f"An unexpected error occurred: {e}")
            best_feedback = "an error occured"

    else:
        message = client_anthropic.messages.create(max_tokens=1024, messages=messages, model="claude-3-5-sonnet-20240620")
        best_feedback = message.content[0].text

    return best_feedback


def critic(review, feedback, pdf_text, critic_prompt, model):
    
    messages = [{
        "role": "user",
        "content": [
      {
        "type": "text",
        "text": critic_prompt
      },
      {
        "type": "text",
        "text": "Here is the ML conference review"
      },
      {
        "type": "text",
        "text": json.dumps(review) #json.dumps(review)
      },
      {
        "type": "text",
        "text": "Here is the feedback about the review"
      },
      {
        "type": "text",
        "text": feedback
      },
      {
        "type": "text",
        "text": "Finally, read the paper this review was written about"
      },
      {
        "type": "text",
        "text": pdf_text
      }
      ]}]

    if model == 'gpt':
        headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {openai_api_key}"
        }
        payload = {
        "model": "gpt-4o",
        "messages": messages,
        "max_tokens": 1000,
        "temperature": 0.2
        }

        try:
            response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
            revised_feedback = response.json()["choices"][0]["message"]["content"]
        except Exception as e:
            print(f"An unexpected error occurred: {e}")
            revised_feedback = "an error occured"

    else:
        message = client_anthropic.messages.create(max_tokens=1024, messages=messages, model="claude-3-5-sonnet-20240620")
        revised_feedback = message.content[0].text

    return revised_feedback

agent_prompt = """
You are given a peer review of a machine learning paper submitted to a top-tier ML conference on OpenReview. Your task is to provide constructive feedback to the reviewer so that it becomes a high-quality review. You will do this by evaluating the review against a checklist and providing specific feedback about where the review fails. 

Here are step-by-step instructions:
1. Read the text of the review and the paper about which the review was written.

2. Evaluate every comment in the review:
   - Focus on comments made in the "weaknesses" or "questions" sections of the review. Ignore the "summary" and "strengths" sections.
   - For each comment, evaluate it against the following checklist. Follow the examples for how to respond.

     Checklist:
     1. Check if the reviewer requests something obviously present in the paper. Only respond if certain of the reviewer's error. If so, quote the relevant paper section verbatim using <quote> </quote> tags and explain how it addresses the reviewer's point. Use only exact quotes and don't comment if uncertain.
        - Example 1: 
          - **Reviewer Comment:** *..In Figure 4, the efficiency experiments have no results for Transformer models, which is a key limitation of the paper.* 
          - **Feedback to reviewer:** You may want to check Section 3, Figure 5 of the paper which has the Transformer results. See: <quote> In Transformers, the proposed technique provides 25% relative improvement in wall-clock time (Figure 5) </quote>.
        - Example 2:
            - **Reviewer Comment:** *Figure 2. Are people forced to select a choice or could they select 'I don't know'? Did you monitor response times to see if the manipulated images required longer times for individuals to pass decisions? In Appendix A, you mention “the response times will also be released upon publication”, however, I do not see further information about this in the paper.*
            - As the reviewer already refers to the key part of the paper with quotes and asks a question that is not answered in the paper, we do not need to give feedback to this comment.

     2. Look for any vague or unjustifiable claims in the review. This results in points that are not actionable or harder to respond to. For such cases, we would like to nudge the reviewer to provide more specific details and justify their claim.
        - Example 1: 
          - **Reviewer Comment:** *A key limitation is that this paper motivates from the TTT perspective, but no TTT experiments are performed and compared, e.g., comparing with (Sun et al., 2020).*
          - **Feedback to reviewer:** In general, a “TTT experiment” is not a broadly known terminology. It would be very helpful if you could describe what specific experiment you are referring to, and why it is important for this paper.
        - Example 2:
            - **Reviewer Comment:** *W1: Many of the key pieces of this work have been considered before separately. I think some additional discussion of related work is needed. W2: It appears that the linear mode connectivity results may be somewhat brittle.*
            - **Feedback to reviewer:** For W1, It would be very helpful to specify or cite what key pieces are considered in earlier works, how they relate to the current work, and specify what is missing. For W2, could you elaborate on why you see the results as brittle and what would be convincing evidence so that the feedback is more actionable?
        - Example 3:
            - **Reviewer Comment:** *The data and model size are not large-scale, thus the paper will not be impactful.*
            - **Feedback to reviewer:** Please consider putting this claim in context. If possible, please discuss why you consider the experiments to be not large-scale using precedents, why it is important to have larger-scale experiments in this context, and how it would inform or improve the manuscript.
     3. If the reviewer claims the paper lacks novelty, ensure they specify why, including references to similar work. If they haven't, we would like to nudge the reviewer to justify the claim, by prompting them to provide the most relevant references, the relationships, and specifying similarities or differences.
        - Example 1: 
            - **Reviewer Comment:** *.. The paper's novelty is limited considering the ICLR standards, as there are very close works [1,2,3]. *
            - **Feedback to reviewer:** Please consider the reasons for why the novelty is limited, and specify what ICLR standards are in this context. In particular, it would be very helpful to all parties involved if you give examples of the closest papers, their similarities, and differences with the methods or results in the current paper.
        - Example 2:
            - **Reviewer Comment:** *.. DASHA is a mash-up between MARINA and existing distributed nonconvex optimization methods. Other than the fact that three variants of DASHA get rid of the uncompressed synchronization in MARINA, this reviewer could not pinpoint a difference between MARINA and DASHA. As such, the main novelty of this work seems to be in terms of theoretical analysis of MARINA when the uncompressed synchronization step is removed. The authors could have done a better job of clarifying where does this novelty lie in the analysis (e.g., pinpointing the key analytical approaches in the lemma that helped improve the analysis)*
            - As the novelty claim is already well-justified and an actionable question is asked, we do not need to give feedback to this review.
     4. Identify any personal attacks or inappropriate remarks made by the reviewer. This can be about the personality, the knowledge, or the experience of the authors. For example, they call the work "incompetent" without justifying why. For this case, we would like to kindly warn the reviewer about their comment and politely suggest they revise their language.
        - Example 1: 
            - **Reviewer Comment:** *.. The authors clearly do not live in the real world and do not care about people or downstream effects of their research. *
            - **Feedback to the reviewer:** We kindly suggest you revise this comment, as it includes remarks about the personalities or intents of the authors.

3. Provide feedback:
    - For each comment that fails according to the checklist, write concise feedback in the following format:
        - Comment: {{the comment of interest}}
        - Feedback: {{your short feedback}}
    - If you do not identify any issues with a comment, do not include it in your feedback list.
    - If you find no issues in the review at all, respond with: "Thanks for your hard work!"

Remember:
- Be concise, limiting your feedback for each comment to 1-2 sentences.
- Do not summarize your feedback at the end or include a preamble at the beginning.
- Do not repeat anything the reviewer already included in their review.
- Do not mention that you are using a checklist or guidelines.
- Do not address the authors at all or provide suggestions to the authors. You are only giving feedback to the reviewer.
"""

critic_prompt = f"""
You are given a list of feedback about a peer review of a machine learning paper submitted to a top-tier ML conference on OpenReview. The aim of the feedback is to guide a reviewer to make the review high-quality. Your task is to edit the feedback for correctness and clarity.

Here are step-by-step instructions:
1. Read the text of the review, the feedback list provided for that review, and the paper about which the review was written.

2. Evaluate every piece of feedback in the feedback list:
   - For each feedback item, it is imperative that you evaluate the correctness of the feedback. If there is a quote in the feedback, ensure that the quote appears VERBATIM in the paper. You need to check every quote and factual claim in the feedback and edit for correctness, as it is imperative all the feedback is correct.
   - For each feedback item, evaluate if it is clear. You should make sure it would not confuse or frustrate the reviewer who reads it.
   - Make sure every comment-feedback pair addresses an issue with the review, otherwise remove it.

3. Edit comments based on evaluations:
    - Do not add any new points unless the previous feedback obviously missed something important.
    - For each comment that needs editing, re-write the feedback concisely in the following format:
        - Comment: {{the comment of interest}}
        - Feedback: {{your short feedback}}
    - If you do not identify any issues with a comment-feedback pair, do not edit it.

4. Remove any comment-feedback pairs where the feedback is that there is no feedback, no edits needed, or the comment is good. The feedback should only be about edits that need to be made.

Remember:
- Be concise, limiting your feedback for each comment to 1-2 sentences.
- Do not summarize your feedback at the end or include a preamble at the beginning.
- Do not repeat anything the reviewer already included in their review.
- Do not mention that you are using a checklist or guidelines.
- Do not address the authors at all or provide suggestions to the authors. You are only giving feedback to the reviewer.

Here are the guidelines that were followed to generate the feedback list originally; you should adhere to these guidelines: {agent_prompt}.
"""

aggregator_prompt = f"""
You are given multiple lists of feedback about a peer review of a machine learning paper submitted to a top-tier ML conference on OpenReview. The aim of the feedback is to guide a reviewer to make the review high-quality. Your task is to aggregate the lists of feedback into one list.

Here are step-by-step instructions:
1. Read the text of the review, the multiple feedback lists provided for that review, and the paper about which the review was written.

2. For all feedback lists, aggregate them into one list with the best comment-feedback pairs from each list.
    - For each comment-feedback pair in the multiple lists that are similar, determine which provides the best feedback and keep only that pair.
    - If there are unique comment-feedback pairs in the multiple lists, criticially determine if it is an essential piece of feedback needed to improve the review. If it us unncessary or redundant, remove the comment-feedback pair.
    - You should end up with one feedback list that has no repeated comments from the review and that is high quality.
    - Return the feedback list in the format you received it in, where the pairs are formatted as:
        - Comment: {{the comment of interest}}
        - Feedback: {{your short feedback}}

Here are the guidelines that were followed to generate the feedback lists originally: {agent_prompt}.
"""

# Set page to wide mode
st.set_page_config(layout="wide")

# Sidebar for navigation
st.sidebar.title('Input review and PDF or OpenReview ID')

option = st.sidebar.radio(
    "Choose one input option:",
    ("Upload PDF with Review", "OpenReview paper ID"))

# # Add a text input box to the sidebar for user input
# user_input = st.sidebar.text_input("Enter OpenReview ID (e.g. xC8xh2RSs2)")

user_input = False

if option == "OpenReview paper ID":
    paper_id = st.text_input("Enter OpenReview ID (e.g. xC8xh2RSs2):")
    paper_id = paper_id.strip()
    reviewer_id = st.text_input("Enter reviewer ID (e.g. gNxe):")
    reviewer_id = reviewer_id.strip()
    model = st.text_input("Which model? ('gpt' or 'claude')")
    iterations = st.text_input("How many iterations to run agent for?")
    if paper_id and reviewer_id and model and iterations:
        upload_file = False
        user_input = True
        submissions = load_ICLR_submissions()
        review, pdf, pdf_text = parse_openreview_id(submissions, paper_id, reviewer_id)
        # st.write(f"Review text: {pdf_text}")

elif option == "Upload PDF with Review":
    user_text = st.text_area("Enter review:")
    uploaded_file = st.file_uploader("Upload PDF", type="pdf")
    model = st.text_input("Which model? ('gpt' or 'claude')")
    iterations = st.text_input("How many iterations to run agent for?")
    if user_text and uploaded_file and model and iterations:
        upload_file = True
        user_input = True
        review, pdf = user_text, uploaded_file
        pdf_text = pdf_to_text(pdf)

        # st.write(f"Review text: {pdf_text}")
        # st.write(f"You entered review: {user_text}")
        # st.write(f"Uploaded file: {uploaded_file.name}")

if user_input:
    annotations = load_annotations()
    
    feedback_list = []
    # Run your pipeline to generate the dataframe based on user input
    iterations = int(iterations)
    if iterations > 1:
        for _ in range(iterations):
            feedback = create_feedback(review, pdf_text, agent_prompt, model)
            feedback_list.append(feedback)
        best_feedback = aggregator(feedback_list, aggregator_prompt, agent_prompt, review, pdf_text, model)
    else:
        best_feedback = create_feedback(review, pdf_text, agent_prompt, model)
    
    revised_feedback = critic(review, best_feedback, pdf_text, critic_prompt, model)
    
    count, match_count = parse_quotes(revised_feedback, pdf_text)
    
    revised_feedback = revised_feedback.replace("<quote>", "'").replace("</quote>", "'")

    st.title(f'Review feedback')

    if not upload_file and annotations['submission_id'].str.contains(str(paper_id)).any():
        st.write(f'We have a human annotation for {paper_id}')
        # Create four columns
        col1, col2, col3, col4 = st.columns(4)
    
        # Column 1: Display review fields
        with col1:
            st.subheader("Review")
            st.write(json.dumps(review))
    
            if not upload_file:
                pdf_url = f"https://openreview.net/pdf?id={paper_id}"
                st.markdown(f"[Click here to view PDF]({pdf_url})")
    
        # Column 2: Display human annotations
        with col2:
            st.subheader('Human annotations')
            if annotations.loc[(annotations['submission_id'] == paper_id), 'annotation_label'].item() == 1:
                sentences = annotations.loc[(annotations['submission_id'] == paper_id), 'annotation_sentences'].item()
                sentences = ast.literal_eval(sentences)
                st.write(sentences)
                # for s in sentences:
                #     st.write(f"- {s}")
            else:
                st.write("No human feedback on this review")
        
        # Column 3: Display feedback
        with col3:
            st.subheader('Best Feedback')
            st.write(best_feedback)
    
        # Column 4: Display revised feedback (from critic)
        with col4:
            st.subheader('Revised Feedback')
            if count > 0: st.write(f"Quotes found verbatim in pdf text: {match_count}/{count}")
            st.write(revised_feedback)
    else:

        # Create three columns
        col1, col2, col3 = st.columns(3)
    
        # Column 1: Display review fields
        with col1:
            st.subheader("Review")
            st.write(json.dumps(review))
    
            if not upload_file:
                pdf_url = f"https://openreview.net/pdf?id={paper_id}"
                st.markdown(f"[Click here to view PDF]({pdf_url})")
    
        # Column 2: Display feedback
        with col2:
            st.subheader('Best Feedback')
            st.write(best_feedback)
    
        # Column 3: Display revised feedback (from critic)
        with col3:
            st.subheader('Revised Feedback')
            if count > 0: st.write(f"Quotes found verbatim in pdf text: {match_count}/{count}")
            st.write(revised_feedback)

else:
    st.title('Please enter OpenReview ID or upload PDF with review to generate feedback')