app.py

import os
from datetime import datetime

import gradio as gr
import pandas as pd

# -----------------------------------------------------------------------------
# Configuration – adjust these paths to point at your data location
# -----------------------------------------------------------------------------
DATA_PATH = "human_judgement/selected_samples.json"  # CSV with columns: question, answer1, answer2
RATINGS_PATH = (
    "human_judgement/human_judgement.csv"  # File where user ratings will be appended
)

# -----------------------------------------------------------------------------
# Helper functions
# -----------------------------------------------------------------------------


def load_data(path: str = DATA_PATH) -> pd.DataFrame:
    """Load the Q/A pairs once and cache them inside gradio runtime."""
    if not os.path.exists(path):
        raise FileNotFoundError(f"Could not find data file at {path}.")
    df = pd.read_json(path, lines=True)
    expected_cols = {"question", "response1", "response2"}
    if not expected_cols.issubset(df.columns):
        raise ValueError(f"CSV file must contain columns: {', '.join(expected_cols)}")
    return df


def load_ratings(path: str = RATINGS_PATH) -> pd.DataFrame:
    """Load the ratings file (creates an empty one if absent)."""
    if os.path.exists(path):
        return pd.read_csv(path)
    return pd.DataFrame(columns=["user_id", "row_index", "choice", "timestamp"])


def save_rating(user_id: str, row_index: int, choice: int, path: str = RATINGS_PATH):
    """Append a single rating row to disk, avoiding accidental duplicates."""
    ratings = load_ratings(path)

    # Prevent duplicate entries for the same user/question pair
    duplicate = (ratings.user_id == user_id) & (ratings.row_index == row_index)
    if duplicate.any():
        return  # already stored, nothing to do

    new_entry = {
        "user_id": user_id,
        "row_index": row_index,
        "choice": choice,  # 1 means answer1 preferred, 2 means answer2 preferred
        "timestamp": datetime.utcnow().isoformat(),
    }
    ratings = pd.concat([ratings, pd.DataFrame([new_entry])], ignore_index=True)
    ratings.to_csv(path, index=False)


def get_next_unrated(df: pd.DataFrame, ratings: pd.DataFrame, user_id: str):
    """Return (row_index, question, answer1, answer2) or None if finished."""
    rated_indices = ratings.loc[ratings.user_id == user_id, "row_index"].tolist()
    unrated_df = df[~df.index.isin(rated_indices)]
    if unrated_df.empty:
        return None
    row = unrated_df.iloc[0]
    return row.name, row.question, row.response1, row.response2


# -----------------------------------------------------------------------------
# Gradio callbacks
# -----------------------------------------------------------------------------


def start_or_resume(user_id: str, state_df):
    """Initialise or resume a session for a given user id."""
    if not user_id.strip():
        return (
            gr.update(visible=True),
            gr.update(visible=False),
            gr.update(visible=False),
            "",
            "",
            "",
            "",
            "Please enter a non‑empty identifier to begin.",
        )

    ratings = load_ratings()
    record = get_next_unrated(state_df, ratings, user_id)
    if record is None:
        # Completed all tasks
        return (
            gr.update(visible=True),
            gr.update(visible=False),
            gr.update(visible=False),
            "",
            "",
            "",
            "",
            "🎉 You have evaluated every item – thank you!",
        )

    idx, q, a1, a2 = record
    return (
        gr.update(visible=True),  # keep user id input visible for reference
        gr.update(visible=True),  # show evaluation section
        gr.update(visible=True),  # enable submit button
        "**" + q + "**",
        a1,
        a2,
        str(idx),
        "",
    )


def submit_preference(user_id: str, row_idx_str: str, choice: str, state_df):
    """Handle a single preference submission and load the next question."""
    if choice not in {"answer1", "answer2"}:
        return gr.update(
            value="Please choose either Answer 1 or Answer 2 before submitting."
        )

    row_idx = int(row_idx_str)
    save_rating(user_id, row_idx, 1 if choice == "answer1" else 2)

    ratings = load_ratings()
    record = get_next_unrated(state_df, ratings, user_id)
    if record is None:
        return "", "", "", "", "🎉 You have evaluated every item – thank you!"

    idx, q, a1, a2 = record
    return "**" + q + "**", a1, a2, str(idx), ""


# -----------------------------------------------------------------------------
# Build Gradio interface
# -----------------------------------------------------------------------------


def build_demo():
    df = load_data()

    with gr.Blocks(title="Question/Answer Preference Rater") as demo:
        gr.Markdown(
            """# Q/A Preference Rater
        Enter your identifier below to start or resume your evaluation session. For every question, select which answer you prefer. Your progress is saved automatically so you can return at any time using the **same identifier**."""
        )

        state_df = gr.State(df)  # keep dataset in memory for callbacks
        state_row_idx = gr.State("")

        # User identifier section
        id_input = gr.Textbox(
            label="User Identifier", placeholder="e.g. Alice", scale=3
        )
        start_btn = gr.Button("Start / Resume", scale=1)

        # Feedback / status message
        info_md = gr.Markdown("", visible=True)

        # Evaluation section (initially hidden)
        with gr.Column(visible=False) as eval_col:
            question_md = gr.Markdown("", label="Question")
            with gr.Row():
                # answer1_box = gr.Textbox(
                #     label="Answer\u00a01", interactive=False, lines=10
                # )
                # answer2_box = gr.Textbox(
                #     label="Answer\u00a02", interactive=False, lines=10
                # )
                answer1_box = gr.Markdown(label="Answer 1")
                answer2_box = gr.Markdown(label="Answer 2")
            choice_radio = gr.Radio(
                ["answer1", "answer2"],
                label="Which answer do you prefer?",
                interactive=True,
            )
        submit_btn = gr.Button("Submit Preference", visible=False)

        # Wire callbacks
        start_btn.click(
            fn=start_or_resume,
            inputs=[id_input, state_df],
            outputs=[
                id_input,
                eval_col,
                submit_btn,
                question_md,
                answer1_box,
                answer2_box,
                state_row_idx,
                info_md,
            ],
        )

        submit_btn.click(
            fn=submit_preference,
            inputs=[id_input, state_row_idx, choice_radio, state_df],
            outputs=[question_md, answer1_box, answer2_box, state_row_idx, info_md],
        )

    return demo


# if __name__ == "__main__":
#     build_demo().launch()
build_demo().launch()