import os import tempfile from datetime import datetime from pathlib import Path import gradio as gr import pandas as pd from huggingface_hub import HfApi, hf_hub_download # ------------------------------------------------------------ # Cloud‑friendly Q/A preference rater for **Hugging Face Spaces** # ------------------------------------------------------------ # This version swaps local CSV persistence for a tiny remote‑dataset # workflow that works on Spaces: # • Ratings are stored in (and loaded from) a lightweight **dataset # repo** on the Hugging Face Hub – no local file system required. # • The dataset repo is set via the `RATINGS_REPO` env‑var. # • You must pass a write‑enabled token (env‑var `HF_TOKEN`) that has # `write` permission on that dataset. # # Quick setup guide # ----------------- # 1. Create a dataset repository to hold the ratings file, e.g.: # https://huggingface.co/datasets//qa‑rater‑data # 2. Inside **Space Settings ▸ Secrets**, add: # • `RATINGS_REPO` → /qa‑rater‑data # • `HF_TOKEN` → a token with *Write* access to that repo # 3. Add `huggingface‑hub` to your `requirements.txt` or # `pip install huggingface‑hub` locally. # 4. Deploy / push your updated Space – ratings will now persist in # the dataset repo instead of the Space’s ephemeral storage. # ------------------------------------------------------------ # ----------------------------------------------------------------------------- # Configuration – constants & styling # ----------------------------------------------------------------------------- DATA_PATH = "human_judgement/selected_samples.json" RATINGS_FILE = "human_judgement/human_judgement.csv" # Name *inside* the dataset repo RATINGS_REPO = os.getenv("RATINGS_REPO") # e.g. "org/qa‑rater‑data" HF_TOKEN = os.getenv("HF_TOKEN") # write token for that repo MAX_HEIGHT_PX = 400 # Max visible height for answer Markdown blocks api = HfApi(token=HF_TOKEN) if HF_TOKEN else None # ----------------------------------------------------------------------------- # Helper functions – data I/O # ----------------------------------------------------------------------------- def load_data(path: str = DATA_PATH) -> pd.DataFrame: """Local read for the static Q/A CSV bundled with the Space repo.""" if not os.path.exists(path): raise FileNotFoundError( f"Could not find data file at {path} – did you upload it?" ) df = pd.read_json(path, lines=True) required = {"question", "response1", "response2"} if not required.issubset(df.columns): raise ValueError(f"CSV must contain columns: {', '.join(required)}") return df # ---------- Rating persistence helpers --------------------------------------- def _download_remote_ratings() -> Path | None: """Try to fetch the current ratings file from the Hub; returns path or None.""" if not RATINGS_REPO: return None try: return Path( hf_hub_download( repo_id=RATINGS_REPO, filename=RATINGS_FILE, repo_type="dataset", token=HF_TOKEN, cache_dir=tempfile.gettempdir(), ) ) except Exception: # File/repo may not exist yet – caller will create empty DF. return None def load_ratings() -> pd.DataFrame: """Return ratings DataFrame from remote repo (or empty if none).""" remote = _download_remote_ratings() if remote and remote.exists(): return pd.read_csv(remote) return pd.DataFrame(columns=["user_id", "row_index", "choice", "timestamp"]) def _upload_remote_ratings(df: pd.DataFrame): """Upload CSV to the dataset repo with a commit per save.""" if not (RATINGS_REPO and api): # Running locally (dev) – save to a temp file for inspection. df.to_csv(RATINGS_FILE, index=False) return with tempfile.TemporaryDirectory() as tmpdir: csv_path = Path(tmpdir) / RATINGS_FILE csv_path.parent.mkdir(parents=True, exist_ok=True) df.to_csv(csv_path, index=False) api.upload_file( path_or_fileobj=str(csv_path), path_in_repo=RATINGS_FILE, repo_id=RATINGS_REPO, repo_type="dataset", commit_message="Add/Update rating", ) def save_rating(user_id: str, row_index: int, choice: int): """Append a rating (deduplicated) and push to the Hub.""" ratings = load_ratings() duplicate = (ratings.user_id == user_id) & (ratings.row_index == row_index) if duplicate.any(): return # already stored new_entry = { "user_id": user_id, "row_index": row_index, "choice": choice, "timestamp": datetime.utcnow().isoformat(), } ratings = pd.concat([ratings, pd.DataFrame([new_entry])], ignore_index=True) _upload_remote_ratings(ratings) def get_next_unrated(df: pd.DataFrame, ratings: pd.DataFrame, user_id: str): rated = ratings.loc[ratings.user_id == user_id, "row_index"].tolist() unrated = df[~df.index.isin(rated)] if unrated.empty: return None row = unrated.iloc[0] return row.name, row.question, row.response1, row.response2 # ----------------------------------------------------------------------------- # Gradio callbacks # ----------------------------------------------------------------------------- def start_or_resume(user_id: str, state_df): if not user_id.strip(): return ( gr.update(value=user_id, visible=True), gr.update(visible=False), # eval_col gr.update(visible=False), # submit_btn "", "", "", "", # q, a1, a2, idx "Please enter a non-empty identifier to begin.", ) ratings = load_ratings() record = get_next_unrated(state_df, ratings, user_id) if record is None: return ( gr.update(value=user_id, visible=True), gr.update(visible=False), gr.update(visible=False), "", "", "", "", "🎉 You have evaluated every item – thank you!", ) idx, q, a1, a2 = record return ( gr.update(value=user_id, visible=True), gr.update(visible=True), # eval_col gr.update(visible=True), # submit_btn "**" + q + "**", a1, a2, str(idx), "", ) def submit_preference(user_id: str, row_idx_str: str, choice: str, state_df): if choice not in {"answer1", "answer2"}: return ( "", "", "", "", "Please choose either Answer 1 or Answer 2 before submitting.", ) row_idx = int(row_idx_str) save_rating(user_id, row_idx, 1 if choice == "answer1" else 2) ratings = load_ratings() record = get_next_unrated(state_df, ratings, user_id) if record is None: return "", "", "", "", "🎉 You have evaluated every item – thank you!" idx, q, a1, a2 = record return "**" + q + "**", a1, a2, str(idx), "" # ----------------------------------------------------------------------------- # Build Gradio interface # ----------------------------------------------------------------------------- def build_demo(): df = load_data() # CSS to constrain very tall answers overflow_css = f""" """ with gr.Blocks(title="Question/Answer Preference Rater") as demo: gr.HTML(overflow_css) gr.Markdown( """# Q/A Preference Rater\nEnter your identifier below to start or resume. For every question, select which answer you prefer. Your progress is saved automatically so you can return at any time using the same identifier.""" ) state_df = gr.State(df) state_row_idx = gr.State("") # Identifier input id_input = gr.Textbox(label="User Identifier", placeholder="e.g. alice") start_btn = gr.Button("Start / Resume") info_md = gr.Markdown("") # Evaluation widgets with gr.Column(visible=False) as eval_col: question_md = gr.Markdown("") with gr.Row(): answer1_md = gr.Markdown(label="Answer 1", elem_classes=["answerbox"]) answer2_md = gr.Markdown(label="Answer 2", elem_classes=["answerbox"]) choice_radio = gr.Radio( ["answer1", "answer2"], label="Which answer do you prefer?" ) submit_btn = gr.Button("Submit Preference", visible=False) # Callbacks wiring start_btn.click( fn=start_or_resume, inputs=[id_input, state_df], outputs=[ id_input, eval_col, submit_btn, question_md, answer1_md, answer2_md, state_row_idx, info_md, ], ) submit_btn.click( fn=submit_preference, inputs=[id_input, state_row_idx, choice_radio, state_df], outputs=[question_md, answer1_md, answer2_md, state_row_idx, info_md], ) return demo if __name__ == "__main__": build_demo().launch()