Update app.py, score_db.py, and requirements.txt

app.py

@@ -1,7 +1,147 @@
+import base64
+import io
+import random
+from io import BytesIO
+
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import numpy as np
+from PIL import Image
+import requests
+from datasets import load_dataset
 import gradio as gr
 
-def greet(name):
-    return "Hello " + name + "!!"
+from score_db import Battle
+from score_db import Model as ModelEnum, Winner
+
+def make_plot(seismic, predicted_image):
+    fig, ax = plt.subplots(1, 1, figsize=(10, 10))
+    ax.imshow(Image.fromarray(seismic), cmap="gray")
+    ax.imshow(predicted_image, cmap="Reds", alpha=0.5, vmin=0, vmax=1)
+    ax.set_axis_off()
+    fig.canvas.draw()
+
+    # Create a bytes buffer to save the plot
+    buf = io.BytesIO()
+    plt.savefig(buf, format='png', bbox_inches='tight')
+    buf.seek(0)
+
+    # Open the PNG image from the buffer and convert it to a NumPy array
+    image = np.array(Image.open(buf))
+    return image
+
+def call_endpoint(model: ModelEnum, img_array, url: str="https://lukasmosser--seisbase-endpoints-predict.modal.run"):
+    response = requests.post(url, json={"img": img_array.tolist(), "model": model})
+
+    if response:
+        # Parse the base64-encoded image data
+        if response.text.startswith("data:image/tiff;base64,"):
+            img_data_out = base64.b64decode(response.text.split(",")[1])
+            predicted_image = np.array(Image.open(BytesIO(img_data_out)))
+            return predicted_image
+
+def select_random_image(dataset):
+    idx = random.randint(0, len(dataset))
+    return idx, np.array(dataset[idx]["seismic"])
+
+def select_random_models():
+    model_a = random.choice(list(ModelEnum))
+    model_b = random.choice(list(ModelEnum))
+    return model_a, model_b
+
+
+# Create a Gradio interface
+with gr.Blocks() as evaluation:
+    gr.Markdown("""
+    ## Seismic Fault Detection Model Evaluation
+    This application allows you to compare the performance of different seismic fault detection models. 
+    Two models are selected randomly, and their predictions are displayed side by side. 
+    You can choose the better model or mark it as a tie. The results are recorded and used to update the model ratings.
+    """)
+
+    battle = gr.State([])
+    radio = gr.Radio(choices=["Less than 5 years", "5 to 20 years", "more than 20 years"], label="How much experience do you have in seismic fault interpretation?")
+    with gr.Row():
+        output_img1 = gr.Image(label="Model A Image")
+        output_img2 = gr.Image(label="Model B Image")
+
+    def show_images():
+        dataset = load_dataset("porestar/crossdomainfoundationmodeladaption-deepfault", split="valid")
+        idx, image_1 = select_random_image(dataset)
+        model_a, model_b = select_random_models()
+        fault_probability_1 = call_endpoint(model_a, image_1)
+        fault_probability_2 = call_endpoint(model_b, image_1)
+
+        img_1 = make_plot(image_1, fault_probability_1)
+        img_2 = make_plot(image_1, fault_probability_2)
+        experience = 1 
+        if radio.value == "5 to 20 years":
+            experience = 2
+        elif radio.value == "more than 20 years":
+            experience = 3
+        battle.value.append(Battle(model_a=model_a, model_b=model_b, winner="tie", judge="None", experience=experience, image_idx=idx))
+        return img_1, img_2
+    
+    # Define the function to make an API call
+    def make_api_call(choice: Winner):
+        api_url = "https://lukasmosser--seisbase-eval-add-battle.modal.run"
+        battle_out = battle.value 
+        battle_out[-1].winner = choice
+        experience = 1 
+        if radio.value == "5 to 20 years":
+            experience = 2
+        elif radio.value == "more than 20 years":
+            experience = 3
+        battle_out[-1].experience = experience
+        response = requests.post(api_url, json=battle_out[-1].dict())
+
+    # Load images on startup
+    evaluation.load(show_images, inputs=[], outputs=[output_img1, output_img2])
+    
+    with gr.Row():
+        btn_winner_a = gr.Button("Winner Model A")
+        btn_tie = gr.Button("Tie")
+        btn_winner_b = gr.Button("Winner Model B")
+
+    # Define button click events
+    btn_winner_a.click(lambda: make_api_call(Winner.model_a), inputs=[], outputs=[]).then(show_images, inputs=[], outputs=[output_img1, output_img2])
+    btn_tie.click(lambda: make_api_call(Winner.tie), inputs=[], outputs=[]).then(show_images, inputs=[], outputs=[output_img1, output_img2])
+    btn_winner_b.click(lambda: make_api_call(Winner.model_b), inputs=[], outputs=[]).then(show_images, inputs=[], outputs=[output_img1, output_img2])
+
+with gr.Blocks() as leaderboard:
+    def get_results():
+        response = requests.get("https://lukasmosser--seisbase-eval-compute-ratings.modal.run")
+        data = response.json()
+
+        models = [entry["model"] for entry in data]
+        elo_ratings = [entry["elo_rating"] for entry in data]
+
+        fig, ax = plt.subplots()
+        ax.barh(models, elo_ratings, color='skyblue')
+        ax.set_xlabel('ELO Rating')
+        ax.set_title('Model ELO Ratings')
+        plt.tight_layout()
+
+        fig.canvas.draw()
+
+        # Create a bytes buffer to save the plot
+        buf = io.BytesIO()
+        plt.savefig(buf, format='png', bbox_inches='tight')
+        buf.seek(0)
+
+        # Open the PNG image from the buffer and convert it to a NumPy array
+        image = np.array(Image.open(buf))
+        return image
+    
+    with gr.Row():
+        elo_ratings = gr.Image(label="ELO Ratings")
+    
+    leaderboard.load(get_results, inputs=[], outputs=[elo_ratings])
+
+demo = gr.TabbedInterface([evaluation, leaderboard], ["Arena", "Leaderboard"])
+
+# Launch the interface
+if __name__ == "__main__":
+    demo.launch(show_error=True)
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()

requirements.txt

@@ -0,0 +1,5 @@
+matplotlib
+numpy
+gradio
+datasets
+requests

score_db.py

@@ -0,0 +1,143 @@
+import csv
+import io
+import json
+import os
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+from typing import List
+
+import pandas as pd
+from fastapi import Response
+from modal import web_endpoint
+import modal
+from pydantic import BaseModel
+
+from rating import compute_mle_elo
+
+# -----------------------
+# Data Model Definition
+# -----------------------
+class ExperienceEnum(int, Enum):
+    novice = 1
+    intermediate = 2
+    expert = 3
+
+class Winner(str, Enum):
+    model_a = "model_a"
+    model_b = "model_b"
+    tie = "tie"
+
+class Model(str, Enum):
+    porestar_deepfault_unet_baseline_1 = "porestar/deepfault-unet-baseline-1"
+    porestar_deepfault_unet_baseline_2 = "porestar/deepfault-unet-baseline-2"
+
+class Battle(BaseModel):
+    model_a: Model
+    model_b: Model
+    winner: Winner
+    judge: str
+    image_idx: int 
+    experience: ExperienceEnum = ExperienceEnum.novice
+    tstamp: str = str(datetime.now())
+
+class EloRating(BaseModel):
+    model: Model
+    elo_rating: float
+
+# -----------------------
+# Modal Configuration
+# -----------------------
+
+# Create a volume to persist data
+data_volume = modal.Volume.from_name("seisbase-data", create_if_missing=True)
+
+JSON_FILE_PATH = Path("/data/battles.json")
+RESULTS_FILE_PATH = Path("/data/ratings.csv")
+
+app_image = modal.Image.debian_slim(python_version="3.10").pip_install("pandas", "scikit-learn", "tqdm", "sympy")
+
+app = modal.App(
+    image=app_image,
+    name="seisbase-eval",
+    volumes={"/data": data_volume},
+)
+
+def ensure_json_file():
+    """Ensure the JSON file exists and is initialized with an empty array if necessary."""
+    if not os.path.exists(JSON_FILE_PATH):
+        JSON_FILE_PATH.parent.mkdir(parents=True, exist_ok=True)
+        with open(JSON_FILE_PATH, "w") as f:
+            json.dump([], f)
+
+def append_to_json_file(data):
+    """Append data to the JSON file."""
+    ensure_json_file()
+    try:
+        with open(JSON_FILE_PATH, "r+") as f:
+            try:
+                battles = json.load(f)
+            except json.JSONDecodeError:
+                # Reset the file if corrupted
+                battles = []
+            battles.append(data)
+            f.seek(0)
+            json.dump(battles, f, indent=4)
+            f.truncate()
+    except Exception as e:
+        raise RuntimeError(f"Failed to append data to JSON file: {e}")
+
+def read_json_file():
+    """Read data from the JSON file."""
+    ensure_json_file()
+    try:
+        with open(JSON_FILE_PATH, "r") as f:
+            try:
+                return json.load(f)
+            except json.JSONDecodeError:
+                return []  # Return an empty list if the file is corrupted
+    except Exception as e:
+        raise RuntimeError(f"Failed to read JSON file: {e}")
+
+@app.function()
+@web_endpoint(method="POST", docs=True)
+def add_battle(battle: Battle):
+    """Add a new battle to the JSON file."""
+    append_to_json_file(battle.dict())
+    return {"status": "success", "battle": battle.dict()}
+
+
+@app.function()
+@web_endpoint(method="GET", docs=True)
+def export_csv():
+    """Fetch all battles and return as CSV."""
+    battles = read_json_file()
+
+    # Create CSV in memory
+    output = io.StringIO()
+    writer = csv.DictWriter(output, fieldnames=["model_a", "model_b", "winner", "judge", "imaged_idx", "experience", "tstamp"])
+    writer.writeheader()
+    writer.writerows(battles)
+
+    csv_data = output.getvalue()
+    return Response(content=csv_data, media_type="text/csv")
+
+@app.function()
+@web_endpoint(method="GET", docs=True)
+def compute_ratings() -> List[EloRating]:
+    """Compute ratings from battles."""
+    battles = pd.read_json(JSON_FILE_PATH, dtype=[str, str, str, str, int, int, str]).sort_values(ascending=True, by=["tstamp"]).reset_index(drop=True)
+    elo_mle_ratings = compute_mle_elo(battles)
+    elo_mle_ratings.to_csv(RESULTS_FILE_PATH)
+    
+    df = pd.read_csv(RESULTS_FILE_PATH)
+    df.columns = ["Model", "Elo rating"]
+    df = df.sort_values("Elo rating", ascending=False).reset_index(drop=True)
+    scores = []
+    for i in range(len(df)):
+        scores.append(EloRating(model=df["Model"][i], elo_rating=df["Elo rating"][i]))
+    return scores
+
+@app.local_entrypoint()
+def main():
+    print("Local entrypoint running. Check endpoints for functionality.")