app.py

import gradio as gr
import pandas as pd

from src.about import (
    CITATION_BUTTON_LABEL,
    CITATION_BUTTON_TEXT,
    EVALUATION_QUEUE_TEXT,
    INTRODUCTION_TEXT,
    LLM_BENCHMARKS_TEXT,
    TITLE,
)

data = {
    "Method": [
        "Human Performance (Handwritten LOTUS Llama-3.1-70B)",
        "Zero-shot Text2SQL (Llama-3.1-70B)",
        "Zero-shot Text2SQL + LM Generation (Llama-3.1-70B)",
        "RAG (E5 + Llama-3.1-70B)",
        "RAG (E5) + LM Rerank (Llama-3.1-70B)",
        "Human Performance (Handwritten LOTUS GPT-4o)",
        "Zero-shot Text2SQL (GPT-4o)",
       "Zero-shot Text2SQL + LM Generation (GPT-4o)",
        "RAG (E5 + GPT-4o)",
        "RAG (E5) + LM Rerank (GPT-4o)",
        "Human Performance (Handwritten LOTUS o3-mini)",
        "Zero-shot Text2SQL (o3-mini)",
        "Zero-shot Text2SQL + LM Generation (o3-mini)",
        "RAG (E5 + o3-mini)",
        "RAG (E5) + LM Rerank (o3-mini)",
        "Zero-shot Text2SQL (Deepseek-R1)",
        "Zero-shot Text2SQL + LM Generation (Deepseek-R1)",
    ],
    # "Model": ["meta-llama/Llama-3.1-70B"] * 5,
    "Execution Accuracy": [55.0, 17.0, 13.0, 0.0, 2.0, 55.0, 18.0, 15.0, 3.0, 3.0, 65.0, 18.0, 30.0, 7.0, 7.0, 12.0, 0.0],
    # "Execution Accuracy": [0.0, 2.0, 55.0, 18.0, 3.0, 3.0, 65.0, 18.0, 7.0, 7.0, 12.0],
}

leaderboard_df = pd.DataFrame(data)

leaderboard_df = leaderboard_df.sort_values(
    "Execution Accuracy", ascending=False
).reset_index(drop=True)
leaderboard_df.insert(0, "Rank", leaderboard_df.index - 2)
leaderboard_df.loc[0, "Rank"] = ""
leaderboard_df.loc[1, "Rank"] = ""
leaderboard_df.loc[2, "Rank"] = ""

def hyperlink_method(method):
    base_url = "https://github.com/TAG-Research/TAG-Bench/tree/main"
    return f'<a href="{base_url}" target="_blank">{method}</a>'

def hyperlink_model(model):
    base_url = "https://huggingface.co/meta-llama/Llama-3.1-70B"
    return f'<a href="{base_url}" target="_blank">{model}</a>'

leaderboard_df["Method"] = leaderboard_df["Method"].apply(hyperlink_method)

def highlight_row(row):
    if row["Rank"] == "":  # First row
        return ["background-color: #d4edda; font-weight: bold;" for _ in row]
    return [""] * len(row)


# Apply the style
leaderboard_df = leaderboard_df.style.apply(highlight_row, axis=1)



# leaderboard_df["Model"] = leaderboard_df["Model"].apply(hyperlink_model)


with gr.Blocks() as demo:
    gr.HTML(
        """
        <div style="text-align: center;">
            <h1 style="font-size: 2.5rem; margin-bottom: 0.5rem;">TAG Leaderboard</h1>
            <p style="font-size: 1.25rem; color: gray;">A benchmark for natural language queries over data</p>
        </div>
        """
    )

    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
            with gr.Row():
                gr.Dataframe(
                    value=leaderboard_df,
                    headers=["Rank", "Method", "Execution Accuracy"],
                    datatype=["str", "html",  "number"],
                    row_count=(5, "dynamic"),
                    wrap=True,
                    elem_id="leaderboard",
                    type="pandas"
                )

        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

        with gr.TabItem("🚀 Submission Instructions ", elem_id="llm-benchmark-tab-table", id=3):
            with gr.Accordion("1️⃣ Required Materials", open=True):
                gr.Markdown(
                    """
                    Ensure the following files are included in your submission:
                    - **output.json**: File containing the evaluation outputs generated by your model. Please refer to [] for format instructions.
                    - **requirements.txt**: A list of dependencies needed to run your model or script.
                    - **README.md**: A detailed description of your submission, including:
                        - Purpose and overview of the submission.
                        - Instructions to reproduce the results.
                        - Any additional notes for evaluators.
                    - **Model/Keys**: Upload your models or API keys to [Hugging Face](https://huggingface.co/) if they are not publicly accessible.
                    
                    **Note**: Submissions missing any of these materials will not be processed.
                    """
                )

            # Section 2: Submission Frequency
            with gr.Accordion("2️⃣ Submission Frequency", open=True):
                gr.Markdown(
                    """
                    - Submissions are accepted **once a month** to ensure sufficient evaluation bandwidth.
                    - Plan your submission timeline accordingly to avoid delays.
                    """
                )

            # Section 3: How to Upload Materials
            with gr.Accordion("3️⃣ How to Upload Materials", open=True):
                gr.Markdown(
                    """
                    Follow these steps to upload your materials:
                    1. Compress all files in the code into a single `.zip` file, or provide a public repository to refer to.
                    2. Email the `.zip` file or repositoty link to our email [email protected].
                    """
                )

            # Section 4: Submission Process
            with gr.Accordion("4️⃣ Submission Process", open=True):
                gr.Markdown(
                    """
                    After uploading your materials:
                    - 
                    - Provide accurate contact information for follow-ups.
                    - Double-check your materials for completeness to avoid processing delays.
                    
                    **Important:** Your submission will be added to the evaluation queue. Depending on the queue size, evaluations may take up to a few weeks.
                    """
                )

            # Footer
            gr.Markdown(
                """
                <div style="text-align: center; margin-top: 2rem;">
                    For further assistance, reach out to [email protected] with questions.
                </div>
                """
            )


demo.launch()