app.py

import gradio as gr
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
import pandas as pd
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import snapshot_download

from src.about import (
    CITATION_BUTTON_LABEL,
    CITATION_BUTTON_TEXT,
    EVALUATION_QUEUE_TEXT,
    INTRODUCTION_TEXT,
    LLM_BENCHMARKS_TEXT,
    LLM_BENCHMARKS_TEXT2,
    TABLE_TEXT,
    TITLE,
    EVALUATION_METRIC_TEXT,
)
from src.display.css_html_js import custom_css
from src.display.utils import (
    BENCHMARK_COLS,
    COLS,
    EVAL_COLS,
    EVAL_TYPES,
    TYPES,
    AutoEvalColumn,
    ModelType,
    fields,
    WeightType,
    Precision,
    NUMERIC_INTERVALS,
    QUOTACOLS,
    QUOTATYPES,
    AutoEvalColumnQuota,
    BENCHMARK_QUOTACOLS
)
from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, DYNAMIC_INFO_REPO, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_PATH, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_leaderboard_df_quota
from src.submission.submit import (
    add_new_eval,
    submit_eval_complete
)
from src.scripts.update_all_request_files import update_dynamic_files
from src.tools.collections import update_collections
from src.tools.datastatics import get_statics
#from src.tools.plots import (
#    create_plot_df,
#    create_scores_df,
#)

def restart_space():
    API.restart_space(repo_id=REPO_ID, token=TOKEN)


def init_space():
    print("begin init space")
    ### Space initialisation
    try:
        print(EVAL_REQUESTS_PATH)
        snapshot_download(
            repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
        )
    except Exception:
        restart_space()
    try:
        print(EVAL_RESULTS_PATH)
        snapshot_download(
            repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
        )
    except Exception:
        restart_space()

    print("QUOTACOLS+COLS",list(set(QUOTACOLS+COLS)))
    raw_data, original_df = get_leaderboard_df(
        results_path=EVAL_RESULTS_PATH,
        requests_path=EVAL_REQUESTS_PATH,
        dynamic_path=DYNAMIC_INFO_FILE_PATH,
        #cols=COLS,
        #benchmark_cols=BENCHMARK_COLS,
        cols=list(set(QUOTACOLS+COLS)),
        benchmark_cols=list(set(BENCHMARK_QUOTACOLS+BENCHMARK_COLS))
    )
    #update_collections(original_df.copy())
    leaderboard_df = original_df.copy()

    raw_data_quota, original_df_quota = get_leaderboard_df(
        results_path=EVAL_RESULTS_PATH,
        requests_path=EVAL_REQUESTS_PATH,
        dynamic_path=DYNAMIC_INFO_FILE_PATH,
        cols=list(set(QUOTACOLS+COLS)),
        benchmark_cols=list(set(BENCHMARK_QUOTACOLS+BENCHMARK_COLS))
    )
    #update_collections(original_df.copy())
    leaderboard_df_quota = original_df_quota.copy()

    #plot_df = create_plot_df(create_scores_df(raw_data))

    (
        finished_eval_queue_df,
        running_eval_queue_df,
        pending_eval_queue_df,
    ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)

    #return leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
    return leaderboard_df, original_df, leaderboard_df_quota, original_df_quota,finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df

leaderboard_df, original_df, leaderboard_df_quota, original_df_quota, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()



# Searching and filtering
#type_query: list,
#precision_query: str,
#size_query: list,
#hide_models: list,
def update_table(
    hidden_df: pd.DataFrame,
    columns: list,
    query: str,
):
    print("query", query)
    #filtered_df = filter_models(df=hidden_df, type_query=type_query, size_query=size_query, precision_query=precision_query, hide_models=hide_models)
    filtered_df = filter_queries(query, hidden_df)
    df = select_columns(filtered_df, columns)
    return df


def load_query(request: gr.Request):  # triggered only once at startup => read query parameter if it exists
    query = request.query_params.get("query") or ""
    return query, query # return one for the "search_bar", one for a hidden component that triggers a reload only if value has changed


def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
    return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]


def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
    always_here_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
    dummy_col = [AutoEvalColumn.dummy.name]
        #AutoEvalColumn.model_type_symbol.name,
        #AutoEvalColumn.model.name,
    # We use COLS to maintain sorting
    filtered_df = df[
        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + dummy_col
    ]
    return filtered_df


def filter_queries(query: str, filtered_df: pd.DataFrame):
    """Added by Abishek"""
    final_df = []
    if query != "":
        queries = [q.strip() for q in query.split(";")]
        for _q in queries:
            _q = _q.strip()
            if _q != "":
                temp_filtered_df = search_table(filtered_df, _q)
                if len(temp_filtered_df) > 0:
                    final_df.append(temp_filtered_df)
        if len(final_df) > 0:
            filtered_df = pd.concat(final_df)
            filtered_df = filtered_df.drop_duplicates(
                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
            )

    return filtered_df

def update_table_q(
    hidden_df: pd.DataFrame,
    columns: list,
    query: str,
):
    filtered_df = filter_queriesq(query, hidden_df)
    df = select_columnsq(filtered_df, columns)
    return df


def search_tableq(df: pd.DataFrame, query: str) -> pd.DataFrame:
    return df[(df[AutoEvalColumnQuota.dummy.name].str.contains(query, case=False))]


def select_columnsq(df: pd.DataFrame, columns: list) -> pd.DataFrame:
    always_here_cols = [c.name for c in fields(AutoEvalColumnQuota) if c.never_hidden]
    dummy_col = [AutoEvalColumnQuota.dummy.name]
    # We use COLS to maintain sorting
    filtered_df = df[
        always_here_cols + [c for c in QUOTACOLS if c in df.columns and c in columns] + dummy_col
    ]
    return filtered_df


def filter_queriesq(query: str, filtered_df: pd.DataFrame):
    """Added by Abishek"""
    final_df = []
    if query != "":
        queries = [q.strip() for q in query.split(";")]
        for _q in queries:
            _q = _q.strip()
            if _q != "":
                temp_filtered_df = search_tableq(filtered_df, _q)
                if len(temp_filtered_df) > 0:
                    final_df.append(temp_filtered_df)
        if len(final_df) > 0:
            filtered_df = pd.concat(final_df)
            filtered_df = filtered_df.drop_duplicates(
                subset=[AutoEvalColumnQuota.model.name, AutoEvalColumnQuota.precision.name, AutoEvalColumnQuota.revision.name]
            )

    return filtered_df

def filter_models(
    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, hide_models: list
) -> pd.DataFrame:
    # Show all models
    if "Private or deleted" in hide_models:
        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
    else:
        filtered_df = df

    if "Contains a merge/moerge" in hide_models:
        filtered_df = filtered_df[filtered_df[AutoEvalColumn.merged.name] == False]

    if "MoE" in hide_models:
        filtered_df = filtered_df[filtered_df[AutoEvalColumn.moe.name] == False]

    if "Flagged" in hide_models:
        filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]

    type_emoji = [t[0] for t in type_query]
    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]

    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
    filtered_df = filtered_df.loc[mask]

    return filtered_df

leaderboard_df = filter_models(
    df=leaderboard_df,
    type_query=[t.to_str(" : ") for t in ModelType],
    size_query=list(NUMERIC_INTERVALS.keys()),
    precision_query=[i.value.name for i in Precision],
    hide_models=[], # Deleted, merges, flagged, MoEs
)

leaderboard_df_quota = filter_models(
    df=leaderboard_df_quota,
    type_query=[t.to_str(" : ") for t in ModelType],
    size_query=list(NUMERIC_INTERVALS.keys()),
    precision_query=[i.value.name for i in Precision],
    hide_models=[], # Deleted, merges, flagged, MoEs
)


demo = gr.Blocks(css=custom_css)
with demo:
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏅 EmbodiedVerse-Open", elem_id="vlm-benchmark-tab-table", id=0):
            #leaderboard = init_leaderboard(LEADERBOARD_DF)
            with gr.Row():
                with gr.Column():
                    with gr.Row():
                        search_bar = gr.Textbox(
                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
                            show_label=False,
                            elem_id="search-bar",
                        )
                    with gr.Row():
                        shown_columns = gr.CheckboxGroup(
                            choices=[
                                c.name
                                for c in fields(AutoEvalColumn)
                                if not c.hidden and not c.never_hidden and not c.dummy
                            ],
                            value=[
                                c.name
                                for c in fields(AutoEvalColumn)
                                if c.displayed_by_default and not c.hidden and not c.never_hidden
                            ],
                            label="Select columns to show",
                            elem_id="column-select",
                            interactive=True,
                        )


            leaderboard_table = gr.components.Dataframe(
                value=leaderboard_df[
                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                    + shown_columns.value
                    + [AutoEvalColumnQuota.dummy.name]
                ],
                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                datatype=TYPES,
                elem_id="leaderboard-table",
                interactive=False,
                visible=True,
                #column_widths=["2%", "33%"]
            )

            # Dummy leaderboard for handling the case when the user uses backspace key
            hidden_leaderboard_table_for_search = gr.components.Dataframe(
                value=original_df[COLS],
                #value=leaderboard_df[COLS],
                headers=COLS,
                datatype=TYPES,
                visible=False,
            )
            search_bar.submit(
                update_table,
                [
                    hidden_leaderboard_table_for_search,
                    shown_columns,
                    search_bar,
                ],
                leaderboard_table,
            )

            # Define a hidden component that will trigger a reload only if a query parameter has been set
            hidden_search_bar = gr.Textbox(value="", visible=False)
            hidden_search_bar.change(
                update_table,
                [
                    hidden_leaderboard_table_for_search,
                    shown_columns,
                    search_bar,
                ],
                leaderboard_table,
            )
            # Check query parameter once at startup and update search bar + hidden component
            demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])

            #for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, hide_models]:
            for selector in [shown_columns]:#, filter_columns_type, filter_columns_precision, filter_columns_size]:#, hide_models]:
                selector.change(
                    update_table,
                    [
                        hidden_leaderboard_table_for_search,
                        shown_columns,
                        search_bar,
                    ],
                    leaderboard_table,
                    queue=True,
                )    
 
        with gr.TabItem("🏅 EmbodiedVerse-Open-Sampled", elem_id="vlm-quota-benchmark-tab-table", id=1):
            #leaderboard = init_leaderboard(LEADERBOARD_DF)
            with gr.Row():
                with gr.Column():
                    with gr.Row():
                        search_bar = gr.Textbox(
                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
                            show_label=False,
                            elem_id="search-bar",
                        )
                    with gr.Row():
                        shown_columns = gr.CheckboxGroup(
                            choices=[
                                c.name
                                for c in fields(AutoEvalColumnQuota)
                                if not c.hidden and not c.never_hidden and not c.dummy
                            ],
                            value=[
                                c.name
                                for c in fields(AutoEvalColumnQuota)
                                if c.displayed_by_default and not c.hidden and not c.never_hidden
                            ],
                            label="Select columns to show",
                            elem_id="column-select",
                            interactive=True,
                        )


            leaderboard_table = gr.components.Dataframe(
                value=leaderboard_df_quota[
                    [c.name for c in fields(AutoEvalColumnQuota) if c.never_hidden]
                    + shown_columns.value
                    + [AutoEvalColumnQuota.dummy.name]
                ],
                headers=[c.name for c in fields(AutoEvalColumnQuota) if c.never_hidden] + shown_columns.value,
                datatype=QUOTATYPES,
                elem_id="leaderboard-table",
                interactive=False,
                visible=True,
                #column_widths=["2%", "33%"]
            )

            # Dummy leaderboard for handling the case when the user uses backspace key
            hidden_leaderboard_table_for_search = gr.components.Dataframe(
                value=original_df_quota[QUOTACOLS],
                headers=QUOTACOLS,
                datatype=QUOTATYPES,
                visible=False,
            )
            search_bar.submit(
                update_table_q,
                [
                    hidden_leaderboard_table_for_search,
                    shown_columns,
                    search_bar,
                ],
                leaderboard_table,
            )

            # Define a hidden component that will trigger a reload only if a query parameter has been set
            hidden_search_bar = gr.Textbox(value="", visible=False)
            hidden_search_bar.change(
                update_table_q,
                [
                    hidden_leaderboard_table_for_search,
                    shown_columns,
                    search_bar,
                ],
                leaderboard_table,
            )
            # Check query parameter once at startup and update search bar + hidden component
            demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])

            #for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, hide_models]:
            for selector in [shown_columns]:#, filter_columns_type, filter_columns_precision, filter_columns_size]: #, hide_models]:
                selector.change(
                    update_table_q,
                    [
                        hidden_leaderboard_table_for_search,
                        shown_columns,
                        search_bar,
                    ],
                    leaderboard_table,
                    queue=True,
                )
            gr.Markdown(EVALUATION_METRIC_TEXT, elem_classes="markdown-text")
        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):
            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
            gr.HTML(TABLE_TEXT)
            gr.Markdown(LLM_BENCHMARKS_TEXT2, elem_classes="markdown-text")
        with gr.TabItem("📤 Submit here!", elem_id="submit-model-tab", id=2):
            # 1. Submit your modelinfos here!
            gr.Markdown("✨ Submit your modelinfos here!")
            with gr.Row():
                model_name = gr.Textbox(label="Model Name")
                revision_commit = gr.Textbox(label="Revision commit")
            
            # 2. Submit your API infos here! (API only)
            gr.Markdown("📧 Submit your API infos here! (API only)")
            with gr.Row():
                model_api_url = gr.Textbox(label="Model online api url")
                model_api_key = gr.Textbox(label="Model online api key")
                online_api_model_name = gr.Textbox(label="Online api model name")
            
            # 3. Submit your inference infos here! (inference only)
            gr.Markdown("📧 Submit your inference infos here! (inference only)")
            with gr.Row():
                with gr.Column():
                    gr.Markdown("upload run.sh file")
                    runsh_file = gr.File(file_types=[".sh"], file_count="single")
                with gr.Column():
                    gr.Markdown("upload model_adapter.py file")
                    adapter_file = gr.File(file_types=[".py"], file_count="single")
            
            # 4. Submit Eval 按钮
            submit_btn = gr.Button("Submit Eval")
            submit_output = gr.HTML(label="", visible=True)
            
            # 绑定事件
            submit_btn.click(
                fn=lambda name, rev, url, key, api_name, runsh, adapter: submit_eval_complete(
                    name, rev, url, key, api_name, runsh, adapter
                ),
                inputs=[model_name, revision_commit, model_api_url, model_api_key, 
                       online_api_model_name, runsh_file, adapter_file],
                outputs=submit_output
            )


    with gr.Row():
        with gr.Accordion("📙 Citation", open=False):
            citation_button = gr.Textbox(
                value=CITATION_BUTTON_TEXT,
                label=CITATION_BUTTON_LABEL,
                lines=20,
                elem_id="citation-button",
                show_copy_button=True,
            )

scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=1800)
scheduler.add_job(update_dynamic_files, "cron", minute=30) # launched every hour on the hour
scheduler.add_job(get_statics, 'cron', hour=12, minute=15, timezone='Asia/Shanghai') # 添加定时任务，每天0:30执行一次
scheduler.start()
demo.queue(default_concurrency_limit=40).launch()