import gradio as gr import numpy as np import pandas as pd from constants import * def get_data(verified, dataset, ipc, label_type, metric_weights=None): if metric_weights is None: metric_weights = [1.0 / len(METRICS) for _ in METRICS] if not isinstance(label_type, list): label_type = [label_type] data = pd.read_csv("data.csv") # filter data with no hlr or ior (no nan) data = data.dropna(subset=["hlr", "ior"]) data["verified"] = data["verified"].apply(lambda x: bool(x)) data["dataset"] = data["dataset"].apply(lambda x: DATASET_LIST[x]) data["ipc"] = data["ipc"].apply(lambda x: IPC_LIST[x]) data["label_type"] = data["label_type"].apply(lambda x: LABEL_TYPE_LIST[x]) if verified: data = data[data["verified"] == verified] data = data[data["dataset"] == dataset] data = data[data["ipc"] == ipc] data = data[data["label_type"].apply(lambda x: x in label_type)] if len(data) == 0: return pd.DataFrame(columns=COLUMN_NAMES) # create a new column for the score data["score"] = data[METRICS[0].lower()] * 0.0 for i, metric in enumerate(METRICS): data["score"] += data[metric.lower()] * metric_weights[i] * METRICS_SIGN[i] data["score"] = (np.exp(-0.01 * data["score"]) - np.exp(-1.0)) / (np.exp(1.0) - np.exp(-1.0)) data = data.sort_values(by="score", ascending=False) data["ranking"] = range(1, len(data) + 1) for metric in METRICS: data[metric.lower()] = data[metric.lower()].apply(lambda x: round(x, 3)) data["score"] = data["score"].apply(lambda x: round(x, 3)) # formatting data["method"] = "[" + data["method"] + "](" + data["method_reference"] + ")" data["verified"] = data["verified"].apply(lambda x: "✅" if x else "") data = data.drop(columns=["method_reference", "dataset", "ipc"]) data = data[['ranking', 'method', 'verified', 'date', 'label_type', 'hlr', 'ior', 'score']] if label_type == "Hard Label": data = data.rename(columns={"ranking": "Ranking", "method": "Method", "date": "Date", "label_type": "Label Type", "hlr": "HLR%↓", "ior": "IOR%↑", "score": "DDRS↑", "verified": "Verified"}) else: data = data.rename(columns={"ranking": "Ranking", "method": "Method", "date": "Date", "label_type": "Label Type", "hlr": "HLR%↓", "ior": "IOR%↑", "score": "DDRS↑", "verified": "Verified"}) return data with gr.Blocks() as leaderboard: gr.HTML(LEADERBOARD_HEADER) gr.Markdown(LEADERBOARD_INTRODUCTION) verified = gr.Checkbox( label="Verified by DD-Ranking Team (Uncheck to view all submissions)", value=True, interactive=True ) dataset = gr.Radio( label="Dataset", choices=DATASET_LIST, value=DATASET_LIST[0], interactive=True, ) ipc = gr.Radio( label="IPC", choices=DATASET_IPC_LIST[dataset.value], value=DATASET_IPC_LIST[dataset.value][0], interactive=True, info=IPC_INFO ) label = gr.CheckboxGroup( label="Label Type", choices=LABEL_TYPE_LIST, value=LABEL_TYPE_LIST, info=LABEL_TYPE_INFO, interactive=True, ) with gr.Accordion("Adjust Score Weights", open=False): gr.Markdown(WEIGHT_ADJUSTMENT_INTRODUCTION, latex_delimiters=[ {'left': '$$', 'right': '$$', 'display': True}, {'left': '$', 'right': '$', 'display': False}, {'left': '\\(', 'right': '\\)', 'display': False}, {'left': '\\[', 'right': '\\]', 'display': True} ]) metric_sliders = [] # for metric in METRICS: # metric_sliders.append(gr.Slider(label=f"Weight for {metric}", minimum=0.0, maximum=1.0, value=0.5, interactive=True)) metric_sliders.append( gr.Slider(label=f"Weight for HLR", minimum=0.0, maximum=1.0, value=0.5, interactive=True)) adjust_btn = gr.Button("Adjust Weights") with gr.Accordion("Metric Definitions", open=False): gr.Markdown(METRIC_DEFINITION_INTRODUCTION, latex_delimiters=[ {'left': '$$', 'right': '$$', 'display': True}, {'left': '$', 'right': '$', 'display': False}, {'left': '\\(', 'right': '\\)', 'display': False}, {'left': '\\[', 'right': '\\]', 'display': True} ]) # metric_weights = [s.value for s in metric_sliders] metric_weights = [metric_sliders[0].value, 1.0 - metric_sliders[0].value] board = gr.components.Dataframe( value=get_data(verified.value, dataset.value, ipc.value, label.value, metric_weights), headers=COLUMN_NAMES, type="pandas", datatype=DATA_TITLE_TYPE, interactive=False, visible=True, max_height=500, ) for component in [verified, dataset, ipc, label]: component.change(lambda v, d, i, l, *m: gr.components.Dataframe( value=get_data(v, d, i, l, [m[0], 1.0 - m[0]]), headers=COLUMN_NAMES, type="pandas", datatype=DATA_TITLE_TYPE, interactive=False, visible=True, max_height=500, ), inputs=[verified, dataset, ipc, label] + metric_sliders, outputs=board) dataset.change(lambda d, i: gr.Radio( label="IPC", choices=DATASET_IPC_LIST[d], value=i if i in DATASET_IPC_LIST[d] else DATASET_IPC_LIST[d][0], interactive=True, info=IPC_INFO ), inputs=[dataset, ipc], outputs=ipc) adjust_btn.click(fn=lambda v, d, i, l, *m: gr.components.Dataframe( value=get_data(v, d, i, l, [m[0], 1.0 - m[0]]), headers=COLUMN_NAMES, type="pandas", datatype=DATA_TITLE_TYPE, interactive=False, visible=True, max_height=500, ), inputs=[verified, dataset, ipc, label] + metric_sliders, outputs=board) citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, elem_id="citation-button", lines=6, show_copy_button=True, ) leaderboard.launch()