app.py

import gradio as gr
from gradio_leaderboard import Leaderboard
import plotly.express as px
from pathlib import Path
import pandas as pd
import numpy as np
abs_path = Path(__file__).parent

def parse_model_args(model_args):
    if "deltazip" in model_args:
        model_args = model_args.split("deltazip")[1]
        model_args = model_args.split(",")[0]
        model_args = model_args.strip(".")
        model_args = model_args.replace(".", "/")
    if "espressor/" in model_args:
        model_args = model_args.split("espressor/")[1]
        model_args = model_args.split(",")[0]
        model_args = model_args.strip(".")
        model_args = model_args.replace(".", "/",1)
        model_args = model_args.split("_")[0]
    else:
        model_args = model_args.split(",")[0]
        model_args = model_args.replace("pretrained=", "")
    return model_args

def parse_model_precision(model_args):
    if "espressor" in model_args:
        if 'W8A8_int8' in model_args:
            precision = 'W8A8_int8'
        else:
            precision = model_args.split("_")[-1]
    else:
        precision = "Default"
    return precision

# Any pandas-compatible data
df = pd.read_csv(str(abs_path / "eval_results.csv"))
# take acc only
df = df[df['metric'] == 'acc']
# dedup
df = df.drop_duplicates(subset=['model', 'task'])
# pivot df, such that the column names are model,task,efficiency
# but keep precision in its original place
df = df.pivot(index='model', columns='task', values='value').reset_index()

df['precision'] = df['model'].apply(lambda x: x.split(":")[-1])
df['model'] = df['model'].apply(lambda x: x.split(":")[0])

# average over all columns starting with 'task_'
df['avg_acc'] = df.filter(like='task_').mean(axis=1)
# keep 2 decimal points for avg_acc, and all tasks_
# rename columns starting with 'task_' by removing 'task_'
df = df.rename(columns=lambda x: x.replace('task_', ''))
numeric_columns = df.select_dtypes(include=[np.number]).columns
df[numeric_columns] = (df[numeric_columns]*100).round(2)

with gr.Blocks() as demo:
    gr.Markdown("""
    # 🥇 Efficient LLM Leaderboard
    """)
    task_options = [col for col in df.columns if col not in ['model', 'precision']]

    with gr.Row():
        selected_tasks = gr.CheckboxGroup(choices=task_options, label="Select Tasks")
    with gr.Row():
        accuracy_plot = gr.Plot(label="Accuracy Plot")
        data_table = gr.Dataframe(value=df, label="Result Table")

    def update_outputs(selected_tasks):
        if not selected_tasks:
            return df[['model', 'precision']], None
        filtered_df = df[['model', 'precision'] + selected_tasks]
        melted_df = filtered_df.melt(id_vars=['model', 'precision'], var_name='task', value_name='accuracy')
        fig = px.bar(melted_df, x='model', y='accuracy', color='precision', barmode='group', facet_col='task')
        return filtered_df, fig

    selected_tasks.change(fn=update_outputs, inputs=selected_tasks, outputs=[data_table, accuracy_plot])


if __name__ == "__main__":
    demo.launch()