File size: 3,039 Bytes
a58e1d4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
import gradio as gr
from gradio_leaderboard import Leaderboard
import plotly.express as px
from pathlib import Path
import pandas as pd
import numpy as np
abs_path = Path(__file__).parent
def parse_model_args(model_args):
if "deltazip" in model_args:
model_args = model_args.split("deltazip")[1]
model_args = model_args.split(",")[0]
model_args = model_args.strip(".")
model_args = model_args.replace(".", "/")
if "espressor/" in model_args:
model_args = model_args.split("espressor/")[1]
model_args = model_args.split(",")[0]
model_args = model_args.strip(".")
model_args = model_args.replace(".", "/",1)
model_args = model_args.split("_")[0]
else:
model_args = model_args.split(",")[0]
model_args = model_args.replace("pretrained=", "")
return model_args
def parse_model_precision(model_args):
if "espressor" in model_args:
if 'W8A8_int8' in model_args:
precision = 'W8A8_int8'
else:
precision = model_args.split("_")[-1]
else:
precision = "Default"
return precision
# Any pandas-compatible data
df = pd.read_csv(str(abs_path / "eval_results.csv"))
# take acc only
df = df[df['metric'] == 'acc']
# dedup
df = df.drop_duplicates(subset=['model', 'task'])
# pivot df, such that the column names are model,task,efficiency
# but keep precision in its original place
df = df.pivot(index='model', columns='task', values='value').reset_index()
df['precision'] = df['model'].apply(lambda x: x.split(":")[-1])
df['model'] = df['model'].apply(lambda x: x.split(":")[0])
# average over all columns starting with 'task_'
df['avg_acc'] = df.filter(like='task_').mean(axis=1)
# keep 2 decimal points for avg_acc, and all tasks_
# rename columns starting with 'task_' by removing 'task_'
df = df.rename(columns=lambda x: x.replace('task_', ''))
numeric_columns = df.select_dtypes(include=[np.number]).columns
df[numeric_columns] = (df[numeric_columns]*100).round(2)
with gr.Blocks() as demo:
gr.Markdown("""
# 🥇 Efficient LLM Leaderboard
""")
task_options = [col for col in df.columns if col not in ['model', 'precision']]
with gr.Row():
selected_tasks = gr.CheckboxGroup(choices=task_options, label="Select Tasks")
with gr.Row():
accuracy_plot = gr.Plot(label="Accuracy Plot")
data_table = gr.Dataframe(value=df, label="Result Table")
def update_outputs(selected_tasks):
if not selected_tasks:
return df[['model', 'precision']], None
filtered_df = df[['model', 'precision'] + selected_tasks]
melted_df = filtered_df.melt(id_vars=['model', 'precision'], var_name='task', value_name='accuracy')
fig = px.bar(melted_df, x='model', y='accuracy', color='precision', barmode='group', facet_col='task')
return filtered_df, fig
selected_tasks.change(fn=update_outputs, inputs=selected_tasks, outputs=[data_table, accuracy_plot])
if __name__ == "__main__":
demo.launch() |