Spaces:
Running
on
Zero
Running
on
Zero
File size: 3,770 Bytes
3427608 b6dc501 b257e01 3427608 b39062c 3427608 a4b32da 4e4bd20 3427608 4e4bd20 3427608 ffc77cb 3427608 a4b32da b6dc501 a4b32da b257e01 a4b32da 3427608 a4b32da 3427608 5cff698 3427608 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 |
import gradio as gr
import os
from serve.gradio_web import *
from serve.gradio_web_video import build_side_by_side_video_ui_anony
from serve.leaderboard import build_leaderboard_tab, build_leaderboard_video_tab, build_leaderboard_contributor
from model.model_manager import ModelManager
from pathlib import Path
from serve.constants import SERVER_PORT, ROOT_PATH, ELO_RESULTS_DIR
def make_default_md():
link_color = "#1976D2" # This color should be clear in both light and dark mode
leaderboard_md = f"""
# 🏅 K-Sort Arena: Efficient and Reliable Benchmarking for Generative Models via K-wise Human Preferences
### [Paper](https://arxiv.org/abs/2408.14468) | [Twitter](https://x.com/_akhaliq/status/1828280979242320014)
- ⚡ For vision tasks, K-wise comparisons can provide much richer info but only take similar time as pairwise comparisons.
- 🎯 Well designed matchmaking algorithm can further save human efforts than random match pairing in normal Arena.
- 📈 Probabilistic modeling can obtain a faster and more stable convergence than Elo scoring system.
"""
return leaderboard_md
def build_combine_demo(models, elo_results_file, leaderboard_table_file):
with gr.Blocks(
title="Play with Open Vision Models",
theme=gr.themes.Default(),
css=block_css,
) as demo:
with gr.Blocks():
md = make_default_md()
md_default = gr.Markdown(md, elem_id="default_leaderboard_markdown")
with gr.Tabs() as tabs_combine:
with gr.Tab("Image Generation", id=0):
with gr.Tabs() as tabs_ig:
with gr.Tab("Generation Leaderboard", id=0):
build_leaderboard_tab()
with gr.Tab("Generation Arena (battle)", id=1):
build_side_by_side_ui_anony(models)
with gr.Tab("Video Generation", id=1):
with gr.Tabs() as tabs_ig:
with gr.Tab("Generation Leaderboard", id=0):
build_leaderboard_video_tab()
with gr.Tab("Generation Arena (battle)", id=1):
build_side_by_side_video_ui_anony(models)
with gr.Tab("Contributor", id=2):
build_leaderboard_contributor()
return demo
def load_elo_results(elo_results_dir):
from collections import defaultdict
elo_results_file = defaultdict(lambda: None)
leaderboard_table_file = defaultdict(lambda: None)
if elo_results_dir is not None:
elo_results_dir = Path(elo_results_dir)
elo_results_file = {}
leaderboard_table_file = {}
for file in elo_results_dir.glob('elo_results_*.pkl'):
if 't2i_generation' in file.name:
elo_results_file['t2i_generation'] = file
# else:
# raise ValueError(f"Unknown file name: {file.name}")
for file in elo_results_dir.glob('*_leaderboard.csv'):
if 't2i_generation' in file.name:
leaderboard_table_file['t2i_generation'] = file
# else:
# raise ValueError(f"Unknown file name: {file.name}")
return elo_results_file, leaderboard_table_file
if __name__ == "__main__":
server_port = int(SERVER_PORT)
root_path = ROOT_PATH
elo_results_dir = ELO_RESULTS_DIR
models = ModelManager()
elo_results_file, leaderboard_table_file = load_elo_results(elo_results_dir)
demo = build_combine_demo(models, elo_results_file, leaderboard_table_file)
demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, share=True)
# demo.launch(server_name="0.0.0.0", server_port=7860, root_path=ROOT_PATH) |