Set up arena

README.md

@@ -1,13 +1,13 @@
 ---
 title: 3D Animation Arena
-emoji: 🌍
-colorFrom: gray
+emoji: 🐠
+colorFrom: green
 colorTo: gray
 sdk: gradio
 sdk_version: 5.27.0
 app_file: app.py
 pinned: false
-short_description: Arena to rank human 3D Animation models
+short_description: Arena to rank 3D animation models
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,362 @@
+import gradio as gr
+from pathlib import Path
+import uuid
+import random
+
+from utils.data_utils import generate_leaderboard
+from utils.plot_utils import plot_ratings
+from utils.utils import simulate, submit_rating, generate_matchup
+from config import MODE, VIDEOS, MODELS, CRITERIA, default_beta
+
+
+head = f"""
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/plotly.js/1.33.1/plotly.min.js"></script>
+    <script>{Path('static/modelViewer.js').read_text()}</script>
+    <script>{Path('static/popup.js').read_text()}</script>
+    <script>{Path('static/plots.js').read_text()}</script>
+"""
+
+with gr.Blocks(title='3D Animation Arena', head=head, css_paths='static/style.css') as arena:
+
+    sessionState = gr.State({
+        'video': None,
+        'modelLeft': None,
+        'modelRight': None,
+        'darkMode': False,
+        'videos': VIDEOS,
+        'currentTab': CRITERIA[0],
+        'uuid': None
+    })
+    
+    frontState = gr.JSON(sessionState, visible=False)
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.HTML('')
+        with gr.Column(scale=12):
+            gr.HTML("<h1 style='text-align:center; font-size:50px'>3D Animation Arena</h1>")
+        with gr.Column(scale=1):
+            toggle_dark = gr.Button(value="Dark Mode")
+
+        def update_toggle_dark(state):
+            state['darkMode'] = not state['darkMode']
+            if state['darkMode']:
+                return gr.update(value="Light Mode"), state
+            else:
+                return gr.update(value="Dark Mode"), state
+
+        toggle_dark.click(
+            inputs=[sessionState],
+            js="""
+            () => {
+                document.body.classList.toggle('dark');
+            }
+            """,
+            fn=update_toggle_dark,
+            outputs=[toggle_dark, sessionState]
+        )
+
+    with gr.Tab(label='Arena'):
+        models = gr.HTML('''
+            <div class="viewer-container">
+                <iframe
+                    id="modelViewerLeft" 
+                    src="https://d39vhmln1nnc4z.cloudfront.net/index.html" 
+                    width="100%" 
+                    height="100%" 
+                    allow="storage-access"
+                ></iframe>
+
+                <iframe 
+                    id="modelViewerRight" 
+                    src="https://d39vhmln1nnc4z.cloudfront.net/index.html" 
+                    width="100%" 
+                    height="100%" 
+                    allow="storage-access"
+                ></iframe>
+            </div>''', 
+            render=False)
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.HTML(f"<h1>1. Choose a video below:</h1>")
+                video = gr.Video(
+                    label='Input Video',
+                    interactive=False,
+                    autoplay=True,
+                    show_download_button=False,
+                    loop=True,
+                    elem_id='gradioVideo',
+                )
+
+                triggerButtons = {}
+                for vid in sessionState.value['videos']:
+                    triggerButtons[vid] = gr.Button(elem_id=f'triggerBtn_{vid}', visible=False)
+                    triggerButtons[vid].click(
+                        fn=lambda vid=vid: gr.update(value=f'https://gradio-model-viewer.s3.eu-west-1.amazonaws.com/sample+videos/{vid}.mp4'),
+                        outputs=[video]
+                    )
+                examples = gr.HTML(visible=False)
+
+            with gr.Column(scale=4):
+                gr.HTML("""
+                    <h1>2. Play around with the models:
+                        <span class="glyphicon glyphicon-question-sign popup-btn btn btn-info btn-lg" data-popup-id="instructionsPopup">
+                            <span class="popup-text" id="instructionsPopup">You can control the playback in both viewers at the same time by using the video, or control both viewers independently by using mouse and GUI!</span>
+                        </span>
+                    </h1>
+                """)
+                with gr.Row():
+                    models.render()
+
+                with gr.Row():
+                    gr.HTML(f"<h1>3. Choose your favorite model for each criteria:</h1>")
+                ratingButtons = {}
+                for criteria in CRITERIA:
+                    with gr.Row():                
+                        with gr.Column():
+                            with gr.Row():
+                                match criteria:
+                                    case 'Global_Appreciation':
+                                        instructions = "Your overall appreciation of the models, including general aesthetics and self-contacts if applicable."
+                                    case 'Ground_Contacts':
+                                        instructions = "The quality of the models' contacts with the ground, including ground penetration and foot sliding."
+                                    case 'Fidelity':
+                                        instructions = "The fidelity of the models compared to the motion of the original video."
+                                    case 'Fluidity':
+                                        instructions = "The smoothness and temporal coherence of the models."
+                                gr.HTML(f"""
+                                    <h2 style='text-align:center;'>{criteria.replace('_', ' ')}
+                                    <span class="glyphicon glyphicon-question-sign popup-btn btn btn-info btn-lg" data-popup-id="{criteria}Popup">
+                                        <span class="popup-text" id="{criteria}Popup">{instructions}</span>
+                                    </span></h2>
+                                """)
+                            with gr.Row():
+                                ratingButtons[criteria] = []
+                                with gr.Column(scale=2):
+                                    ratingButtons[criteria].append(gr.Button('Left Model', variant='primary', interactive=False))
+                                with gr.Column(scale=1, min_width=2):
+                                    ratingButtons[criteria].append(gr.Button('Skip', min_width=2, interactive=False))
+                                with gr.Column(scale=2):
+                                    ratingButtons[criteria].append(gr.Button('Right Model', variant='primary', interactive=False))
+    
+
+    # Leaderboard per criteria
+    with gr.Tab(label='Leaderboards') as leaderboard_tab:
+
+        if MODE == 'testing':
+            # Simulation controls
+            with gr.Row():
+                simulate_btn = gr.Button('Simulate Matches', variant='primary')
+                add_model_btn = gr.Button('Add Model', variant='secondary')
+            with gr.Row():
+                gr.Markdown('''
+                ## Probability of each model to be chosen is updated after each vote following: \
+                $$ p_i = \\frac{e^{-\\frac{Matches_i}{\\beta}}}{\\sum_{j=1}^{N} e^{-\\frac{Matches_j}{\\beta}}} $$  
+                    ''')
+                iterate = gr.Number(label='Number of iterations', value=100, minimum=1, maximum=2000, precision=0, interactive=True)
+                beta = gr.Number(label='Beta', value=default_beta, minimum=1, maximum=1000, precision=0, step=10, interactive=True)
+        else:
+            beta = gr.Number(label='Beta', value=default_beta, render=False)
+
+        leaderboards = {}
+        tabs = {}
+        for criteria in CRITERIA:
+            with gr.Tab(label=criteria.replace('_', ' ')) as tabs[criteria]:
+                with gr.Row():
+                    gr.HTML(f"<h2 style='text-align:center;'>{criteria.replace('_', ' ')}</h2>")
+                with gr.Row():
+                    leaderboards[criteria] = gr.Dataframe(value=None, row_count=(len(MODELS), 'fixed'), headers=['Model', 'Elo', 'Wins', 'Matches', 'Win Rate'], interactive=False)
+
+        # Plots
+        if MODE == 'testing':
+            with gr.Row():
+                elo_plot = gr.Plot(value=None, label='Elo Ratings', format='plotly', elem_id='plot')
+            with gr.Row():
+                wr_plot = gr.Plot(value=None, label='Win Rates', format='plotly', elem_id='plot')
+            with gr.Row():
+                matches_plot = gr.Plot(value=None, label='Matches played', format='plotly', elem_id='plot')
+        elif MODE == 'production':
+            elo_plot = gr.Plot(value=None, label='Elo Ratings', format='plotly', elem_id='plot', visible=False)
+            wr_plot = gr.Plot(value=None, label='Win Rates', format='plotly', elem_id='plot', visible=False)
+            matches_plot = gr.Plot(value=None, label='Matches played', format='plotly', elem_id='plot', visible=False)
+
+    with gr.Tab(label='About'):
+        gr.Markdown('''
+        ## Thank you for using the 3D Animation Arena!
+                        
+        This app is designed to compare different models based on human preferences, inspired by dylanebert's [3D Arena](https://huggingface.co/spaces/dylanebert/3d-arena) on Hugging Face. 
+        Current rankings often use metrics to assess the quality of a model, but these metrics may not always reflect the complexity behind human preferences.
+
+        The current models competing in the arena are:
+        -  4DHumans (https://github.com/shubham-goel/4D-Humans)
+        - CLIFF (https://github.com/haofanwang/CLIFF)
+        - GVHMR (https://github.com/zju3dv/GVHMR)
+        - HybrIK (https://github.com/jeffffffli/HybrIK)
+        - WHAM (https://github.com/yohanshin/WHAM)
+                
+        All inferences are precomputed following the code in the associated GitHub repository. 
+        Some post-inference modifications have been made to some models in order to make the comparison possible. 
+        These modifications include:
+        * Adjusting height to a common ground
+        * Fixing the root depth of certain models, when depth was extremely jittery
+        * Fixing the root position of certain models, when no root position was available
+
+        All models use the SMPL body model to discard the influence of the body model on the comparison.
+        These choices were made without any intention to favor or harm any model. 
+        All matchups are generated randomly, don't hesitate to rate the same videos multiple times as the matchups will probably be different!
+
+        ---
+
+        If you have comments, complaints or suggestions, please contact me at [email protected]. 
+        New models and videos will be added over time, feel free to share your ideas! Keep in mind that I will not add raw inferences from other people to keep it fair.  
+        ''')
+
+
+    # Event handlers
+    def randomize_videos(state):
+        state['uuid'] = str(uuid.uuid4())
+        random.shuffle(state['videos'])
+        gallery = "<div class='gallery'>"
+        for vid in state['videos']:
+            gallery += f"""
+                <button class="btn btn-info thumbnail-btn" onclick="(function() {{
+                    let gradioVideo = document.getElementById('gradioVideo');
+                    let videoComponent = gradioVideo ? gradioVideo.querySelector('video') : null;
+                    if (videoComponent && !videoComponent.src.includes('{vid}')) {{
+                        Array.from(document.getElementsByClassName('thumbnail-btn')).forEach(btn => btn.disabled = true);
+                    }}
+                    document.getElementById('triggerBtn_{vid}').click();
+                    }})()">
+                    <video class="thumbnail" preload="" loop muted onmouseenter="this.play()" onmouseleave="this.pause()">
+                        <source src="https://gradio-model-viewer.s3.eu-west-1.amazonaws.com/sample+videos/{vid}.mp4">
+                    </video>
+                </button>
+            """
+        gallery += "</div>"
+        return state, gallery
+
+    async def display_leaderboards():
+        return [await generate_leaderboard(criteria) for criteria in CRITERIA]
+    
+    arena.load(
+        inputs=[sessionState],
+        fn=lambda state: randomize_videos(state),
+        outputs=[sessionState, examples],
+    ).then(
+        inputs=[],
+        fn=lambda: gr.update(visible=True),
+        outputs=[examples]
+    ).then(
+        inputs=[gr.State(CRITERIA[0])],
+        fn=plot_ratings,  
+        outputs=[elo_plot, wr_plot, matches_plot]
+    ).then(
+        inputs=[],
+        fn=display_leaderboards,
+        outputs=[leaderboards[criteria] for criteria in CRITERIA]
+    )
+
+    async def update_models(video, state):
+        leaderboard = await generate_leaderboard(CRITERIA[0])
+        video_name = video.split('/')[-1].split('.')[0]
+        modelLeft, modelRight = generate_matchup(leaderboard=leaderboard, beta=beta.value)
+
+        state['video'] = video_name
+        state['modelLeft'] = MODELS[modelLeft]
+        state['modelRight'] = MODELS[modelRight]
+
+        return state, state
+
+    video.change(
+        inputs=[video, sessionState],
+        fn=update_models,
+        outputs=[sessionState, frontState]
+    )
+
+    # Weird workaround to run JS function on state change, from https://github.com/gradio-app/gradio/issues/3525#issuecomment-2348596861
+    frontState.change(
+        inputs=[frontState],
+        js='(state) => updateViewers(state)',
+        fn=lambda state: None,
+    ).then(
+        inputs=None,
+        fn=lambda: tuple(gr.update(interactive=True) for _ in sum(ratingButtons.values(), [])),
+        outputs= sum(ratingButtons.values(), [])
+    )
+
+    leaderboard_tab.select(
+        inputs=None,
+        js='() => resetPlots()',
+        fn=None,
+    ).then(
+        fn=lambda: [gr.update(value=None) for _ in range(3)],
+        outputs=[elo_plot, wr_plot, matches_plot]
+    ).then(
+        inputs=[sessionState],
+        fn=lambda state: plot_ratings(state['currentTab']),
+        outputs=[elo_plot, wr_plot, matches_plot]
+    )
+
+    async def process_rating(state, i, criteria):
+        return gr.update(value=await submit_rating(
+            criteria=criteria, 
+            winner=state['modelLeft'] if i == 0 else state['modelRight'] if i == 2 else None, 
+            loser=state['modelRight'] if i == 0 else state['modelLeft'] if i == 2 else None,
+            uuid=state['uuid']
+        ))
+
+    def update_tab(state, criteria):
+        state['currentTab'] = criteria
+        return state
+    
+    for criteria in CRITERIA:
+        for i, button in enumerate(ratingButtons[criteria]):
+            button.click(
+            #     fn=lambda i=i, criteria=criteria: gr.Info(f'{"You chose Left Model for " if i == 0 else "You chose Right Model for " if i == 2 else "You skipped "} {criteria.replace("_", " ")}!'),
+            # ).then(
+                fn=lambda: tuple(gr.update(interactive=False) for _ in range(len(ratingButtons[criteria]))),
+                outputs=ratingButtons[criteria]
+            ).then(
+                inputs=[sessionState, gr.State(i), gr.State(criteria)],
+                fn=process_rating,
+                outputs=[leaderboards[criteria]],
+            )
+
+        tabs[criteria].select(
+            fn=lambda: [gr.update(value=None) for _ in range(3)],
+            outputs=[elo_plot, wr_plot, matches_plot]
+        ).then(
+            inputs=[gr.State(criteria)],
+            fn=plot_ratings,  
+            outputs=[elo_plot, wr_plot, matches_plot]
+        ).then(
+            inputs=[sessionState, gr.State(criteria)],
+            fn=update_tab,
+            outputs=[sessionState]
+        )
+
+
+    if MODE == 'testing':
+        for criteria in CRITERIA:
+            simulate_btn.click(
+                inputs=[iterate, beta, gr.State(criteria)],
+                fn=simulate,
+                outputs=[leaderboards[criteria]],
+            ).then(fn=lambda: [gr.update(value=None) for _ in range(3)],
+                outputs=[elo_plot, wr_plot, matches_plot]
+            ).then(
+                inputs=[gr.State(criteria)],
+                fn=plot_ratings,  
+                outputs=[elo_plot, wr_plot, matches_plot]
+            )
+
+        add_model_btn.click(
+            fn=lambda: MODELS.append(f'model_{len(MODELS)}'),
+        )
+
+if __name__ == '__main__':
+    gr.set_static_paths(['static'])
+    arena.queue(default_concurrency_limit=50).launch(inbrowser=True, allowed_paths=['static/'])

config.py

@@ -0,0 +1,61 @@
+import os
+
+MODE =  os.getenv('ARENA_MODE') # 'testing' or 'production'
+
+default_beta = 500
+
+MODELS = [
+    '4DHumans',
+    'CLIFF',
+    'GVHMR',
+    'HybrIK',
+    'WHAM',
+    'TokenHMR',
+    'STAF',
+    'CameraHMR'
+]
+
+VIDEOS = [
+    'backflip',
+    'ballet_dance',
+    'ballet_jump',
+    'basketball_dunk',
+    'boxing',
+    'breakdance',
+    'bridge',
+    'capoeira',
+    'contorsionist',
+    'dance_feathers',
+    'dance_modern1',
+    'dance_modern2',
+    'dance_modern3',
+    'dance_road',
+    'dance_tiktok',
+    'highkick',
+    'hiphop',
+    'parkour',
+    'pillars',
+    'skateboard',
+    'spinkick',
+    'trampoline',
+    'wall_jump',
+    'yoga',
+    'cartoon_fall',
+    'cutting_tree',
+    'fencing',
+    'ferocity',
+    'ice_skating',
+    'moonwalk',
+    'npc',
+    'running',
+    'sitting',
+    'stretching',
+    'tennis',
+]
+
+CRITERIA = [
+    'Global_Appreciation',
+    'Ground_Contacts',
+    'Fidelity',
+    'Fluidity',
+]

requirements.txt

@@ -0,0 +1,7 @@
+pandas
+plotly
+numpy
+gradio==5.16.0
+huggingface_hub==0.28.1
+aioboto3==14.1.0
+pydantic==2.10.3

static/modelViewer.js

@@ -0,0 +1,58 @@
+function updateViewers(state) {
+    let video = state.video;
+    let modelLeft = state.modelLeft;
+    let modelRight = state.modelRight;
+
+    let gradioVideo = document.getElementById("gradioVideo");
+    let videoComponent = gradioVideo ? gradioVideo.querySelector("video") : null;
+
+    if (videoComponent && document.getElementById("modelViewerLeft") && document.getElementById("modelViewerRight")) {
+
+        videoComponent.setAttribute("muted", true);
+        document.getElementById("modelViewerLeft").contentWindow.postMessage({ action: "loadModel", modelUrl: `https://gradio-model-viewer.s3.eu-west-1.amazonaws.com/models/${modelLeft}/${video}.glb` }, "*");
+        document.getElementById("modelViewerRight").contentWindow.postMessage({ action: "loadModel", modelUrl: `https://gradio-model-viewer.s3.eu-west-1.amazonaws.com/models/${modelRight}/${video}.glb` }, "*");
+
+        let loadedCount = 0;
+
+        window.addEventListener("message", (event) => {
+            if (event.data.status === "modelLoaded") {
+                loadedCount++;
+                if (loadedCount === 2) {
+                    videoComponent.addEventListener("play", syncModelViewers);
+                    videoComponent.addEventListener("pause", syncModelViewers);
+                    videoComponent.addEventListener("timeupdate", syncModelViewers);
+
+                    Array.from(document.getElementsByClassName('thumbnail-btn')).forEach(btn => btn.disabled = false);
+                }
+            }
+            else if (event.data.status === "modelLoadError") {
+                Array.from(document.getElementsByClassName('thumbnail-btn')).forEach(btn => btn.disabled = false);
+            }
+        });
+
+    }
+}
+
+function syncModelViewers(event) {
+    let videoComponent = event.target;
+    let modelViewerLeft = document.getElementById("modelViewerLeft");
+    let modelViewerRight = document.getElementById("modelViewerRight");
+
+    if (!modelViewerLeft || !modelViewerRight) return;
+
+    switch (event.type) {
+        case "play":
+            modelViewerLeft.contentWindow.postMessage({ action: "playAnimation" }, "*");
+            modelViewerRight.contentWindow.postMessage({ action: "playAnimation" }, "*");
+            break;
+        case "pause":
+            modelViewerLeft.contentWindow.postMessage({ action: "pauseAnimation" }, "*");
+            modelViewerRight.contentWindow.postMessage({ action: "pauseAnimation" }, "*");
+            break;
+        case "timeupdate":
+            let currentTime = videoComponent.currentTime;
+            modelViewerLeft.contentWindow.postMessage({ action: "setAnimationTime", currentTime }, "*");
+            modelViewerRight.contentWindow.postMessage({ action: "setAnimationTime", currentTime }, "*");    
+            break;
+    }
+}

static/plots.js

@@ -0,0 +1,6 @@
+// Fix plot display issues with gradio Tabs
+function resetPlots() {
+    document.querySelectorAll(".js-plotly-plot").forEach(plot => {
+        Plotly.relayout(plot, { autosize: true });
+    });
+};

static/popup.js

@@ -0,0 +1,24 @@
+function togglePopup(event, show) {
+    var popupBtn = event.target;
+    var popupId = popupBtn.getAttribute('data-popup-id');
+    var popup = document.getElementById(popupId);
+
+    popup.parentElement.parentElement.parentElement.parentElement.parentElement.style.overflow = 'visible';
+    
+    popup.classList.toggle("show");
+    popupBtn.parentElement.style.overflow = 'visible';
+
+    event.stopPropagation();
+}
+
+document.addEventListener("mouseover", function(event) {
+    if (event.target.classList.contains('popup-btn')) {
+        togglePopup(event, true);
+    }
+});
+
+document.addEventListener("mouseout", function(event) {
+    if (event.target.classList.contains('popup-btn')) {
+        togglePopup(event, false);
+    }
+});

static/style.css

@@ -0,0 +1,135 @@
+.viewer-container {
+    display: flex;
+    gap: 20px; 
+    justify-content: center;
+}
+model-viewer {
+    width: 50%;
+    height: 500px;
+    background-color: #eeeeee;
+}
+iframe {
+    width: 50%;
+    height: 500px;
+    background-color: #eeeeee;
+    border: none;
+    border-radius: 5px;
+}
+
+.js-plotly-plot {
+    width: 100%;
+    display: flex;
+    justify-content: center;
+}
+
+@font-face {
+    font-family: 'Glyphicons Halflings';
+    
+    src: url('https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/fonts/glyphicons-halflings-regular.eot');
+    src: url('https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/fonts/glyphicons-halflings-regular.woff2') format('woff2'), url('https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/fonts/glyphicons-halflings-regular.woff') format('woff'), url('https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/fonts/glyphicons-halflings-regular.ttf') format('truetype'), url('https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg');
+}
+.glyphicon {
+    position: relative;
+    top: 2px;
+    display: inline-block;
+    font-family: 'Glyphicons Halflings';
+    font-style: normal;
+    font-weight: normal;
+    line-height: 1;
+    
+    -webkit-font-smoothing: antialiased;
+    -moz-osx-font-smoothing: grayscale;
+}
+.glyphicon-question-sign:before {
+    content: "\e085";
+}
+
+.popup-btn {
+    position: relative;
+    display: inline-block;
+    cursor: pointer;
+    -webkit-user-select: none;
+    -moz-user-select: none;
+    -ms-user-select: none;
+    user-select: none;
+    overflow: visible !important;
+    font-size: 20px;
+    border: none;
+    text-align: center;
+    text-decoration: none;
+    padding-left: 10px;
+}
+
+.popup-text {
+    visibility: hidden;
+    width: 300px;
+    background-color: #e4e4e7;
+    color: #000;
+    text-align: center;
+    border-radius: 10px;
+    padding: 20px 10px;
+    position: absolute;
+    z-index: 1;
+    bottom: 150%;
+    left: 50%;
+    margin-left: -145px;
+    overflow: visible !important;
+    font-size: 17px;
+    font-family: Arial, sans-serif;
+    box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
+    transition: visibility 0.3s, opacity 0.3s ease-in-out;
+    opacity: 0;
+}
+
+.popup-text.dark {
+    background-color: #52525b;
+    color: #fff;
+}
+
+/* Popup arrow */
+.popup-text::after {
+    content: "";
+    position: absolute;
+    top: 100%;
+    left: 50%;
+    margin-left: -5px;
+    border-width: 5px;
+    border-style: solid;
+    border-color: #e4e4e7 transparent transparent transparent;
+}
+
+.popup-text.dark::after {
+    border-color: #52525b transparent transparent transparent;
+}
+
+.popup-text.show {
+    visibility: visible;
+    opacity: 1;
+}
+
+.thumbnail-btn {
+    height:75px; 
+    width: auto;
+    overflow: hidden;
+    display: flex;
+}
+.thumbnail-btn:hover {
+    cursor: pointer;
+    transform: scale(1.05);
+    transition: transform 0.5s;
+}
+
+.thumbnail {
+    width: auto;
+    height: 100%;
+    object-fit: cover;
+    border-radius: 5px;
+}
+
+.gallery {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 10px;
+    justify-content: flex-start;
+    align-items: flex-start;
+}

utils/data_utils.py

@@ -0,0 +1,49 @@
+from .s3_utils import read_from_s3
+from config import MODELS
+import pandas as pd
+
+async def generate_leaderboard(criteria : str) -> pd.DataFrame:
+    """
+    Generate the leaderboard from saved data.
+
+    Args:
+        criteria (str): The criteria corresponding to the leaderboard.
+    
+    Returns:
+        pd.DataFrame: The leaderboard.
+    """
+    try:
+        leaderboard = await read_from_s3(f'leaderboard_{criteria}.csv')
+        if leaderboard is None:
+            raise Exception
+    except:
+        leaderboard = pd.DataFrame({
+            'Model': pd.Series(dtype='str'),
+            'Elo': pd.Series(dtype='int'),
+            'Wins': pd.Series(dtype='int'),
+            'Matches': pd.Series(dtype='int'),
+            'Win Rate': pd.Series(dtype='float')
+        })
+
+    for model in MODELS:
+        if model not in leaderboard['Model'].values:
+            leaderboard = pd.concat([leaderboard, pd.DataFrame([{'Model': model, 'Elo': 1500, 'Wins': 0, 'Matches': 0, 'Win Rate': 0.0}])], ignore_index=True)
+    leaderboard = leaderboard.sort_values('Elo', ascending=False).reset_index(drop=True)
+    leaderboard['Win Rate'] = leaderboard['Win Rate'].apply(lambda x: round(x, 2))
+
+    return leaderboard
+
+async def generate_data() -> pd.DataFrame:
+    """
+    Generate the data for the matches.
+
+    Returns:
+        pd.DataFrame: The data for the matches.
+    """
+    try :
+        data = await read_from_s3('data.csv')
+        if data is None:
+            raise Exception
+    except:
+        data = pd.DataFrame(columns=['Criteria', 'Model', 'Opponent', 'Won', 'Elo', 'Win Rate', 'Matches', 'Timestamp', 'UUID'])
+    return data

utils/emoji_utils.py

@@ -0,0 +1,14 @@
+def add_emoji(model : str) -> str:
+    match model:
+        case '4DHumans':
+            return '\U0001F3A5 4DHumans'
+        case 'CLIFF':
+            return '\U0001F3D4 CLIFF'
+        case 'GVHMR':
+            return '\U0001F4CD GVHMR'
+        case 'HybrIK':
+            return '\U0001F9BE HybrIK'
+        case 'WHAM':
+            return '\U0001F3AF WHAM'
+        case _:
+            return model

utils/plot_utils.py

@@ -0,0 +1,28 @@
+import plotly.express as px
+from .data_utils import generate_data
+
+async def plot_ratings(criteria : str):
+    """
+    Plot different ratings of the models for a given criteria on a line graph.
+
+    Args:
+        criteria (str): The criteria to plot the ratings for.
+    """
+    data = await generate_data()
+
+    elo_data = data[data['Criteria'] == criteria].reset_index(drop=True)
+    elo_data.reset_index(inplace=True)
+    elo_data['index'] = elo_data['index'].apply(lambda x: int(x/2))
+    elo_fig = px.line(elo_data, x='index', y='Elo', color='Model', title='Elo Ratings Over Total Votes', labels={'index': 'Total Votes', 'Elo': 'Elo Rating'})
+
+    wr_data = data[data['Criteria'] == criteria].reset_index(drop=True)
+    wr_data.reset_index(inplace=True)
+    wr_data['index'] = wr_data['index'].apply(lambda x: int(x/2))
+    wr_fig = px.line(wr_data, x='index', y='Win Rate', color='Model', title='Win Rates Over Total Votes', labels={'index': 'Total Votes', 'Win Rate': 'Win Rate'})
+
+    matches_data = data[data['Criteria'] == criteria].reset_index(drop=True)
+    matches_data.reset_index(inplace=True)
+    matches_data['index'] = matches_data['index'].apply(lambda x: int(x/2))
+    matches_fig = px.line(matches_data, x='index', y='Matches', color='Model', title='Matches Played Over Total Votes', labels={'index': 'Total Votes', 'Matches': 'Matches Played'})
+    
+    return elo_fig, wr_fig, matches_fig

utils/s3_utils.py

@@ -0,0 +1,123 @@
+import aioboto3
+import pandas as pd
+from io import StringIO
+from typing import Optional, Union
+import os
+
+AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
+S3_BUCKET = os.getenv('S3_BUCKET')
+S3_VIDEO_PATH = 'sample videos'
+S3_MODEL_PATH = 'models'
+S3_DATA_PATH = '3d_animation_arena/results'
+
+async def download_from_s3(file_key : str, target_dir: str, bucket : str = S3_BUCKET) -> Optional[str]:
+    """
+    Downloads a file from an S3 bucket.
+
+    Args:
+        file_key (str): The key of the file in the S3 bucket, including extension.
+        target_dir (str): The path to the directory to save the downloaded file.
+        bucket (str, optional): The name of the S3 bucket.
+
+    Returns:
+        Optional[str]: The path to the file or None if the download fails.
+    """
+    session = aioboto3.Session()
+    target_path = os.path.join(target_dir, file_key)
+
+    async with session.client(
+        's3',
+        aws_access_key_id=AWS_ACCESS_KEY_ID,
+        aws_secret_access_key=AWS_SECRET_ACCESS_KEY
+    ) as s3_client:
+        try:
+            os.makedirs(target_dir, exist_ok=True)
+            if os.path.exists(target_path):
+                print(f'{file_key} already exists in {target_dir}')
+                return target_path
+            with open(target_path, 'wb') as f:
+                match file_key.split('.')[-1]:
+                    case 'mp4':
+                        await s3_client.download_fileobj(bucket, os.path.join(S3_VIDEO_PATH, file_key), f)
+                    case 'glb'|'obj'|'stl'|'gltf'|'splat'|'ply':
+                        await s3_client.download_fileobj(bucket, os.path.join(S3_MODEL_PATH, file_key), f)
+                    case _:
+                        print(f"Unsupported file type: {file_key}")
+                        raise ValueError(f"Unsupported file type: {file_key}")
+            return target_path
+        except Exception as e:
+            print(f'Error downloading {file_key} from bucket {bucket}: {e}')
+            raise e
+            
+
+async def read_from_s3(file_key : str, bucket : str = S3_BUCKET) -> Optional[Union[pd.DataFrame, str]]:
+    """
+    Reads a file from an S3 bucket based on its file extension and returns the appropriate data type.
+
+    Args:
+        file_key (str): The key of the file in the S3 bucket.
+        bucket (str, optional): The name of the S3 bucket.
+
+    Returns:
+        Optional[Union[pd.DataFrame, str]]: 
+            - A pandas DataFrame if the file is a CSV.
+            - A temporary file path (str) if the file is a GLB.
+            - A presigned URL (str) if the file is an MP4.
+            - None if the file type is unsupported.
+    """
+    session = aioboto3.Session()
+    async with session.client(
+        's3',
+        aws_access_key_id=AWS_ACCESS_KEY_ID,
+        aws_secret_access_key=AWS_SECRET_ACCESS_KEY
+    ) as s3_client:
+        try:
+            match file_key.split('.')[-1]:
+                case 'csv':
+                    response = await s3_client.get_object(Bucket=bucket, Key=os.path.join(S3_DATA_PATH, file_key))
+                    content = await response['Body'].read()
+                    result = pd.read_csv(StringIO(content.decode("utf-8")))
+                    return result
+                case _:
+                    print(f"Unsupported file type for reading: {file_key}")
+                    raise ValueError(f"Unsupported file type for reading: {file_key}")
+        except Exception as e:
+            print(f'Error reading {file_key} from bucket {bucket}: {e}')
+            raise e
+
+
+async def write_to_s3(file_key : str, dataframe: pd.DataFrame, bucket : str = S3_BUCKET) -> None:
+    """
+    Writes a pandas DataFrame to an S3 bucket as a CSV file.
+
+    Args:
+        file_key (str): The key (file name) under which the file will be stored in the S3 bucket.
+        dataframe (pd.DataFrame): The pandas DataFrame to write to the S3 bucket.
+        bucket (str, optional): The name of the S3 bucket.
+
+    Raises:
+        Exception: Reraises any exception encountered during the write process.
+    """
+    session = aioboto3.Session()
+    async with session.client(
+        's3',
+        aws_access_key_id=AWS_ACCESS_KEY_ID,
+        aws_secret_access_key=AWS_SECRET_ACCESS_KEY
+    ) as s3_client:
+        try:
+            match file_key.split('.')[-1]:
+                case 'csv':
+                    csv_buffer = StringIO()
+                    dataframe.to_csv(csv_buffer, index=False)
+                    await s3_client.put_object(
+                        Bucket=bucket, 
+                        Key=os.path.join(S3_DATA_PATH, file_key), 
+                        Body=csv_buffer.getvalue()
+                    )
+                case _:
+                    print(f"Unsupported file type for writing: {file_key}")
+                    raise ValueError(f"Unsupported file type for writing: {file_key}")
+        except Exception as e:
+            print(f'Error writing {file_key} to bucket {bucket}: {e}')
+            raise e

utils/utils.py

@@ -0,0 +1,176 @@
+from typing import Tuple
+import pandas as pd
+import numpy as np
+import time
+import asyncio
+from utils.s3_utils import write_to_s3
+from utils.data_utils import generate_leaderboard, generate_data
+
+submit_lock = asyncio.Lock()
+
+def update_ratings(R_win : int, R_lose : int, k : int = 32) -> Tuple[int, int]:
+    """
+    Update the ratings of two players after a match.
+
+    Args:
+        R_win (int): The rating of the winning player.
+        R_lose (int): The rating of the losing player.
+        k (int, optional): The k-factor. Defaults to 32.
+
+    Returns:
+        Tuple[int, int]: The updated ratings of the winning and losing players.
+    """
+    E_win = 1 / (1 + 10 ** ((R_lose - R_win) / 480))
+    E_lose = 1 / (1 + 10 ** ((R_win - R_lose) / 480))
+    return int(R_win + k * (1 - E_win)), int(R_lose + k * (0 - E_lose))
+
+def generate_matchup(leaderboard : pd.DataFrame, beta : int) -> tuple[str, str]:
+    """
+    Generate a pseudo-random matchup between two models.
+
+    Args:
+        leaderboard (pd.DataFrame): The leaderboard of models
+        beta (int): The damping factor for the Elo update.
+
+    Returns:
+        model1 (str): The first model.
+        model2 (str): The second model.
+    """
+    if leaderboard['Matches'].sum() == 0:
+        return np.random.choice(leaderboard.index, 2, replace=False)
+    weights = [np.exp(-leaderboard.at[model, 'Matches'] / beta) for model in leaderboard.index]
+    weights = weights / np.sum(weights) # Normalize weights
+    selected = np.random.choice(leaderboard.index, 2, replace=False, p=weights)
+    np.random.shuffle(selected)
+    model1, model2 = selected
+    return model1, model2
+
+async def simulate(iter : int, beta : int, criteria : str) -> pd.DataFrame:
+    """
+    Simulate matches between random models.
+
+    Args:
+        iter (int): The number of matches to simulate.
+        beta (int): The damping factor for the Elo update.
+        criteria (str): The criteria for the rating.
+
+    Returns:
+        leaderboard (pd.DataFrame): Updated leaderboard after simulation
+    """
+    data = await generate_data()
+
+    leaderboard = await generate_leaderboard(criteria)
+    leaderboard.set_index('Model', inplace=True)
+
+    for _ in range(iter):
+        # Generate random matchups
+        timestamp = time.time()
+        model1, model2 = generate_matchup(leaderboard, beta)
+        R1, R2 = leaderboard.at[model1, 'Elo'], leaderboard.at[model2, 'Elo']
+        R1_new, R2_new = update_ratings(R1, R2)
+
+        # Update leaderboard
+        leaderboard.at[model1, 'Elo'], leaderboard.at[model2, 'Elo'] = R1_new, R2_new
+        leaderboard.at[model1, 'Wins'] += 1
+        leaderboard.at[model1, 'Matches'] += 1
+        leaderboard.at[model2, 'Matches'] += 1
+        leaderboard.at[model1, 'Win Rate'] = np.round(leaderboard.at[model1, 'Wins'] / leaderboard.at[model1, 'Matches'], 2)
+        leaderboard.at[model2, 'Win Rate'] = np.round(leaderboard.at[model2, 'Wins'] / leaderboard.at[model2, 'Matches'], 2)
+
+        # Save match data
+        data.loc[len(data)] = {
+            'Criteria': criteria,
+            'Model': model1, 
+            'Opponent': model2, 
+            'Won': True, 
+            'Elo': leaderboard.at[model1, 'Elo'], 
+            'Win Rate': leaderboard.at[model1, 'Win Rate'],
+            'Matches': leaderboard.at[model1, 'Matches'],
+            'Timestamp': timestamp,
+            'UUID': None
+        }
+
+        data.loc[len(data)] = {
+            'Criteria': criteria,
+            'Model': model2, 
+            'Opponent': model1, 
+            'Won': False, 
+            'Elo': leaderboard.at[model2, 'Elo'], 
+            'Win Rate': leaderboard.at[model2, 'Win Rate'],
+            'Matches': leaderboard.at[model2, 'Matches'],
+            'Timestamp': timestamp,
+            'UUID': None
+        }
+
+    leaderboard = leaderboard.sort_values('Elo', ascending=False).reset_index(drop=False)
+
+    await asyncio.gather(
+        write_to_s3(f'leaderboard_{criteria}.csv', leaderboard), 
+        write_to_s3('data.csv', data)
+    )
+
+    return leaderboard
+
+
+async def submit_rating(criteria : str, winner : str, loser : str, uuid : str) -> None:
+    """
+    Submit a rating for a match.
+
+    Args:
+        criteria (str): The criteria for the rating.
+        winner (str): The winning model.
+        loser (str): The losing model.
+        uuid (str): The UUID of the session.
+    """
+    async with submit_lock:
+        data = await generate_data()
+
+        leaderboard = await generate_leaderboard(criteria)
+        leaderboard.set_index('Model', inplace=True)
+
+        if winner is None or loser is None:
+            return leaderboard
+
+        timestamp = time.time()
+        R_win, R_lose = leaderboard.at[winner, 'Elo'], leaderboard.at[loser, 'Elo']
+        R_win_new, R_lose_new = update_ratings(R_win, R_lose)
+
+        # Update leaderboard
+        leaderboard.loc[[winner, loser], 'Elo'] = [R_win_new, R_lose_new]
+        leaderboard.at[winner, 'Wins'] += 1
+        leaderboard.loc[[winner, loser], 'Matches'] += [1, 1]
+        leaderboard.loc[[winner, loser], 'Win Rate'] = (
+            leaderboard.loc[[winner, loser], 'Wins'] / leaderboard.loc[[winner, loser], 'Matches']
+        ).apply(lambda x: round(x, 2))
+
+        # Save match data
+        data.loc[len(data)] = {
+            'Criteria': criteria,
+            'Model': winner, 
+            'Opponent': loser, 
+            'Won': True, 
+            'Elo': leaderboard.at[winner, 'Elo'], 
+            'Win Rate': leaderboard.at[winner, 'Win Rate'],
+            'Matches': leaderboard.at[winner, 'Matches'],
+            'Timestamp': timestamp,
+            'UUID': uuid
+        }
+        
+        data.loc[len(data)] = {
+            'Criteria': criteria,
+            'Model': loser, 
+            'Opponent': winner, 
+            'Won': False, 
+            'Elo': leaderboard.at[loser, 'Elo'], 
+            'Win Rate': leaderboard.at[loser, 'Win Rate'],
+            'Matches': leaderboard.at[loser, 'Matches'],
+            'Timestamp': timestamp,
+            'UUID': uuid
+        }
+
+        leaderboard = leaderboard.sort_values('Elo', ascending=False).reset_index(drop=False)
+        await asyncio.gather(
+            write_to_s3(f'leaderboard_{criteria}.csv', leaderboard),
+            write_to_s3('data.csv', data)
+        )
+        return leaderboard