initial commit

README.md

@@ -1,14 +1,43 @@
 ---
 title: IDEA Bench Arena
-emoji: 🌖
-colorFrom: gray
-colorTo: gray
+emoji: 📉
+colorFrom: blue
+colorTo: green
 sdk: gradio
 sdk_version: 5.9.1
 app_file: app.py
 pinned: false
-license: cc-by-nc-4.0
-short_description: Official ARENA of IDEA-Bench
+license: cc-by-4.0
+short_description: Official arena of IDEA-Bench.
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Installation
+
+- for cuda 11.8
+```bash
+conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia
+pip3 install -U xformers --index-url https://download.pytorch.org/whl/cu118
+pip install -r requirements.txt
+```
+- for cuda 12.1
+```bash
+conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia
+pip install -r requirements.txt
+```
+
+## Start Hugging Face UI
+```bash
+python app.py
+```
+
+## Start Log server
+```bash
+uvicorn serve.log_server:app --reload --port 22005 --host 0.0.0.0
+```
+
+## Update leaderboard
+```bash
+cd arena_elo && bash update_leaderboard.sh
+```
+
+Paper: [https://arxiv.org/abs/2412.11767](https://arxiv.org/abs/2412.11767)

app.py

@@ -0,0 +1,65 @@
+import gradio as gr
+import os
+# os.system("pip install -r requirements.txt -U")
+# os.system("pip uninstall -y apex")
+# os.system("pip uninstall -y flash-attn")
+# os.system("FLASH_ATTENTION_FORCE_BUILD=TRUE pip install flash-attn")
+from serve.gradio_web import *
+from serve.leaderboard import build_leaderboard_tab
+from model.model_manager import ModelManager
+from pathlib import Path
+from serve.constants import SERVER_PORT, ROOT_PATH, ELO_RESULTS_DIR   
+
+def build_combine_demo(models, elo_results_file, leaderboard_table_file):
+
+    with gr.Blocks(
+        title="Play with Open Vision Models",
+        theme=gr.themes.Default(),
+        css=block_css,
+    ) as demo:
+        with gr.Tabs() as tabs_combine:
+            with gr.Tab("Image Generation", id=0):
+                with gr.Tabs() as tabs_ig:
+                    with gr.Tab("Generation Arena (battle)", id=0):
+                        build_side_by_side_ui_anony(models)
+
+                    with gr.Tab("Generation Arena (side-by-side)", id=1):
+                        build_side_by_side_ui_named(models)
+
+                    if elo_results_file:
+                        with gr.Tab("Generation Leaderboard", id=2):
+                            build_leaderboard_tab(elo_results_file['t2i_generation'], leaderboard_table_file['t2i_generation'])
+            
+            with gr.Tab("About Us", id=3): 
+                build_about()
+
+    return demo
+
+
+def load_elo_results(elo_results_dir):
+    from collections import defaultdict
+    elo_results_file = defaultdict(lambda: None)
+    leaderboard_table_file = defaultdict(lambda: None)
+    if elo_results_dir is not None:
+        elo_results_dir = Path(elo_results_dir)
+        elo_results_file = {}
+        leaderboard_table_file = {}
+        for file in elo_results_dir.glob('elo_results*.pkl'):
+            elo_results_file['t2i_generation'] = file
+        for file in elo_results_dir.glob('*leaderboard.csv'):
+            leaderboard_table_file['t2i_generation'] = file
+            
+    return elo_results_file, leaderboard_table_file
+    
+if __name__ == "__main__":
+
+    server_port = int(SERVER_PORT)
+    root_path = ROOT_PATH
+    elo_results_dir = ELO_RESULTS_DIR
+    
+    models = ModelManager()
+    
+    elo_results_file, leaderboard_table_file = load_elo_results(elo_results_dir)
+    demo = build_combine_demo(models, elo_results_file, leaderboard_table_file)
+    demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+    demo.launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)

arena_elo/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 WildVision-Bench
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

arena_elo/README.md

@@ -0,0 +1,46 @@
+## Computing the Elo Ratings
+
+
+```bash
+apt-get -y install pkg-config
+pip install -r requirements.txt
+```
+
+
+### to update the leaderboard
+
+```bash
+export LOGDIR="/path/to/your/logdir"
+bash update_elo_rating.sh
+```
+
+### to inspect the leaderboard status
+```bash
+python -m elo_rating.inspect_elo_rating_pkl
+```
+
+### to inspect the collected data status and cost
+```bash
+export LOGDIR="/path/to/your/logdir"
+python -m elo_rating.inspect_cost
+```
+
+### to upload the battle data to hugging face🤗 
+```bash
+export HUGGINGFACE_TOKEN="your_huggingface_token"
+bash get_latest_data.sh
+python -m elo_rating.upload_battle_data --repo_id "WildVision/wildvision-bench" --log_dir "./vision-arena-logs/"
+```
+
+### to upload the chat data to hugging face🤗 
+```bash
+export HUGGINGFACE_TOKEN="your_huggingface_token"
+bash get_latest_data.sh
+python -m elo_rating.upload_chat_data --repo_id "WildVision/wildvision-bench" --log_dir "./vision-arena-logs/"
+```
+
+
+### to get the collected data
+```bash
+python -m 
+

arena_elo/cut_off_date.txt

@@ -0,0 +1 @@
+20241226

arena_elo/elo_rating/basic_stats.py

@@ -0,0 +1,227 @@
+import argparse
+import code
+import datetime
+import json
+import os
+from pytz import timezone
+import time
+
+import pandas as pd  # pandas>=2.0.3
+import plotly.express as px
+import plotly.graph_objects as go
+from tqdm import tqdm
+
+NUM_SERVERS = 1
+LOG_ROOT_DIR = os.getenv("LOGDIR", None)
+if LOG_ROOT_DIR is None:
+    raise ValueError("LOGDIR environment variable not set, please set it by `export LOGDIR=...`")
+
+def get_log_files(max_num_files=None):
+    log_root = os.path.expanduser(LOG_ROOT_DIR)
+    filenames = []
+    if NUM_SERVERS == 1:
+        for filename in os.listdir(log_root):
+            if filename.endswith("-conv.json"):
+                filepath = f"{log_root}/{filename}"
+                name_tstamp_tuple = (filepath, os.path.getmtime(filepath))
+                filenames.append(name_tstamp_tuple)
+    else:
+        for i in range(NUM_SERVERS):
+            for filename in os.listdir(f"{log_root}/server{i}"):
+                if filename.endswith("-conv.json"):
+                    filepath = f"{log_root}/server{i}/{filename}"
+                    name_tstamp_tuple = (filepath, os.path.getmtime(filepath))
+                    filenames.append(name_tstamp_tuple)
+    # sort by tstamp
+    filenames = sorted(filenames, key=lambda x: x[1])
+    filenames = [x[0] for x in filenames]
+
+    max_num_files = max_num_files or len(filenames)
+    filenames = filenames[-max_num_files:]
+    return filenames
+
+
+def load_log_files(filename):
+    data = []
+    for retry in range(5):
+        try:
+            lines = open(filename).readlines()
+            break
+        except FileNotFoundError:
+            time.sleep(2)
+
+    for l in lines:
+        row = json.loads(l)
+        data.append(
+            dict(
+                type=row["type"],
+                tstamp=row["tstamp"],
+                model=row.get("model", ""),
+                models=row.get("models", ["", ""]),
+            )
+        )
+    return data
+
+
+def load_log_files_parallel(log_files, num_threads=16):
+    data_all = []
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(load_log_files, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
+
+
+def get_anony_vote_df(df):
+    anony_vote_df = df[
+        df["type"].isin(["leftvote", "rightvote", "tievote", "bothbad_vote"])
+    ]
+    anony_vote_df = anony_vote_df[anony_vote_df["models"].apply(lambda x: x[0] == "")]
+    return anony_vote_df
+
+
+def merge_counts(series, on, names):
+    ret = pd.merge(series[0], series[1], on=on)
+    for i in range(2, len(series)):
+        ret = pd.merge(ret, series[i], on=on)
+    ret = ret.reset_index()
+    old_names = list(ret.columns)[-len(series) :]
+    rename = {old_name: new_name for old_name, new_name in zip(old_names, names)}
+    ret = ret.rename(columns=rename)
+    return ret
+
+
+def report_basic_stats(log_files):
+    df_all = load_log_files_parallel(log_files)
+    df_all = pd.DataFrame(df_all)
+    now_t = df_all["tstamp"].max()
+    df_1_hour = df_all[df_all["tstamp"] > (now_t - 3600)]
+    df_1_day = df_all[df_all["tstamp"] > (now_t - 3600 * 24)]
+    anony_vote_df_all = get_anony_vote_df(df_all)
+
+    # Chat trends
+    chat_dates = [
+        datetime.datetime.fromtimestamp(x, tz=timezone("US/Pacific")).strftime(
+            "%Y-%m-%d"
+        )
+        for x in df_all[df_all["type"] == "chat"]["tstamp"]
+    ]
+    chat_dates_counts = pd.value_counts(chat_dates)
+    vote_dates = [
+        datetime.datetime.fromtimestamp(x, tz=timezone("US/Pacific")).strftime(
+            "%Y-%m-%d"
+        )
+        for x in anony_vote_df_all["tstamp"]
+    ]
+    vote_dates_counts = pd.value_counts(vote_dates)
+    chat_dates_bar = go.Figure(
+        data=[
+            go.Bar(
+                name="Anony. Vote",
+                x=vote_dates_counts.index,
+                y=vote_dates_counts,
+                text=[f"{val:.0f}" for val in vote_dates_counts],
+                textposition="auto",
+            ),
+            go.Bar(
+                name="Chat",
+                x=chat_dates_counts.index,
+                y=chat_dates_counts,
+                text=[f"{val:.0f}" for val in chat_dates_counts],
+                textposition="auto",
+            ),
+        ]
+    )
+    chat_dates_bar.update_layout(
+        barmode="stack",
+        xaxis_title="Dates",
+        yaxis_title="Count",
+        height=300,
+        width=1200,
+    )
+
+    # Model call counts
+    model_hist_all = df_all[df_all["type"] == "chat"]["model"].value_counts()
+    model_hist_1_day = df_1_day[df_1_day["type"] == "chat"]["model"].value_counts()
+    model_hist_1_hour = df_1_hour[df_1_hour["type"] == "chat"]["model"].value_counts()
+    model_hist = merge_counts(
+        [model_hist_all, model_hist_1_day, model_hist_1_hour],
+        on="model",
+        names=["All", "Last Day", "Last Hour"],
+    )
+    model_hist_md = model_hist.to_markdown(index=False, tablefmt="github")
+
+    # Action counts
+    action_hist_all = df_all["type"].value_counts()
+    action_hist_1_day = df_1_day["type"].value_counts()
+    action_hist_1_hour = df_1_hour["type"].value_counts()
+    action_hist = merge_counts(
+        [action_hist_all, action_hist_1_day, action_hist_1_hour],
+        on="type",
+        names=["All", "Last Day", "Last Hour"],
+    )
+    action_hist_md = action_hist.to_markdown(index=False, tablefmt="github")
+
+    # Anony vote counts
+    anony_vote_hist_all = anony_vote_df_all["type"].value_counts()
+    anony_vote_df_1_day = get_anony_vote_df(df_1_day)
+    anony_vote_hist_1_day = anony_vote_df_1_day["type"].value_counts()
+    # anony_vote_df_1_hour = get_anony_vote_df(df_1_hour)
+    # anony_vote_hist_1_hour = anony_vote_df_1_hour["type"].value_counts()
+    anony_vote_hist = merge_counts(
+        [anony_vote_hist_all, anony_vote_hist_1_day],
+        on="type",
+        names=["All", "Last Day"],
+    )
+    anony_vote_hist_md = anony_vote_hist.to_markdown(index=False, tablefmt="github")
+
+    # Last 24 hours
+    chat_1_day = df_1_day[df_1_day["type"] == "chat"]
+    num_chats_last_24_hours = []
+    base = df_1_day["tstamp"].min()
+    for i in range(24, 0, -1):
+        left = base + (i - 1) * 3600
+        right = base + i * 3600
+        num = ((chat_1_day["tstamp"] >= left) & (chat_1_day["tstamp"] < right)).sum()
+        num_chats_last_24_hours.append(num)
+    times = [
+        datetime.datetime.fromtimestamp(
+            base + i * 3600, tz=timezone("US/Pacific")
+        ).strftime("%Y-%m-%d %H:%M:%S %Z")
+        for i in range(24, 0, -1)
+    ]
+    last_24_hours_df = pd.DataFrame({"time": times, "value": num_chats_last_24_hours})
+    last_24_hours_md = last_24_hours_df.to_markdown(index=False, tablefmt="github")
+
+    # Last update datetime
+    last_updated_tstamp = now_t
+    last_updated_datetime = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y-%m-%d %H:%M:%S %Z")
+
+    # code.interact(local=locals())
+
+    return {
+        "chat_dates_bar": chat_dates_bar,
+        "model_hist_md": model_hist_md,
+        "action_hist_md": action_hist_md,
+        "anony_vote_hist_md": anony_vote_hist_md,
+        "num_chats_last_24_hours": last_24_hours_md,
+        "last_updated_datetime": last_updated_datetime,
+    }
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    basic_stats = report_basic_stats(log_files)
+
+    print(basic_stats["action_hist_md"] + "\n")
+    print(basic_stats["model_hist_md"] + "\n")
+    print(basic_stats["anony_vote_hist_md"] + "\n")
+    print(basic_stats["num_chats_last_24_hours"] + "\n")

arena_elo/elo_rating/clean_battle_data.py

@@ -0,0 +1,342 @@
+"""
+Clean chatbot arena battle log.
+
+Usage:
+python3 clean_battle_data.py --mode conv_release
+"""
+import argparse
+import datetime
+import json
+import os
+import sys
+from pytz import timezone
+import time
+import PIL
+from PIL import ImageFile
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+from tqdm import tqdm
+
+from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
+from .utils import detect_language, get_time_stamp_from_date
+
+VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote"]
+IDENTITY_WORDS = [
+    "vicuna",
+    "lmsys",
+    "koala",
+    "uc berkeley",
+    "open assistant",
+    "laion",
+    "chatglm",
+    "chatgpt",
+    "gpt-4",
+    "openai",
+    "anthropic",
+    "claude",
+    "bard",
+    "palm",
+    "lamda",
+    "google",
+    "llama",
+    "qianwan",
+    "alibaba",
+    "mistral",
+    "zhipu",
+    "KEG lab",
+    "01.AI",
+    "AI2",
+    "Tülu",
+    "Tulu",
+    "NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.",
+    "$MODERATION$ YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES.",
+    "API REQUEST ERROR. Please increase the number of max tokens.",
+    "**API REQUEST ERROR** Reason: The response was blocked.",
+    "**API REQUEST ERROR**",
+]
+
+for i in range(len(IDENTITY_WORDS)):
+    IDENTITY_WORDS[i] = IDENTITY_WORDS[i].lower()
+
+
+def remove_html(raw):
+    if raw.startswith("<h3>"):
+        return raw[raw.find(": ") + 2 : -len("</h3>\n")]
+    if raw.startswith("### Model A: ") or raw.startswith("### Model B: "):
+        return raw[13:]
+    return raw
+
+
+def to_openai_format(messages):
+    roles = ["user", "assistant"]
+    ret = []
+    for i, x in enumerate(messages):
+        ret.append({"role": roles[i % 2], "content": x[1]})
+    return ret
+
+
+def replace_model_name(old_name, tstamp):
+    replace_dict = {
+        "bard": "palm-2",
+        "claude-v1": "claude-1",
+        "claude-instant-v1": "claude-instant-1",
+        "oasst-sft-1-pythia-12b": "oasst-pythia-12b",
+        "claude-2": "claude-2.0",
+        "PlayGroundV2": "PlayGround V2",
+        "PlayGroundV2.5": "PlayGround V2.5",
+    }
+    if old_name in ["gpt-4", "gpt-3.5-turbo"]:
+        if tstamp > 1687849200:
+            return old_name + "-0613"
+        else:
+            return old_name + "-0314"
+    if old_name in replace_dict:
+        return replace_dict[old_name]
+    return old_name
+
+
+def read_file(filename):
+    data = []
+    for retry in range(5):
+        try:
+            # lines = open(filename).readlines()
+            for l in open(filename):
+                row = json.loads(l)
+                if row["type"] in VOTES:
+                    data.append(row)
+            break
+        except FileNotFoundError:
+            time.sleep(2)
+        except json.JSONDecodeError:
+            print(f"Error in reading {filename}")
+            print(row)
+            exit(0)
+    return data
+
+
+def read_file_parallel(log_files, num_threads=16):
+    data_all = []
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
+
+def load_image(image_path):
+    try:
+        return PIL.Image.open(image_path)
+    except:
+        return None
+
+def clean_battle_data(log_files, exclude_model_names, ban_ip_list=None, sanitize_ip=False, mode="simple"):
+    data = read_file_parallel(log_files, num_threads=16)
+
+    convert_type = {
+        "leftvote": "model_a",
+        "rightvote": "model_b",
+        "tievote": "tie",
+        "bothbad_vote": "tie (bothbad)",
+    }
+    
+    all_models = set()
+    all_ips = dict()
+    ct_anony = 0
+    ct_invalid = 0
+    ct_leaked_identity = 0
+    ct_banned = 0
+    battles = []
+    for row in tqdm(data, desc="Cleaning"):
+        if row["models"][0] is None or row["models"][1] is None:
+            print(f"Invalid model names: {row['models']}")
+            continue
+
+        # Resolve model names
+        models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
+        if "model_name" in row["states"][0]:
+            models_hidden = [
+                row["states"][0]["model_name"],
+                row["states"][1]["model_name"],
+            ]
+            if models_hidden[0] is None:
+                models_hidden = models_public
+        else:
+            models_hidden = models_public
+
+        if (models_public[0] == "" and models_public[1] != "") or (
+            models_public[1] == "" and models_public[0] != ""
+        ):
+            ct_invalid += 1
+            print(f"Invalid model names: {models_public}")
+            continue
+
+        if row["anony"]:
+            anony = True
+            models = models_hidden
+            ct_anony += 1
+        else:
+            anony = False
+            models = models_public
+            if not models_public == models_hidden:
+                print(f"Model names mismatch: {models_public} vs {models_hidden}")
+                ct_invalid += 1
+                continue
+
+        def preprocess_model_name(m):
+            if m == "Playground v2":
+                return 'playground_PlayGroundV2_generation'
+            if m == "Playground v2.5":
+                return 'playground_PlayGroundV2.5_generation'
+            return m
+        models = [preprocess_model_name(m) for m in models]
+
+        # valid = True
+        # for _model in models:
+        #     print(_model)
+        #     input()
+        #     try:
+        #         platform, model_name, task = _model.split("_")
+        #     except ValueError:
+        #         print(f"Invalid model names: {_model}")
+        #         valid = False
+        #         break
+        #     if not (platform.lower() in ["playground", "imagenhub", 'fal'] and (task == "generation" or task == "text2image")):
+        #         valid = False
+        #         break
+        # if not valid:
+        #     ct_invalid += 1
+        #     print(f"Invalid model names: {models} for t2i_generation")
+        #     continue
+        # for i, _model in enumerate(models):
+        #     platform, model_name, task = _model.split("_")
+        #     models[i] = model_name
+
+        models = [replace_model_name(m, row["tstamp"]) for m in models]
+        
+        # Exclude certain models
+        if exclude_model_names and any(x in exclude_model_names for x in models):
+            ct_invalid += 1
+            continue
+
+        if mode == "conv_release":
+            # assert the two images are the same
+            date = datetime.datetime.fromtimestamp(row["tstamp"], tz=timezone("US/Pacific")).strftime("%Y-%m-%d") # 2024-02-29
+            image_path_format = f"{LOG_ROOT_DIR}/{date}-convinput_images/input_image_"
+            image_path_0 = image_path_format + str(row["states"][0]["conv_id"]) + ".png"
+            image_path_1 = image_path_format + str(row["states"][1]["conv_id"]) + ".png"
+            if not os.path.exists(image_path_0) or not os.path.exists(image_path_1):
+                print(f"Image not found for {image_path_0} or {image_path_1}")
+                ct_invalid += 1
+                continue
+            
+            image_0 = load_image(image_path_0)
+            image_1 = load_image(image_path_1)
+            if image_0 is None or image_1 is None:
+                print(f"Image not found for {image_path_0} or {image_path_1}")
+                ct_invalid += 1
+                continue
+            if image_0.tobytes() != image_1.tobytes():
+                print(f"Image not the same for {image_path_0} and {image_path_1}")
+                ct_invalid += 1
+                continue
+        
+        
+        # question_id = row["states"][0]["conv_id"]
+
+        ip = row["ip"]
+        if ip not in all_ips:
+            all_ips[ip] = {"ip": ip, "count": 0, "sanitized_id": len(all_ips)}
+        all_ips[ip]["count"] += 1
+        if sanitize_ip:
+            user_id = f"arena_user_{all_ips[ip]['sanitized_id']}"
+        else:
+            user_id = f"{all_ips[ip]['ip']}"
+
+        if ban_ip_list is not None and ip in ban_ip_list:
+            ct_banned += 1
+            print(f"User {user_id} is banned")
+            continue
+
+        # Save the results
+        battles.append(
+            dict(
+                model_a=models[0],
+                model_b=models[1],
+                winner=convert_type[row["type"]],
+                judge=f"arena_user_{user_id}",
+                anony=anony,
+                tstamp=row["tstamp"],
+            )
+        )
+
+        all_models.update(models_hidden)
+    battles.sort(key=lambda x: x["tstamp"])
+    last_updated_tstamp = battles[-1]["tstamp"]
+
+    last_updated_datetime = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y-%m-%d %H:%M:%S %Z")
+
+    print(
+        f"#votes: {len(data)}, #invalid votes: {ct_invalid}, "
+        f"#leaked_identity: {ct_leaked_identity} "
+        f"#banned: {ct_banned} "
+    )
+    print(f"#battles: {len(battles)}, #anony: {ct_anony}")
+    print(f"#models: {len(all_models)}, {all_models}")
+    print(f"last-updated: {last_updated_datetime}")
+
+    if ban_ip_list is not None:
+        for ban_ip in ban_ip_list:
+            if ban_ip in all_ips:
+                del all_ips[ban_ip]
+    print("Top 30 IPs:")
+    print(sorted(all_ips.values(), key=lambda x: x["count"], reverse=True)[:30])
+    return battles
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    parser.add_argument(
+        "--mode", type=str, choices=["simple", "conv_release"], default="simple"
+    )
+    parser.add_argument("--exclude-model-names", type=str, nargs="+")
+    parser.add_argument("--ban-ip-file", type=str)
+    parser.add_argument("--sanitize-ip", action="store_true", default=False)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    ban_ip_list = json.load(open(args.ban_ip_file)) if args.ban_ip_file else None
+
+    battles = clean_battle_data(
+        log_files, args.exclude_model_names or [], ban_ip_list, args.sanitize_ip, args.mode,
+    )
+    last_updated_tstamp = battles[-1]["tstamp"]
+    cutoff_date = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y%m%d")
+
+    if args.mode == "simple":
+        for x in battles:
+            for key in [
+                "conversation_a",
+                "conversation_b",
+                "question_id",
+            ]:
+                if key in x:
+                    del x[key]
+        print("Samples:")
+        for i in range(min(4, len(battles))):
+            print(battles[i])
+        output = f"clean_battle_{cutoff_date}.json"
+    elif args.mode == "conv_release":
+        output = f"clean_battle_conv_{cutoff_date}.json"
+
+    with open(output, "w") as fout:
+        json.dump(battles, fout, indent=2, ensure_ascii=False)
+    print(f"Write cleaned data to {output}")
+    
+    with open("cut_off_date.txt", "w") as fout:
+        fout.write(cutoff_date)

arena_elo/elo_rating/elo_analysis.py

@@ -0,0 +1,395 @@
+import argparse
+from collections import defaultdict
+import datetime
+import json
+import math
+import pickle
+from pytz import timezone
+
+import numpy as np
+import pandas as pd
+import plotly.express as px
+from tqdm import tqdm
+
+from .model_registry import get_model_info
+from .basic_stats import get_log_files
+from .clean_battle_data import clean_battle_data
+        
+pd.options.display.float_format = "{:.2f}".format
+
+
+def compute_elo(battles, K=4, SCALE=400, BASE=10, INIT_RATING=1000):
+    rating = defaultdict(lambda: INIT_RATING)
+
+    for rd, model_a, model_b, winner in battles[
+        ["model_a", "model_b", "winner"]
+    ].itertuples():
+        ra = rating[model_a]
+        rb = rating[model_b]
+        ea = 1 / (1 + BASE ** ((rb - ra) / SCALE))
+        eb = 1 / (1 + BASE ** ((ra - rb) / SCALE))
+        if winner == "model_a":
+            sa = 1
+        elif winner == "model_b":
+            sa = 0
+        elif winner == "tie" or winner == "tie (bothbad)":
+            sa = 0.5
+        else:
+            raise Exception(f"unexpected vote {winner}")
+        rating[model_a] += K * (sa - ea)
+        rating[model_b] += K * (1 - sa - eb)
+
+    return dict(rating)
+
+
+def get_bootstrap_result(battles, func_compute_elo, num_round=1000):
+    rows = []
+    for i in tqdm(range(num_round), desc="bootstrap"):
+        tmp_battles = battles.sample(frac=1.0, replace=True)
+        print(f"Bootstrap round {i}: {tmp_battles.shape}")
+        rows.append(func_compute_elo(tmp_battles))
+    df = pd.DataFrame(rows)
+    return df[df.median().sort_values(ascending=False).index]
+
+
+def compute_elo_mle_with_tie(df, SCALE=400, BASE=10, INIT_RATING=1000):
+    print(f"Columns in df: {df.columns}")
+    print(f"First few rows: {df.head()}")
+
+    from sklearn.linear_model import LogisticRegression
+
+    models = pd.concat([df["model_a"], df["model_b"]]).unique()
+    models = pd.Series(np.arange(len(models)), index=models)
+
+    # duplicate battles
+    df = pd.concat([df, df], ignore_index=True)
+    p = len(models.index)
+    n = df.shape[0]
+
+    X = np.zeros([n, p])
+    X[np.arange(n), models[df["model_a"]]] = +math.log(BASE)
+    X[np.arange(n), models[df["model_b"]]] = -math.log(BASE)
+
+    # one A win => two A win
+    Y = np.zeros(n)
+    Y[df["winner"] == "model_a"] = 1.0
+
+    # one tie => one A win + one B win
+    # find tie + tie (both bad) index
+    tie_idx = (df["winner"] == "tie") | (df["winner"] == "tie (bothbad)")
+    tie_idx[len(tie_idx) // 2 :] = False
+    Y[tie_idx] = 1.0
+
+    lr = LogisticRegression(fit_intercept=False)
+    lr.fit(X, Y)
+
+    elo_scores = SCALE * lr.coef_[0] + INIT_RATING
+    # calibrate llama-13b to 800 if applicable
+    if "llama-13b" in models.index:
+        elo_scores += 800 - elo_scores[models["llama-13b"]]
+    return pd.Series(elo_scores, index=models.index).sort_values(ascending=False)
+
+
+def get_median_elo_from_bootstrap(bootstrap_df):
+    median = dict(bootstrap_df.quantile(0.5))
+    median = {k: int(v + 0.5) for k, v in median.items()}
+    return median
+
+
+def compute_pairwise_win_fraction(battles, model_order, limit_show_number=None):
+    # Times each model wins as Model A
+    a_win_ptbl = pd.pivot_table(
+        battles[battles["winner"] == "model_a"],
+        index="model_a",
+        columns="model_b",
+        aggfunc="size",
+        fill_value=0,
+    )
+
+    # Table counting times each model wins as Model B
+    b_win_ptbl = pd.pivot_table(
+        battles[battles["winner"] == "model_b"],
+        index="model_a",
+        columns="model_b",
+        aggfunc="size",
+        fill_value=0,
+    )
+
+    # Table counting number of A-B pairs
+    num_battles_ptbl = pd.pivot_table(
+        battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0
+    )
+
+    # Computing the proportion of wins for each model as A and as B
+    # against all other models
+    row_beats_col_freq = (a_win_ptbl + b_win_ptbl.T) / (
+        num_battles_ptbl + num_battles_ptbl.T
+    )
+
+    if model_order is None:
+        prop_wins = row_beats_col_freq.mean(axis=1).sort_values(ascending=False)
+        model_order = list(prop_wins.keys())
+
+    if limit_show_number is not None:
+        model_order = model_order[:limit_show_number]
+
+    # Arrange ordering according to proprition of wins
+    row_beats_col = row_beats_col_freq.loc[model_order, model_order]
+    return row_beats_col
+
+
+def visualize_leaderboard_table(rating):
+    models = list(rating.keys())
+    models.sort(key=lambda k: -rating[k])
+
+    emoji_dict = {
+        1: "🥇",
+        2: "🥈",
+        3: "🥉",
+    }
+
+    md = ""
+    md += "| Rank | Model | Elo Rating | Description |\n"
+    md += "| --- | --- | --- | --- |\n"
+    for i, model in enumerate(models):
+        rank = i + 1
+        minfo = get_model_info(model)
+        emoji = emoji_dict.get(rank, "")
+        md += f"| {rank} | {emoji} [{model}]({minfo.link}) | {rating[model]:.0f} | {minfo.description} |\n"
+
+    return md
+
+
+def visualize_pairwise_win_fraction(battles, model_order):
+    row_beats_col = compute_pairwise_win_fraction(battles, model_order)
+    fig = px.imshow(
+        row_beats_col,
+        color_continuous_scale="RdBu",
+        text_auto=".2f",
+        height=700,
+        width=700,
+    )
+    fig.update_layout(
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        xaxis_side="top",
+        title_y=0.07,
+        title_x=0.5,
+    )
+    fig.update_traces(
+        hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Fraction of A Wins: %{z}<extra></extra>"
+    )
+
+    return fig
+
+
+def visualize_battle_count(battles, model_order):
+    ptbl = pd.pivot_table(
+        battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0
+    )
+    battle_counts = ptbl + ptbl.T
+    fig = px.imshow(
+        battle_counts.loc[model_order, model_order],
+        text_auto=True,
+        height=700,
+        width=700,
+    )
+    fig.update_layout(
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        xaxis_side="top",
+        title_y=0.07,
+        title_x=0.5,
+    )
+    fig.update_traces(
+        hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Count: %{z}<extra></extra>"
+    )
+    return fig
+
+
+def visualize_average_win_rate(battles, limit_show_number):
+    row_beats_col_freq = compute_pairwise_win_fraction(
+        battles, None, limit_show_number=limit_show_number
+    )
+    fig = px.bar(
+        row_beats_col_freq.mean(axis=1).sort_values(ascending=False),
+        text_auto=".2f",
+        height=500,
+        width=700,
+    )
+    fig.update_layout(
+        yaxis_title="Average Win Rate", xaxis_title="Model", showlegend=False
+    )
+    return fig
+
+
+def visualize_bootstrap_elo_rating(df, df_final, limit_show_number):
+    bars = (
+        pd.DataFrame(
+            dict(
+                lower=df.quantile(0.025),
+                rating=df_final,
+                upper=df.quantile(0.975),
+            )
+        )
+        .reset_index(names="model")
+        .sort_values("rating", ascending=False)
+    )
+    bars = bars[:limit_show_number]
+    bars["error_y"] = bars["upper"] - bars["rating"]
+    bars["error_y_minus"] = bars["rating"] - bars["lower"]
+    bars["rating_rounded"] = np.round(bars["rating"], 2)
+    fig = px.scatter(
+        bars,
+        x="model",
+        y="rating",
+        error_y="error_y",
+        error_y_minus="error_y_minus",
+        text="rating_rounded",
+        height=500,
+        width=700,
+    )
+    fig.update_layout(xaxis_title="Model", yaxis_title="Rating")
+    return fig
+
+
+def report_elo_analysis_results(battles_json, rating_system="bt", num_bootstrap=100, anony_only=True):
+    battles = pd.DataFrame(battles_json)
+    print(battles)
+    print(f"Data before filtering: {battles.shape}")
+    if battles.empty:
+        raise ValueError("The battles dataframe is empty. Check the data loading or filtering steps.")
+    
+    battles = battles.sort_values(ascending=True, by=["tstamp"])
+    # Only use anonymous votes
+    if anony_only:
+        battles = battles[battles["anony"]].reset_index(drop=True)
+    if battles.empty:
+        raise ValueError("No anonymous battles found. Check the filtering logic.")
+
+    battles_no_ties = battles[~battles["winner"].str.contains("tie")]
+    print(f"Data after filtering: {battles_no_ties.shape}")
+    if battles_no_ties.empty:
+        raise ValueError("No valid battles after removing ties. Check the filtering logic.")
+    # Online update
+    elo_rating_online = compute_elo(battles)
+
+    if rating_system == "bt":
+        bootstrap_df = get_bootstrap_result(
+            battles, compute_elo_mle_with_tie, num_round=num_bootstrap
+        )
+        elo_rating_final = compute_elo_mle_with_tie(battles)
+    elif rating_system == "elo":
+        bootstrap_df = get_bootstrap_result(
+            battles, compute_elo, num_round=num_bootstrap
+        )
+        elo_rating_median = get_median_elo_from_bootstrap(bootstrap_df)
+        elo_rating_final = elo_rating_median
+
+    model_order = list(elo_rating_final.keys())
+    model_order.sort(key=lambda k: -elo_rating_final[k])
+
+    limit_show_number = 25  # limit show number to make plots smaller
+    model_order = model_order[:limit_show_number]
+
+    # leaderboard_table_df: elo rating, variance, 95% interval, number of battles
+    leaderboard_table_df = pd.DataFrame(
+        {
+            "rating": elo_rating_final,
+            "variance": bootstrap_df.var(),
+            "rating_q975": bootstrap_df.quantile(0.975),
+            "rating_q025": bootstrap_df.quantile(0.025),
+            "num_battles": battles["model_a"].value_counts()
+            + battles["model_b"].value_counts(),
+        }
+    )
+
+    # Plots
+    leaderboard_table = visualize_leaderboard_table(elo_rating_final)
+    win_fraction_heatmap = visualize_pairwise_win_fraction(battles_no_ties, model_order)
+    battle_count_heatmap = visualize_battle_count(battles_no_ties, model_order)
+    average_win_rate_bar = visualize_average_win_rate(
+        battles_no_ties, limit_show_number
+    )
+    bootstrap_elo_rating = visualize_bootstrap_elo_rating(
+        bootstrap_df, elo_rating_final, limit_show_number
+    )
+
+    last_updated_tstamp = battles["tstamp"].max()
+    last_updated_datetime = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y-%m-%d %H:%M:%S %Z")
+
+    return {
+        "rating_system": rating_system,
+        "elo_rating_online": elo_rating_online,
+        "elo_rating_final": elo_rating_final,
+        "leaderboard_table": leaderboard_table,
+        "win_fraction_heatmap": win_fraction_heatmap,
+        "battle_count_heatmap": battle_count_heatmap,
+        "average_win_rate_bar": average_win_rate_bar,
+        "bootstrap_elo_rating": bootstrap_elo_rating,
+        "last_updated_datetime": last_updated_datetime,
+        "last_updated_tstamp": last_updated_tstamp,
+        "bootstrap_df": bootstrap_df,
+        "leaderboard_table_df": leaderboard_table_df,
+    }
+
+
+def pretty_print_elo_rating(rating):
+    model_order = list(rating.keys())
+    model_order.sort(key=lambda k: -rating[k])
+    for i, model in enumerate(model_order):
+        print(f"{i+1:2d}, {model:25s}, {rating[model]:.0f}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--clean-battle-file", type=str)
+    parser.add_argument("--max-num-files", type=int)
+    parser.add_argument("--num-bootstrap", type=int, default=100)
+    parser.add_argument(
+        "--rating-system", type=str, choices=["bt", "elo"], default="bt"
+    )
+    parser.add_argument("--exclude-tie", action="store_true", default=False)
+    args = parser.parse_args()
+
+    np.random.seed(42)
+
+    if args.clean_battle_file:
+        # Read data from a cleaned battle files
+        battles = pd.read_json(args.clean_battle_file)
+    else:
+        # Read data from all log files
+        log_files = get_log_files(args.max_num_files)
+        battles = clean_battle_data(log_files)
+
+    anony_results = report_elo_analysis_results(
+        battles, rating_system=args.rating_system, num_bootstrap=args.num_bootstrap, anony_only=True
+    )
+    full_results = report_elo_analysis_results(
+        battles, rating_system=args.rating_system, num_bootstrap=args.num_bootstrap, anony_only=False
+    )
+    
+
+    print("# Online Elo")
+    pretty_print_elo_rating(anony_results["elo_rating_online"])
+    print("# Median")
+    pretty_print_elo_rating(anony_results["elo_rating_final"])
+    print(f"Annoy last update : {anony_results['last_updated_datetime']}")
+    print(f"Full last update : {full_results['last_updated_datetime']}")
+
+    last_updated_tstamp = full_results["last_updated_tstamp"]
+    cutoff_date = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y%m%d")
+
+
+    results = {
+        "anony": anony_results,
+        "full": full_results,
+    }
+    with open("cut_off_date.txt", "w") as fout:
+        fout.write(cutoff_date)
+    with open(f"elo_results_{cutoff_date}.pkl", "wb") as fout:
+        pickle.dump(results, fout)

arena_elo/elo_rating/filter_clean_battle_data.py

@@ -0,0 +1,34 @@
+import json
+import argparse
+import pandas as pd
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--task_name", type=str, default="image_editing", choices=["image_editing", "t2i_generation", "video_generation"])
+    parser.add_argument("--clean-battle-file", type=str)
+    parser.add_argument("--model-info-file", type=str)
+
+    args = parser.parse_args()
+
+
+    if args.clean_battle_file:
+        print(args.clean_battle_file)
+        with open(args.clean_battle_file, 'r') as f:
+            battle_data = json.load(f)
+
+    if args.model_info_file:
+        with open(args.model_info_file, 'r') as f:
+            model_info = json.load(f)
+
+    valid_models = set(model_info.keys())
+
+    filtered_battle_data = [
+        entry for entry in battle_data
+        if entry['model_a'] in valid_models and entry['model_b'] in valid_models
+    ]
+
+    with open(f"filtered_battle_{args.task_name}.json", 'w') as f:
+        json.dump(filtered_battle_data, f, indent=2)
+
+    print(f"Filtered data saved to filtered_battle_{args.task_name}.json")

arena_elo/elo_rating/generate_leaderboard.py

@@ -0,0 +1,88 @@
+import csv
+import fire
+import json
+import pandas as pd
+import pickle
+
+
+def main(
+        model_info_file: str,
+        elo_rating_pkl: str,
+        output_csv: str
+):
+    model_info = {}
+    with open(model_info_file, 'r', newline='', encoding='utf-8') as file:
+        csv_reader = csv.reader(file)
+        header = next(csv_reader)
+        for row in csv_reader:
+            if len(row) == 8:  # Ensure that all columns are present in the row
+                name, upload_date, description, parameter_count, creator, result_path, license, link = row
+                model_info[name] = {
+                    "upload_date": upload_date,
+                    "description": description,
+                    "parameter_count": parameter_count,
+                    "creator": creator,
+                    "result_path": result_path,
+                    "license": license,
+                    "link": link
+                }
+
+    with open(elo_rating_pkl, "rb") as fin:
+        elo_rating_results = pickle.load(fin)
+
+    anony_elo_rating_results = elo_rating_results["anony"]
+    full_elo_rating_results = elo_rating_results["full"]
+    anony_leaderboard_data = anony_elo_rating_results["leaderboard_table_df"]
+    full_leaderboard_data = full_elo_rating_results["leaderboard_table_df"]
+
+    fields = ["key", "Model", "Arena Elo rating (anony)", "Arena Elo rating (full)", "license", "creator", "link"]
+    # set creator and license to empty for now
+    all_models = anony_leaderboard_data.index.tolist()
+
+    for model in all_models:
+        if not model in model_info:
+            model_info[model] = {}
+            model_info[model]["license"] = "N/A"
+            model_info[model]["creator"] = "N/A"
+            model_info[model]["link"] = "N/A"
+            print(f"Model {model} not found in model_info.json")
+            #continue # 没有 model_info 不显示
+        model_info[model]["Model"] = model
+        model_info[model]["key"] = model
+
+        if model in anony_leaderboard_data.index:
+            model_info[model]["Arena Elo rating (anony)"] = anony_leaderboard_data.loc[model, "rating"]
+        else:
+            model_info[model]["Arena Elo rating (anony)"] = 0
+
+        if model in full_elo_rating_results["leaderboard_table_df"].index:
+            model_info[model]["Arena Elo rating (full)"] = full_leaderboard_data.loc[model, "rating"]
+        else:
+            model_info[model]["Arena Elo rating (full)"] = 0
+        # if model in anony_leaderboard_data.index:
+        #     model_info[model]["Arena Elo rating"] = anony_leaderboard_data.loc[model, "rating"]
+        # else:
+        #     model_info[model]["Arena Elo rating"] = 0
+
+    final_model_info = {}
+    for model in model_info:
+        if "Model" in model_info[model]:
+            final_model_info[model] = model_info[model]
+    model_info = final_model_info
+
+    exclude_keys = ['starting_from']
+    for key in exclude_keys:
+        for model in model_info:
+            if key in model_info[model]:
+                del model_info[model][key]
+    df = pd.DataFrame(model_info).T
+    df = df[fields]
+    # sort by anony rating
+    df = df.sort_values(by=["Arena Elo rating (anony)"], ascending=False)
+    df.to_csv(output_csv, index=False)
+    print("Leaderboard data saved to", output_csv)
+    print(df)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

arena_elo/elo_rating/inspect_conv_rating.py

@@ -0,0 +1,234 @@
+import argparse
+import code
+import datetime
+import json
+import os
+from pytz import timezone
+import time
+
+import pandas as pd
+from tqdm import tqdm
+import csv
+
+import base64
+from icecream import ic
+from openai import OpenAI
+
+# Function to encode the image
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+def get_log_files(max_num_files=None):
+    dates = []
+    for month in [2, 3]:
+        for day in range(1, 32):
+            dates.append(f"2024-{month:02d}-{day:02d}")
+
+    num_servers = 1
+    filenames = []
+    for d in dates:
+        for i in range(num_servers):
+            # name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
+            name = os.path.expanduser(f"vision-arena-logs/{d}-conv.json")
+            if os.path.exists(name):
+                filenames.append(name)
+    max_num_files = max_num_files or len(filenames)
+    filenames = filenames[-max_num_files:]
+    return filenames
+
+
+def pretty_print_conversation(messages):
+    for role, msg in messages:
+        print(f"[[{role}]]: {msg}")
+
+
+def get_gpt4v_response(client, img_bs64=None, text_prompt="", use_vision=False):
+    if use_vision:
+        response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": text_prompt},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{img_bs64}"
+                        }
+                    },
+                ],
+            }
+        ],
+        max_tokens=100,
+        )
+    else:
+        response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": text_prompt},
+                ],
+            }
+        ],
+        max_tokens=100,
+        )
+    return response.choices[0].message.content
+
+task_template_map = {
+    "image_caption": "Give me the semantic alignment score between the given image and the given caption: \"{generated_sentence}\" on a scale of 0-100. Only reply the score value.",
+    "vqa": "Rate the answer correctness regarding the question within the context of the given image on a scale of 0-100. Only reply the score value.",
+    "pair_rate_old": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nGiven the instruction and the image, please compare the correctness of responses A and B. Reply with \"leftvote\" if you find A better, \"rightvote\" if B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both responses are equally satisfactory. If you are unable to make a decision, please reply with \"NA\".",
+    "pair_rate_wexplanation": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.",
+    "pair_rate": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. Reply with \"leftvote\" if you find assistant A better, \"rightvote\" if assistant B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both assistants provide equally satisfactory answers. If you are unable to make a decision, please reply with \"NA\"."
+}
+
+def inspect_convs(log_files):
+    ic(log_files)
+    data = []
+    total_vote = 0
+    correct_vote = 0
+        
+    client = OpenAI()
+    with open('all_pairvote_log_wgpt_prtchatbot.csv', 'w', newline='') as csvfile:
+        # fieldnames = ['tstamp', 'type', 'model_1', 'model_2', 'template_name_1', 'template_name_2', 'system_message_1', 'system_message_2', 'role_1', 'role_2', 'instruction_1', 'instruction_2', 'message_1', 'message_2', 'offset_1', 'offset_2', 'conv_id_1', 'conv_id_2', 'model_name_1', 'model_name_2', 'ip']
+        fieldnames = ['tstamp', 'type', 'models', 'states', 'ip', 'gpt_vote']
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        
+        # Write the header
+        writer.writeheader()
+
+        for filename in tqdm(log_files, desc="read files"):
+            for retry in range(5):
+                try:
+                    lines = open(filename).readlines()
+                    break
+                except FileNotFoundError:
+                    time.sleep(2)
+            
+            for l in lines:
+                row = json.loads(l)
+                
+                if "states" not in row:
+                    continue
+                if row["type"] not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
+                    continue
+                
+                model_names = row["states"][0]["model_name"], row["states"][1]["model_name"]
+                
+
+                # Iterate through each state and write the relevant information
+                if not len(row["states"][0]['messages']): continue
+                # ic(row["states"][0]['messages'][1][1])
+
+                if row["states"][0]['messages'][1][1] is None or row["states"][1]['messages'][1][1] is None or "NETWORK ERROR" in row["states"][0]['messages'][1][1] or "NETWORK ERROR" in row["states"][1]['messages'][1][1]: continue
+                total_vote += 1
+                # row = {
+                #     'tstamp': row['tstamp'],
+                #     'type': row['type'],
+                #     'model_1': row['models'][0],
+                #     'model_2': row['models'][1],
+                #     'template_name_1': row["states"][0]['template_name'],
+                #     'system_message_1': row["states"][0]['system_message'],
+                #     'template_name_2': row["states"][1]['template_name'],
+                #     'system_message_2': row["states"][1]['system_message'],
+                #     'role_1': row["states"][0]['roles'],
+                #     'role_2': row["states"][1]['roles'],
+                #     'instruction_1': row["states"][0]['messages'][0][1],
+                #     'instruction_2': row["states"][1]['messages'][0][1],
+                #     'message_1': row["states"][0]['messages'][1][1],
+                #     'message_2': row["states"][1]['messages'][1][1],
+                #     'offset_1': row["states"][0]['offset'],
+                #     'offset_2': row["states"][1]['offset'],
+                #     'conv_id_1': row["states"][0]['conv_id'],
+                #     'conv_id_2': row["states"][1]['conv_id'],
+                #     'model_name_1': row["states"][0]['model_name'],
+                #     'model_name_2': row["states"][1]['model_name'],
+                #     'ip': row['ip']
+                # }
+                # writer.writerow(row)
+                # Convert complex objects to JSON strings
+                # TODO: check two image are the same
+                conv_id = row["states"][0]['conv_id']
+                image_path = os.path.join("/local/home/yujielu/project/Arena-Elo/vision-arena-logs", os.path.basename(filename)[:-5]+"input_images", f"input_image_{conv_id}.png")
+                if not os.path.exists(image_path):
+                    response = "NA"
+                    ic(image_path)
+                else:
+                    base64_image = encode_image(image_path)
+                    left_response = row["states"][0]['messages'][1][1]
+                    right_response = row["states"][1]['messages'][1][1]
+                    sep = "-" * 20
+                    instruction = row["states"][0]['messages'][0][1]
+                    generated_sentence = f"[The Start of Assistant A’s Answer]\n{left_response}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{right_response}\n[The End of Assistant B’s Answer]"
+                    text_prompt = task_template_map["pair_rate"].format(instruction=instruction, generated_sentence=generated_sentence)
+                    # ic(text_prompt)
+                    try:
+                        response = get_gpt4v_response(client, img_bs64=base64_image, text_prompt=text_prompt, use_vision=True)
+                    except:
+                        ic(">>> skip")
+                        response = "NA"
+                    
+                    # response = get_gpt4v_response(client, img_bs64=base64_image, text_prompt=text_prompt, use_vision=True)
+                    ic(row['type'], response)
+                    if response.strip() not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
+                        response = "NA"
+                    # ic(generated_sentence)
+                    
+                    # if row['type'] == "leftvote":
+                    #     row['type'] = "A"
+                    # elif row['type'] == "rightvote":
+                    #     row['type'] = "B"
+                    # elif row['type'] in ["bothbad_vote", "tievote"]:
+                    #     row['type'] = "C"
+                    if row['type'] == response.strip():
+                        correct_vote += 1
+                row['models'] = json.dumps(row['models'])
+                row['states'] = json.dumps(row['states'], ensure_ascii=False)
+                row['gpt_vote'] = response
+                
+                # Write the modified row to the CSV file
+                writer.writerow(row)
+                # if row["type"] == "leftvote":
+                #     winner, loser = model_names[0], model_names[1]
+                #     winner_conv, loser_conv = row["states"][0], row["states"][1]
+                # elif row["type"] == "rightvote":
+                #     loser, winner = model_names[0], model_names[1]
+                #     loser_conv, winner_conv = row["states"][0], row["states"][1]
+
+                # if loser == "llava-v1.5-13b" and winner == "llava-v1.5-13b":
+                #     print("=" * 20)
+                #     print(f"Winner: {winner}")
+                #     pretty_print_conversation(winner_conv["messages"])
+                #     print(f"Loser: {loser}")
+                #     pretty_print_conversation(loser_conv["messages"])
+                #     print("=" * 20)
+                #     input()
+                # if row['type'] == 'bothbad_vote':
+                #     from icecream import ic
+                #     ic(model_names)
+                # if row["type"] == "bothbad_vote" and "gpt-4-vision-preview" in model_names:
+                #    print("=" * 20)
+                #    print(f"Model A: {model_names[0]}")
+                #    pretty_print_conversation(row["states"][0]["messages"])
+                #    print(f"Model B: {model_names[1]}")
+                #    pretty_print_conversation(row["states"][1]["messages"])
+                #    print("=" * 20)
+                #    input()
+                # if correct_vote >= 300: break
+    ic(total_vote, correct_vote)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    
+
+            
+    inspect_convs(log_files)

arena_elo/elo_rating/inspect_cost.py

@@ -0,0 +1,177 @@
+import fire
+import time
+import json
+from collections import defaultdict
+from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
+from .utils import detect_language, get_time_stamp_from_date, get_input_image_path, load_image_from_path
+from tqdm import tqdm
+VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote", "chat"]
+
+
+def remove_html(raw):
+    if raw.startswith("<h3>"):
+        return raw[raw.find(": ") + 2 : -len("</h3>\n")]
+    if raw.startswith("### Model A: ") or raw.startswith("### Model B: "):
+        return raw[13:]
+    return raw
+
+
+def read_file(filename):
+    data = []
+    for retry in range(5):
+        try:
+            # lines = open(filename).readlines()
+            for l in open(filename):
+                row = json.loads(l)
+                if row["type"] in VOTES:
+                    data.append(row)
+            break
+        except FileNotFoundError:
+            time.sleep(2)
+    return data
+
+
+def read_file_parallel(log_files, num_threads=16):
+    data_all = []
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
+
+def num_tokens(s:str):
+    if s is None:
+        return 0
+    return len(s) / 4
+
+def main(
+):
+    log_files = get_log_files()
+    data = read_file_parallel(log_files)
+    
+    all_model_counts = defaultdict(int)
+    all_model_input_tokens_counts = defaultdict(list)
+    all_model_output_tokens_counts = defaultdict(list)
+    all_model_image_sizes = defaultdict(list)
+    chat_battle_counts = defaultdict(int)
+    for row in tqdm(data, desc="counting"):
+        if row['type'] == "chat":
+            chat_battle_counts["chat"] += 1
+            all_model_counts[row['model']] += 1
+            tstamp = row["tstamp"]
+            conv_id = row["state"]["conv_id"]
+            
+            image = load_image_from_path(get_input_image_path(tstamp, conv_id))
+            if image is None:
+                image_size = None
+            else:
+                image_size = load_image_from_path(get_input_image_path(tstamp, conv_id)).size
+            all_model_image_sizes[row['model']].append(image_size)
+            try:
+                for message in row["state"]["messages"][row["state"]["offset"] :: 2]:
+                    all_model_input_tokens_counts[row['model']].append(num_tokens(message[1]))
+                for message in row["state"]["messages"][row["state"]["offset"] + 1 :: 2]:
+                    all_model_output_tokens_counts[row['model']].append(num_tokens(message[1]))
+            except Exception as e:
+                print(row)
+                raise e
+            
+        else:
+            chat_battle_counts[row['type']] += 1
+            if row["models"][0] is None or row["models"][1] is None:
+                continue
+
+            # Resolve model names
+            models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
+            if "model_name" in row["states"][0]:
+                models_hidden = [
+                    row["states"][0]["model_name"],
+                    row["states"][1]["model_name"],
+                ]
+                if models_hidden[0] is None:
+                    models_hidden = models_public
+            else:
+                models_hidden = models_public
+
+            if (models_public[0] == "" and models_public[1] != "") or (
+                models_public[1] == "" and models_public[0] != ""
+            ):
+                continue
+
+            if models_public[0] == "" or models_public[0] == "Model A":
+                anony = True
+                models = models_hidden
+            else:
+                anony = False
+                models = models_public
+                if not models_public == models_hidden:
+                    continue
+            
+            all_model_counts[models[0]] += 1
+            all_model_counts[models[1]] += 1
+            tstamp = row["tstamp"]
+            conv_id1 = row["states"][0]["conv_id"]
+            conv_id2 = row["states"][1]["conv_id"]
+            
+            image1 = load_image_from_path(get_input_image_path(tstamp, conv_id1))
+            image2 = load_image_from_path(get_input_image_path(tstamp, conv_id2))
+            all_model_image_sizes[models[0]].append(None if image1 is None else image1.size)
+            all_model_image_sizes[models[1]].append(None if image2 is None else image2.size)
+            
+            for message in row["states"][0]["messages"][row["states"][0]["offset"] :: 2]:
+                all_model_input_tokens_counts[models[0]].append(num_tokens(message[1]))
+            for message in row["states"][0]["messages"][row["states"][0]["offset"] + 1 :: 2]:
+                all_model_output_tokens_counts[models[0]].append(num_tokens(message[1]))
+            for message in row["states"][1]["messages"][row["states"][1]["offset"] :: 2]:
+                all_model_input_tokens_counts[models[1]].append(num_tokens(message[1]))
+            for message in row["states"][1]["messages"][row["states"][1]["offset"] + 1 :: 2]:
+                all_model_output_tokens_counts[models[1]].append(num_tokens(message[1]))
+
+    print("### Chat battle counts (requests)")
+    print(json.dumps(chat_battle_counts, indent=4))
+    
+    print("### Model counts (requests)")
+    print(json.dumps(all_model_counts, indent=4))
+    
+    print("### Model Avg input tokens counts (tokens)")
+    average_input_tokens_counts = {}
+    for model, counts in all_model_input_tokens_counts.items():
+        average_input_tokens_counts[model] = sum(counts) / len(counts)
+    print(json.dumps(average_input_tokens_counts, indent=4))
+    
+    print("### Model AVg output tokens counts (tokens)")
+    average_output_tokens_counts = {}
+    for model, counts in all_model_output_tokens_counts.items():
+        average_output_tokens_counts[model] = sum(counts) / len(counts)
+    print(json.dumps(average_output_tokens_counts, indent=4))
+    
+    print("### Model Avg image sizes (height, width)")
+    average_image_sizes = {}
+    for model, sizes in all_model_image_sizes.items():
+        avg_height = sum([size[0] for size in sizes if size is not None]) / len(sizes)
+        avg_width = sum([size[1] for size in sizes if size is not None]) / len(sizes)
+        average_image_sizes[model] = (avg_height, avg_width)
+    print(json.dumps(average_image_sizes, indent=4))
+    
+    print("### GPT-4V estimated cost (USD)")
+    gpt_4v_name = "gpt-4-vision-preview"
+    gpt_4v_cost = {}
+    gpt_4v_cost['input'] = sum(all_model_input_tokens_counts[gpt_4v_name]) / 1000 * 0.01
+    gpt_4v_cost['output'] = sum(all_model_output_tokens_counts[gpt_4v_name]) / 1000 * 0.03
+    
+    all_image_cost = 0
+    for size in all_model_image_sizes[gpt_4v_name]:
+        if size is None:
+            continue
+        all_image_tokens = (size[0] // 512 + 1) * (size[1] // 512 + 1) * 170 + 85
+        all_image_cost += all_image_tokens / 1000 * 0.01
+    gpt_4v_cost['image'] = all_image_cost
+    print(json.dumps(gpt_4v_cost, indent=4))
+        
+    
+    
+    
+if __name__ == "__main__":
+    fire.Fire(main)

arena_elo/elo_rating/inspect_elo_rating_pkl.py

@@ -0,0 +1,33 @@
+import pickle
+import plotly.graph_objects as go
+
+def output_figure(data, figure_name="battle_count_heatmap", label="annoy"):
+    fig = data[label][figure_name]
+    fig.update_layout(
+        height=700,
+        width=700,
+        title={'text': f'{figure_name}', 'x': 0.5, 'y': 0.07},
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        # coloraxis_colorscale=[[0.0, '#0d0887'], [1.0, '#f0f921']],
+        margin={'t': 60}
+    )
+    fig.write_image(f"{figure_name}.png")
+
+with open("./results/latest/elo_results.pkl",'rb') as f:
+    data = pickle.load(f)
+    print()
+    df = data["anony"]["leaderboard_table_df"]
+    # sort by rating 
+    print(data["anony"].keys())
+    
+    for figure_name in [ 'win_fraction_heatmap', 'battle_count_heatmap',]:
+        output_figure(data, figure_name, "anony")
+        
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df) 
+    df = data["full"]["leaderboard_table_df"]
+    # sort by rating 
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df)
+    print('done') 

arena_elo/elo_rating/model_registry.py

@@ -0,0 +1,578 @@
+"""Additional information of the models."""
+from collections import namedtuple, OrderedDict
+from typing import List
+
+
+ModelInfo = namedtuple("ModelInfo", ["simple_name", "link", "description"])
+
+
+model_info = OrderedDict()
+
+
+def register_model_info(
+    full_names: List[str], simple_name: str, link: str, description: str
+):
+    info = ModelInfo(simple_name, link, description)
+
+    for full_name in full_names:
+        model_info[full_name] = info
+
+
+def get_model_info(name: str) -> ModelInfo:
+    if name in model_info:
+        return model_info[name]
+    else:
+        # To fix this, please use `register_model_info` to register your model
+        return ModelInfo(
+            name, "", "Register the description at arena.model/model_registry.py"
+        )
+
+
+register_model_info(
+    [
+        "IEITYuan/Yuan2-2B-Janus-hf",
+        "IEITYuan/Yuan2-2B-hf",
+        "IEITYuan/Yuan2-51B-hf",
+        "IEITYuan/Yuan2-102B-hf",
+    ],
+    "IEIT-Yuan2",
+    "https://github.com/IEIT-Yuan/Yuan-2.0",
+    "Yuan2.0 is a new generation Fundamental Large Language Model developed by IEIT System.",
+)
+
+register_model_info(
+    ["mixtral-8x7b-instruct-v0.1", "mistral-7b-instruct"],
+    "Mixtral of experts",
+    "https://mistral.ai/news/mixtral-of-experts/",
+    "A Mixture-of-Experts model by Mistral AI",
+)
+
+register_model_info(
+    ["gemini-pro"],
+    "Gemini",
+    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
+    "Gemini by Google",
+)
+
+register_model_info(
+    ["gemini-pro-vision"],
+    "Gemini",
+    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
+    "Gemini by Google",
+)
+
+register_model_info(
+    ["solar-10.7b-instruct-v1.0"],
+    "SOLAR-10.7B-Instruct",
+    "https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0",
+    "A model trained using depth up-scaling by Upstage AI",
+)
+
+register_model_info(
+    ["gpt-4-turbo"],
+    "GPT-4-Turbo",
+    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
+    "GPT-4-Turbo by OpenAI",
+)
+
+register_model_info(
+    ["gpt-4-vision-preview"],
+    "gpt-4-vision-preview",
+    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
+    "GPT-4(V) by OpenAI",
+)
+
+register_model_info(
+    ["gpt-3.5-turbo", "gpt-3.5-turbo-0314", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-1106"],
+    "GPT-3.5",
+    "https://platform.openai.com/docs/models/gpt-3-5",
+    "GPT-3.5-Turbo by OpenAI",
+)
+
+register_model_info(
+    ["gpt-4", "gpt-4-0314", "gpt-4-0613"],
+    "GPT-4",
+    "https://openai.com/research/gpt-4",
+    "GPT-4 by OpenAI",
+)
+
+register_model_info(
+    ["claude-2.1", "claude-2.0"],
+    "Claude",
+    "https://www.anthropic.com/index/claude-2",
+    "Claude 2 by Anthropic",
+)
+
+register_model_info(
+    ["claude-1"],
+    "Claude",
+    "https://www.anthropic.com/index/introducing-claude",
+    "Claude 1 by Anthropic",
+)
+
+register_model_info(
+    ["claude-instant-1", "claude-instant-1.2"],
+    "Claude Instant",
+    "https://www.anthropic.com/index/introducing-claude",
+    "Claude Instant by Anthropic",
+)
+
+register_model_info(
+    ["pplx-70b-online", "pplx-7b-online"],
+    "pplx-online-llms",
+    "https://blog.perplexity.ai/blog/introducing-pplx-online-llms",
+    "Online LLM API by Perplexity AI",
+)
+
+register_model_info(
+    ["openhermes-2.5-mistral-7b"],
+    "OpenHermes-2.5-Mistral-7B",
+    "https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B",
+    "a mistral-based model fine-tuned on 1M GPT-4 outputs",
+)
+
+register_model_info(
+    ["starling-lm-7b-alpha"],
+    "Starling-LM-7B-alpha",
+    "https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha",
+    "an open model trained using RLAIF by Berkeley",
+)
+
+register_model_info(
+    ["tulu-2-dpo-70b"],
+    "Tulu 2",
+    "https://huggingface.co/allenai/tulu-2-dpo-70b",
+    "an instruction and RLHF model by UW/AllenAI",
+)
+
+register_model_info(
+    ["yi-34b-chat", "yi-6b-chat"],
+    "Yi-Chat",
+    "https://huggingface.co/01-ai/Yi-34B-Chat",
+    "A large language model by 01 AI",
+)
+
+register_model_info(
+    ["llama-2-70b-chat", "llama-2-34b-chat", "llama-2-13b-chat", "llama-2-7b-chat"],
+    "Llama 2",
+    "https://ai.meta.com/llama/",
+    "open foundation and fine-tuned chat models by Meta",
+)
+
+register_model_info(
+    [
+        "vicuna-33b",
+        "vicuna-33b-v1.3",
+        "vicuna-13b",
+        "vicuna-13b-v1.3",
+        "vicuna-7b",
+        "vicuna-7b-v1.3",
+    ],
+    "Vicuna",
+    "https://lmsys.org/blog/2023-03-30-vicuna/",
+    "a chat assistant fine-tuned on user-shared conversations by LMSYS",
+)
+
+register_model_info(
+    ["chatglm3-6b", "chatglm2-6b", "chatglm-6b"],
+    "ChatGLM",
+    "https://chatglm.cn/blog",
+    "an open bilingual dialogue language model by Tsinghua University",
+)
+
+register_model_info(
+    ["openchat-3.5"],
+    "OpenChat 3.5",
+    "https://github.com/imoneoi/openchat",
+    "an open model fine-tuned on Mistral-7B using C-RLFT",
+)
+
+register_model_info(
+    ["tenyxchat-7b-v1"],
+    "TenyxChat-7B",
+    "https://huggingface.co/tenyx/TenyxChat-7B-v1",
+    "an open model DPO trained on top of OpenChat-3.5 using Tenyx fine-tuning",
+)
+
+register_model_info(
+    ["zephyr-7b-beta", "zephyr-7b-alpha"],
+    "Zephyr",
+    "https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha",
+    "a chatbot fine-tuned from Mistral by Hugging Face",
+)
+
+register_model_info(
+    ["notus-7b-v1"],
+    "Notus",
+    "https://huggingface.co/argilla/notus-7b-v1",
+    "a chatbot fine-tuned from Zephyr SFT by Argilla",
+)
+
+register_model_info(
+    ["catppt"],
+    "CatPPT",
+    "https://huggingface.co/rishiraj/CatPPT",
+    "a chatbot fine-tuned from a SLERP merged model by Rishiraj Acharya",
+)
+
+register_model_info(
+    ["TinyLlama"],
+    "TinyLlama",
+    "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "The TinyLlama project is an open endeavor to pretrain a 1.1B Llama model on 3 trillion tokens.",
+)
+
+register_model_info(
+    ["qwen-14b-chat"],
+    "Qwen",
+    "https://huggingface.co/Qwen/Qwen-14B-Chat",
+    "a large language model by Alibaba Cloud",
+)
+
+register_model_info(
+    ["codellama-34b-instruct", "codellama-13b-instruct", "codellama-7b-instruct"],
+    "Code Llama",
+    "https://ai.meta.com/blog/code-llama-large-language-model-coding/",
+    "open foundation models for code by Meta",
+)
+
+register_model_info(
+    ["wizardlm-70b", "wizardlm-30b", "wizardlm-13b"],
+    "WizardLM",
+    "https://github.com/nlpxucan/WizardLM",
+    "an instruction-following LLM using evol-instruct by Microsoft",
+)
+
+register_model_info(
+    ["wizardcoder-15b-v1.0"],
+    "WizardLM",
+    "https://github.com/nlpxucan/WizardLM/tree/main/WizardCoder",
+    "Empowering Code Large Language Models with Evol-Instruct",
+)
+
+register_model_info(
+    ["mpt-7b-chat", "mpt-30b-chat"],
+    "MPT-Chat",
+    "https://www.mosaicml.com/blog/mpt-30b",
+    "a chatbot fine-tuned from MPT by MosaicML",
+)
+
+register_model_info(
+    ["guanaco-33b", "guanaco-65b"],
+    "Guanaco",
+    "https://github.com/artidoro/qlora",
+    "a model fine-tuned with QLoRA by UW",
+)
+
+register_model_info(
+    ["gpt4all-13b-snoozy"],
+    "GPT4All-Snoozy",
+    "https://github.com/nomic-ai/gpt4all",
+    "a finetuned LLaMA model on assistant style data by Nomic AI",
+)
+
+register_model_info(
+    ["koala-13b"],
+    "Koala",
+    "https://bair.berkeley.edu/blog/2023/04/03/koala",
+    "a dialogue model for academic research by BAIR",
+)
+
+register_model_info(
+    ["RWKV-4-Raven-14B"],
+    "RWKV-4-Raven",
+    "https://huggingface.co/BlinkDL/rwkv-4-raven",
+    "an RNN with transformer-level LLM performance",
+)
+
+register_model_info(
+    ["alpaca-13b"],
+    "Alpaca",
+    "https://crfm.stanford.edu/2023/03/13/alpaca.html",
+    "a model fine-tuned from LLaMA on instruction-following demonstrations by Stanford",
+)
+
+register_model_info(
+    ["oasst-pythia-12b"],
+    "OpenAssistant (oasst)",
+    "https://open-assistant.io",
+    "an Open Assistant for everyone by LAION",
+)
+
+register_model_info(
+    ["oasst-sft-7-llama-30b"],
+    "OpenAssistant (oasst)",
+    "https://open-assistant.io",
+    "an Open Assistant for everyone by LAION",
+)
+
+register_model_info(
+    ["palm-2"],
+    "PaLM 2 Chat",
+    "https://cloud.google.com/vertex-ai/docs/release-notes#May_10_2023",
+    "PaLM 2 for Chat (chat-bison@001) by Google",
+)
+
+register_model_info(
+    ["llama-7b", "llama-13b"],
+    "LLaMA",
+    "https://arxiv.org/abs/2302.13971",
+    "open and efficient foundation language models by Meta",
+)
+
+register_model_info(
+    ["open-llama-7b-v2-open-instruct", "open-llama-7b-open-instruct"],
+    "Open LLaMa (Open Instruct)",
+    "https://medium.com/vmware-data-ml-blog/starter-llm-for-the-enterprise-instruction-tuning-openllama-7b-d05fc3bbaccc",
+    "Open LLaMa fine-tuned on instruction-following data by VMware",
+)
+
+register_model_info(
+    ["dolly-v2-12b"],
+    "Dolly",
+    "https://www.databricks.com/blog/2023/04/12/dolly-first-open-commercially-viable-instruction-tuned-llm",
+    "an instruction-tuned open large language model by Databricks",
+)
+
+register_model_info(
+    ["stablelm-tuned-alpha-7b"],
+    "StableLM",
+    "https://github.com/stability-AI/stableLM",
+    "Stability AI language models",
+)
+
+register_model_info(
+    ["codet5p-6b"],
+    "CodeT5p-6b",
+    "https://huggingface.co/Salesforce/codet5p-6b",
+    "Code completion model released by Salesforce",
+)
+
+register_model_info(
+    ["fastchat-t5-3b", "fastchat-t5-3b-v1.0"],
+    "FastChat-T5",
+    "https://huggingface.co/lmsys/fastchat-t5-3b-v1.0",
+    "a chat assistant fine-tuned from FLAN-T5 by LMSYS",
+)
+
+register_model_info(
+    ["phoenix-inst-chat-7b"],
+    "Phoenix-7B",
+    "https://huggingface.co/FreedomIntelligence/phoenix-inst-chat-7b",
+    "a multilingual chat assistant fine-tuned from Bloomz to democratize ChatGPT across languages by CUHK(SZ)",
+)
+
+register_model_info(
+    ["realm-7b-v1"],
+    "ReaLM",
+    "https://github.com/FreedomIntelligence/ReaLM",
+    "A chatbot fine-tuned from LLaMA2 with data generated via iterative calls to UserGPT and ChatGPT by CUHK(SZ) and SRIBD.",
+)
+
+register_model_info(
+    ["billa-7b-sft"],
+    "BiLLa-7B-SFT",
+    "https://huggingface.co/Neutralzz/BiLLa-7B-SFT",
+    "an instruction-tuned bilingual LLaMA with enhanced reasoning ability by an independent researcher",
+)
+
+register_model_info(
+    ["h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2"],
+    "h2oGPT-GM-7b",
+    "https://huggingface.co/h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2",
+    "an instruction-tuned OpenLLaMA with enhanced conversational ability by H2O.ai",
+)
+
+register_model_info(
+    ["baize-v2-7b", "baize-v2-13b"],
+    "Baize v2",
+    "https://github.com/project-baize/baize-chatbot#v2",
+    "A chatbot fine-tuned from LLaMA with ChatGPT self-chat data and Self-Disillation with Feedback (SDF) by UCSD and SYSU.",
+)
+
+register_model_info(
+    [
+        "airoboros-l2-7b-2.1",
+        "airoboros-l2-13b-2.1",
+        "airoboros-c34b-2.1",
+        "airoboros-l2-70b-2.1",
+    ],
+    "airoboros",
+    "https://huggingface.co/jondurbin/airoboros-l2-70b-2.1",
+    "an instruction-tuned LlaMa model tuned with 100% synthetic instruction-response pairs from GPT4",
+)
+
+register_model_info(
+    [
+        "spicyboros-7b-2.2",
+        "spicyboros-13b-2.2",
+        "spicyboros-70b-2.2",
+    ],
+    "spicyboros",
+    "https://huggingface.co/jondurbin/spicyboros-70b-2.2",
+    "de-aligned versions of the airoboros models",
+)
+
+register_model_info(
+    ["Robin-7b-v2", "Robin-13b-v2", "Robin-33b-v2"],
+    "Robin-v2",
+    "https://huggingface.co/OptimalScale/robin-7b-v2-delta",
+    "A chatbot fine-tuned from LLaMA-7b, achieving competitive performance on chitchat, commonsense reasoning and instruction-following tasks, by OptimalScale, HKUST.",
+)
+
+register_model_info(
+    ["manticore-13b-chat"],
+    "Manticore 13B Chat",
+    "https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg",
+    "A chatbot fine-tuned from LlaMa across several CoT and chat datasets.",
+)
+
+register_model_info(
+    ["redpajama-incite-7b-chat"],
+    "RedPajama-INCITE-7B-Chat",
+    "https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Chat",
+    "A chatbot fine-tuned from RedPajama-INCITE-7B-Base by Together",
+)
+
+register_model_info(
+    [
+        "falcon-7b",
+        "falcon-7b-instruct",
+        "falcon-40b",
+        "falcon-40b-instruct",
+        "falcon-180b",
+        "falcon-180b-chat",
+    ],
+    "Falcon",
+    "https://huggingface.co/tiiuae/falcon-180B",
+    "TII's flagship series of large language models",
+)
+
+register_model_info(
+    ["tigerbot-7b-sft"],
+    "Tigerbot",
+    "https://huggingface.co/TigerResearch/tigerbot-7b-sft",
+    "TigerBot is a large-scale language model (LLM) with multiple languages and tasks.",
+)
+
+register_model_info(
+    ["internlm-chat-7b", "internlm-chat-7b-8k"],
+    "InternLM",
+    "https://huggingface.co/internlm/internlm-chat-7b",
+    "InternLM is a multi-language large-scale language model (LLM), developed by SHLAB.",
+)
+
+register_model_info(
+    ["Qwen-7B-Chat"],
+    "Qwen",
+    "https://huggingface.co/Qwen/Qwen-7B-Chat",
+    "Qwen is a multi-language large-scale language model (LLM), developed by Damo Academy.",
+)
+
+register_model_info(
+    ["Llama2-Chinese-13b-Chat", "LLama2-Chinese-13B"],
+    "Llama2-Chinese",
+    "https://huggingface.co/FlagAlpha/Llama2-Chinese-13b-Chat",
+    "Llama2-Chinese is a multi-language large-scale language model (LLM), developed by FlagAlpha.",
+)
+
+register_model_info(
+    ["Chinese-Alpaca-2-7B", "Chinese-Alpaca-2-13B"],
+    "Chinese-Alpaca",
+    "https://huggingface.co/hfl/chinese-alpaca-2-13b",
+    "New extended Chinese vocabulary beyond Llama-2, open-sourcing the Chinese LLaMA-2 and Alpaca-2 LLMs.",
+)
+
+register_model_info(
+    ["Vigogne-2-7B-Instruct", "Vigogne-2-13B-Instruct"],
+    "Vigogne-Instruct",
+    "https://huggingface.co/bofenghuang/vigogne-2-7b-instruct",
+    "Vigogne-Instruct is a French large language model (LLM) optimized for instruction-following, developed by Bofeng Huang",
+)
+
+register_model_info(
+    ["Vigogne-2-7B-Chat", "Vigogne-2-13B-Chat"],
+    "Vigogne-Chat",
+    "https://huggingface.co/bofenghuang/vigogne-2-7b-chat",
+    "Vigogne-Chat is a French large language model (LLM) optimized for instruction-following and multi-turn dialogues, developed by Bofeng Huang",
+)
+
+register_model_info(
+    ["stable-vicuna-13B-HF"],
+    "stable-vicuna",
+    "https://huggingface.co/TheBloke/stable-vicuna-13B-HF",
+    "StableVicuna is a Vicuna model fine-tuned using RLHF via PPO on various conversational and instructional datasets.",
+)
+
+register_model_info(
+    ["deluxe-chat-v1", "deluxe-chat-v1.1", "deluxe-chat-v1.2"],
+    "DeluxeChat",
+    "",
+    "Deluxe Chat",
+)
+
+register_model_info(
+    [
+        "Xwin-LM-7B-V0.1",
+        "Xwin-LM-13B-V0.1",
+        "Xwin-LM-70B-V0.1",
+        "Xwin-LM-7B-V0.2",
+        "Xwin-LM-13B-V0.2",
+    ],
+    "Xwin-LM",
+    "https://github.com/Xwin-LM/Xwin-LM",
+    "Chat models developed by Xwin-LM team",
+)
+
+register_model_info(
+    ["lemur-70b-chat"],
+    "Lemur-Chat",
+    "https://huggingface.co/OpenLemur/lemur-70b-chat-v1",
+    "an openly accessible language model optimized for both natural language and coding capabilities ",
+)
+
+register_model_info(
+    ["Mistral-7B-OpenOrca"],
+    "Open-Orca",
+    "https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca",
+    "A fine-tune of [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) using [OpenOrca dataset](https://huggingface.co/datasets/Open-Orca/OpenOrca)",
+)
+
+register_model_info(
+    ["dolphin-2.2.1-mistral-7b"],
+    "dolphin-mistral",
+    "https://huggingface.co/ehartford/dolphin-2.2.1-mistral-7b",
+    "An uncensored fine-tuned Mistral 7B",
+)
+
+register_model_info(
+    [
+        "AquilaChat-7B",
+        "AquilaChat2-7B",
+        "AquilaChat2-34B",
+    ],
+    "Aquila-Chat",
+    "https://huggingface.co/BAAI/AquilaChat2-34B",
+    "Chat models developed by BAAI team",
+)
+
+register_model_info(
+    ["xDAN-L1-Chat-RL-v1"],
+    "xDAN-L1-Chat",
+    "https://huggingface.co/xDAN-AI/xDAN-L1-Chat-RL-v1",
+    "A large language chat model created by xDAN-AI.",
+)
+
+register_model_info(
+    ["MetaMath-70B-V1.0", "MetaMath-7B-V1.0"],
+    "MetaMath",
+    "https://huggingface.co/meta-math",
+    "MetaMath is a finetune of Llama2 on [MetaMathQA](https://huggingface.co/datasets/meta-math/MetaMathQA) that specializes in mathematical reasoning.",
+)
+
+register_model_info(
+    ["Yuan2-2B-hf", "Yuan2-51B-hf", "Yuan2-102B-hf"],
+    "IEIYuan",
+    "https://huggingface.co/IEITYuan",
+    "Yuan2 is a Basemodel developed by IEI.",
+)

arena_elo/elo_rating/upload_battle_data.py

@@ -0,0 +1,193 @@
+import fire
+import json
+import os
+import datasets
+import datetime
+from pathlib import Path
+from datetime import datetime
+from PIL import Image
+
+datasets.config.DEFAULT_MAX_BATCH_SIZE = 500
+def create_hf_dataset(data_file: str, split="test"):
+    hf_dataset = datasets.Dataset.from_list(
+        data_file,
+        features=datasets.Features(
+            {
+                "question_id": datasets.Value("string"),
+                "model": datasets.Value("string"),
+                "conversation": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "language": datasets.Value("string"),
+                "image": datasets.Image(),
+                "turn": datasets.Value("int32"),
+            }
+        ),
+        split=split,
+    )
+    return hf_dataset
+
+def create_hf_battle_dataset(data_file: str, split="test"):
+    hf_dataset = datasets.Dataset.from_list(
+        data_file,
+        features=datasets.Features(
+            {
+                "question_id": datasets.Value("string"),
+                "model_a": datasets.Value("string"),
+                "model_b": datasets.Value("string"),
+                "conversation_a": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "conversation_b": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "language": datasets.Value("string"),
+                "image": datasets.Image(),
+                "turn": datasets.Value("int32"),
+                "anony": datasets.Value("bool"),
+            }
+        ),
+        split=split,
+    )
+    return hf_dataset
+                           
+                                            
+
+
+def load_image(path:str):
+    try:
+        return Image.open(path)
+    except Exception as e:
+        print(f"Error loading image {path}: {e}")
+        return None
+
+def get_date_from_time_stamp(unix_timestamp: int):
+    # Create a datetime object from the Unix timestamp
+    dt = datetime.fromtimestamp(unix_timestamp)
+
+    # Convert the datetime object to a string with the desired format
+    date_str = dt.strftime("%Y-%m-%d")
+    return date_str
+
+def load_battle_image(battle, log_dir):
+    image_path = Path(log_dir) / f"{get_date_from_time_stamp(battle['tstamp'])}-convinput_images" / f"input_image_{battle['question_id']}.png"
+    return load_image(image_path)
+    
+
+def main(
+    data_file: str = "./results/latest/clean_battle_conv.json",
+    repo_id: str = "DongfuTingle/wildvision-bench",
+    log_dir: str = os.getenv("LOGDIR", "./vision-arena-logs/"),
+    mode="battle",
+    token = os.environ.get("HUGGINGFACE_TOKEN", None)
+):
+    with open(data_file, "r") as f:
+        data = json.load(f)
+    
+    
+    
+    has_image_stats = {
+        "has_image": 0,
+        "no_image": 0,
+    }
+    if mode == "keep_bad_only":
+        # anony only
+        data = [d for d in data if d["anony"]]
+        
+        new_data = []
+        for battle in data:
+            image = load_battle_image(battle, log_dir)
+            if image is None:
+                has_image_stats["no_image"] += 1
+                # we don't keep the data without image
+                continue
+            has_image_stats["has_image"] += 1
+            
+            if battle["winner"] in ["model_a", "model_b"]:
+                if battle["winner"] == "model_a":
+                    worse_model = "model_b"
+                    worse_conv = "conversation_b"
+                if battle["winner"] == "model_b":
+                    worse_model = "model_a"
+                    worse_conv = "conversation_a"
+                    
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle[worse_model],
+                    "conversation": battle[worse_conv],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+            elif battle["winner"] == "tie (bothbad)":
+                
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle["model_a"],
+                    "conversation": battle["conversation_a"],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle["model_b"],
+                    "conversation": battle["conversation_b"],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+                
+        split = "test"
+        hf_dataset = create_hf_dataset(new_data, "test")
+    
+    elif mode == "battle":
+        new_data = []
+        for battle in data:
+            image = load_battle_image(battle, log_dir)
+            if image is None:
+                has_image_stats["no_image"] += 1
+                continue
+            has_image_stats["has_image"] += 1
+            new_data.append({
+                "question_id": battle["question_id"],
+                "model_a": battle["model_a"],
+                "model_b": battle["model_b"],
+                "conversation_a": battle["conversation_a"],
+                "conversation_b": battle["conversation_b"],
+                "language": battle["language"],
+                "image": image,
+                "turn": battle["turn"],
+                "anony": battle["anony"],
+            })
+        split = "test"
+        hf_dataset = create_hf_battle_dataset(new_data, "test")
+    else:
+        raise ValueError(f"Invalid mode: {mode}")
+
+    print(f"Stats: {has_image_stats}")
+    print(hf_dataset)
+    print(f"Uploading to part {repo_id}:{split}...")
+    hf_dataset.push_to_hub(
+        repo_id=repo_id,
+        config_name=mode,
+        split=split,
+        token=token,
+        commit_message=f"Add vision-arena {split} dataset",
+    )
+    
+    print("Done!")
+    
+    
+if __name__ == "__main__":
+    fire.Fire(main)

arena_elo/elo_rating/utils.py

@@ -0,0 +1,83 @@
+from datetime import datetime
+import pytz
+import PIL
+import os 
+
+def detect_language(text: str) -> str:
+    """Detect the langauge of a string."""
+    import polyglot  # pip3 install polyglot pyicu pycld2
+    from polyglot.detect import Detector
+    from polyglot.detect.base import logger as polyglot_logger
+    import pycld2
+
+    polyglot_logger.setLevel("ERROR")
+
+    try:
+        lang_code = Detector(text).language.name
+    except (pycld2.error, polyglot.detect.base.UnknownLanguage):
+        lang_code = "unknown"
+    return lang_code
+
+
+def get_time_stamp_from_date(date_str:str):
+    """
+    Convert a date string to a Unix timestamp
+    Args:
+        date_str (str): The input date string in the format 'YYYY-MM-DD-HH:MM-TZ', e.g. '2024-02-10-14:00-PT'
+    """
+    
+    # Convert the date string into a format that Python's datetime can understand
+    # and specify the correct timezone for PT, which is 'US/Pacific'
+    date_format = "%Y-%m-%d-%H:%M-%Z"
+
+    # Parse the date string into a datetime object
+    # Note: PT is not directly recognized by pytz, so we manually map it to 'US/Pacific'
+    timezone_map = {
+        "PT": "US/Pacific",
+    }
+
+    # Extract the timezone abbreviation
+    tz_abbr = date_str.split("-")[-1]
+    # Map the abbreviation to a pytz timezone
+    tz_info = pytz.timezone(timezone_map[tz_abbr])
+
+    # Remove the timezone abbreviation for parsing
+    date_str_parsed = date_str.rsplit("-", 1)[0]
+
+    # Create a datetime object with the corresponding timezone
+    dt = datetime.strptime(date_str_parsed, "%Y-%m-%d-%H:%M").replace(tzinfo=tz_info)
+
+    # Convert the datetime object to a Unix timestamp
+    unix_timestamp = dt.timestamp()
+    return unix_timestamp
+
+def get_date_from_time_stamp(unix_timestamp: int):
+    # Create a datetime object from the Unix timestamp
+    dt = datetime.fromtimestamp(unix_timestamp)
+
+    # Convert the datetime object to a string with the desired format
+    date_str = dt.strftime("%Y-%m-%d %H:%M:%S %Z")
+    return date_str
+
+
+def get_input_image_path(tstamp, conv_id):
+    # from tstamp to date e.g. 2024-02-10
+    date_str = datetime.fromtimestamp(tstamp, tz=pytz.timezone("US/Pacific")).strftime("%Y-%m-%d")
+    LOGDIR = os.getenv("LOGDIR")
+    return f"{LOGDIR}/{date_str}-convinput_images/input_image_{conv_id}.png"
+
+def load_image_from_path(image_path):
+    # Load the image from the specified
+    # path using the Python Imaging Library (PIL)
+    try:
+        image = PIL.Image.open(image_path)
+        return image
+    except FileNotFoundError:
+        print(f"Image not found at path: {image_path}")
+        return None
+    except PIL.UnidentifiedImageError:
+        print(f"Unidentified image format at path: {image_path}")
+        return None
+    
+
+    

arena_elo/evaluator/convert_to_evaluator_data.py

@@ -0,0 +1,134 @@
+import argparse
+import json
+import os
+import time
+from pytz import timezone
+from tqdm import tqdm
+import base64
+from icecream import ic
+from PIL import Image
+
+
+# Function to encode the image
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+def get_log_files(max_num_files=None):
+    dates = []
+    for month in [2, 3]:
+        for day in range(1, 32):
+            dates.append(f"2024-{month:02d}-{day:02d}")
+
+    num_servers = 1
+    filenames = []
+    for d in dates:
+        for i in range(num_servers):
+            # name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
+            name = os.path.expanduser(f"vision-arena-logs/{d}-conv.json")
+            if os.path.exists(name):
+                filenames.append(name)
+    max_num_files = max_num_files or len(filenames)
+    filenames = filenames[-max_num_files:]
+    return filenames
+
+
+def pretty_print_conversation(messages):
+    for role, msg in messages:
+        print(f"[[{role}]]: {msg}")
+
+task_template_map = {
+    "image_caption": "Give me the semantic alignment score between the given image and the given caption: \"{generated_sentence}\" on a scale of 0-100. Only reply the score value.",
+    "vqa": "Rate the answer correctness regarding the question within the context of the given image on a scale of 0-100. Only reply the score value.",
+    "pair_rate_old": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nGiven the instruction and the image, please compare the correctness of responses A and B. Reply with \"leftvote\" if you find A better, \"rightvote\" if B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both responses are equally satisfactory. If you are unable to make a decision, please reply with \"NA\".",
+    "pair_rate_wexplanation": "<image>[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.",
+    "pair_rate": "<image>[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. Reply with \"leftvote\" if you find assistant A better, \"rightvote\" if assistant B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both assistants provide equally satisfactory answers. If you are unable to make a decision, please reply with \"NA\"."
+}
+
+def inspect_convs(log_files):
+    json_data = []
+
+    ic(log_files)
+    total_vote = 0
+
+    for filename in tqdm(log_files, desc="read files"):
+        for retry in range(5):
+            try:
+                lines = open(filename).readlines()
+                break
+            except FileNotFoundError:
+                time.sleep(2)
+        
+        for l in lines:
+            row = json.loads(l)
+            
+            if "states" not in row:
+                continue
+            if row["type"] not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
+                continue
+            
+            model_names = row["states"][0]["model_name"], row["states"][1]["model_name"]
+            
+
+            # Iterate through each state and write the relevant information
+            if not len(row["states"][0]['messages']): continue
+            # ic(row["states"][0]['messages'][1][1])
+
+            if row["states"][0]['messages'][1][1] is None or row["states"][1]['messages'][1][1] is None or "NETWORK ERROR" in row["states"][0]['messages'][1][1] or "NETWORK ERROR" in row["states"][1]['messages'][1][1]: continue
+            total_vote += 1
+
+            conv_id = row["states"][0]['conv_id']
+            image_path = os.path.join("/local/home/yujielu/project/Arena-Elo/vision-arena-logs", os.path.basename(filename)[:-5]+"input_images", f"input_image_{conv_id}.png")
+            if not os.path.exists(image_path) :
+                continue
+            try:
+                image = Image.open(image_path).convert("RGB")
+            except:
+                continue
+
+            left_response = row["states"][0]['messages'][1][1]
+            right_response = row["states"][1]['messages'][1][1]
+            instruction = row["states"][0]['messages'][0][1]
+            generated_sentence = f"[The Start of Assistant A’s Answer]\n{left_response}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{right_response}\n[The End of Assistant B’s Answer]"
+            text_prompt = task_template_map["pair_rate"].format(instruction=instruction, generated_sentence=generated_sentence)
+            
+            user_input = text_prompt
+            # Create the conversation structure
+            conversation = [
+                {
+                    "from": "human",
+                    "value": user_input
+                },
+                {
+                    "from": "gpt",
+                    "value": row["type"]
+                }
+            ]
+            
+            # Create the JSON object for each row
+            json_obj = {
+                "id": conv_id,
+                "image": image_path,
+                "conversations": conversation
+            }
+            
+            # Append the JSON object to the list
+            json_data.append(json_obj)
+
+        # Write the JSON data to a file
+        with open('output_evaluator_data.json', 'w') as json_file:
+            json.dump(json_data, json_file, indent=2)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    
+
+            
+    inspect_convs(log_files)
+
+
+

arena_elo/evaluator/rating_analysis.ipynb

@@ -0,0 +1,321 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1338\n",
+      "1044\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "import pandas as pd\n",
+    "import json\n",
+    "\n",
+    "# Replace 'your_file_name.csv' with the path to your CSV file\n",
+    "file_name = 'all_pairvote_log_wgpt.csv'\n",
+    "\n",
+    "# Load the CSV file into a DataFrame\n",
+    "df = pd.read_csv(file_name)\n",
+    "\n",
+    "# Define a function to parse JSON data\n",
+    "def parse_json(data):\n",
+    "    try:\n",
+    "        # Parse the JSON data\n",
+    "        return json.loads(data)\n",
+    "    except ValueError as e:\n",
+    "        # Return None or an empty dictionary if the data cannot be parsed\n",
+    "        return None\n",
+    "\n",
+    "# Apply the parse_json function to the 'models' and 'states' columns\n",
+    "df['models'] = df['models'].apply(parse_json)\n",
+    "df['states'] = df['states'].apply(parse_json)\n",
+    "# row[\"states\"][0]['messages'][0][1]\n",
+    "\n",
+    "# Now df contains the parsed JSON data in the 'models' and 'states' columns\n",
+    "# print(df.head())\n",
+    "print(len(df))\n",
+    "# filter_vote_df = df[df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\"])]#, \"tievote\", \"bothbad_vote\"\n",
+    "# \\#1\n",
+    "filter_vote_df = df[df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])]\n",
+    "# \\#2\n",
+    "# filter_vote_df = df\n",
+    "filter_vote_df.loc[~filter_vote_df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\"]), \"gpt_vote\"] = \"tie\"\n",
+    "filter_vote_df.loc[~filter_vote_df[\"type\"].isin([\"leftvote\", \"rightvote\"]), \"type\"] = \"tie\"\n",
+    "# \\#3\n",
+    "#[df[\"gpt_vote\"].isin([\"leftvote\", \"rightvote\"]) & df[\"type\"].isin([\"leftvote\", \"rightvote\"])]\n",
+    "filtered_df = filter_vote_df[filter_vote_df[\"states\"].apply(lambda x: len(x[0]['messages'][0][1]) > 10)]\n",
+    "print(len(filtered_df))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Confusion Matrix:\n",
+      "[[300  61  34]\n",
+      " [102 269  27]\n",
+      " [ 99 111  41]]\n",
+      "\n",
+      "Accuracy: 0.5842911877394636\n"
+     ]
+    }
+   ],
+   "source": [
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Assuming df is your DataFrame\n",
+    "\n",
+    "# True labels\n",
+    "y_true = filtered_df[\"type\"]\n",
+    "\n",
+    "# Predictions\n",
+    "y_pred = filtered_df[\"gpt_vote\"]\n",
+    "\n",
+    "# Compute the confusion matrix\n",
+    "# conf_matrix = confusion_matrix(y_true, y_pred, labels=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])\n",
+    "conf_matrix = confusion_matrix(y_true, y_pred, labels=[\"leftvote\", \"rightvote\", \"tie\"])\n",
+    "\n",
+    "# Compute the accuracy\n",
+    "accuracy = accuracy_score(y_true, y_pred)\n",
+    "\n",
+    "print(\"Confusion Matrix:\")\n",
+    "print(conf_matrix)\n",
+    "\n",
+    "print(\"\\nAccuracy:\", accuracy)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0YAAAJwCAYAAACtcHEcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB0yUlEQVR4nO3deZyN5f/H8feZfcaYMZaxMwyGkTWJURSKKFshS/YtREQoe4oKaVFpsaRBZUlRUrLLUtYsk2VkaYSxjGEYZq7fH37O12ksc8aMM+N+PR+P83jMXPd9X+dzxrnHec913ddtM8YYAQAAAICFubm6AAAAAABwNYIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAWsWTJElWsWFE+Pj6y2Ww6c+ZMuvY/ffp02Ww2HTx4MF37zcpsNptGjhzp6jIApALBCECafPjhh7LZbHrwwQddXcodu/Zh7vfff7/h9kceeUT33XffXa7q3hQdHa3evXurVKlS8vPzk5+fn8LDw9WrVy9t377dYd+RI0fKZrPZH9f2HTp0qOLi4iTJYfutHitWrLhtbZcvX1Z4eLhsNpvGjx9vb+/Tp49sNpv27dt302NfffVV2Wy2FK/hRlasWKFmzZopX7588vLyUnBwsJ566inNnz//tsfeidjYWLVo0UK+vr6aPHmyZs6cqWzZsmXoc95NISEhstlsqlu37g23f/rpp/b3w83O9VtZt26dRo4cme5hEkDm4eHqAgBkTZGRkQoJCdHGjRu1b98+lShRwtUlIZNbtGiRWrZsKQ8PD7Vp00YVKlSQm5ub9uzZo/nz5+ujjz5SdHS0ihYt6nDcRx99JH9/f8XHx2vp0qV6/fXX9euvv2rt2rWaOXOmw75ffPGFfv755xTtZcqUuW1977//vg4dOpSivU2bNnr//fc1a9YsDR8+/IbHzp49W+XKlVP58uVv+RwjRozQ6NGjVbJkSXXv3l1FixZVbGysfvjhBz399NOKjIxU69atb1trWmzatEnnzp3Ta6+9dtPwcKeee+45Pfvss/L29s6Q/m/Hx8dHy5cv17Fjx5QvXz6HbZGRkfLx8dHFixfT1Pe6des0atQodejQQTly5Ej1cQkJCfLw4OMWkCUYAHDSgQMHjCQzf/58kydPHjNy5MhUHXf58mVz6dKlDK7OedOmTTOSzKZNm264vVatWqZs2bJ3uap7y759+0y2bNlMmTJlzD///JNi++XLl827775rDh06ZG8bMWKEkWROnDjhsG+zZs2MJLNu3boU/fTq1cuk5b+2f//91wQGBprRo0cbSebtt9922F6iRAlTunTpGx67bt06I8mMGzfuls/xzTffGEnmmWeeMYmJiSm2L1myxHz//fdO155aM2bMuOX7PKsrWrSoqVOnjgkICDCTJk1y2Hb48GHj5uZmnn766TT/DN5++20jyURHR99236SkJJOQkOD0cwBwLabSAXBaZGSkgoKC1LBhQz3zzDOKjIxMsc/BgwftU5ImTZqk0NBQeXt7a9euXZKkPXv26JlnnlHOnDnl4+OjKlWq6LvvvnPo49oUt7Vr16p///7KkyePsmXLpqZNm+rEiRMO+y5cuFANGzZUgQIF5O3trdDQUL322mtKSkpK99d/7bVNnz49xbb/Xk9wbTrYX3/9pbZt2yowMFB58uTRsGHDZIzR4cOH1bhxYwUEBChfvnyaMGGCQ3+JiYkaPny47r//fgUGBipbtmx6+OGHtXz58hvWNH78eH3yySf2n/cDDzygTZs23fL1/P7777LZbJoxY0aKbT/99JNsNpsWLVokSTp37pxefPFFhYSEyNvbW8HBwXrssce0efPmWz7HW2+9pfPnz2vatGnKnz9/iu0eHh7q06ePChcufMt+JKl27dqSrk7LSy+DBw9WWFiY2rZte8Ptbdq00Z49e274OmfNmiWbzaZWrVrd8jmGDRumnDlzaurUqfL09EyxvV69enryySft3x8/flydO3dW3rx55ePjowoVKqT4N0rtv/sjjzyi9u3bS5IeeOAB2Ww2dejQQdLVKWjXvr7eI488okceecSh7f3331fZsmXl5+enoKAgValSRbNmzbJvv9k1Rh9++KHKli0rb29vFShQQL169UoxJe3alNVdu3bp0UcflZ+fnwoWLKi33nrrZj/SFHx8fNSsWTOHmqSrI3pBQUGqV69eimO2b9+uDh06qHjx4vLx8VG+fPnUqVMnxcbG2vcZOXKkBg4cKEkqVqyYfUretddps9nUu3dvRUZG2l/nkiVL7Nuu/U5ISEhQ6dKlVbp0aSUkJNj7P3XqlPLnz6+IiIgM+Z0FIHUY2wXgtMjISDVr1kxeXl5q1aqVPvroI23atEkPPPBAin2nTZumixcvqlu3bvL29lbOnDm1c+dO1ahRQwULFtTgwYOVLVs2ff3112rSpInmzZunpk2bOvTxwgsvKCgoSCNGjNDBgwc1adIk9e7dW1999ZV9n+nTp8vf31/9+/eXv7+/fv31Vw0fPlxxcXF6++23U/W6zp49q5MnT6Zov3z5spM/oZRatmypMmXKaNy4cVq8eLHGjBmjnDlzasqUKapdu7befPNNRUZGasCAAXrggQdUs2ZNSVJcXJw+++wztWrVSl27dtW5c+f0+eefq169etq4caMqVqzo8DyzZs3SuXPn1L17d9lsNr311ltq1qyZDhw4cMMP45JUpUoVFS9eXF9//bX9w/M1X331lcMHyh49emju3Lnq3bu3wsPDFRsbqzVr1mj37t2qXLnyTV//okWLVKJEiXS5Jm3//v2SpFy5ct1xX5K0ceNGzZgxQ2vWrJHNZrvhPm3atNGoUaM0a9Ysh9eZlJSkr7/+Wg8//LCKFCly0+fYu3ev9uzZo06dOil79uy3rSkhIUGPPPKI9u3bp969e6tYsWL65ptv1KFDB505c0Z9+/Z12P92/+6vvvqqwsLC9Mknn2j06NEqVqyYQkNDU/kTuurTTz9Vnz599Mwzz6hv3766ePGitm/frg0bNtxy+t/IkSM1atQo1a1bV88//7yioqLsvzPWrl3r8L48ffq06tevr2bNmqlFixaaO3euBg0apHLlyumJJ55IVZ2tW7fW448/rv3799tf46xZs/TMM8/c8Bz4+eefdeDAAXXs2FH58uXTzp079cknn2jnzp1av369bDabmjVrpr/++kuzZ8/WO++8o9y5c0uS8uTJY+/n119/1ddff63evXsrd+7cCgkJSfFcvr6+mjFjhmrUqKFXX31VEydOlCT16tVLZ8+e1fTp0+Xu7p6q1wkgA7h6yApA1vL7778bSebnn382xhiTnJxsChUqZPr27euwX3R0tJFkAgICzPHjxx221alTx5QrV85cvHjR3pacnGwiIiJMyZIl7W3XprjVrVvXJCcn29v79etn3N3dzZkzZ+xtFy5cSFFr9+7djZ+fn8Pz3Mi157nV4/qpdNde27Rp01L0JcmMGDHC/v216WDdunWzt125csUUKlTI2Gw2h+lXp0+fNr6+vqZ9+/YO+/53+uHp06dN3rx5TadOnVLUlCtXLnPq1Cl7+8KFC42k207RGjJkiPH09HQ49tKlSyZHjhwOzxMYGGh69ep1y77+6+zZs0aSadKkSYptp0+fNidOnLA/rv93vPazi4qKMidOnDDR0dFmypQpxtvb2+TNm9ecP38+RX/OTqVLTk42VatWNa1atTLG/O/n+N+pdMYY88ADD5hChQqZpKQke9uSJUuMJDNlypRbPs+1f4d33nknVXVNmjTJSDJffvmlvS0xMdFUr17d+Pv7m7i4OId6U/PvfrMpo0WLFnV4z11Tq1YtU6tWLfv3jRs3vu2U0mvPcW262fHjx42Xl5d5/PHHHX5uH3zwgZFkpk6d6vB8kswXX3xhb7t06ZLJly+fefrpp2/5vNdeR8OGDc2VK1dMvnz5zGuvvWaMMWbXrl1Gklm5cuUNfwY3+t0xe/ZsI8msWrXK3narqXSSjJubm9m5c+cNt13/O8GYq+ebm5ubWbVqlX2K5X+n/wG4+5hKB8ApkZGRyps3rx599FFJV6eJtGzZUnPmzLnhFJCnn37a4a+qp06d0q+//qoWLVro3LlzOnnypE6ePKnY2FjVq1dPe/fu1dGjRx366Natm8Nf8h9++GElJSXp77//trf5+vrav77W78MPP6wLFy5oz549qXptkydP1s8//5zicbsL6lOjS5cu9q/d3d1VpUoVGWPUuXNne3uOHDkUFhamAwcOOOzr5eUlSUpOTtapU6d05coVValS5YbTulq2bKmgoCD79w8//LAkOfR5Iy1bttTly5cdVkZbunSpzpw5o5YtWzrUuGHDBv3zzz+pfen2FeT8/f1TbHvkkUeUJ08e+2Py5Mkp9gkLC1OePHlUrFgxde/eXSVKlNDixYvl5+eX6hpuZvr06dqxY4fefPPN2+7btm1bHTlyRKtWrbK3zZo1S15eXmrevPktj732M0jNaJEk/fDDD8qXL5/D9DxPT0/16dNH8fHxWrlypcP+af13d0aOHDl05MiR207NvN4vv/yixMREvfjii3Jz+99Hjq5duyogIECLFy922N/f399hOqOXl5eqVq3q1Otwd3dXixYtNHv2bElXf2cVLlzY/jP5r+t/d1y8eFEnT55UtWrVJOm2U0SvV6tWLYWHh6dq35EjR6ps2bJq3769evbsqVq1aqlPnz6pfi4AGYNgBCDVkpKSNGfOHD366KOKjo7Wvn37tG/fPj344IP6999/tWzZshTHFCtWzOH7ffv2yRijYcOGOXwgzpMnj0aMGCHp6rUV1/vvFKVrHwBPnz5tb9u5c6eaNm2qwMBABQQEKE+ePPYPWGfPnk3V66tatarq1q2b4nH9B860+u9rCAwMlI+Pj31KzvXt178uSZoxY4bKly8vHx8f5cqVS3ny5NHixYtv+LpS87O6kQoVKqh06dIO0xO/+uor5c6d235Nj3T1WqE///xThQsXVtWqVTVy5Mjbfmi9Fgbi4+NTbJsyZYp+/vlnffnllzc9ft68efr555+1YsUK7du3T3/++afuv//+Wz7n9eLj43Xs2DH749r1aXFxcRoyZIgGDhyYqmubnn32Wbm7u9uvX7l48aIWLFigJ5544rbvkYCAAElXQ3tq/P333ypZsqRDmJD+t7re9X8UkNL+7+6MQYMGyd/fX1WrVlXJkiXVq1cvrV279pbHXKszLCzMod3Ly0vFixdP8ToKFSqUYjpjUFCQ06+jdevW2rVrl7Zt26ZZs2bp2Wefvek0yVOnTqlv377KmzevfH197SFcSv3vDinl77pb8fLy0tSpUxUdHa1z585p2rRpN60PwN3DNUYAUu3XX39VTEyM5syZozlz5qTYHhkZqccff9yh7fq/xkpXRz0kacCAATe8EFpSiqW/bzbn3hgjSTpz5oxq1aqlgIAAjR49WqGhofLx8dHmzZs1aNAg+3Oml5t9gLnVRdM3eg23e12S9OWXX6pDhw5q0qSJBg4cqODgYLm7u2vs2LH2a22c7fNmWrZsqddff10nT55U9uzZ9d1336lVq1YOSw23aNFCDz/8sBYsWKClS5fq7bff1ptvvqn58+ff9BqQwMBA5c+fX3/++WeKbdeuObrVDUFr1qyZIkA6Y/z48Ro1apT9+6JFi+rgwYMaP368EhMT1bJlS/vzHzlyRNLVQHHw4EEVKFDAPmJ3baGJefPmafLkyfr+++917tw5tWnT5rY1lC5dWpK0Y8eONL+OW7mTf/dbvZ+v77dMmTKKiorSokWLtGTJEs2bN08ffvihhg8f7vDzvRN38jqu9+CDDyo0NFQvvviioqOjb3kNVIsWLbRu3ToNHDhQFStWlL+/v5KTk1W/fn2nfnf893fd7fz000+SrgbsvXv3OhWsAGQMghGAVIuMjFRwcPANpzvNnz9fCxYs0Mcff3zLDwjFixeXdHVaUHrdS2XFihWKjY3V/Pnz7YsWSOm7atn1rv01/r+rav33r9/pYe7cuSpevLjmz5/v8AH22uhaemrZsqVGjRqlefPmKW/evIqLi9Ozzz6bYr/8+fOrZ8+e6tmzp44fP67KlSvr9ddfv+XF8Q0bNtRnn32mjRs3qmrVqule+620a9dODz30kP37a+/PQ4cO6fTp0ypbtmyKY9544w298cYb2rJli8MCF23atNGSJUv0448/atasWQoICNBTTz112xpKlSqlsLAwLVy4UO++++4NpxVer2jRotq+fbuSk5MdRo2uTQv9772e7kRQUNANb1r6999/28/Xa7Jly6aWLVuqZcuWSkxMVLNmzfT6669ryJAh8vHxueHrkKSoqCiHvhITExUdHZ1h91OSpFatWmnMmDEqU6ZMikVKrjl9+rSWLVumUaNGOdyjau/evSn2Tc8Rne3bt2v06NHq2LGjtm7dqi5dumjHjh0KDAxMt+cA4Dym0gFIlYSEBM2fP19PPvmknnnmmRSP3r1769y5cymW3P6v4OBgPfLII5oyZYpiYmJSbP/vMtypce2vzNf/VTkxMVEffvih032lRkBAgHLnzu1wrYmkDHm+G722DRs26Lfffkv35ypTpozKlSunr776Sl999ZXy58/vEDSTkpJSTC0KDg5WgQIFdOnSpVv2/fLLL8vPz0+dOnXSv//+m2K7syMCzihevLjD1MgaNWpIkvr06aMFCxY4PKZMmSJJ6tChgxYsWJDir/hNmjSRn5+fPvzwQ/34449q1qzZDQPBjYwaNUqxsbHq0qWLrly5kmL70qVL7cuiN2jQQMeOHXOY2njlyhW9//778vf3V61atdL0s7iR0NBQrV+/XomJifa2RYsW6fDhww77Xb98tXR1Olh4eLiMMTddubFu3bry8vLSe++95/Bv/Pnnn+vs2bNq2LBhur2O/+rSpYtGjBiRYgn8693o/JKkSZMmpdg3W7ZsklL+QcRZly9fVocOHVSgQAG9++67mj59uv7991/169fvjvoFcOcYMQKQKt99953OnTunRo0a3XB7tWrVlCdPHkVGRjpcrH8jkydP1kMPPaRy5cqpa9euKl68uP7991/99ttvOnLkiLZt2+ZUbREREQoKClL79u3Vp08f2Ww2zZw5M0M/bHfp0kXjxo1Tly5dVKVKFa1atUp//fVXuj/Pk08+qfnz56tp06Zq2LChoqOj9fHHHys8PPyG1+zcqZYtW2r48OHy8fFR586dHUYrzp07p0KFCumZZ55RhQoV5O/vr19++UWbNm265YdPSSpZsqRmzZqlVq1aKSwsTG3atFGFChVkjFF0dLRmzZolNzc3FSpUKN1f081Urlw5xRLj16bUlS1bVk2aNElxjL+/v5o0aWK/zig10+iuadmypXbs2KHXX39dW7ZsUatWrVS0aFHFxsZqyZIlWrZsmb3fbt26acqUKerQoYP++OMPhYSEaO7cuVq7dq0mTZqU6kUcUqNLly6aO3eu6tevrxYtWmj//v368ssvUyzn/fjjjytfvnyqUaOG8ubNq927d+uDDz5Qw4YNb1pPnjx5NGTIEI0aNUr169dXo0aNFBUVpQ8//FAPPPDATe8blR6KFi3qcE+xGwkICFDNmjX11ltv6fLlyypYsKCWLl16w9Hma9e1vfrqq3r22Wfl6empp556yh6YUmvMmDHaunWrli1bpuzZs6t8+fIaPny4hg4dqmeeeUYNGjRwqj8A6YdgBCBVIiMj5ePjo8cee+yG293c3NSwYUNFRkam+Mvyf4WHh+v333/XqFGjNH36dMXGxio4OFiVKlVymM6SWrly5dKiRYv00ksvaejQoQoKClLbtm1Vp06dm17HdKeGDx+uEydOaO7cufr666/1xBNP6Mcff1RwcHC6Pk+HDh107NgxTZkyRT/99JPCw8P15Zdf6ptvvtGKFSvS9bmkqx/ehw4dqgsXLqQIuH5+furZs6eWLl2q+fPnKzk5WSVKlNCHH36o559//rZ9N27cWDt27NCECRO0dOlSTZ06VTabTUWLFlXDhg3Vo0cPVahQId1fU3pr06aNZs2apfz58zssTJEaY8aMUe3atfXee+/po48+0qlTpxQUFKRq1app4cKF9j88+Pr6asWKFRo8eLBmzJihuLg4hYWFadq0aTe8GeudqFevniZMmKCJEyfqxRdfVJUqVezn0/W6d++uyMhITZw4UfHx8SpUqJD69OmjoUOH3rL/kSNHKk+ePPrggw/Ur18/5cyZU926ddMbb7xx03tr3U2zZs3SCy+8oMmTJ8sYo8cff1w//vijChQo4LDfAw88oNdee00ff/yxlixZouTkZEVHRzsVjDZv3qw33nhDvXv3tq/sKV29wfDChQvVtWtX7dy5Uzly5EivlwfACTaTkX9SBQAAAIAsgGuMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFjePXmDV99KvV1dAuByvy0c6+oSAJcrXSC7q0sAALiYTyoTDyNGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACwv0wSj/fv3a+jQoWrVqpWOHz8uSfrxxx+1c+dOF1cGAAAA4F6XKYLRypUrVa5cOW3YsEHz589XfHy8JGnbtm0aMWKEi6sDAAAAcK/LFMFo8ODBGjNmjH7++Wd5eXnZ22vXrq3169e7sDIAAAAAVpApgtGOHTvUtGnTFO3BwcE6efKkCyoCAAAAYCWZIhjlyJFDMTExKdq3bNmiggULuqAiAAAAAFaSKYLRs88+q0GDBunYsWOy2WxKTk7W2rVrNWDAALVr187V5QEAAAC4x2WKYPTGG2+odOnSKly4sOLj4xUeHq6aNWsqIiJCQ4cOdXV5AAAAAO5xNmOMcXUR1xw+fFg7duxQfHy8KlWqpJIlS6apH99KvdO5MiDr+W3hWFeXALhc6QLZXV0CAMDFfDxSt1+mGDEaPXq0Lly4oMKFC6tBgwZq0aKFSpYsqYSEBI0ePdrV5QEAAAC4x2WKESN3d3fFxMQoODjYoT02NlbBwcFKSkpyqj9GjABGjACJESMAQBYbMTLGyGazpWjftm2bcubM6YKKAAAAAFhJKvNTxggKCpLNZpPNZlOpUqUcwlFSUpLi4+PVo0cPF1YIAAAAwApcGowmTZokY4w6deqkUaNGKTAw0L7Ny8tLISEhql69ugsrBAAAAGAFLg1G7du3lyQVK1ZMERER8vT0dGU5AAAAACzKpcHomlq1aikpKUnz5s3T7t27JUlly5ZVo0aN5O7u7uLqAAAAANzrMkUw2rdvnxo0aKCjR48qLCxMkjR27FgVLlxYixcvVmhoqIsrBAAAAHAvyxSr0vXp00ehoaE6fPiwNm/erM2bN+vQoUMqVqyY+vTp4+ryAAAAANzjMsWI0cqVK7V+/XqHpblz5cqlcePGqUaNGi6sDAAAAIAVZIpg5O3trXPnzqVoj4+Pl5eXlwsqwn91bf6Quj7zsIoWuBpedx84pjc++VFL1+6SJHl7eWhc/2ZqXu9+eXt56JffdqvvG1/p+Kn//bsWzhekd19pqVpVSik+4ZIiv9+gYe9/p6SkZJe8JiA9nDp5XJGfva+tG9fp0qWLylegkJ4fMEKhYeGSpA2rf9Uvi+bpwN49ij93Vm9+FKmQEmEurhrIeHNmRWrGtM918uQJlQorrcGvDFO58uVdXRZwV3EeZC2ZYirdk08+qW7dumnDhg0yxsgYo/Xr16tHjx5q1KiRq8uDpKP/ntGw9xcqos1bqtHmba3Y+Je+eaebyhTPJ0l6a8DTaljzPrV5+XM93mWS8ucJ1JwJXezHu7nZNP+95+Xl6aFHO0xQ1+Ez1bbRgxr+fENXvSTgjsWfi9PwFzvL3d1DQ954VxM/+1rPde+nbNkD7PtcupigsPsqqnWXF1xYKXB3LfnxB41/a6y69+ylOd8sUFhYaT3fvbNiY2NdXRpw13AeZD2ZIhi99957Cg0NVfXq1eXj4yMfHx/VqFFDJUqU0Lvvvuvq8iDph1V/6qc1u7T/0AntO3RcIyd/r/gLl1S1fDEF+PuoQ5PqGjRxvlZu+ktbdh9WtxFfqnrFUFUtFyJJqlu9jMoUz6dOr87Q9r+OaunaXRr94WJ1b1FTnh6sPIis6buvZihXnrzqOXCESpS+T8H5C6pClWrKV6CQfZ+ajzXUM891VbnKVV1YKXB3zZwxTc2eaaEmTZ9WaIkSGjpilHx8fPTt/HmuLg24azgPsp5MEYxy5MihhQsXKioqSnPnztXcuXMVFRWlBQsWONz0FZmDm5tNzevdr2y+XtqwPVqVyhSRl6eHfl0fZd/nr4P/6lDMKT1Yvpgk6cHyxfTnvn8cptb9vG63ArP7Kjw0/11/DUB6+P23VSpeqowmjh6krs0f06AerbXshwWuLgtwqcuJidq9a6eqVY+wt7m5ualatQht37bFhZUBdw/nQdaUKa4xWrNmjR566CGVLFlSJUuWdOrYS5cu6dKlSw5tJjlJNjdGIdJb2RIFtGLGS/Lx8lB8wiW1fOlT7TlwTBVKFdKlxMs6G5/gsP/x2DjlzXV1SlHeXAE6Hut4HdnxU3FXt+UOkKIEZDnHY47q5+/nqeHTbdS0dUftj9qlaZPHy8PDU7Uef9LV5QEucfrMaSUlJSlXrlwO7bly5VJ09AEXVQXcXZwHWVOmGDGqXbu2ihUrpldeeUW7du1y6tixY8cqMDDQ4XHl3z8yqFJr++vgv3rw2bGq2W68Pv1mjT4d/ZxK//81RoAVJZtkFStZWq0691KxEqVVt2Ez1WnQRD8vYpoEAABZTaYIRv/8849eeuklrVy5Uvfdd58qVqyot99+W0eOHLntsUOGDNHZs2cdHh55778LVVvP5StJOnD4pLbsPqzh73+nHX8dVa9Wj+hYbJy8vTwV6O/rsH9wrgD9G3t1VOjf2DgF58ruuD3n1dGkf0/G3Z0XAKSzoJy5VbBIMYe2gkWK6eTxYy6qCHC9oBxBcnd3T3GBeWxsrHLnzu2iqoC7i/Mga8oUwSh37tzq3bu31q5dq/3796t58+aaMWOGQkJCVLt27Vse6+3trYCAAIcH0+juDjebTd5eHtqy+5ASL1/Row/+bwnikkWDVSR/Tm3YHi1J2rA9WveVKKA8Qf72fepUK62z5xK0+wAfIpE1hZWtoJgjfzu0xRz5W3nyct0crMvTy0tlwstqw/rf7G3JycnasOE3la9QyYWVAXcP50HWlCmuMbpesWLFNHjwYFWoUEHDhg3TypUrXV0SJI1+oZF+WrtTh2NOK3s2H7V8oopqVimpp3p+qLj4i5r+7W9686VmOnX2vM6dv6iJg5pr/bYD2rjjoCTpl992a/eBY/p8THu9+u63ypsrQCN6PakpX69S4uUrrn1xQBo1eLq1hvftpAWzpqp6rce0L2qnlv2wQF1ffNW+T3zcWZ08fkynY09Ikv75/yCVI2cu5cjJXw1xb3qufUcNe2WQypa9T/eVK68vZ85QQkKCmjRt5urSgLuG8yDrsRljjKuLuGbt2rWKjIzU3LlzdfHiRTVu3Fht2rRR/fr1nerHt1LvDKrQuj4a0VqPVg1TvtwBOht/UX/uPaoJ037Rrxv2SPrfDV5b1P//G7yu262+Y7/Sv9ctuFAkf5DefeVZ1by/pM5fvKTI7zdq6HsLucFrBvlt4VhXl2AJf6xfrdmff6BjRw8rT74CevKZNqrToKl9+4qfvtdH40elOO6Z57qqebvud7NUSypdIPvtd0KGmB35pf3GlmGly2jQK0NVvnwFV5cF3FWcB5mDTyqHgjJFMBoyZIjmzJmjf/75R4899pjatGmjxo0by8/PL039EYwAghEgEYwAAKkPRpliKt2qVas0cOBAtWjRggvSAAAAANx1mSIYjR07VhEREfLwcCznypUrWrdunWrWrOmiygAAAABYQaZYle7RRx/VqVOnUrSfPXtWjz76qAsqAgAAAGAlmSIYGWNks9lStMfGxipbtmwuqAgAAACAlbh0Kl2zZleXK7TZbOrQoYO8vb3t25KSkrR9+3ZFRES4qjwAAAAAFuHSYBQYGCjp6ohR9uzZ5evra9/m5eWlatWqqWvXrq4qDwAAAIBFuCwY9e/fXx988IGyZcumgwcP6rPPPpO/v7+rygEAAABgYS67xuj9999XfHy8pKvLdV+4cMFVpQAAAACwOJeNGIWEhOi9997T448/LmOMfvvtNwUFBd1wX5brBgAAAJCRbMYY44on/vbbb9WjRw8dP35cNptNNyvDZrMpKSnJqb59K/VOjxKBLO23hWNdXQLgcqULZHd1CQAAF/NJ5VCQy0aMmjRpoiZNmig+Pl4BAQGKiopScHCwq8oBAAAAYGEuXZVOkvz9/bV8+XIVK1ZMHh4uLwcAAACABWWKG7zWqlVLf//9t4YOHapWrVrp+PHjkqQff/xRO3fudHF1AAAAAO51mSIYrVy5UuXKldOGDRs0f/58+2p127Zt04gRI1xcHQAAAIB7XaYIRoMHD9aYMWP0888/y8vLy95eu3ZtrV+/3oWVAQAAALCCTBGMduzYoaZNm6ZoDw4O1smTJ11QEQAAAAAryRTBKEeOHIqJiUnRvmXLFhUsWNAFFQEAAACwkkwRjJ599lkNGjRIx44dk81mU3JystauXasBAwaoXbt2ri4PAAAAwD0uUwSjN954Q6VLl1bhwoUVHx+v8PBwPfzww4qIiNDQoUNdXR4AAACAe5zNGGNcXcQ1hw8f1o4dOxQfH69KlSqpZMmSaerHt1LvdK4MyHp+WzjW1SUALle6QHZXlwAAcDGfVN4q1WV3VO3fv/8tt1+/Gt3EiRMzuhwAAAAAFuayYLRly5ZU7Wez2TK4EgAAAABW57JgtHz5clc9NQAAAAA4yBSLLwAAAACAKxGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFiezRhjXF1Eevt66z+uLgFwufZD57u6BMDlTn/b29UlAABczMcjdfsxYgQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8ghGAAAAACyPYAQAAADA8jJNMFq9erXatm2r6tWr6+jRo5KkmTNnas2aNS6uDAAAAMC9LlMEo3nz5qlevXry9fXVli1bdOnSJUnS2bNn9cYbb7i4OgAAAAD3ukwRjMaMGaOPP/5Yn376qTw9Pe3tNWrU0ObNm11YGQAAAAAryBTBKCoqSjVr1kzRHhgYqDNnztz9ggAAAABYSqYIRvny5dO+fftStK9Zs0bFixd3QUUAAAAArCRTBKOuXbuqb9++2rBhg2w2m/755x9FRkZqwIABev75511dHgAAAIB7nIerC5CkwYMHKzk5WXXq1NGFCxdUs2ZNeXt7a8CAAXrhhRdcXR4AAACAe5zNGGNcXcQ1iYmJ2rdvn+Lj4xUeHi5/f/809fP11n/SuTIg62k/dL6rSwBc7vS3vV1dAgDAxXxSORSUKabSderUSefOnZOXl5fCw8NVtWpV+fv76/z58+rUqZOrywMAAABwj8sUwWjGjBlKSEhI0Z6QkKAvvvjCBRUBAAAAsBKXXmMUFxcnY4yMMTp37px8fHzs25KSkvTDDz8oODjYhRUCAAAAsAKXBqMcOXLIZrPJZrOpVKlSKbbbbDaNGjXKBZUBAAAAsBKXBqPly5fLGKPatWtr3rx5ypkzp32bl5eXihYtqgIFCriwQgAAAABW4NJgVKtWLUlSdHS0ChcuLDe3THHJEwAAAACLyRT3MSpatKjOnDmjzz//XLt375YklS1bVp06dVJgYKCLqwMAAABwr8sUQzS///67QkND9c477+jUqVM6deqUJk6cqNDQUG3evNnV5QEAAAC4x2WKEaN+/fqpUaNG+vTTT+XhcbWkK1euqEuXLnrxxRe1atUqF1cIAAAA4F6WKYLR77//7hCKJMnDw0Mvv/yyqlSp4sLKAAAAAFhBpphKFxAQoEOHDqVoP3z4sLJnz+6CigAAAABYSaYIRi1btlTnzp311Vdf6fDhwzp8+LDmzJmjLl26qFWrVq4uDwAAAMA9LlNMpRs/frxsNpvatWunK1euSJI8PT31/PPPa9y4cS6uDgAAAMC9zmaMMa4u4poLFy5o//79kqTQ0FD5+fmlqZ+vt/6TnmUBWVL7ofNdXQLgcqe/7e3qEgAALuaTyqGgTDFi9OWXX6pZs2by8/NTuXLlXF0ObuLgrm1a8/1X+if6L507HatWA15T+AMP2bcbY/TrN9P0+7LFung+XkXC7lOjLv2UK38hSdLp48e0Yv4XOvDnFsWfOaXsOXOrwkN1VatZW3l4eLrqZQGpNqD5/WpSvbhKFQpSQuIVbdh9TK9OX6e9R8847Pdg6Xwa+Vw1PRCWV0nJRtsPnNBTw7/TxcQkSVLF0Dwa06G67i95dfu36/Zr0GdrdP7iZRe8KiBjzJkVqRnTPtfJkydUKqy0Br8yTOXKl3d1WcBdxXmQtWSKa4z69eun4OBgtW7dWj/88IOSkpJcXRJuIPHSReUrGqonO/W94fbV383R+h/nq1GXfur++ofy8vHRjDde1uXEREnSyX8OySQbNe7aXy9MmKYn2vXUpl++1y+zP7ubLwNIs4fvK6CPF+9QrQFz9eSwhfLwcNOi1xrJz/t/f2N6sHQ+LRz1lJZtOaSH+3+jh/p9rY8X7VBy8tXB+fw5s2nxmMbaH3NWNV/6Ro1HfKfwIjn1ab86rnpZQLpb8uMPGv/WWHXv2UtzvlmgsLDSer57Z8XGxrq6NOCu4TzIejJFMIqJidGcOXNks9nUokUL5c+fX7169dK6detcXRquU6rSg6r7bGeFV304xTZjjH77Ya5qNXtOZR54SPmKhurpXkN07vRJ7d60RpJUsmJVNes5SCUqPKCceQuoTJUaeujJFtq1cfXdfilAmjQe8b2+XLZHuw+d0o7oWHV75xcVCQ5QpRLB9n3e6vKQPvx+u8bP3azdh05p79EzmrdmnxKvJEuSnnggRJevJOvFj1Zq79Ez+mPvcb0weYWa1iih4vkDXfXSgHQ1c8Y0NXumhZo0fVqhJUpo6IhR8vHx0bfz57m6NOCu4TzIejJFMPLw8NCTTz6pyMhIHT9+XO+8844OHjyoRx99VKGhoa4uD6lw+niM4s+cUmi5++1tPn7+KlSijA7v3XnT4y5eOC9ff5ZkR9YUkM1bknQ6/qIkKU+gr6qWzqcTZxK0/O2ndXBmJy0d21QR4fntx3h7uuvylSRdf3VnQuLVRWeu3w/Iqi4nJmr3rp2qVj3C3ubm5qZq1SK0fdsWF1YG3D2cB1lTpghG1/Pz81O9evX0xBNPqGTJkjp48KCrS0IqxJ85JUnyDwxyaM8WGGTf9l+xx45q/ZIFeqDuUxleH5DebDbp7a4Pa93Of7Tr76vv8WL5AiRJr7auqqk/7VLjEd9p6/4T+uH1JgotcHU0aMX2I8ob5Kd+zSrJ08NNObJ5a0yH6pKkfDmzuebFAOno9JnTSkpKUq5cuRzac+XKpZMnT7qoKuDu4jzImjJNMLpw4YIiIyPVoEEDFSxYUJMmTVLTpk21c+fNRxsk6dKlS4qLi3N4XE68dJeqRlrFnTqhL954WfdVq6UqdZ50dTmA0yY9X0tli+ZUu7d+sre52WySpM+X/KmZv+zWtgMn9fJna/TXkdNq/1i4JGn3oVPq+s4y9WlaUafm9dDBLzvp4LE4HTt9XiY50ywSCgCA5WSKVemeffZZLVq0SH5+fmrRooWGDRum6tWrp+rYsWPHatSoUQ5tz3Tvr+Y9XsqIUnET/jlySpLiz55W9qD//XXk/NnTyhdSwmHfuFMnNXV0fxUuVVaNuvHvhKznnR411eCBENUdPF9HY8/b22NOX/169yHHUdKow6dVOI+//fuvVv6lr1b+peAcvjp/8YqMMerTpKKij529Oy8AyEBBOYLk7u6e4gLz2NhY5c6d20VVAXcX50HWlClGjNzd3fX1118rJiZGH3zwQapDkSQNGTJEZ8+edXg06cR9K+62oOD88s+RUwd2bLa3XbxwXkf27VbhkmXtbXGnTmjq6H4qUKyUmvUcJDe3TPEWBFLtnR411ah6cdV/9Vv9/e85h21//3tO/8TGq1QhxymlJQrm0KHjjvtK0vEzCTp/8bKeqVlSFy8nadnWwxlaO3A3eHp5qUx4WW1Y/5u9LTk5WRs2/KbyFSq5sDLg7uE8yJoyxYhRvXr1VKdOHbm7uzu0JyYmas6cOWrXrt1Nj/X29pa3t7dDm6dXfIbUaXWXLibo1LGj9u/PHI9RzMF98vXPrhy586p6g2e0YsFM5cxfUEHB+bXsq6nKHpRbZf7/Xkdxp07o81H9lCN3XtV/rofOx/3vr+PZ/3/ECcjMJj1fSy1rlVLzMYsVf+Gy8ua4ehPqsxcu2e9R9M68LRrapqp2RJ/UtgMn1bZOaYUVClLrsT/a++nxZDmt331M8QmXVadSYb3RMULDZvyms+cTXfK6gPT2XPuOGvbKIJUte5/uK1deX86coYSEBDVp2szVpQF3DedB1mMzxrh8Uru7u7tiYmIUHBzs0B4bG6vg4GCn72v09dZ/0rM8/L/onVs1dXS/FO2VatVTs56D/3eD118W6eKFeBUJK6enOr+o3AUKS5I2r1iiBR+9ecO+X/tqeYbWbkXth853dQn3nIRFNx6N7vrOL/py2R779wOeqazuDcspKLuPdkSf1KvT1mndrhj79s/611X9KiHy9/VU1JHTmjR/i2Yvj8rw+q3o9LfMIHCV2ZFf2m9sGVa6jAa9MlTly1dwdVnAXcV5kDn4pHIoKE3BaPXq1ZoyZYr279+vuXPnqmDBgpo5c6aKFSumhx56yNnu5Obmpn///Vd58uRxaN+2bZseffRRnTp141XNboZgBBCMAIlgBABIfTByeirdvHnz9Nxzz6lNmzbasmWLLl26ugLc2bNn9cYbb+iHH35IdV+VKlWSzWaTzWZTnTp15OHxv3KSkpIUHR2t+vXrO1siAAAAADjF6WA0ZswYffzxx2rXrp3mzJljb69Ro4bGjBnjVF9NmjSRJG3dulX16tWTv///Vm3y8vJSSEiInn76aWdLBAAAAACnOB2MoqKiVLNmzRTtgYGBOnPmjFN9jRgxQpIUEhKili1bysfHx9lyAAAAAOCOOR2M8uXLp3379ikkJMShfc2aNSpevHiaimjfvr2kq6vQHT9+XMnJyQ7bixQpkqZ+AQAAACA1nA5GXbt2Vd++fTV16lTZbDb9888/+u233zRgwAANGzYsTUXs3btXnTp10rp16xzajTGy2WxOr0oHAAAAAM5wOhgNHjxYycnJqlOnji5cuKCaNWvK29tbAwYM0AsvvJCmIjp06CAPDw8tWrRI+fPnl81mS1M/AAAAAJAWab6PUWJiovbt26f4+HiFh4c7LJzgrGzZsumPP/5Q6dKl09zH9ViuG2C5bkBiuW4AQOqX63ZztuNOnTrp3Llz8vLyUnh4uKpWrSp/f3+dP39enTp1crY7SVJ4eLhOnjyZpmMBAAAA4E45HYxmzJihhISEFO0JCQn64osvUt1PXFyc/fHmm2/q5Zdf1ooVKxQbG+uwLS4uztkSAQAAAMApqb7GKC4uTsYYGWN07tw5h6W1k5KS9MMPPyg4ODjVT5wjRw6Ha4mMMapTp47DPiy+AAAAAOBuSHUwuhZkbDabSpUqlWK7zWbTqFGjUv3Ey5cvT/W+AAAAAJCRUh2Mli9fLmOMateurXnz5ilnzpz2bV5eXipatKgKFCiQ6ieuVauWc5UCAAAAQAZJdTC6FmSio6NVuHBhubk5fXnSTW3fvv2G7TabTT4+PipSpIi8vb3T7fkAAAAA4HpO38eoaNGiOnPmjD7//HPt3r1bklS2bFl16tRJgYGBaSqiYsWKt7x3kaenp1q2bKkpU6Y4XNsEAAAAAOnB6WGf33//XaGhoXrnnXd06tQpnTp1ShMnTlRoaKg2b96cpiIWLFigkiVL6pNPPtHWrVu1detWffLJJwoLC9OsWbP0+eef69dff9XQoUPT1D8AAAAA3IrTN3h9+OGHVaJECX366afy8Lg64HTlyhV16dJFBw4c0KpVq5wuomrVqnrttddUr149h/affvpJw4YN08aNG/Xtt9/qpZde0v79+2/bHzd4BbjBKyBxg1cAQOpv8Or0VLrff//dIRRJkoeHh15++WVVqVLF2e4kSTt27FDRokVTtBctWlQ7duyQdHW6XUxMTJr6BwAAAIBbcXoqXUBAgA4dOpSi/fDhw8qePXuaiihdurTGjRunxMREe9vly5c1btw4lS5dWpJ09OhR5c2bN039AwAAAMCtOD1i1LJlS3Xu3Fnjx49XRESEJGnt2rUaOHCgWrVqlaYiJk+erEaNGqlQoUIqX768pKujSElJSVq0aJEk6cCBA+rZs2ea+gcAAACAW3E6GI0fP142m03t2rXTlStXJF1dNe7555/XuHHj0lRERESEoqOjFRkZqb/++kuS1Lx5c7Vu3do+CvXcc8+lqW8AAAAAuJ1UL75QpUoVdenSRa1bt1ZAQIAuXLhgXwghNDRUfn5+GVqoM1h8AWDxBUBi8QUAQAYsvlChQgW9/PLLeumll/T000+rU6dOeuSRR9JYnvTdd9/piSeekKenp7777rtb7tuoUaM0Pw8AAAAA3I5Ty3VfuHBBX3/9taZPn67Vq1erWLFi6tSpk9q3b6+CBQs69cRubm46duyYgoOD5eZ28zUgbDabkpKSnOqbESOAESNAYsQIAJD6ESOnVqXz8/NThw4dtGLFCv3111969tlnNWXKFIWEhKhhw4aaPz/1H8SSk5MVHBysy5cv65FHHtGePXuUnJyc4uFsKAIAAAAAZzm9XPc1oaGhGjNmjA4ePKjZs2dr/fr1at68udP9eHp6aseOHbccNQIAAACAjHRHaWTFihXq0KGDOnTooKSkJHXt2jVN/bRt21afffbZnZQCAAAAAGnm9HLdR44c0fTp0zV9+nQdOHBADz/8sD788EM1b95cvr6+aSriypUrmjp1qn755Rfdf//9ypYtm8P2iRMnpqlfAAAAAEiNVAejr7/+WlOnTtWyZcsUHBys9u3bq1OnTipRosQdF/Hnn3+qcuXKkmS/j9E1NpvtjvsHAAAAgFtJdTBq27atGjZsqAULFqhBgwbpek3Q8uXL060vAAAAAHBWqoPRkSNHFBwcnJG1AAAAAIBLpHrYh1AEAAAA4F7FGtkAAAAALI9gBAAAAMDyCEYAAAAALM/p+xhdk5iYqOPHjys5OdmhvUiRIndcFAAAAADcTU4Ho71796pTp05at26dQ7sxRjabTUlJSelWHAAAAADcDU4How4dOsjDw0OLFi1S/vz5uQErAAAAgCzP6WC0detW/fHHHypdunRG1AMAAAAAd53Tiy+Eh4fr5MmTGVELAAAAALiE08HozTff1Msvv6wVK1YoNjZWcXFxDg8AAAAAyGqcnkpXt25dSVKdOnUc2ll8AQAAAEBW5XQwWr58eUbUAQAAAAAu43QwqlWrVkbUAQAAAAAuk+YbvF64cEGHDh1SYmKiQ3v58uXvuCgAAAAAuJucDkYnTpxQx44d9eOPP95wO9cYAQAAAMhqnF6V7sUXX9SZM2e0YcMG+fr6asmSJZoxY4ZKliyp7777LiNqBAAAAIAM5fSI0a+//qqFCxeqSpUqcnNzU9GiRfXYY48pICBAY8eOVcOGDTOiTgAAAADIME6PGJ0/f17BwcGSpKCgIJ04cUKSVK5cOW3evDl9qwMAAACAu8DpYBQWFqaoqChJUoUKFTRlyhQdPXpUH3/8sfLnz5/uBQIAAABARnN6Kl3fvn0VExMjSRoxYoTq16+vyMhIeXl5afr06eldHwAAAABkOKeDUdu2be1f33///fr777+1Z88eFSlSRLlz507X4gAAAADgbkjzfYyu8fPzU+XKldOjFgAAAABwCaeDkTFGc+fO1fLly3X8+HElJyc7bJ8/f366FQcAAAAAd4PTwejFF1/UlClT9Oijjypv3ryy2WwZURcAAAAA3DVOB6OZM2dq/vz5atCgQUbUAwAAAAB3ndPLdQcGBqp48eIZUQsAAAAAuITTwWjkyJEaNWqUEhISMqIeAAAAALjrnJ5K16JFC82ePVvBwcEKCQmRp6enw/bNmzenW3EAAAAAcDc4HYzat2+vP/74Q23btmXxBQAAAAD3BKeD0eLFi/XTTz/poYceyoh6AAAAAOCuc/oao8KFCysgICAjagEAAAAAl3A6GE2YMEEvv/yyDh48mAHlAAAAAMDdZzPGGGcOCAoK0oULF3TlyhX5+fmlWHzh1KlT6VpgWly84uoKAAAAAGQGPqm8eMjpa4wmTZrk7CEAAAAAkKk5PWKUFTBiBAAAAEDKwBGjQ4cO3XJ7kSJFnO0SAAAAAFzK6REjNze3W967KCkp6Y6LulOMGAEAAACQMnDEaMuWLQ7fX758WVu2bNHEiRP1+uuvO9sdAAAAALhcul1jtHjxYr399ttasWJFenR3RxgxAgAAACClfsTI6fsY3UxYWJg2bdqUXt0BAAAAwF3j9FS6uLg4h++NMYqJidHIkSNVsmTJdCsMAAAAAO4Wp4NRjhw5Uiy+YIxR4cKFNWfOnHQrDAAAAADuFqeD0fLlyx2+d3NzU548eVSiRAl5eDjdHQAAAAC4HDd4BQAAAHDPSvflur/77rtU7deoUaPUdgkAAAAAmUKqR4zc3BwXsLPZbPrvoTabjRu8AgAAAMg00n257uTkZIeHn5+f9u3b59CWGUIRAAAAADgr3e5jBAAAAABZFcEIAAAAgOURjAAAAABYXpqDkc1mS3GjVwAAAADIilK9Kl1QUJBDEDpz5owCAgJSrFZ36tSp9K0wDViVDgAAAICUAfcxmjRpUhpLAQAAAIDMLdUjRlkJI0YAAAAApAy4jxEAAAAA3KsIRgAAAAAsj2AEAAAAwPIIRgAAAAAsj2AEAAAAwPJSvVz39Y4cOaLvvvtOhw4dUmJiosO2iRMnpkthAAAAAHC3OB2Mli1bpkaNGql48eLas2eP7rvvPh08eFDGGFWuXDkjagQAAACADOX0VLohQ4ZowIAB2rFjh3x8fDRv3jwdPnxYtWrVUvPmzTOiRgAAAADIUE7f4DV79uzaunWrQkNDFRQUpDVr1qhs2bLatm2bGjdurIMHD2ZQqanHDV4BAAAASBl4g9ds2bLZryvKnz+/9u/fb9928uRJZ7sDAAAAAJdz+hqjatWqac2aNSpTpowaNGigl156STt27ND8+fNVrVq1jKgRAAAAADKU01PpDhw4oPj4eJUvX17nz5/XSy+9pHXr1qlkyZKaOHGiihYtmlG1phpT6QAAAABIqZ9K53QwygoIRgAAAACk1AejNN3HSJISExN1/PhxJScnO7QXKVIkrV0CAAAAgEs4HYz++usvde7cWevWrXNoN8bIZrMpKSkp3YoDAAAAgLvB6WDUsWNHeXh4aNGiRcqfP79sNltG1AUAAAAAd43T1xhly5ZNf/zxh0qXLp1RNd0xrjECAAAAIGXgfYzCw8O5XxEAAACAe4rTwejNN9/Uyy+/rBUrVig2NlZxcXEODwAAAADIapyeSufmdjVL/ffaosy0+AJT6QAAAABIGbhc9/Lly509BAAAAAAyNW7wCgAAAOCelWGLL0jS6tWr1bZtW0VEROjo0aOSpJkzZ2rNmjVp6Q4AAAAAXMrpYDRv3jzVq1dPvr6+2rx5sy5duiRJOnv2rN544410LxAAAAAAMprTwWjMmDH6+OOP9emnn8rT09PeXqNGDW3evDldiwMAAACAu8HpYBQVFaWaNWumaA8MDNSZM2fSVMTMmTNVo0YNFShQQH///bckadKkSVq4cGGa+gMAAAAAZzgdjPLly6d9+/alaF+zZo2KFy/udAEfffSR+vfvrwYNGujMmTP25b5z5MihSZMmOd0fAAAAADjL6WDUtWtX9e3bVxs2bJDNZtM///yjyMhIDRgwQM8//7zTBbz//vv69NNP9eqrr8rd3d3eXqVKFe3YscPp/gAAAADAWU7fx2jw4MFKTk5WnTp1dOHCBdWsWVPe3t4aMGCAXnjhBacLiI6OVqVKlVK0e3t76/z58073BwAAAADOcioYJSUlae3aterVq5cGDhyoffv2KT4+XuHh4fL3909TAcWKFdPWrVtVtGhRh/YlS5aoTJkyaeoTAAAAAJzhVDByd3fX448/rt27dytHjhwKDw+/4wL69++vXr166eLFizLGaOPGjZo9e7bGjh2rzz777I77BwAAAIDbcXoq3X333acDBw6oWLFi6VJAly5d5Ovrq6FDh+rChQtq3bq1ChQooHfffVfPPvtsujwHAAAAANyKzRhjnDlgyZIlGjJkiF577TXdf//9ypYtm8P2gICANBdz4cIFxcfHKzg4OM19SNLFK3d0OAAAAIB7hE8qh4KcDkZubv9byM5ms9m/NsbIZrPZl9tOrdq1a2v+/PnKkSOHQ3tcXJyaNGmiX3/91an+JIIRAAAAgKtSG4ycnkq3fPlyZw+5pRUrVigxMTFF+8WLF7V69ep0fS4AAAAAuBGng1GtWrVuuu3PP/9MdT/bt2+3f71r1y4dO3bM/n1SUpKWLFmiggULOlseAAAAADjN6al0/3Xu3DnNnj1bn332mf74449UT6Vzc3OzT8W7UQm+vr56//331alTJ6drYiodAAAAACkDp9Jds2rVKn3++eeaN2+eChQooGbNmmny5MmpPj46OlrGGBUvXlwbN25Unjx57Nu8vLwUHBwsd3f3tJYHAAAAAKnmVDA6duyYpk+frs8//1xxcXFq0aKFLl26pG+//dbpexpdu6FrcnKyU8cBAAAAQHpzu/0uVz311FMKCwvT9u3bNWnSJP3zzz96//3306WI/fv364UXXlDdunVVt25d9enTR/v370+XvgEAAADgdlIdjH788Ud17txZo0aNUsOGDdNtmttPP/2k8PBwbdy4UeXLl1f58uW1YcMGlS1bVj///HO6PAfurjmzIvXEY7X1QKVyavNsc+24bqENwCo4D2B1nAMA50FWk+pgtGbNGp07d07333+/HnzwQX3wwQc6efLkHRcwePBg9evXTxs2bNDEiRM1ceJEbdiwQS+++KIGDRp0x/3j7lry4w8a/9ZYde/ZS3O+WaCwsNJ6vntnxcbGuro04K7hPIDVcQ4AnAdZkdOr0p0/f15fffWVpk6dqo0bNyopKUkTJ05Up06dlD17dqcL8PHx0Y4dO1SyZEmH9r/++kvly5fXxYsXne6TVelcp82zzVX2vnJ6ZehwSVevIXu8Ti21av2cOnft5uLqgLuD8wBWxzkAcB5kJqldlS7VI0bXZMuWTZ06ddKaNWu0Y8cOvfTSSxo3bpyCg4PVqFEjZ7tTnjx5tHXr1hTtW7duVXBwsNP9wXUuJyZq966dqlY9wt7m5uamatUitH3bFhdWBtw9nAewOs4BgPMgq3I6GF0vLCxMb731lo4cOaLZs2enqY+uXbuqW7duevPNN7V69WqtXr1a48aNU/fu3dW1a9fbHn/p0iXFxcU5PC5dupSmWnBnTp85raSkJOXKlcuhPVeuXOky7RLICjgPYHWcAwDnQVaV5vsYXc/d3V1NmjRRkyZNnD522LBhyp49uyZMmKAhQ4ZIkgoUKKCRI0eqT58+tz1+7NixGjVqlEPbq8NGaOjwkU7XAgAAAMCa0iUY3QmbzaZ+/fqpX79+OnfunCQ5da3SkCFD1L9/f4c24+6drjUidYJyBMnd3T3FRYWxsbHKnTu3i6oC7i7OA1gd5wDAeZBV3dFUuvQwZswYRUdHS7oaiJxdwMHb21sBAQEOD29vgpEreHp5qUx4WW1Y/5u9LTk5WRs2/KbyFSq5sDLg7uE8gNVxDgCcB1mVy4PRN998oxIlSigiIkIffvgh8y6zuOfad9T8uV/ru28X6MD+/RozeqQSEhLUpGkzV5cG3DWcB7A6zgGA8yArcnq57oywc+dORUZGas6cOTpy5Igee+wxtWnTRk2aNJGfn5/T/bFct2vNjvxSM6Z9rpMnTyisdBkNemWoypev4OqygLuK8wBWxzkAcB5kFqldrjtTBKPrrV27VrNmzdI333yjixcvKi4uzuk+CEYAAAAApAy8j1FGy5Ytm3x9feXl5aXLly+7uhwAAAAAFpApglF0dLRef/11lS1bVlWqVNGWLVs0atQoHTt2zNWlAQAAALAAly/XXa1aNW3atEnly5dXx44d1apVKxUsWNDVZQEAAACwEJcHozp16mjq1KkKDw93dSkAAAAALCrTLL6QmJio6OhohYaGysPjzvIaiy8AAAAAkLLQ4gsJCQnq3Lmz/Pz8VLZsWR06dEiS9MILL2jcuHEurg4AAACAFbg8GA0ePFjbtm3TihUr5OPjY2+vW7euvvrqKxdWBgAAAMAqXH6N0bfffquvvvpK1apVk81ms7eXLVtW+/fvd2FlAAAAAKzC5SNGJ06cUHBwcIr28+fPOwQlAAAAAMgoLg9GVapU0eLFi+3fXwtDn332mapXr+6qsgAAAABYiMun0r3xxht64okntGvXLl25ckXvvvuudu3apXXr1mnlypWuLg8AAACABbh8xOihhx7S1q1bdeXKFZUrV05Lly5VcHCwfvvtN91///2uLg8AAACABWSa+xilJ+5jBAAAAEBK/X2MXDKVLi4uTgEBAfavb+XafgAAAACQUVwyYuTu7q6YmBgFBwfLzc3thqvPGWNks9mUlJTkdP+MGAEAAACQMvmI0a+//qqcOXNKkqZNm6bChQvL3d3dYZ/k5GQdOnTIFeUBAAAAsBiXX2N0/ejR9WJjYxUcHMyIEQAAAIA0S+2IkctXpbs2Ze6/4uPj5ePj44KKAAAAAFiNy+5j1L9/f0lXb+g6bNgw+fn52bclJSVpw4YNqlixoouqAwAAAGAlLgtGW7ZskXR1xGjHjh3y8vKyb/Py8lKFChU0YMAAV5UHAAAAwEJcfo1Rx44d9e6776brstxcYwQAAABASv01Ri4PRhmBYAQAAABAykKLLwAAAACAqxGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5RGMAAAAAFgewQgAAACA5WW6YHTx4kVXlwAAAADAYjJFMEpOTtZrr72mggULyt/fXwcOHJAkDRs2TJ9//rmLqwMAAABwr8sUwWjMmDGaPn263nrrLXl5ednb77vvPn322WcurAwAAACAFWSKYPTFF1/ok08+UZs2beTu7m5vr1Chgvbs2ePCygAAAABYQaYIRkePHlWJEiVStCcnJ+vy5csuqAgAAACAlWSKYBQeHq7Vq1enaJ87d64qVarkgooAAAAAWImHqwuQpOHDh6t9+/Y6evSokpOTNX/+fEVFRemLL77QokWLXF0eAAAAgHuczRhjXF2EJK1evVqjR4/Wtm3bFB8fr8qVK2v48OF6/PHHne7r4pUMKBAAAABAluOTyqGgTBOM0hPBCAAAAICU+mCUKa4xKl68uGJjY1O0nzlzRsWLF3dBRQAAAACsJFMEo4MHDyopKSlF+6VLl3T06FEXVAQAAADASly6+MJ3331n//qnn35SYGCg/fukpCQtW7ZMISEhLqgMAAAAgJW49BojN7erA1Y2m03/LcPT01MhISGaMGGCnnzySaf65RojAAAAAFLqrzFy6YhRcnKyJKlYsWLatGmTcufO7cpyAAAAAFgUq9IBAAAAuGdlqVXpJGnlypV66qmnVKJECZUoUUKNGjXS6tWrXV0WAAAAAAvIFMHoyy+/VN26deXn56c+ffqoT58+8vX1VZ06dTRr1ixXlwcAAADgHpcpptKVKVNG3bp1U79+/RzaJ06cqE8//VS7d+92qj+m0gEAAACQUj+VLlMEI29vb+3cuVMlSpRwaN+3b5/uu+8+Xbx40an+CEYAAAAApCx2jVHhwoW1bNmyFO2//PKLChcu7IKKAAAAAFiJS5frvuall15Snz59tHXrVkVEREiS1q5dq+nTp+vdd991cXUAAAAA7nWZYiqdJC1YsEATJkywX09UpkwZDRw4UI0bN3a6L6bSAQAAAJCy2DVG6Y1gBAAAAEDKYtcYdenSRStWrHB1GQAAAAAsKlMEoxMnTqh+/foqXLiwBg4cqK1bt7q6JAAAAAAWkmmm0p0+fVrffPONZs2apdWrV6t06dJq06aNWrdurZCQEKf6YiodAAAAACmLX2N05MgRzZ49W1OnTtXevXt15YpzSYdgBAAAAEDKYtcYXe/y5cv6/ffftWHDBh08eFB58+Z1dUkAAAAA7nGZJhgtX75cXbt2Vd68edWhQwcFBARo0aJFOnLkiKtLAwAAAHCPyxRT6QoWLKhTp06pfv36atOmjZ566il5e3unuT+m0gEAAACQstg1Rp9++qmaN2+uHDly3HK/I0eOqECBAnJzu/VAF8EIAAAAgJTFglFqBQQEaOvWrSpevPgt9yMYAQAAAJCy8OILt5KFMhwAAACALCRLBSMAAAAAyAgEIwAAAACWRzACAAAAYHlZKhjZbDZXlwAAAADgHpSlghGLLwAAAADICFlque7Dhw+rQIECcnd3v+V+LNcNAAAAQMoC9zFq1qxZqvedP3++U30TjFxrzqxIzZj2uU6ePKFSYaU1+JVhKle+vKvLAu4qzgNYHecAwHmQWWT6+xgFBgbaHwEBAVq2bJl+//13+/Y//vhDy5YtU2BgoKtKRBos+fEHjX9rrLr37KU53yxQWFhpPd+9s2JjY11dGnDXcB7A6jgHAM6DrChTTKUbNGiQTp06pY8//tg+TS4pKUk9e/ZUQECA3n77baf6Y8TIddo821xl7yunV4YOlyQlJyfr8Tq11Kr1c+rctZuLqwPuDs4DWB3nAMB5kJlk+hGj602dOlUDBgxwuHbI3d1d/fv319SpU11YGZxxOTFRu3ftVLXqEfY2Nzc3VasWoe3btriwMuDu4TyA1XEOAJwHWVWmCEZXrlzRnj17UrTv2bNHycnJtzz20qVLiouLc3hcunQpo0rFLZw+c1pJSUnKlSuXQ3uuXLl08uRJF1UF3F2cB7A6zgGA8yCryhTBqGPHjurcubMmTpyoNWvWaM2aNZowYYK6dOmijh073vLYsWPHOlyvFBgYqLffHHuXKgcAAABwL0jljLuMNX78eOXLl08TJkxQTEyMJCl//vwaOHCgXnrppVseO2TIEPXv39+hzbh7Z1ituLmgHEFyd3dPcVFhbGyscufO7aKqgLuL8wBWxzkAcB5kVZlixMjNzU0vv/yyjh49qjNnzujMmTM6evSoXn755dves8jb21sBAQEOD29vgpEreHp5qUx4WW1Y/5u9LTk5WRs2/KbyFSq5sDLg7uE8gNVxDgCcB1lVphgxul5AQICrS8AdeK59Rw17ZZDKlr1P95Urry9nzlBCQoKaNE39fauArI7zAFbHOQBwHmRFmSYYzZ07V19//bUOHTqkxMREh22bN292UVVwVv0nGuj0qVP68IP3dPLkCYWVLqMPp3ymXAwbw0I4D2B1nAMA50FWlCnuY/Tee+/p1VdfVYcOHfTJJ5+oY8eO2r9/vzZt2qRevXrp9ddfd6o/7mMEAAAAQEr9fYwyRTAqXbq0RowYoVatWil79uzatm2bihcvruHDh+vUqVP64IMPnOqPYAQAAABAymI3eD106JAiIq7eAMvX11fnzp2TJD333HOaPXu2K0sDAAAAYAGZIhjly5dPp06dkiQVKVJE69evlyRFR0crEwxoAQAAALjHZYpgVLt2bX333XeSrt7stV+/fnrsscfUsmVLNW3a1MXVAQAAALjXZYprjJKTk5WcnCwPj6sTAOfMmaN169apZMmS6t69u7y8vJzqj2uMAAAAAEhZbPGF9EYwAgAAACClPhhlmvsYnT59Wp9//rl2794tSQoPD1fHjh2VM2dOF1cGAAAA4F6XKUaMVq1apUaNGikgIEBVqlSRJP3xxx86c+aMvv/+e9WsWdOp/hgxAgAAACBlsal05cqVU/Xq1fXRRx/J3d1dkpSUlKSePXtq3bp12rFjh1P9EYwAAAAASFksGPn6+mrr1q0KCwtzaI+KilLFihWVkJDgVH8EIwAAAABSFrvBa+XKle3XFl1v9+7dqlChggsqAgAAAGAlLlt8Yfv27fav+/Tpo759+2rfvn2qVq2aJGn9+vWaPHmyxo0b56oSAQAAAFiEy6bSubm5yWaz6XZPb7PZlJSU5FTfTKUDAAAAIGWB5bqjo6Nd9dQAAAAA4CBTLL6Q3hgxAgAAACBlgRGj/9q7d6+WL1+u48ePKzk52WHb8OHDXVQVAAAAACvIFCNGn376qZ5//nnlzp1b+fLlk81ms2+z2WzavHmzU/0xYgQAAABAymL3MSpatKh69uypQYMGpUt/BCMAAAAAUha7j9Hp06fVvHlzV5cBAAAAwKIyRTBq3ry5li5d6uoyAAAAAFiUy6bSvffee/avz58/r4kTJ6phw4YqV66cPD09Hfbt06ePU30zlQ4AAACAlAWuMSpWrFiq9rPZbDpw4IBTfROMAAAAAEhZIBhlJIIRAAAAACmLLb4wevRoXbhwIUV7QkKCRo8e7YKKAAAAAFhJphgxcnd3V0xMjIKDgx3aY2NjFRwcrKSkJKf6Y8QIAAAAgJTFRoyMMQ43db1m27ZtypkzpwsqAgAAAGAlqcxPGSMoKEg2m002m02lSpVyCEdJSUmKj49Xjx49XFghAAAAACtw6VS6GTNmyBijTp06adKkSQoMDLRv8/LyUkhIiKpXr+50v0ylAwAAACBlsVXpVq5cqYiIiBT3L0orghEAAAAAKYsFI+nq1Llvv/1Wu3fvliSVLVtWjRo1kru7u9N9EYwAAAAASFksGO3bt08NGjTQ0aNHFRYWJkmKiopS4cKFtXjxYoWGhjrVH8EIAAAAgJTFglGDBg1kjFFkZKR9FbrY2Fi1bdtWbm5uWrx4sVP9EYwAAAAASFksGGXLlk3r169XuXLlHNq3bdumGjVqKD4+3qn+CEYAAAAApCx2HyNvb2+dO3cuRXt8fLy8vLxcUBEAAAAAK8kUwejJJ59Ut27dtGHDBhljZIzR+vXr1aNHDzVq1MjV5QEAAAC4x2WKqXRnzpxR+/bt9f3339uX7L58+bIaN26s6dOnO9zfKDWYSgcAAABAymLXGF2zb98+7dq1S5IUHh6uEiVKpKkfghEAAAAAKfXBKJW7ZbzPP/9c77zzjvbu3StJKlmypF588UV16dLFxZUBAAAAuNdlimA0fPhwTZw4US+88IKqV68uSfrtt9/Ur18/HTp0SKNHj3ZxhQAAAADuZZliKl2ePHn03nvvqVWrVg7ts2fP1gsvvKCTJ0861R9T6QAAAABIWWy57suXL6tKlSop2u+//35duULKAQAAAJCxMkUweu655/TRRx+laP/kk0/Upk0bF1QEAAAAwEpcdo1R//797V/bbDZ99tlnWrp0qapVqyZJ2rBhgw4dOqR27dq5qkQAAAAAFuGya4weffTRVO1ns9n066+/OtU31xgBAAAAkLLofYzSC8EIAAAAgJTFFl8AAAAAAFciGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMsjGAEAAACwPIIRAAAAAMuzGWOMq4vAveXSpUsaO3ashgwZIm9vb1eXA7gE5wHAeQBInAdZCcEI6S4uLk6BgYE6e/asAgICXF0O4BKcBwDnASBxHmQlTKUDAAAAYHkEIwAAAACWRzACAAAAYHkEI6Q7b29vjRgxggsMYWmcBwDnASBxHmQlLL4AAAAAwPIYMQIAAABgeQQjAAAAAJZHMAIAAABgeQQjC3rkkUf04osvpnr/b7/9ViVKlJC7u7tTxwGZmc1m07fffpvq/VesWCGbzaYzZ85kWE1AZsH7HZmRs59fUiskJESTJk266faDBw/KZrNp69at6f7cHTp0UJMmTdK9X6QNwQi31b17dz3zzDM6fPiwXnvttQw7iadPn64cOXKke7/AjcTExOiJJ55I1z5HjhypihUrpmufEh9SkfH++4EzIiJCMTExCgwMvGs1ZOSHT0Dic4bk/B8FrcbD1QUgc4uPj9fx48dVr149FShQwNXlAOkiMTFR+fLlc3UZQKbl5eXFOQLAchgxsrhLly5pwIABKliwoLJly6YHH3xQK1askHT1r9TZs2eXJNWuXVs2m02PPPKIZsyYoYULF8pms8lms2nFihWKiIjQoEGDHPo+ceKEPD09tWrVKknS6dOn1a5dOwUFBcnPz09PPPGE9u7da3+ujh076uzZs/Z+R44cedsagdR45JFH1Lt3b7344ovKnTu36tWrl+KvZuvWrVPFihXl4+OjKlWq6Ntvv73hX6//+OMPValSRX5+foqIiFBUVJSkq3+JHDVqlLZt22Z/D0+fPl2tW7dWy5YtHfq4fPmycufOrS+++ELS1fd4nz59FBwcLB8fHz300EPatGmTpKt/RX/00UclSUFBQbLZbOrQoYMkKTk5WWPHjlWxYsXk6+urChUqaO7cuRnwE8S9rEOHDlq5cqXeffddh/fuf0cp16xZo4cffli+vr4qXLiw+vTpo/Pnz0uSXnnlFT344IMp+q5QoYJGjx4t6er7dfTo0SpUqJC8vb1VsWJFLVmyxL5vsWLFJEmVKlWy/39zzWeffaYyZcrIx8dHpUuX1ocffpgBPwlkBVeuXFHv3r0VGBio3Llza9iwYbp255m0fs6QpAsXLqhTp07Knj27ihQpok8++STFc+/Zs0cRERHy8fHRfffdp5UrV9q3JSUlqXPnzvbfx2FhYXr33Xcdjk9KSlL//v2VI0cO5cqVSy+//LJSe9ecTz75RAUKFFBycrJDe+PGjdWpUyf79x999JFCQ0Pl5eWlsLAwzZw5074tJCREktS0aVPZbDb795K0cOFCVa5cWT4+PipevLhGjRqlK1eupKq2e4qB5dSqVcv07dvXGGNMly5dTEREhFm1apXZt2+fefvtt423t7f566+/zKVLl0xUVJSRZObNm2diYmLM2bNnTYsWLUz9+vVNTEyMiYmJMZcuXTIffPCBKVKkiElOTrY/z/vvv+/Q1qhRI1OmTBmzatUqs3XrVlOvXj1TokQJk5iYaC5dumQmTZpkAgIC7P2eO3futjUCqVGrVi3j7+9vBg4caPbs2WP27NljJJkFCxYYY4w5e/asyZkzp2nbtq3ZuXOn+eGHH0ypUqWMJLNlyxZjjDHLly83ksyDDz5oVqxYYXbu3GkefvhhExERYYwx5sKFC+all14yZcuWtb+HL1y4YBYtWmR8fX3t72djjPn++++Nr6+viYuLM8YY06dPH1OgQAHzww8/mJ07d5r27duboKAgExsba65cuWLmzZtnJJmoqCgTExNjzpw5Y4wxZsyYMaZ06dJmyZIlZv/+/WbatGnG29vbrFix4u79cJHlnTlzxlSvXt107drV/t795ZdfjCRz+vRpY4wx+/btM9myZTPvvPOO+euvv8zatWtNpUqVTIcOHYwxxvz5559Gktm3b5+932tte/fuNcYYM3HiRBMQEGBmz55t9uzZY15++WXj6elp/12+ceNGI8n88ssvJiYmxsTGxhpjjPnyyy9N/vz5zbx588yBAwfMvHnzTM6cOc306dPv4k8JmcG13+V9+/Y1e/bsMV9++aXx8/Mzn3zyiTEm7Z8zihYtanLmzGkmT55s9u7da8aOHWvc3NzMnj17jDHGREdHG0mmUKFCZu7cuWbXrl2mS5cuJnv27ObkyZPGGGMSExPN8OHDzaZNm8yBAwfstX311Vf2+t98800TFBRk5s2bZ3bt2mU6d+5ssmfPbho3bnzb137q1Cnj5eVlfvnlF3tbbGysQ9v8+fONp6enmTx5somKijITJkww7u7u5tdffzXGGHP8+HEjyUybNs3ExMSY48ePG2OMWbVqlQkICDDTp083+/fvN0uXLjUhISFm5MiRd/gvlvUQjCzoWjD6+++/jbu7uzl69KjD9jp16pghQ4YYY4w5ffq0kWSWL19u396+ffsUJ/Hx48eNh4eHWbVqlb2tevXqZtCgQcYYY/766y8jyaxdu9a+/eTJk8bX19d8/fXXxhhjpk2bZgIDAx36TU2NwO3UqlXLVKpUyaHt+mD00UcfmVy5cpmEhAT79k8//fSGwej6/5QWL15sJNmPGzFihKlQoYLD81y+fNnkzp3bfPHFF/a2Vq1amZYtWxpjjImPjzeenp4mMjLSvj0xMdEUKFDAvPXWWw7Pfe1DqjHGXLx40fj5+Zl169Y5PF/nzp1Nq1atnPjpAI5/MDMm5Xuuc+fOplu3bg7HrF692ri5udnf/xUqVDCjR4+2bx8yZIh58MEH7d8XKFDAvP766w59PPDAA6Znz57GmP99+Lx2zl0TGhpqZs2a5dD22muvmerVq6fptSLrqlWrlilTpozDH2EHDRpkypQpk+bPGcZcDUZt27a1f5+cnGyCg4PNRx99ZIz533tz3Lhx9n0uX75sChUqZN58882b1turVy/z9NNP27/Pnz+//ff69X2kJhgZY0zjxo1Np06d7N9PmTLFFChQwCQlJRljjImIiDBdu3Z1OKZ58+amQYMG9u+v/7/vmjp16pg33njDoW3mzJkmf/78qarrXsJUOgvbsWOHkpKSVKpUKfn7+9sfK1eu1P79+53qK0+ePHr88ccVGRkpSYqOjtZvv/2mNm3aSJJ2794tDw8Ph6kWuXLlUlhYmHbv3n1XaoS13X///TfdFhUVpfLly8vHx8feVrVq1RvuW758efvX+fPnlyQdP378pn17eHioRYsW9nPj/PnzWrhwof3c2L9/vy5fvqwaNWrYj/H09FTVqlVveW7s27dPFy5c0GOPPeZwbnzxxRecG0h327Zt0/Tp0x3ea/Xq1VNycrKio6MlSW3atNGsWbMkScYYzZ492/4+j4uL0z///OPwPpekGjVq3PJ9fv78ee3fv1+dO3d2eO4xY8bwPreoatWqyWaz2b+vXr269u7dq127dqXpc8Y11/9ut9lsypcvX4rf7dWrV7d/7eHhoSpVqjj0PXnyZN1///3KkyeP/P399cknn+jQoUOSpLNnzyomJsahvmt9pFabNm00b948Xbp0SZIUGRmpZ599Vm5uVz/O79692+lzTLp6fo8ePdrhHOvatatiYmJ04cKFVNd3L2DxBQuLj4+Xu7u7/vjjD7m7uzts8/f3d7q/Nm3aqE+fPnr//fc1a9YslStXTuXKlctUNcK6smXLli79eHp62r++9p/zf+d8/1ebNm1Uq1YtHT9+XD///LN8fX1Vv379O6ojPj5ekrR48WIVLFjQYZu3t/cd9Q38V3x8vLp3764+ffqk2FakSBFJUqtWrTRo0CBt3rxZCQkJOnz4cIrr69LyvJL06aefpriG6b//JwB34vrf7dLV3++3+91+vTlz5mjAgAGaMGGCqlevruzZs+vtt9/Whg0b0q3Gp556SsYYLV68WA888IBWr16td9555477jY+P16hRo9SsWbMU267/g6EVEIwsrFKlSkpKStLx48f18MMPp/o4Ly8vJSUlpWhv3LixunXrpiVLlmjWrFlq166dfVuZMmV05coVbdiwQREREZKk2NhYRUVFKTw8/Kb9prVGwBlhYWH68ssvdenSJXuouLb4gTNudm5ERESocOHC+uqrr/Tjjz+qefPm9v+Er10ku3btWhUtWlTS1cUZNm3aZF8+2cvLS5Ic+g4PD5e3t7cOHTqkWrVqOV0rcL2bvXevqVy5snbt2qUSJUrcdJ9ChQqpVq1aioyMVEJCgh577DEFBwdLkgICAlSgQAGtXbvW4f26du1a++jsjd7nefPmVYECBXTgwAH76BOs7b9BY/369SpZsqTCw8PT9DnDGevXr1fNmjUlXV0E4o8//lDv3r0lXX0vR0REqGfPnvb9rx/VDAwMVP78+bVhw4YUfVSuXDlVz+/j46NmzZopMjJS+/btU1hYmMOxZcqU0dq1a9W+fXt729q1a+2vX7oaAP/7M6hcubKioqJueX5bBcHIwkqVKqU2bdqoXbt2mjBhgipVqqQTJ05o2bJlKl++vBo2bHjD40JCQvTTTz8pKipKuXLlUmBgoDw9PZUtWzY1adJEw4YN0+7du9WqVSv7MSVLllTjxo3VtWtXTZkyRdmzZ9fgwYNVsGBBNW7c2N5vfHy8li1bpgoVKsjPzy/NNQLOaN26tV599VV169ZNgwcP1qFDhzR+/HhJcpiycTshISGKjo7W1q1bVahQIWXPnt0etFq3bq2PP/5Yf/31l5YvX24/Jlu2bHr++ec1cOBA5cyZU0WKFNFbb72lCxcuqHPnzpKkokWLymazadGiRWrQoIF8fX2VPXt2DRgwQP369VNycrIeeughnT17VmvXrlVAQIDDf4zA7YSEhGjDhg06ePCg/P39U/ylfNCgQapWrZp69+6tLl26KFu2bNq1a5d+/vlnffDBB/b92rRpoxEjRigxMTHFX7IHDhyoESNGKDQ0VBUrVtS0adO0detW+zTT4OBg+fr6asmSJSpUqJB8fHwUGBioUaNGqU+fPgoMDFT9+vV16dIl/f777zp9+rT69++f8T8cZCqHDh1S//791b17d23evFnvv/++JkyYkObPGX5+fql+7smTJ6tkyZIqU6aM3nnnHZ0+fdq+IlzJkiX1xRdf6KefflKxYsU0c+ZMbdq0yb7aoiT17dtX48aNU8mSJVW6dGlNnDjR6fvTtWnTRk8++aR27typtm3bOmwbOHCgWrRooUqVKqlu3br6/vvvNX/+fP3yyy/2fUJCQrRs2TLVqFFD3t7eCgoK0vDhw/Xkk0+qSJEieuaZZ+Tm5qZt27bpzz//1JgxY5yqL8tz9UVOuPuuv8j22ioqISEhxtPT0+TPn980bdrUbN++3Rhz48UXjh8/bh577DHj7++fYtsPP/xgJJmaNWumeN5Tp06Z5557zgQGBhpfX19Tr169FCvL9ejRw+TKlctIMiNGjEhVjcDt/PfCcmNSXoC6du1aU758eePl5WXuv/9+M2vWLCPJvirRjRZA2LJli5FkoqOjjTFXF0R4+umnTY4cOewr/1yza9cuI8kULVrU4cJhY4xJSEgwL7zwgsmdO7fx9vY2NWrUMBs3bnTYZ/To0SZfvnzGZrOZ9u3bG2OuXiA8adIkExYWZjw9PU2ePHlMvXr1zMqVK+/o5wXriYqKMtWqVTO+vr729+5/3+8bN260/+7Pli2bKV++fIrFFE6fPm28vb2Nn5+fw0qMxhiTlJRkRo4caQoWLGg8PT1NhQoVzI8//uiwz6effmoKFy5s3NzcTK1ateztkZGRpmLFisbLy8sEBQWZmjVrmvnz56f7zwGZW61atUzPnj1Njx49TEBAgAkKCjKvvPKK/XdqWj9nFC1a1LzzzjsO+1WoUMG+/driC7NmzTJVq1Y1Xl5eJjw83L7amzFXf/936NDBBAYGmhw5cpjnn3/eDB482GFBnsuXL5u+ffuagIAAkyNHDtO/f3/Trl27VC++YMzV8yh//vxGktm/f3+K7R9++KEpXry48fT0NKVKlXJY+McYY7777jtTokQJ4+HhYYoWLWpvX7JkiYmIiDC+vr4mICDAVK1a1b7an5XYjEnlAuoAYCGRkZH2e174+vq6uhwAAJDBmEoHAJK++OILFS9eXAULFtS2bds0aNAgtWjRglAEAIBFEIwAQNKxY8c0fPhwHTt2TPnz51fz5s31+uuvu7osAIAFHDp0yGGRhP/atWuXfQVIZBym0gEAAAAudOXKFR08ePCm20NCQuThwXhGRiMYAQAAALA8N1cXAAAAAACuRjACAAAAYHkEIwAAAACWRzACAAAAYHkEIwAAAACWRzACAKTZsWPH1LdvX5UoUUI+Pj7KmzevatSooY8++kgXLlyw7xcSEiKbzSabzaZs2bKpcuXK+uabb1Jsu9GjQ4cOt6xh7dq18vDwUMWKFe1tTz31lOrXr3/D/VevXi2bzabt27en2FauXDn16NHjhsfNnDlT3t7eOnny5G1+KpLNZtO333572/0AAJkHwQgAkCYHDhxQpUqVtHTpUr3xxhvasmWLfvvtN7388statGiRfvnlF4f9R48erZiYGG3ZskUPPPCAWrZsqXXr1mnTpk2KiYlRTEyM5s2bJ0mKioqyt7377rs3reHMmTNq166d6tSp49DeuXNn/fzzzzpy5EiKY6ZNm6YqVaqofPnyKbZ17txZc+bMUUJCwg2Pa9SokXLnzp2qnw8AIGshGAEA0qRnz57y8PDQ77//rhYtWqhMmTIqXry4GjdurMWLF+upp55y2D979uzKly+fSpUqpcmTJ8vX11fff/+98uTJo3z58ilfvnzKmTOnJCk4ONjeFhgYeNMaevToodatW6t69eoO7U8++aTy5Mmj6dOnO7THx8frm2++UefOnW/YX9u2bZWQkGAPaNdER0drxYoV9uM++ugjhYaGysvLS2FhYZo5c6Z935CQEElS06ZNZbPZ7N9L0sKFC1W5cmX5+PioePHiGjVqlK5cuXLT1wcAuHsIRgAAp8XGxmrp0qXq1auXsmXLdsN9bDbbTY/38PCQp6enEhMT01zDtGnTdODAAY0YMeKG/bdr107Tp0/X9fcx/+abb5SUlKRWrVrdsM/cuXOrcePGmjp1qkP79OnTVahQIT3++ONasGCB+vbtq5deekl//vmnunfvro4dO2r58uWSpE2bNtnri4mJsX+/evVqtWvXTn379tWuXbs0ZcoUTZ8+Xa+//nqafwYAgPRDMAIAOG3fvn0yxigsLMyhPXfu3PL395e/v78GDRp0w2MTExM1duxYnT17VrVr107T8+/du1eDBw/Wl19+KQ8Pjxvu06lTJ+3fv18rV660t02bNk1PP/30LUehOnfurBUrVig6OlqSZIzRjBkz1L59e7m5uWn8+PHq0KGDevbsqVKlSql///5q1qyZxo8fL0nKkyePJClHjhzKly+f/ftRo0Zp8ODBat++vYoXL67HHntMr732mqZMmZKmnwEAIH0RjAAA6Wbjxo3aunWrypYtq0uXLjlsGzRokPz9/eXn56c333xT48aNU8OGDW/b57Wg5e/vrx49eigpKUmtW7fWqFGjVKpUqZseV7p0aUVERNhHf/bt26fVq1ffdBrdNY899pgKFSqkadOmSZKWLVumQ4cOqWPHjpKk3bt3q0aNGg7H1KhRQ7t3775lv9u2bdPo0aMdXk/Xrl0VExPjsFAFAMA1bvxnNgAAbqFEiRKy2WyKiopyaC9evLgkydfXN8UxAwcOVIcOHeTv76+8efPecqrd9bZu3Wr/OiAgQOfOndPvv/+uLVu2qHfv3pKk5ORkGWPk4eGhpUuX2keiOnfurBdeeEGTJ0/WtGnTFBoaqlq1at3y+dzc3NShQwfNmDFDI0eO1LRp0/Too4/aX1taxcfHa9SoUWrWrFmKbT4+PnfUNwDgzjFiBABwWq5cufTYY4/pgw8+0Pnz51N1TO7cuVWiRAnly5cv1aFIuhrCrj2Cg4MVEBCgHTt2aOvWrfZHjx49FBYWpq1bt+rBBx+0H9uiRQu5ublp1qxZ+uKLL9SpU6dUPXfHjh11+PBhzZ8/XwsWLHAYZSpTpozWrl3rsP/atWsVHh5u/97T01NJSUkO+1SuXFlRUVEOr+faw82N/44BwNUYMQIApMmHH36oGjVqqEqVKho5cqTKly8vNzc3bdq0SXv27NH999+fIc/r5uam++67z6EtODhYPj4+Kdr9/f3VsmVLDRkyRHFxcbe9J9I1xYoVU+3atdWtWzd5e3s7jPIMHDhQLVq0UKVKlVS3bl19//33mj9/vsPy5CEhIVq2bJlq1Kghb29vBQUFafjw4XryySdVpEgRPfPMM3Jzc9O2bdv0559/asyYMWn/gQAA0gV/ogIApEloaKi2bNmiunXrasiQIapQoYKqVKmi999/XwMGDNBrr73m6hIlXZ1Od/r0adWrV08FChRw+rjWrVs7THVr0qSJ3n33XY0fP15ly5bVlClTNG3aND3yyCP2fSZMmKCff/5ZhQsXVqVKlSRJ9erV06JFi7R06VI98MADqlatmt555x0VLVo03V4rACDtbOb6dUwBAAAAwIIYMQIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgeQQjAAAAAJZHMAIAAABgef8HWA2Jk61Jt7wAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 1000x700 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import seaborn as sns\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.metrics import confusion_matrix\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Assuming df is your DataFrame\n",
+    "\n",
+    "# True labels and predictions\n",
+    "y_true = filtered_df[\"type\"]\n",
+    "y_pred = filtered_df[\"gpt_vote\"]\n",
+    "\n",
+    "# Compute the confusion matrix\n",
+    "conf_matrix = confusion_matrix(y_true, y_pred, labels=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])\n",
+    "\n",
+    "# Create a pandas DataFrame from the confusion matrix\n",
+    "conf_matrix_df = pd.DataFrame(conf_matrix, index=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"], columns=[\"leftvote\", \"rightvote\", \"tievote\", \"bothbad_vote\"])\n",
+    "\n",
+    "# Plotting the heatmap\n",
+    "plt.figure(figsize=(10, 7))\n",
+    "sns.heatmap(conf_matrix_df, annot=True, fmt=\"d\", cmap=\"Blues\", cbar=False)\n",
+    "plt.title(\"Arena Human vs GPT-4V Confusion Matrix\")\n",
+    "plt.xlabel(\"GPT-4V Vote\")\n",
+    "plt.ylabel(\"Arena Human Vote\")\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy: 0.5842911877394636\n",
+      "F1 Score (Macro): 0.514392348541452\n",
+      "F1 Score (Micro): 0.5842911877394636\n",
+      "F1 Score (Weighted): 0.5536668839130223\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sklearn.metrics import accuracy_score, f1_score\n",
+    "\n",
+    "# Assuming df is your DataFrame and it contains 'type' as true labels and 'gpt_vote' as predictions\n",
+    "y_true = filtered_df['type']\n",
+    "y_pred = filtered_df['gpt_vote']\n",
+    "\n",
+    "# Calculate accuracy\n",
+    "accuracy = accuracy_score(y_true, y_pred)\n",
+    "print(f'Accuracy: {accuracy}')\n",
+    "\n",
+    "# Calculate F1 score, here using 'macro' average to treat all classes equally\n",
+    "f1 = f1_score(y_true, y_pred, average='macro')\n",
+    "print(f'F1 Score (Macro): {f1}')\n",
+    "\n",
+    "# If you want to calculate F1 score with other averages, for example 'micro' or 'weighted', you can do:\n",
+    "f1_micro = f1_score(y_true, y_pred, average='micro')\n",
+    "print(f'F1 Score (Micro): {f1_micro}')\n",
+    "\n",
+    "f1_weighted = f1_score(y_true, y_pred, average='weighted')\n",
+    "print(f'F1 Score (Weighted): {f1_weighted}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Cohen's Kappa Score: 0.3442144615665177\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sklearn.metrics import cohen_kappa_score\n",
+    "\n",
+    "# Assuming df is your DataFrame and it contains 'type' as true labels and 'gpt_vote' as predictions\n",
+    "y_true = filtered_df['type']\n",
+    "y_pred = filtered_df['gpt_vote']\n",
+    "\n",
+    "# Calculate Cohen's Kappa score\n",
+    "kappa = cohen_kappa_score(y_true, y_pred)\n",
+    "print(f'Cohen\\'s Kappa Score: {kappa}')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy Score: 0.5842911877394636\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sklearn.metrics import accuracy_score\n",
+    "accuracy = accuracy_score(y_true, y_pred)\n",
+    "print(f'Accuracy Score: {accuracy}')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Pearson Correlation Coefficient: 0.2880096104357029\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Assuming filtered_df is your DataFrame and it contains 'type' and 'gpt_vote' columns\n",
+    "# Convert 'type' and 'gpt_vote' to categorical codes\n",
+    "filtered_df['type_int'] = pd.factorize(filtered_df['type'])[0]\n",
+    "filtered_df['gpt_vote_int'] = pd.factorize(filtered_df['gpt_vote'])[0]\n",
+    "\n",
+    "# Now you can calculate Pearson correlation between these new integer columns\n",
+    "pearson_correlation = filtered_df['type_int'].corr(filtered_df['gpt_vote_int'])\n",
+    "print(f'Pearson Correlation Coefficient: {pearson_correlation}')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "otask",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

arena_elo/get_latest_data.sh

@@ -0,0 +1,17 @@
+
+# set LOGDIR to default if not set before
+if [ -z "$LOGDIR" ]; then
+    export LOGDIR="./vision-arena-logs"
+fi
+mkdir -p results
+
+
+# # for battle data
+python -m elo_rating.clean_battle_data --model_infos_file "./model_infos.json" --mode conv_release
+battle_cutoff_date=`cat cut_off_date.txt` && rm cut_off_date.txt && echo "Battle data last updated on $battle_cutoff_date"
+
+mkdir -p ./results/latest
+mkdir -p ./results/$battle_cutoff_date && mv ./clean_battle_conv_$battle_cutoff_date.json ./results/$battle_cutoff_date/clean_battle_conv.json
+cp ./results/$battle_cutoff_date/clean_battle_conv.json ./results/latest/clean_battle_conv.json
+
+echo "Battle data last updated on $battle_cutoff_date" >> ./results/latest/latest_updated_date.txt

arena_elo/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "arena_elo"
+version = "0.2.35"
+description = "Elo rating system for WildVision Bench Arena"
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache Software License",
+]
+dependencies = [
+    "numpy", "prompt_toolkit>=3.0.0", "uvicorn","polyglot", "pyicu", "pycld2", "morfessor", "scikit-learn",
+    "pytz", "tqdm", "pandas", "plotly", "fire", "Pillow"
+]
+
+[project.urls]
+"Homepage" = "https://github.com/WildVision-Bench/Arena-Elo"
+"Bug Tracker" = "https://github.com/WildVision-Bench/Arena-Elo/issues"
+
+[tool.setuptools.packages.find]
+exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]
+
+[tool.wheel]
+exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]

arena_elo/requirements.txt

@@ -0,0 +1,28 @@
+-e git+https://github.com/WildVision-Bench/Arena-Elo.git@9dc2fa8543a2e9eda3d5bc01c2212fdfcdd4bfb5#egg=arena_elo
+click==8.1.7
+fire==0.5.0
+h11==0.14.0
+joblib==1.3.2
+Morfessor==2.0.6
+numpy==1.26.4
+packaging==23.2
+pandas==2.2.0
+pillow==10.2.0
+plotly==5.18.0
+polyglot==16.7.4
+prompt-toolkit==3.0.43
+pycld2==0.41
+PyICU==2.12
+python-dateutil==2.8.2
+pytz==2024.1
+scikit-learn==1.4.0
+scipy==1.12.0
+six==1.16.0
+tenacity==8.2.3
+termcolor==2.4.0
+threadpoolctl==3.2.0
+tqdm==4.66.2
+typing_extensions==4.9.0
+tzdata==2024.1
+uvicorn==0.27.1
+wcwidth==0.2.13

arena_elo/results/20241224/clean_battle.json

@@ -0,0 +1,210 @@
+[
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030427.6669
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030452.0238
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030464.2602
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030476.2328
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030495.2955
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030503.418
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030511.3926
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034259.9984
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034275.6871
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034284.7354
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034293.468
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034303.2042
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034314.1941
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034326.5092
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034331.6963
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034336.5346
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034351.9521
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034366.1775
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034380.5877
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034384.3087
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034389.1583
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034405.9359
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034412.3533
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034419.0118
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034425.6972
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034432.5891
+  }
+]

arena_elo/results/20241224/elo_results.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73f08bd5db0423e0fec2bcc78b8d8153ff04a93d88cd4877136d08ba50ea3a84
+size 44938

arena_elo/results/20241224/leaderboard.csv

@@ -0,0 +1,3 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),license,creator,link
+ChatDiT,ChatDiT,1026.7303773982676,1026.7303773982676,-,Tongyi Lab,https://github.com/ali-vilab/ChatDiT
+FLUX-1,FLUX-1,973.2696226017323,973.2696226017323,-,Black Forest Labs,https://www.basedlabs.ai/tools/flux1

arena_elo/results/20241226/clean_battle.json

@@ -0,0 +1,482 @@
+[
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030427.6669
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030452.0238
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030464.2602
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030476.2328
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030495.2955
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030503.418
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030511.3926
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034259.9984
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034275.6871
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034284.7354
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034293.468
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034303.2042
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034314.1941
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034326.5092
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034331.6963
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034336.5346
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034351.9521
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034366.1775
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034380.5877
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034384.3087
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034389.1583
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034405.9359
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034412.3533
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034419.0118
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034425.6972
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034432.5891
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092762.0
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092774.618
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092797.2067
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092804.6699
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092810.2635
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735093113.5724
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735093133.2436
+  },
+  {
+    "model_a": "GPT-4o + Stable Diffusion 3 Medium",
+    "model_b": "GPT-4o + OmniGen",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735187628.4881
+  },
+  {
+    "model_a": "GPT-4o + Stable Diffusion 3 Medium",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735187649.4872
+  },
+  {
+    "model_a": "GPT-4o + Emu2",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735197562.2637
+  },
+  {
+    "model_a": "GPT-4o + FLUX.1 [dev]",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735197586.8438
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "GPT-4o + FLUX.1 [dev]",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735201758.7145
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202083.631
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202099.4377
+  },
+  {
+    "model_a": "GPT-4o + OmniGen",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735202132.8592
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202545.8694
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202565.5723
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202573.0118
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735203523.809
+  },
+  {
+    "model_a": "GPT-4o + OmniGen",
+    "model_b": "GPT-4o + DALLE-3",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735205600.7414
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "GPT-4o + DALLE-3",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735207454.8251
+  },
+  {
+    "model_a": "GPT-4o + OmniGen",
+    "model_b": "GPT-4o + Stable Diffusion 3 Medium",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735207466.0131
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + Emu2",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215923.1589
+  },
+  {
+    "model_a": "GPT-4o + PixArt-Sigma",
+    "model_b": "GPT-4o + DALLE-3",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215935.7597
+  },
+  {
+    "model_a": "GPT-4o + OmniGen",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215942.7093
+  },
+  {
+    "model_a": "GPT-4o + PixArt-Sigma",
+    "model_b": "GPT-4o + OmniGen",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215949.7965
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215962.6898
+  },
+  {
+    "model_a": "GPT-4o + Stable Diffusion 3 Medium",
+    "model_b": "GPT-4o + DALLE-3",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215968.9052
+  },
+  {
+    "model_a": "GPT-4o + FLUX.1 [dev]",
+    "model_b": "GPT-4o + Stable Diffusion 3 Medium",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215976.5079
+  },
+  {
+    "model_a": "GPT-4o + Emu2",
+    "model_b": "GPT-4o + Stable Diffusion 3 Medium",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215982.9709
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215993.2305
+  },
+  {
+    "model_a": "GPT-4o + Stable Diffusion 3 Medium",
+    "model_b": "GPT-4o + FLUX.1 [dev]",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215999.8713
+  },
+  {
+    "model_a": "GPT-4o + PixArt-Sigma",
+    "model_b": "GPT-4o + FLUX.1 [dev]",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735216012.8216
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735216021.653
+  }
+]

arena_elo/results/20241226/elo_results.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f83c57c40a52e7cc67b47144cc16b5b21417baad81c270e84728cc223da347
+size 59910

arena_elo/results/20241226/leaderboard.csv

@@ -0,0 +1,9 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),license,creator,link
+GPT-4o + FLUX.1 [dev],GPT-4o + FLUX.1 [dev],1175.6821194184254,1081.7818725315192,FLUX.1 [dev] Non-Commercial License,Black Forest Labs,https://huggingface.co/black-forest-labs/FLUX.1-dev
+ChatDiT,ChatDiT,1162.3247570727874,1220.8633537889925,MIT License,Tongyi Lab,https://github.com/ali-vilab/ChatDiT
+GPT-4o + Emu2,GPT-4o + Emu2,1134.6891079598486,1152.7284045727401,Apache License 2.0,BAAI,https://huggingface.co/BAAI/Emu2
+FLUX-1,FLUX-1,1107.9942010514312,1165.8807578999263,N/A,N/A,N/A
+GPT-4o + Stable Diffusion 3 Medium,GPT-4o + Stable Diffusion 3 Medium,1066.9600535574334,1033.9863968056318,Stability AI Community License,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-3-medium
+GPT-4o + PixArt-Sigma,GPT-4o + PixArt-Sigma,933.0714315244284,886.7730412848159,CreativeML Open RAIL++-M License,Huawei Noah's Ark Lab,https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS
+GPT-4o + DALLE-3,GPT-4o + DALLE-3,804.061447868383,852.071446157934,OpenAI Terms of Use,OpenAI,https://openai.com/index/dall-e-3/
+GPT-4o + OmniGen,GPT-4o + OmniGen,615.2168815472633,605.9147269584403,MIT License,BAAI,https://huggingface.co/spaces/Shitao/OmniGen

arena_elo/results/latest/clean_battle.json

@@ -0,0 +1,482 @@
+[
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030427.6669
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030452.0238
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030464.2602
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030476.2328
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030495.2955
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030503.418
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735030511.3926
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034259.9984
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034275.6871
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034284.7354
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034293.468
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034303.2042
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034314.1941
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034326.5092
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034331.6963
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034336.5346
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034351.9521
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034366.1775
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034380.5877
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034384.3087
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034389.1583
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034405.9359
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034412.3533
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034419.0118
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034425.6972
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735034432.5891
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092762.0
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092774.618
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092797.2067
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092804.6699
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735092810.2635
+  },
+  {
+    "model_a": "FLUX-1",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735093113.5724
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "FLUX-1",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735093133.2436
+  },
+  {
+    "model_a": "GPT-4o + Stable Diffusion 3 Medium",
+    "model_b": "GPT-4o + OmniGen",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735187628.4881
+  },
+  {
+    "model_a": "GPT-4o + Stable Diffusion 3 Medium",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735187649.4872
+  },
+  {
+    "model_a": "GPT-4o + Emu2",
+    "model_b": "ChatDiT",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735197562.2637
+  },
+  {
+    "model_a": "GPT-4o + FLUX.1 [dev]",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735197586.8438
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "GPT-4o + FLUX.1 [dev]",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735201758.7145
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202083.631
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202099.4377
+  },
+  {
+    "model_a": "GPT-4o + OmniGen",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735202132.8592
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202545.8694
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202565.5723
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735202573.0118
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": false,
+    "tstamp": 1735203523.809
+  },
+  {
+    "model_a": "GPT-4o + OmniGen",
+    "model_b": "GPT-4o + DALLE-3",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735205600.7414
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "GPT-4o + DALLE-3",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735207454.8251
+  },
+  {
+    "model_a": "GPT-4o + OmniGen",
+    "model_b": "GPT-4o + Stable Diffusion 3 Medium",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735207466.0131
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "GPT-4o + Emu2",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215923.1589
+  },
+  {
+    "model_a": "GPT-4o + PixArt-Sigma",
+    "model_b": "GPT-4o + DALLE-3",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215935.7597
+  },
+  {
+    "model_a": "GPT-4o + OmniGen",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215942.7093
+  },
+  {
+    "model_a": "GPT-4o + PixArt-Sigma",
+    "model_b": "GPT-4o + OmniGen",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215949.7965
+  },
+  {
+    "model_a": "GPT-4o + DALLE-3",
+    "model_b": "ChatDiT",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215962.6898
+  },
+  {
+    "model_a": "GPT-4o + Stable Diffusion 3 Medium",
+    "model_b": "GPT-4o + DALLE-3",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215968.9052
+  },
+  {
+    "model_a": "GPT-4o + FLUX.1 [dev]",
+    "model_b": "GPT-4o + Stable Diffusion 3 Medium",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215976.5079
+  },
+  {
+    "model_a": "GPT-4o + Emu2",
+    "model_b": "GPT-4o + Stable Diffusion 3 Medium",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215982.9709
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215993.2305
+  },
+  {
+    "model_a": "GPT-4o + Stable Diffusion 3 Medium",
+    "model_b": "GPT-4o + FLUX.1 [dev]",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735215999.8713
+  },
+  {
+    "model_a": "GPT-4o + PixArt-Sigma",
+    "model_b": "GPT-4o + FLUX.1 [dev]",
+    "winner": "model_b",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735216012.8216
+  },
+  {
+    "model_a": "ChatDiT",
+    "model_b": "GPT-4o + PixArt-Sigma",
+    "winner": "model_a",
+    "judge": "arena_user_127.0.0.1",
+    "anony": true,
+    "tstamp": 1735216021.653
+  }
+]

arena_elo/results/latest/elo_results.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f83c57c40a52e7cc67b47144cc16b5b21417baad81c270e84728cc223da347
+size 59910

arena_elo/results/latest/leaderboard.csv

@@ -0,0 +1,9 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),license,creator,link
+GPT-4o + FLUX.1 [dev],GPT-4o + FLUX.1 [dev],1175.6821194184254,1081.7818725315192,FLUX.1 [dev] Non-Commercial License,Black Forest Labs,https://huggingface.co/black-forest-labs/FLUX.1-dev
+ChatDiT,ChatDiT,1162.3247570727874,1220.8633537889925,MIT License,Tongyi Lab,https://github.com/ali-vilab/ChatDiT
+GPT-4o + Emu2,GPT-4o + Emu2,1134.6891079598486,1152.7284045727401,Apache License 2.0,BAAI,https://huggingface.co/BAAI/Emu2
+FLUX-1,FLUX-1,1107.9942010514312,1165.8807578999263,N/A,N/A,N/A
+GPT-4o + Stable Diffusion 3 Medium,GPT-4o + Stable Diffusion 3 Medium,1066.9600535574334,1033.9863968056318,Stability AI Community License,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-3-medium
+GPT-4o + PixArt-Sigma,GPT-4o + PixArt-Sigma,933.0714315244284,886.7730412848159,CreativeML Open RAIL++-M License,Huawei Noah's Ark Lab,https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS
+GPT-4o + DALLE-3,GPT-4o + DALLE-3,804.061447868383,852.071446157934,OpenAI Terms of Use,OpenAI,https://openai.com/index/dall-e-3/
+GPT-4o + OmniGen,GPT-4o + OmniGen,615.2168815472633,605.9147269584403,MIT License,BAAI,https://huggingface.co/spaces/Shitao/OmniGen

arena_elo/simple_test.py

@@ -0,0 +1,16 @@
+import pickle
+with open("./results/latest/elo_results.pkl",'rb') as f:
+    data = pickle.load(f)
+    print()
+    df = data["anony"]["leaderboard_table_df"]
+    # sort by rating 
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df)
+
+    print()
+
+    df = data["full"]["leaderboard_table_df"]
+    # sort by rating 
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df)
+    print('done') 

arena_elo/update_elo_rating.sh

@@ -0,0 +1,30 @@
+# set LOGDIR to default if not set before
+if [ -z "$LOGDIR" ]; then
+    echo "LOGDIR is not set. Using default '../logs/vote_log'"
+    export LOGDIR="../logs/vote_log"
+fi
+
+mkdir -p results
+
+# for battle data
+python -m elo_rating.clean_battle_data
+battle_cutoff_date=`cat cut_off_date.txt` && rm cut_off_date.txt && echo "Battle data last updated on $battle_cutoff_date"
+
+mkdir -p ./results/$battle_cutoff_date
+
+cp clean_battle_$battle_cutoff_date.json ./results/latest/clean_battle.json
+mv clean_battle_$battle_cutoff_date.json ./results/$battle_cutoff_date/clean_battle.json
+
+python -m elo_rating.elo_analysis --clean-battle-file ./results/$battle_cutoff_date/clean_battle.json
+mv ./elo_results_$battle_cutoff_date.pkl ./results/$battle_cutoff_date/elo_results.pkl
+
+# generate the leaderboard
+
+python -m elo_rating.generate_leaderboard \
+    --model_info_file "../model/model_registry.csv" \
+    --elo_rating_pkl "./results/$battle_cutoff_date/elo_results.pkl" \
+    --output_csv "./results/$battle_cutoff_date/leaderboard.csv"
+
+mkdir -p ./results/latest
+cp ./results/$battle_cutoff_date/leaderboard.csv ./results/latest/leaderboard.csv
+cp ./results/$battle_cutoff_date/elo_results.pkl ./results/latest/elo_results.pkl

logs/vote_log/2024-12-24-conv.json

@@ -0,0 +1,26 @@
+{"tstamp": 1735030427.6669, "type": "rightvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735030452.0238, "type": "leftvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735030464.2602, "type": "leftvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735030476.2328, "type": "leftvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735030495.2955, "type": "bothbad_vote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735030503.418, "type": "bothbad_vote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735030511.3926, "type": "leftvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034259.9984, "type": "bothbad_vote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034275.6871, "type": "leftvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034284.7354, "type": "leftvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034293.468, "type": "leftvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034303.2042, "type": "rightvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034314.1941, "type": "leftvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034326.5092, "type": "leftvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034331.6963, "type": "rightvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034336.5346, "type": "bothbad_vote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034351.9521, "type": "rightvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034366.1775, "type": "rightvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034380.5877, "type": "leftvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034384.3087, "type": "rightvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034389.1583, "type": "leftvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034405.9359, "type": "rightvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034412.3533, "type": "rightvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034419.0118, "type": "leftvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034425.6972, "type": "rightvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735034432.5891, "type": "rightvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}

logs/vote_log/2024-12-25-conv.json

@@ -0,0 +1,7 @@
+{"tstamp": 1735092762.0, "type": "leftvote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735092774.618, "type": "bothbad_vote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735092797.2067, "type": "leftvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735092804.6699, "type": "rightvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735092810.2635, "type": "leftvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735093113.5724, "type": "rightvote", "models": ["FLUX-1", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735093133.2436, "type": "bothbad_vote", "models": ["ChatDiT", "FLUX-1"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}

logs/vote_log/2024-12-26-conv.json

@@ -0,0 +1,27 @@
+{"tstamp": 1735187628.4881, "type": "leftvote", "models": ["GPT-4o + Stable Diffusion 3 Medium", "GPT-4o + OmniGen"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735187649.4872, "type": "rightvote", "models": ["GPT-4o + Stable Diffusion 3 Medium", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735197562.2637, "type": "leftvote", "models": ["GPT-4o + Emu2", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735197586.8438, "type": "leftvote", "models": ["GPT-4o + FLUX.1 [dev]", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735201758.7145, "type": "leftvote", "models": ["ChatDiT", "GPT-4o + FLUX.1 [dev]"], "states": [{}, {}], "anony": false, "ip": "127.0.0.1"}
+{"tstamp": 1735202083.631, "type": "rightvote", "models": ["GPT-4o + DALLE-3", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": false, "ip": "127.0.0.1"}
+{"tstamp": 1735202099.4377, "type": "leftvote", "models": ["GPT-4o + DALLE-3", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": false, "ip": "127.0.0.1"}
+{"tstamp": 1735202132.8592, "type": "rightvote", "models": ["GPT-4o + OmniGen", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735202545.8694, "type": "rightvote", "models": ["GPT-4o + DALLE-3", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": false, "ip": "127.0.0.1"}
+{"tstamp": 1735202565.5723, "type": "leftvote", "models": ["GPT-4o + DALLE-3", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": false, "ip": "127.0.0.1"}
+{"tstamp": 1735202573.0118, "type": "bothbad_vote", "models": ["GPT-4o + DALLE-3", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": false, "ip": "127.0.0.1"}
+{"tstamp": 1735203523.809, "type": "bothbad_vote", "models": ["GPT-4o + DALLE-3", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": false, "ip": "127.0.0.1"}
+{"tstamp": 1735205600.7414, "type": "rightvote", "models": ["GPT-4o + OmniGen", "GPT-4o + DALLE-3"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735207454.8251, "type": "leftvote", "models": ["ChatDiT", "GPT-4o + DALLE-3"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735207466.0131, "type": "rightvote", "models": ["GPT-4o + OmniGen", "GPT-4o + Stable Diffusion 3 Medium"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215923.1589, "type": "rightvote", "models": ["GPT-4o + DALLE-3", "GPT-4o + Emu2"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215935.7597, "type": "leftvote", "models": ["GPT-4o + PixArt-Sigma", "GPT-4o + DALLE-3"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215942.7093, "type": "bothbad_vote", "models": ["GPT-4o + OmniGen", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215949.7965, "type": "leftvote", "models": ["GPT-4o + PixArt-Sigma", "GPT-4o + OmniGen"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215962.6898, "type": "rightvote", "models": ["GPT-4o + DALLE-3", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215968.9052, "type": "bothbad_vote", "models": ["GPT-4o + Stable Diffusion 3 Medium", "GPT-4o + DALLE-3"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215976.5079, "type": "bothbad_vote", "models": ["GPT-4o + FLUX.1 [dev]", "GPT-4o + Stable Diffusion 3 Medium"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215982.9709, "type": "rightvote", "models": ["GPT-4o + Emu2", "GPT-4o + Stable Diffusion 3 Medium"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215993.2305, "type": "leftvote", "models": ["ChatDiT", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735215999.8713, "type": "bothbad_vote", "models": ["GPT-4o + Stable Diffusion 3 Medium", "GPT-4o + FLUX.1 [dev]"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735216012.8216, "type": "rightvote", "models": ["GPT-4o + PixArt-Sigma", "GPT-4o + FLUX.1 [dev]"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735216021.653, "type": "leftvote", "models": ["ChatDiT", "GPT-4o + PixArt-Sigma"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}

logs/vote_log/2024-12-27-conv.json

@@ -0,0 +1,4 @@
+{"tstamp": 1735286354.5764, "type": "rightvote", "models": ["GPT-4o + PixArt-Sigma", "GPT-4o + OmniGen"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735286365.2329, "type": "bothbad_vote", "models": ["GPT-4o + Stable Diffusion 3 Medium", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735286374.6751, "type": "leftvote", "models": ["GPT-4o + Emu2", "ChatDiT"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}
+{"tstamp": 1735286382.1211, "type": "leftvote", "models": ["GPT-4o + FLUX.1 [dev]", "GPT-4o + Emu2"], "states": [{}, {}], "anony": true, "ip": "127.0.0.1"}

logs/vote_log/gr_web_image_generation_multi.log

@@ -0,0 +1,450 @@
+2024-12-26 11:49:30 | INFO | stdout | An error occurred while loading the CSV file: 'utf-8' codec can't decode byte 0xa1 in position 1240: invalid start byte
+2024-12-26 11:49:31 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 11:49:31 | INFO | stdout | 
+2024-12-26 11:49:31 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 11:49:38 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 11:49:38 | INFO | stdout | Rerunning server... use `close()` to stop if you need to change `launch()` parameters.
+2024-12-26 11:49:38 | INFO | stdout | ----
+2024-12-26 11:49:38 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 11:49:38 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 65, in <module>
+2024-12-26 11:49:38 | ERROR | stderr |     demo.launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 11:49:38 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2619, in launch
+2024-12-26 11:49:38 | ERROR | stderr |     raise ValueError(
+2024-12-26 11:49:38 | ERROR | stderr | ValueError: When localhost is not accessible, a shareable link must be created. Please set share=True or check your proxy settings to allow access to localhost.
+2024-12-26 11:49:42 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-26 11:49:42 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 11:49:42 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-26 11:49:42 | ERROR | stderr |     lock.acquire()
+2024-12-26 11:49:42 | ERROR | stderr | KeyboardInterrupt:
+2024-12-26 12:07:12 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 12:07:12 | INFO | stdout | 
+2024-12-26 12:07:12 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 12:33:48 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 12:34:09 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 12:43:31 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 12:43:31 | INFO | stdout | Rerunning server... use `close()` to stop if you need to change `launch()` parameters.
+2024-12-26 12:43:31 | INFO | stdout | ----
+2024-12-26 12:43:31 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 12:43:31 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 65, in <module>
+2024-12-26 12:43:31 | ERROR | stderr |     demo.launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 12:43:31 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2619, in launch
+2024-12-26 12:43:31 | ERROR | stderr |     raise ValueError(
+2024-12-26 12:43:31 | ERROR | stderr | ValueError: When localhost is not accessible, a shareable link must be created. Please set share=True or check your proxy settings to allow access to localhost.
+2024-12-26 14:01:35 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 14:01:36 | INFO | stdout | 
+2024-12-26 14:01:36 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 14:05:04 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 14:05:04 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 14:05:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 14:05:04 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 14:05:04 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 14:05:04 | ERROR | stderr | 
+2024-12-26 14:05:04 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 14:05:04 | ERROR | stderr | 
+2024-12-26 14:05:04 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 14:05:04 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 14:05:04 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 14:05:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 14:05:04 | ERROR | stderr |     self.block_thread()
+2024-12-26 14:05:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 14:05:04 | ERROR | stderr |     self.server.close()
+2024-12-26 14:05:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 14:05:04 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 14:05:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 14:05:04 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 14:05:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 14:05:04 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 14:05:04 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 14:17:14 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 14:17:14 | INFO | stdout | 
+2024-12-26 14:17:14 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 14:21:13 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 14:21:13 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 14:21:13 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 14:21:13 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 14:21:13 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 14:21:13 | ERROR | stderr | 
+2024-12-26 14:21:13 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 14:21:13 | ERROR | stderr | 
+2024-12-26 14:21:13 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 14:21:13 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 14:21:13 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 14:21:13 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 14:21:13 | ERROR | stderr |     self.block_thread()
+2024-12-26 14:21:13 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 14:21:13 | ERROR | stderr |     self.server.close()
+2024-12-26 14:21:13 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 14:21:13 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 14:21:13 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 14:21:13 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 14:21:13 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 14:21:13 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 14:21:13 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 15:14:17 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 15:14:17 | INFO | stdout | 
+2024-12-26 15:14:17 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 15:19:22 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 15:19:46 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 15:20:35 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 15:20:35 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/queueing.py", line 625, in process_events
+2024-12-26 15:20:35 | ERROR | stderr |     response = await route_utils.call_process_api(
+2024-12-26 15:20:35 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/route_utils.py", line 322, in call_process_api
+2024-12-26 15:20:35 | ERROR | stderr |     output = await app.get_blocks().process_api(
+2024-12-26 15:20:35 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2047, in process_api
+2024-12-26 15:20:35 | ERROR | stderr |     result = await self.call_function(
+2024-12-26 15:20:35 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 1594, in call_function
+2024-12-26 15:20:35 | ERROR | stderr |     prediction = await anyio.to_thread.run_sync(  # type: ignore
+2024-12-26 15:20:35 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
+2024-12-26 15:20:35 | ERROR | stderr |     return await get_async_backend().run_sync_in_worker_thread(
+2024-12-26 15:20:35 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2505, in run_sync_in_worker_thread
+2024-12-26 15:20:35 | ERROR | stderr |     return await future
+2024-12-26 15:20:35 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 1005, in run
+2024-12-26 15:20:35 | ERROR | stderr |     result = context.run(func, *args)
+2024-12-26 15:20:35 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/utils.py", line 869, in wrapper
+2024-12-26 15:20:35 | ERROR | stderr |     response = f(*args, **kwargs)
+2024-12-26 15:20:35 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/utils.py", line 164, in refresh_side_by_side
+2024-12-26 15:20:35 | ERROR | stderr |     state0, state1, prompt, input_images, output_images_A, output_images_B = models.get_result_of_random_case(model_name_A, model_name_B)
+2024-12-26 15:20:35 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/model/model_manager.py", line 215, in get_result_of_random_case
+2024-12-26 15:20:35 | ERROR | stderr |     output_images_A = model_A.get_result(case_name)
+2024-12-26 15:20:35 | ERROR | stderr | AttributeError: 'NoneType' object has no attribute 'get_result'
+2024-12-26 16:22:06 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 16:22:07 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 16:22:07 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 16:22:07 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 16:22:07 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 16:22:07 | ERROR | stderr | 
+2024-12-26 16:22:07 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 16:22:07 | ERROR | stderr | 
+2024-12-26 16:22:07 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 16:22:07 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 16:22:07 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 16:22:07 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 16:22:07 | ERROR | stderr |     self.block_thread()
+2024-12-26 16:22:07 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 16:22:07 | ERROR | stderr |     self.server.close()
+2024-12-26 16:22:07 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 16:22:07 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 16:22:07 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 16:22:07 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 16:22:07 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 16:22:07 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 16:22:07 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 16:28:41 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 16:28:41 | INFO | stdout | 
+2024-12-26 16:28:41 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 16:29:18 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 16:33:49 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 16:33:49 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 16:33:49 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 16:33:49 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 16:33:49 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 16:33:49 | ERROR | stderr | 
+2024-12-26 16:33:49 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 16:33:49 | ERROR | stderr | 
+2024-12-26 16:33:49 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 16:33:49 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 16:33:49 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 16:33:49 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 16:33:49 | ERROR | stderr |     self.block_thread()
+2024-12-26 16:33:49 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 16:33:49 | ERROR | stderr |     self.server.close()
+2024-12-26 16:33:49 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 16:33:49 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 16:33:49 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 16:33:49 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 16:33:49 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 16:33:49 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 16:33:49 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 16:34:06 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 16:34:06 | INFO | stdout | 
+2024-12-26 16:34:06 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 16:34:43 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 16:34:59 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 16:35:32 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 16:42:25 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 16:42:45 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 16:42:53 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-26 16:58:43 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-26 17:33:20 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 18:04:14 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 18:04:26 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 19:58:21 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 19:58:21 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 19:58:21 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 19:58:21 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 19:58:21 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 19:58:21 | ERROR | stderr | 
+2024-12-26 19:58:21 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 19:58:21 | ERROR | stderr | 
+2024-12-26 19:58:21 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 19:58:21 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 19:58:21 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 19:58:21 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 19:58:21 | ERROR | stderr |     self.block_thread()
+2024-12-26 19:58:21 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 19:58:21 | ERROR | stderr |     self.server.close()
+2024-12-26 19:58:21 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 19:58:21 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 19:58:21 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 19:58:21 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 19:58:21 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 19:58:21 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 19:58:21 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:07:22 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 20:07:22 | INFO | stdout | 
+2024-12-26 20:07:22 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 20:09:29 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 20:09:29 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:09:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 20:09:29 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 20:09:29 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:09:29 | ERROR | stderr | 
+2024-12-26 20:09:29 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 20:09:29 | ERROR | stderr | 
+2024-12-26 20:09:29 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:09:29 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 20:09:29 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 20:09:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 20:09:29 | ERROR | stderr |     self.block_thread()
+2024-12-26 20:09:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 20:09:29 | ERROR | stderr |     self.server.close()
+2024-12-26 20:09:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 20:09:29 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 20:09:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 20:09:29 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 20:09:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 20:09:29 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 20:09:29 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:09:29 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-26 20:09:29 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:09:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-26 20:09:29 | ERROR | stderr |     lock.acquire()
+2024-12-26 20:09:29 | ERROR | stderr | KeyboardInterrupt:
+2024-12-26 20:09:34 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 20:09:34 | INFO | stdout | 
+2024-12-26 20:09:34 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 20:10:51 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 20:10:51 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:10:51 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 20:10:51 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 20:10:51 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:10:51 | ERROR | stderr | 
+2024-12-26 20:10:51 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 20:10:51 | ERROR | stderr | 
+2024-12-26 20:10:51 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:10:51 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 20:10:51 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 20:10:51 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 20:10:51 | ERROR | stderr |     self.block_thread()
+2024-12-26 20:10:51 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 20:10:51 | ERROR | stderr |     self.server.close()
+2024-12-26 20:10:51 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 20:10:51 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 20:10:51 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 20:10:51 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 20:10:51 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 20:10:51 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 20:10:51 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:10:51 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-26 20:10:51 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:10:51 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-26 20:10:51 | ERROR | stderr |     lock.acquire()
+2024-12-26 20:10:51 | ERROR | stderr | KeyboardInterrupt:
+2024-12-26 20:11:17 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 20:11:17 | INFO | stdout | 
+2024-12-26 20:11:17 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 20:11:33 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 20:11:33 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:11:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 20:11:33 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 20:11:33 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:11:33 | ERROR | stderr | 
+2024-12-26 20:11:33 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 20:11:33 | ERROR | stderr | 
+2024-12-26 20:11:33 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:11:33 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 20:11:33 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 20:11:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 20:11:33 | ERROR | stderr |     self.block_thread()
+2024-12-26 20:11:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 20:11:33 | ERROR | stderr |     self.server.close()
+2024-12-26 20:11:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 20:11:33 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 20:11:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 20:11:33 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 20:11:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 20:11:33 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 20:11:33 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:11:33 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-26 20:11:33 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:11:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-26 20:11:33 | ERROR | stderr |     lock.acquire()
+2024-12-26 20:11:33 | ERROR | stderr | KeyboardInterrupt:
+2024-12-26 20:13:30 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 20:13:30 | INFO | stdout | 
+2024-12-26 20:13:30 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 20:18:33 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 20:18:33 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:18:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 20:18:33 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 20:18:33 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:18:33 | ERROR | stderr | 
+2024-12-26 20:18:33 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 20:18:33 | ERROR | stderr | 
+2024-12-26 20:18:33 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:18:33 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 20:18:33 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 20:18:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 20:18:33 | ERROR | stderr |     self.block_thread()
+2024-12-26 20:18:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 20:18:33 | ERROR | stderr |     self.server.close()
+2024-12-26 20:18:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 20:18:33 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 20:18:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 20:18:33 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 20:18:33 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 20:18:33 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 20:18:33 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:18:40 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 20:18:40 | INFO | stdout | 
+2024-12-26 20:18:40 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 20:23:29 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 20:23:29 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:23:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 20:23:29 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 20:23:29 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:23:29 | ERROR | stderr | 
+2024-12-26 20:23:29 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 20:23:29 | ERROR | stderr | 
+2024-12-26 20:23:29 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:23:29 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 20:23:29 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 20:23:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 20:23:29 | ERROR | stderr |     self.block_thread()
+2024-12-26 20:23:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 20:23:29 | ERROR | stderr |     self.server.close()
+2024-12-26 20:23:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 20:23:29 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 20:23:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 20:23:29 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 20:23:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 20:23:29 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 20:23:29 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:23:33 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 20:23:34 | INFO | stdout | 
+2024-12-26 20:23:34 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 20:24:01 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 20:24:01 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:24:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 20:24:01 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 20:24:01 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:24:01 | ERROR | stderr | 
+2024-12-26 20:24:01 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 20:24:01 | ERROR | stderr | 
+2024-12-26 20:24:01 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:24:01 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 20:24:01 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 20:24:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 20:24:01 | ERROR | stderr |     self.block_thread()
+2024-12-26 20:24:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 20:24:01 | ERROR | stderr |     self.server.close()
+2024-12-26 20:24:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 20:24:01 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 20:24:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 20:24:01 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 20:24:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 20:24:01 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 20:24:01 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:24:01 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-26 20:24:01 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:24:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-26 20:24:01 | ERROR | stderr |     lock.acquire()
+2024-12-26 20:24:01 | ERROR | stderr | KeyboardInterrupt:
+2024-12-26 20:25:01 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 20:25:01 | INFO | stdout | 
+2024-12-26 20:25:01 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 20:25:23 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 20:25:35 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 20:25:42 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-26 20:25:49 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 20:26:02 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 20:26:08 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-26 20:26:16 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-26 20:26:22 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 20:26:33 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 20:26:39 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-26 20:26:52 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-26 20:27:01 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-26 20:27:19 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 20:27:19 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:27:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 20:27:19 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 20:27:19 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:27:19 | ERROR | stderr | 
+2024-12-26 20:27:19 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 20:27:19 | ERROR | stderr | 
+2024-12-26 20:27:19 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:27:19 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 20:27:19 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 20:27:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 20:27:19 | ERROR | stderr |     self.block_thread()
+2024-12-26 20:27:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 20:27:19 | ERROR | stderr |     self.server.close()
+2024-12-26 20:27:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 20:27:19 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 20:27:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 20:27:19 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 20:27:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 20:27:19 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 20:27:19 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:27:20 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-26 20:27:20 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:27:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-26 20:27:20 | ERROR | stderr |     lock.acquire()
+2024-12-26 20:27:20 | ERROR | stderr | KeyboardInterrupt:
+2024-12-26 20:27:58 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-26 20:27:58 | INFO | stdout | 
+2024-12-26 20:27:58 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-26 20:28:20 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-26 20:28:20 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:28:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-26 20:28:20 | ERROR | stderr |     time.sleep(0.1)
+2024-12-26 20:28:20 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:28:20 | ERROR | stderr | 
+2024-12-26 20:28:20 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-26 20:28:20 | ERROR | stderr | 
+2024-12-26 20:28:20 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:28:20 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-26 20:28:20 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-26 20:28:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-26 20:28:20 | ERROR | stderr |     self.block_thread()
+2024-12-26 20:28:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-26 20:28:20 | ERROR | stderr |     self.server.close()
+2024-12-26 20:28:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-26 20:28:20 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-26 20:28:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-26 20:28:20 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-26 20:28:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-26 20:28:20 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-26 20:28:20 | ERROR | stderr | KeyboardInterrupt
+2024-12-26 20:28:20 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-26 20:28:20 | ERROR | stderr | Traceback (most recent call last):
+2024-12-26 20:28:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-26 20:28:20 | ERROR | stderr |     lock.acquire()
+2024-12-26 20:28:20 | ERROR | stderr | KeyboardInterrupt:
+2024-12-27 15:58:24 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-27 15:58:24 | INFO | stdout | 
+2024-12-27 15:58:24 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-27 15:59:14 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-27 15:59:25 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-27 15:59:34 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-27 15:59:42 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-27 16:00:57 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-27 16:00:57 | INFO | stdout | Rerunning server... use `close()` to stop if you need to change `launch()` parameters.
+2024-12-27 16:00:57 | INFO | stdout | ----
+2024-12-27 16:00:57 | ERROR | stderr | Traceback (most recent call last):
+2024-12-27 16:00:57 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/huggingface/IDEA-Bench-Arena/app.py", line 65, in <module>
+2024-12-27 16:00:57 | ERROR | stderr |     demo.launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-27 16:00:57 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2619, in launch
+2024-12-27 16:00:57 | ERROR | stderr |     raise ValueError(
+2024-12-27 16:00:57 | ERROR | stderr | ValueError: When localhost is not accessible, a shareable link must be created. Please set share=True or check your proxy settings to allow access to localhost.

logs/vote_log/gr_web_image_generation_multi.log.2024-12-25

@@ -0,0 +1,797 @@
+2024-12-24 14:01:42 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:01:42 | INFO | stdout | 
+2024-12-24 14:01:42 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:02:18 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 14:02:18 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/queueing.py", line 625, in process_events
+2024-12-24 14:02:18 | ERROR | stderr |     response = await route_utils.call_process_api(
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/route_utils.py", line 322, in call_process_api
+2024-12-24 14:02:18 | ERROR | stderr |     output = await app.get_blocks().process_api(
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2047, in process_api
+2024-12-24 14:02:18 | ERROR | stderr |     result = await self.call_function(
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 1594, in call_function
+2024-12-24 14:02:18 | ERROR | stderr |     prediction = await anyio.to_thread.run_sync(  # type: ignore
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
+2024-12-24 14:02:18 | ERROR | stderr |     return await get_async_backend().run_sync_in_worker_thread(
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2505, in run_sync_in_worker_thread
+2024-12-24 14:02:18 | ERROR | stderr |     return await future
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 1005, in run
+2024-12-24 14:02:18 | ERROR | stderr |     result = context.run(func, *args)
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/utils.py", line 869, in wrapper
+2024-12-24 14:02:18 | ERROR | stderr |     response = f(*args, **kwargs)
+2024-12-24 14:02:18 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/vote_utils.py", line 53, in rightvote_last_response_igm
+2024-12-24 14:02:18 | ERROR | stderr |     vote_last_response_igm(
+2024-12-24 14:02:18 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/vote_utils.py", line 32, in vote_last_response_igm
+2024-12-24 14:02:18 | ERROR | stderr |     append_json_item_on_log_server(data, get_conv_log_filename())
+2024-12-24 14:02:18 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/utils.py", line 171, in append_json_item_on_log_server
+2024-12-24 14:02:18 | ERROR | stderr |     response = requests.post(url, data={'json_str': json_item, 'file_name': log_file})
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/api.py", line 115, in post
+2024-12-24 14:02:18 | ERROR | stderr |     return request("post", url, data=data, json=json, **kwargs)
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/api.py", line 59, in request
+2024-12-24 14:02:18 | ERROR | stderr |     return session.request(method=method, url=url, **kwargs)
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/sessions.py", line 575, in request
+2024-12-24 14:02:18 | ERROR | stderr |     prep = self.prepare_request(req)
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/sessions.py", line 484, in prepare_request
+2024-12-24 14:02:18 | ERROR | stderr |     p.prepare(
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/models.py", line 367, in prepare
+2024-12-24 14:02:18 | ERROR | stderr |     self.prepare_url(url, params)
+2024-12-24 14:02:18 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/models.py", line 438, in prepare_url
+2024-12-24 14:02:18 | ERROR | stderr |     raise MissingSchema(
+2024-12-24 14:02:18 | ERROR | stderr | requests.exceptions.MissingSchema: Invalid URL '/logs/append_json': No scheme supplied. Perhaps you meant https:///logs/append_json?
+2024-12-24 14:06:36 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:06:36 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:06:36 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:06:36 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:06:36 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:06:36 | ERROR | stderr | 
+2024-12-24 14:06:36 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:06:36 | ERROR | stderr | 
+2024-12-24 14:06:36 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:06:36 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:06:36 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:06:36 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:06:36 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:06:36 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:06:36 | ERROR | stderr |     self.server.close()
+2024-12-24 14:06:36 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:06:36 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:06:36 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:06:36 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:06:36 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:06:36 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:06:36 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:06:41 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:06:41 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 77, in <module>
+2024-12-24 14:06:41 | ERROR | stderr |     demo = build_combine_demo(models, elo_results_file, leaderboard_table_file)
+2024-12-24 14:06:41 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 27, in build_combine_demo
+2024-12-24 14:06:41 | ERROR | stderr |     build_side_by_side_ui_named(models)
+2024-12-24 14:06:41 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/gradio_web.py", line 226, in build_side_by_side_ui_named
+2024-12-24 14:06:41 | ERROR | stderr |     leftvote_btn = gr.Button(
+2024-12-24 14:06:41 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/component_meta.py", line 179, in wrapper
+2024-12-24 14:06:41 | ERROR | stderr |     return fn(self, **kwargs)
+2024-12-24 14:06:41 | ERROR | stderr | TypeError: Button.__init__() got an unexpected keyword argument 'style'
+2024-12-24 14:13:32 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:13:32 | INFO | stdout | 
+2024-12-24 14:13:32 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:14:04 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:14:04 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:14:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:14:04 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:14:04 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:14:04 | ERROR | stderr | 
+2024-12-24 14:14:04 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:14:04 | ERROR | stderr | 
+2024-12-24 14:14:04 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:14:04 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:14:04 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:14:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:14:04 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:14:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:14:04 | ERROR | stderr |     self.server.close()
+2024-12-24 14:14:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:14:04 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:14:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:14:04 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:14:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:14:04 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:14:04 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:14:04 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:14:04 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:14:04 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:14:04 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:14:04 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:14:09 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:14:09 | INFO | stdout | 
+2024-12-24 14:14:09 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:16:39 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:16:39 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:16:39 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:16:39 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:16:39 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:16:39 | ERROR | stderr | 
+2024-12-24 14:16:39 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:16:39 | ERROR | stderr | 
+2024-12-24 14:16:39 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:16:39 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:16:39 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:16:39 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:16:39 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:16:39 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:16:39 | ERROR | stderr |     self.server.close()
+2024-12-24 14:16:39 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:16:39 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:16:39 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:16:39 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:16:39 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:16:39 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:16:39 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:16:44 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:16:44 | INFO | stdout | 
+2024-12-24 14:16:44 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:17:37 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:17:38 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:17:38 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:17:38 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:17:38 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:17:38 | ERROR | stderr | 
+2024-12-24 14:17:38 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:17:38 | ERROR | stderr | 
+2024-12-24 14:17:38 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:17:38 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:17:38 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:17:38 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:17:38 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:17:38 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:17:38 | ERROR | stderr |     self.server.close()
+2024-12-24 14:17:38 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:17:38 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:17:38 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:17:38 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:17:38 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:17:38 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:17:38 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:17:39 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:17:39 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:17:39 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:17:39 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:17:39 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:17:44 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:17:44 | INFO | stdout | 
+2024-12-24 14:17:44 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:17:54 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:17:54 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:17:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:17:54 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:17:54 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:17:54 | ERROR | stderr | 
+2024-12-24 14:17:54 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:17:54 | ERROR | stderr | 
+2024-12-24 14:17:54 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:17:54 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:17:54 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:17:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:17:54 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:17:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:17:54 | ERROR | stderr |     self.server.close()
+2024-12-24 14:17:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:17:54 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:17:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:17:54 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:17:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:17:54 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:17:54 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:17:54 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:17:54 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:17:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:17:54 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:17:54 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:18:02 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:18:02 | INFO | stdout | 
+2024-12-24 14:18:02 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:19:08 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:19:09 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:19:09 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:19:09 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:19:09 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:19:09 | ERROR | stderr | 
+2024-12-24 14:19:09 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:19:09 | ERROR | stderr | 
+2024-12-24 14:19:09 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:19:09 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:19:09 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:19:09 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:19:09 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:19:09 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:19:09 | ERROR | stderr |     self.server.close()
+2024-12-24 14:19:09 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:19:09 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:19:09 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:19:09 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:19:09 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:19:09 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:19:09 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:19:09 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:19:09 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:19:09 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:19:09 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:19:09 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:19:14 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:19:14 | INFO | stdout | 
+2024-12-24 14:19:14 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:19:25 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:19:25 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:19:25 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:19:25 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:19:25 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:19:25 | ERROR | stderr | 
+2024-12-24 14:19:25 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:19:25 | ERROR | stderr | 
+2024-12-24 14:19:25 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:19:25 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:19:25 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:19:25 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:19:25 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:19:25 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:19:25 | ERROR | stderr |     self.server.close()
+2024-12-24 14:19:25 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:19:25 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:19:25 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:19:25 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:19:25 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:19:25 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:19:25 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:19:26 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:19:26 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:19:26 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:19:26 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:19:26 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:19:30 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:19:30 | INFO | stdout | 
+2024-12-24 14:19:30 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:19:53 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:19:54 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:19:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:19:54 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:19:54 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:19:54 | ERROR | stderr | 
+2024-12-24 14:19:54 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:19:54 | ERROR | stderr | 
+2024-12-24 14:19:54 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:19:54 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:19:54 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:19:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:19:54 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:19:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:19:54 | ERROR | stderr |     self.server.close()
+2024-12-24 14:19:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:19:54 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:19:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:19:54 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:19:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:19:54 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:19:54 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:19:54 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:19:54 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:19:54 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:19:54 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:19:54 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:20:25 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:20:25 | INFO | stdout | 
+2024-12-24 14:20:25 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:25:19 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:25:19 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:25:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:25:19 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:25:19 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:25:19 | ERROR | stderr | 
+2024-12-24 14:25:19 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:25:19 | ERROR | stderr | 
+2024-12-24 14:25:19 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:25:19 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:25:19 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:25:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:25:19 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:25:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:25:19 | ERROR | stderr |     self.server.close()
+2024-12-24 14:25:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:25:19 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:25:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:25:19 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:25:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:25:19 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:25:19 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:28:13 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:28:14 | INFO | stdout | 
+2024-12-24 14:28:14 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:33:40 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:33:40 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:33:40 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:33:40 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:33:40 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:33:40 | ERROR | stderr | 
+2024-12-24 14:33:40 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:33:40 | ERROR | stderr | 
+2024-12-24 14:33:40 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:33:40 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:33:40 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:33:40 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:33:40 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:33:40 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:33:40 | ERROR | stderr |     self.server.close()
+2024-12-24 14:33:40 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:33:40 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:33:40 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:33:40 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:33:40 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:33:40 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:33:40 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:33:45 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:33:45 | INFO | stdout | 
+2024-12-24 14:33:45 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:34:55 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:34:56 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:34:56 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:34:56 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:34:56 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:34:56 | ERROR | stderr | 
+2024-12-24 14:34:56 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:34:56 | ERROR | stderr | 
+2024-12-24 14:34:56 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:34:56 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:34:56 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:34:56 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:34:56 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:34:56 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:34:56 | ERROR | stderr |     self.server.close()
+2024-12-24 14:34:56 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:34:56 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:34:56 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:34:56 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:34:56 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:34:56 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:34:56 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:34:56 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:34:56 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:34:56 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:34:56 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:34:56 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:35:01 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:35:01 | INFO | stdout | 
+2024-12-24 14:35:01 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:35:20 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:35:20 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:35:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:35:20 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:35:20 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:35:20 | ERROR | stderr | 
+2024-12-24 14:35:20 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:35:20 | ERROR | stderr | 
+2024-12-24 14:35:20 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:35:20 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:35:20 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:35:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:35:20 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:35:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:35:20 | ERROR | stderr |     self.server.close()
+2024-12-24 14:35:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:35:20 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:35:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:35:20 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:35:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:35:20 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:35:20 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:35:20 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:35:20 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:35:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:35:20 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:35:20 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:35:26 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:35:26 | INFO | stdout | 
+2024-12-24 14:35:26 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:35:34 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:35:34 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:35:34 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:35:34 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:35:34 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:35:34 | ERROR | stderr | 
+2024-12-24 14:35:34 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:35:34 | ERROR | stderr | 
+2024-12-24 14:35:34 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:35:34 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:35:34 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:35:34 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:35:34 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:35:34 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:35:34 | ERROR | stderr |     self.server.close()
+2024-12-24 14:35:34 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:35:34 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:35:34 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:35:34 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:35:34 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:35:34 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:35:34 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:35:34 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:35:34 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:35:34 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:35:34 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:35:34 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:35:39 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:35:39 | INFO | stdout | 
+2024-12-24 14:35:39 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:36:28 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:36:28 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:36:28 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:36:28 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:36:28 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:36:28 | ERROR | stderr | 
+2024-12-24 14:36:28 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:36:28 | ERROR | stderr | 
+2024-12-24 14:36:28 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:36:28 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:36:28 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:36:28 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:36:28 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:36:28 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:36:28 | ERROR | stderr |     self.server.close()
+2024-12-24 14:36:28 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:36:28 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:36:28 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:36:28 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:36:28 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:36:28 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:36:28 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:36:29 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:36:29 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:36:29 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:36:29 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:36:29 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:36:33 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:36:33 | INFO | stdout | 
+2024-12-24 14:36:33 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:36:41 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:36:41 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:36:41 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:36:41 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:36:41 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:36:41 | ERROR | stderr | 
+2024-12-24 14:36:41 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:36:41 | ERROR | stderr | 
+2024-12-24 14:36:41 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:36:41 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:36:41 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:36:41 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:36:41 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:36:41 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:36:41 | ERROR | stderr |     self.server.close()
+2024-12-24 14:36:41 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:36:41 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:36:41 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:36:41 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:36:41 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:36:41 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:36:41 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:36:41 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:36:41 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:36:41 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:36:41 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:36:41 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:37:47 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:37:47 | INFO | stdout | 
+2024-12-24 14:37:47 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:40:00 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 14:40:00 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:40:00 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 14:40:00 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 14:40:00 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:40:00 | ERROR | stderr | 
+2024-12-24 14:40:00 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 14:40:00 | ERROR | stderr | 
+2024-12-24 14:40:00 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:40:00 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 14:40:00 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 14:40:00 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 14:40:00 | ERROR | stderr |     self.block_thread()
+2024-12-24 14:40:00 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 14:40:00 | ERROR | stderr |     self.server.close()
+2024-12-24 14:40:00 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 14:40:00 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 14:40:00 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 14:40:00 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 14:40:00 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 14:40:00 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 14:40:00 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 14:40:00 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 14:40:00 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:40:00 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 14:40:00 | ERROR | stderr |     lock.acquire()
+2024-12-24 14:40:00 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 14:40:05 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 14:40:05 | INFO | stdout | 
+2024-12-24 14:40:05 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 14:40:16 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-24 14:40:16 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:40:16 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/queueing.py", line 625, in process_events
+2024-12-24 14:40:16 | ERROR | stderr |     response = await route_utils.call_process_api(
+2024-12-24 14:40:16 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/route_utils.py", line 322, in call_process_api
+2024-12-24 14:40:16 | ERROR | stderr |     output = await app.get_blocks().process_api(
+2024-12-24 14:40:16 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2047, in process_api
+2024-12-24 14:40:16 | ERROR | stderr |     result = await self.call_function(
+2024-12-24 14:40:16 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 1594, in call_function
+2024-12-24 14:40:16 | ERROR | stderr |     prediction = await anyio.to_thread.run_sync(  # type: ignore
+2024-12-24 14:40:16 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
+2024-12-24 14:40:16 | ERROR | stderr |     return await get_async_backend().run_sync_in_worker_thread(
+2024-12-24 14:40:16 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2505, in run_sync_in_worker_thread
+2024-12-24 14:40:16 | ERROR | stderr |     return await future
+2024-12-24 14:40:16 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 1005, in run
+2024-12-24 14:40:16 | ERROR | stderr |     result = context.run(func, *args)
+2024-12-24 14:40:16 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/utils.py", line 869, in wrapper
+2024-12-24 14:40:16 | ERROR | stderr |     response = f(*args, **kwargs)
+2024-12-24 14:40:16 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/vote_utils.py", line 65, in bothbad_vote_last_response_igm
+2024-12-24 14:40:16 | ERROR | stderr |     vote_last_response_igm(
+2024-12-24 14:40:16 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/vote_utils.py", line 27, in vote_last_response_igm
+2024-12-24 14:40:16 | ERROR | stderr |     "models": [x.name for x in states],
+2024-12-24 14:40:16 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/vote_utils.py", line 27, in <listcomp>
+2024-12-24 14:40:16 | ERROR | stderr |     "models": [x.name for x in states],
+2024-12-24 14:40:16 | ERROR | stderr | AttributeError: 'NoneType' object has no attribute 'name'
+2024-12-24 14:42:01 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 14:42:01 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/queueing.py", line 625, in process_events
+2024-12-24 14:42:01 | ERROR | stderr |     response = await route_utils.call_process_api(
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/route_utils.py", line 322, in call_process_api
+2024-12-24 14:42:01 | ERROR | stderr |     output = await app.get_blocks().process_api(
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2047, in process_api
+2024-12-24 14:42:01 | ERROR | stderr |     result = await self.call_function(
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 1594, in call_function
+2024-12-24 14:42:01 | ERROR | stderr |     prediction = await anyio.to_thread.run_sync(  # type: ignore
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
+2024-12-24 14:42:01 | ERROR | stderr |     return await get_async_backend().run_sync_in_worker_thread(
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2505, in run_sync_in_worker_thread
+2024-12-24 14:42:01 | ERROR | stderr |     return await future
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 1005, in run
+2024-12-24 14:42:01 | ERROR | stderr |     result = context.run(func, *args)
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/utils.py", line 869, in wrapper
+2024-12-24 14:42:01 | ERROR | stderr |     response = f(*args, **kwargs)
+2024-12-24 14:42:01 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/vote_utils.py", line 41, in leftvote_last_response_igm
+2024-12-24 14:42:01 | ERROR | stderr |     vote_last_response_igm(
+2024-12-24 14:42:01 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/vote_utils.py", line 32, in vote_last_response_igm
+2024-12-24 14:42:01 | ERROR | stderr |     append_json_item_on_log_server(data, get_conv_log_filename())
+2024-12-24 14:42:01 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/serve/utils.py", line 206, in append_json_item_on_log_server
+2024-12-24 14:42:01 | ERROR | stderr |     response = requests.post(url, data={'json_str': json_item, 'file_name': log_file})
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/api.py", line 115, in post
+2024-12-24 14:42:01 | ERROR | stderr |     return request("post", url, data=data, json=json, **kwargs)
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/api.py", line 59, in request
+2024-12-24 14:42:01 | ERROR | stderr |     return session.request(method=method, url=url, **kwargs)
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/sessions.py", line 575, in request
+2024-12-24 14:42:01 | ERROR | stderr |     prep = self.prepare_request(req)
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/sessions.py", line 484, in prepare_request
+2024-12-24 14:42:01 | ERROR | stderr |     p.prepare(
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/models.py", line 367, in prepare
+2024-12-24 14:42:01 | ERROR | stderr |     self.prepare_url(url, params)
+2024-12-24 14:42:01 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/requests/models.py", line 438, in prepare_url
+2024-12-24 14:42:01 | ERROR | stderr |     raise MissingSchema(
+2024-12-24 14:42:01 | ERROR | stderr | requests.exceptions.MissingSchema: Invalid URL '/logs/append_json': No scheme supplied. Perhaps you meant https:///logs/append_json?
+2024-12-24 15:03:08 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 15:03:08 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:03:08 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 15:03:08 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 15:03:08 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 15:03:08 | ERROR | stderr | 
+2024-12-24 15:03:08 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 15:03:08 | ERROR | stderr | 
+2024-12-24 15:03:08 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:03:08 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 15:03:08 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 15:03:08 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 15:03:08 | ERROR | stderr |     self.block_thread()
+2024-12-24 15:03:08 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 15:03:08 | ERROR | stderr |     self.server.close()
+2024-12-24 15:03:08 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 15:03:08 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 15:03:08 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 15:03:08 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 15:03:08 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 15:03:08 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 15:03:08 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 15:03:13 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 15:03:13 | INFO | stdout | 
+2024-12-24 15:03:13 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 15:04:38 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 15:05:17 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 15:05:17 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:05:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 15:05:17 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 15:05:17 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 15:05:17 | ERROR | stderr | 
+2024-12-24 15:05:17 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 15:05:17 | ERROR | stderr | 
+2024-12-24 15:05:17 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:05:17 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 15:05:17 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 15:05:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 15:05:17 | ERROR | stderr |     self.block_thread()
+2024-12-24 15:05:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 15:05:17 | ERROR | stderr |     self.server.close()
+2024-12-24 15:05:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 15:05:17 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 15:05:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 15:05:17 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 15:05:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 15:05:17 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 15:05:17 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 15:05:17 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 15:05:17 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:05:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 15:05:17 | ERROR | stderr |     lock.acquire()
+2024-12-24 15:05:17 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 15:07:04 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 15:07:04 | INFO | stdout | 
+2024-12-24 15:07:04 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 15:07:09 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 15:08:19 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 15:08:19 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:08:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 15:08:19 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 15:08:19 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 15:08:19 | ERROR | stderr | 
+2024-12-24 15:08:19 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 15:08:19 | ERROR | stderr | 
+2024-12-24 15:08:19 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:08:19 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 15:08:19 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 15:08:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 15:08:19 | ERROR | stderr |     self.block_thread()
+2024-12-24 15:08:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 15:08:19 | ERROR | stderr |     self.server.close()
+2024-12-24 15:08:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 15:08:19 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 15:08:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 15:08:19 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 15:08:19 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 15:08:19 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 15:08:19 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 15:08:20 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/anaconda3/envs/tiger/lib/python3.10/threading.py'>
+2024-12-24 15:08:20 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:08:20 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1567, in _shutdown
+2024-12-24 15:08:20 | ERROR | stderr |     lock.acquire()
+2024-12-24 15:08:20 | ERROR | stderr | KeyboardInterrupt:
+2024-12-24 15:12:22 | INFO | stdout | {'t2i_generation': PosixPath('arena_elo/results/latest/elo_results_t2i_generation.pkl'), 'video_generation': PosixPath('arena_elo/results/latest/elo_results_video_generation.pkl'), 'image_editing': PosixPath('arena_elo/results/latest/elo_results_image_editing.pkl')}
+2024-12-24 15:12:22 | INFO | stdout | {'t2i_generation': PosixPath('arena_elo/results/latest/t2i_generation_leaderboard.csv'), 'video_generation': PosixPath('arena_elo/results/latest/video_generation_leaderboard.csv'), 'image_editing': PosixPath('arena_elo/results/latest/image_editing_leaderboard.csv')}
+2024-12-24 15:15:46 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:15:46 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 79, in <module>
+2024-12-24 15:15:46 | ERROR | stderr |     input()
+2024-12-24 15:15:46 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 15:26:57 | INFO | stdout | arena_elo/results/latest/t2i_generation_leaderboard.csv
+2024-12-24 15:26:57 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 15:26:57 | INFO | stdout | 
+2024-12-24 15:26:57 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 15:56:17 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 15:56:17 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:56:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 15:56:17 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 15:56:17 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 15:56:17 | ERROR | stderr | 
+2024-12-24 15:56:17 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 15:56:17 | ERROR | stderr | 
+2024-12-24 15:56:17 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 15:56:17 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 15:56:17 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 15:56:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 15:56:17 | ERROR | stderr |     self.block_thread()
+2024-12-24 15:56:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 15:56:17 | ERROR | stderr |     self.server.close()
+2024-12-24 15:56:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 15:56:17 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 15:56:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 15:56:17 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 15:56:17 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 15:56:17 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 15:56:17 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 16:52:26 | INFO | stdout | arena_elo/results/latest/t2i_generation_leaderboard.csv
+2024-12-24 16:52:27 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 16:52:27 | INFO | stdout | 
+2024-12-24 16:52:27 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 16:53:47 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 16:54:12 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 16:54:24 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 16:54:36 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 16:54:55 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-24 16:55:03 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-24 16:55:11 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 16:55:45 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 16:55:46 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 16:55:46 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 16:55:46 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 16:55:46 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 16:55:46 | ERROR | stderr | 
+2024-12-24 16:55:46 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 16:55:46 | ERROR | stderr | 
+2024-12-24 16:55:46 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 16:55:46 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 16:55:46 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 16:55:46 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 16:55:46 | ERROR | stderr |     self.block_thread()
+2024-12-24 16:55:46 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 16:55:46 | ERROR | stderr |     self.server.close()
+2024-12-24 16:55:46 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 16:55:46 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 16:55:46 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 16:55:46 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 16:55:46 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 16:55:46 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 16:55:46 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 17:57:24 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-24 17:57:24 | INFO | stdout | 
+2024-12-24 17:57:24 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-24 17:57:39 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-24 17:57:55 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 17:58:04 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 17:58:13 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 17:58:23 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 17:58:34 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 17:58:46 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 17:58:51 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 17:58:56 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-24 17:59:11 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 17:59:26 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 17:59:40 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 17:59:44 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 17:59:49 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 18:00:05 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 18:00:12 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 18:00:19 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-24 18:00:25 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 18:00:32 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-24 18:00:46 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-24 18:00:47 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 18:00:47 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-24 18:00:47 | ERROR | stderr |     time.sleep(0.1)
+2024-12-24 18:00:47 | ERROR | stderr | KeyboardInterrupt
+2024-12-24 18:00:47 | ERROR | stderr | 
+2024-12-24 18:00:47 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-24 18:00:47 | ERROR | stderr | 
+2024-12-24 18:00:47 | ERROR | stderr | Traceback (most recent call last):
+2024-12-24 18:00:47 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 78, in <module>
+2024-12-24 18:00:47 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-24 18:00:47 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-24 18:00:47 | ERROR | stderr |     self.block_thread()
+2024-12-24 18:00:47 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-24 18:00:47 | ERROR | stderr |     self.server.close()
+2024-12-24 18:00:47 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-24 18:00:47 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-24 18:00:47 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-24 18:00:47 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-24 18:00:47 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-24 18:00:47 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-24 18:00:47 | ERROR | stderr | KeyboardInterrupt
+2024-12-25 10:12:09 | INFO | stdout | * Running on local URL:  http://127.0.0.1:7860
+2024-12-25 10:12:09 | INFO | stdout | 
+2024-12-25 10:12:09 | INFO | stdout | To create a public link, set `share=True` in `launch()`.
+2024-12-25 10:12:41 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-25 10:12:54 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-25 10:13:17 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-25 10:13:24 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-25 10:13:30 | INFO | gradio_web_server_image_generation_multi | leftvote (named). ip: 127.0.0.1
+2024-12-25 10:18:33 | INFO | gradio_web_server_image_generation_multi | rightvote (named). ip: 127.0.0.1
+2024-12-25 10:18:53 | INFO | gradio_web_server_image_generation_multi | bothbad_vote (named). ip: 127.0.0.1
+2024-12-25 10:25:51 | INFO | stdout | Keyboard interruption in main thread... closing server.
+2024-12-25 10:25:52 | ERROR | stderr | Traceback (most recent call last):
+2024-12-25 10:25:52 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2869, in block_thread
+2024-12-25 10:25:52 | ERROR | stderr |     time.sleep(0.1)
+2024-12-25 10:25:52 | ERROR | stderr | KeyboardInterrupt
+2024-12-25 10:25:52 | ERROR | stderr | 
+2024-12-25 10:25:52 | ERROR | stderr | During handling of the above exception, another exception occurred:
+2024-12-25 10:25:52 | ERROR | stderr | 
+2024-12-25 10:25:52 | ERROR | stderr | Traceback (most recent call last):
+2024-12-25 10:25:52 | ERROR | stderr |   File "/Users/jasi/Documents/ali-vilab/modelscope/IDEA-Bench-Arena/app.py", line 64, in <module>
+2024-12-25 10:25:52 | ERROR | stderr |     demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, show_error=True)
+2024-12-25 10:25:52 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2774, in launch
+2024-12-25 10:25:52 | ERROR | stderr |     self.block_thread()
+2024-12-25 10:25:52 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/blocks.py", line 2873, in block_thread
+2024-12-25 10:25:52 | ERROR | stderr |     self.server.close()
+2024-12-25 10:25:52 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/site-packages/gradio/http_server.py", line 69, in close
+2024-12-25 10:25:52 | ERROR | stderr |     self.thread.join(timeout=5)
+2024-12-25 10:25:52 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1100, in join
+2024-12-25 10:25:52 | ERROR | stderr |     self._wait_for_tstate_lock(timeout=max(timeout, 0))
+2024-12-25 10:25:52 | ERROR | stderr |   File "/opt/anaconda3/envs/tiger/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
+2024-12-25 10:25:52 | ERROR | stderr |     if lock.acquire(block, timeout):
+2024-12-25 10:25:52 | ERROR | stderr | KeyboardInterrupt