ui updates (#50)

- big ui update (5ba04356379684b78ac2e957803ea23d5be95114)

app.py

@@ -23,7 +23,7 @@ with gr.Blocks() as block:
     gr.Markdown(LEADERBOARD_INTRODUCTION)
     
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        # Table 1
+        # Table 1, the main leaderboard of overall scores
         with gr.TabItem("📊 MMEB (V2)", elem_id="qa-tab-table1", id=1):
             with gr.Row():
                 with gr.Accordion("Citation", open=False):
@@ -92,10 +92,11 @@ with gr.Blocks() as block:
             )
             refresh_button2.click(fn=v2.refresh_data, outputs=data_component2)
 
-        # table 2
+        # table 2, image scores only
         with gr.TabItem("🖼️ Image", elem_id="qa-tab-table1", id=2):
+            gr.Markdown(v2.TABLE_INTRODUCTION_I)
             data_component3 = gr.components.Dataframe(
-                value=df2[v2.COLUMN_NAMES_I],
+                value=v2.rank_models(df2[v2.COLUMN_NAMES_I], 'Image-Overall'),
                 headers=v2.COLUMN_NAMES_I,
                 type="pandas",
                 datatype=v2.DATA_TITLE_TYPE_I,
@@ -104,10 +105,11 @@ with gr.Blocks() as block:
                 max_height=2400, 
             )
 
-        # table 3
+        # table 3, video scores only
         with gr.TabItem("💽 Video", elem_id="qa-tab-table1", id=3):
+            gr.Markdown(v2.TABLE_INTRODUCTION_V)
             data_component4 = gr.components.Dataframe(
-                value=df2[v2.COLUMN_NAMES_V],
+                value=v2.rank_models(df2[v2.COLUMN_NAMES_V], 'Video-Overall'),
                 headers=v2.COLUMN_NAMES_V,
                 type="pandas",
                 datatype=v2.DATA_TITLE_TYPE_V,
@@ -116,10 +118,11 @@ with gr.Blocks() as block:
                 max_height=2400, 
             )
 
-        # table 4
+        # table 4, visual document scores only
         with gr.TabItem("📑 Visual Doc", elem_id="qa-tab-table1", id=4):
+            gr.Markdown(v2.TABLE_INTRODUCTION_D)
             data_component5 = gr.components.Dataframe(
-                value=df2[v2.COLUMN_NAMES_D],
+                value=v2.rank_models(df2[v2.COLUMN_NAMES_D], 'VisDoc'),
                 headers=v2.COLUMN_NAMES_D,
                 type="pandas",
                 datatype=v2.DATA_TITLE_TYPE_D,

scores/LamRA-Ret-Qwen2.5VL-7b.json

@@ -1,7 +1,10 @@
 {
     "metadata": {
         "model_name": "LamRA-Ret-Qwen2.5VL-7b",
-        "report_generated_date": "2025-06-09T07:00:24.383583"
+        "report_generated_date": "2025-06-09T07:00:24.383583", 
+        "model_size": 8.29, 
+        "url": "https://huggingface.co/code-kunkun/LamRA-Ret-Qwen2.5VL-7b", 
+        "data_source": "TIGER-Lab"
     },
     "metrics": {
         "image": {

scores/LamRA-Ret.json

@@ -1,7 +1,10 @@
 {
     "metadata": {
         "model_name": "LamRA-Ret",
-        "report_generated_date": "2025-06-09T07:03:51.413144"
+        "report_generated_date": "2025-06-09T07:03:51.413144", 
+        "model_size": 8.29, 
+        "url": "https://huggingface.co/code-kunkun/LamRA-Ret", 
+        "data_source": "TIGER-Lab"
     },
     "metrics": {
         "image": {

scores/VLM2Vec-V1-Qwen2VL-2B.json

@@ -1,7 +1,10 @@
 {
     "metadata": {
         "model_name": "VLM2Vec-V1-Qwen2VL-2B",
-        "report_generated_date": "2025-06-09T07:08:50.537181"
+        "report_generated_date": "2025-06-09T07:08:50.537181", 
+        "model_size": 2.21, 
+        "url": "https://huggingface.co/TIGER-Lab/VLM2Vec-Qwen2VL-2B", 
+        "data_source": "TIGER-Lab"
     },
     "metrics": {
         "image": {

scores/VLM2Vec-V1-Qwen2VL-7B.json

@@ -1,7 +1,10 @@
 {
     "metadata": {
         "model_name": "VLM2Vec-V1-Qwen2VL-7B",
-        "report_generated_date": "2025-06-08T08:08:07.905654"
+        "report_generated_date": "2025-06-08T08:08:07.905654", 
+        "model_size": 8.29, 
+        "url": "https://huggingface.co/TIGER-Lab/VLM2Vec-Qwen2VL-7B", 
+        "data_source": "TIGER-Lab"
     },
     "metrics": {
         "image": {

scores/VLM2Vec-V2.0-Qwen2VL-2B.json

@@ -1,7 +1,10 @@
 {
     "metadata": {
         "model_name": "VLM2Vec-V2.0-Qwen2VL-2B",
-        "report_generated_date": "2025-06-09T07:05:59.773788"
+        "report_generated_date": "2025-06-09T07:05:59.773788", 
+        "model_size": 2.21, 
+        "url": "https://huggingface.co/VLM2Vec/VLM2Vec-V2.0", 
+        "data_source": "TIGER-Lab"
     },
     "metrics": {
         "image": {

scores/colpali-v1.3.json

@@ -1,7 +1,10 @@
 {
     "metadata": {
         "model_name": "colpali-v1.3",
-        "report_generated_date": "2025-06-09T07:08:13.841120"
+        "report_generated_date": "2025-06-09T07:08:13.841120", 
+        "model_size": 2.92, 
+        "url": "https://huggingface.co/vidore/colpali-v1.3", 
+        "data_source": "TIGER-Lab"
     },
     "metrics": {
         "image": {

scores/gme-Qwen2-VL-2B-Instruct.json

@@ -1,7 +1,10 @@
 {
     "metadata": {
         "model_name": "gme-Qwen2-VL-2B-Instruct",
-        "report_generated_date": "2025-06-09T07:04:30.518891"
+        "report_generated_date": "2025-06-09T07:04:30.518891", 
+        "model_size": 2.21, 
+        "url": "https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct", 
+        "data_source": "TIGER-Lab"
     },
     "metrics": {
         "image": {

scores/gme-Qwen2-VL-7B-Instruct.json

@@ -1,7 +1,10 @@
 {
     "metadata": {
         "model_name": "gme-Qwen2-VL-7B-Instruct",
-        "report_generated_date": "2025-06-09T07:05:25.508931"
+        "report_generated_date": "2025-06-09T07:05:25.508931", 
+        "model_size": 8.29, 
+        "url": "https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct", 
+        "data_source": "TIGER-Lab"
     },
     "metrics": {
         "image": {

utils.py

@@ -57,26 +57,9 @@ SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction
 
 ## ⚠ Please note that you need to submit the JSON file with the following format:
 
-### **TO SUBMIT V1 ONLY (Depreciating)**
-```json
-[
-    {
-        "Model": "<Model Name>",
-        "URL": "<Model URL>" or null,
-        "Model Size(B)": 1000 or null,
-        "Data Source": "Self-Reported",
-        "V1-Overall": 50.0,
-        "I-CLS": 50.0,
-        "I-QA": 50.0,
-        "I-RET": 50.0,
-        "I-VG": 50.0
-    }, 
-]
-```
-
-### ***Important Notes: We will be releasing MMEB-V2 soon!***
-### ***In V2, the detailed scores of each dataset will be included, and our code will automatically generate the results and calculate the overall scores.***
-### **A V2 Submission would look like this: (TO BE RELEASED SOON)**
+### ***Important Notes: We have released MMEB-V2 and will deprecate MMEB-V1 soon. All further submissions should be made using the V2 format (see following).***
+### ***In V2, the detailed scores of each dataset will be included, and our code will automatically generate the results and calculate the overall scores. See the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for more information.***
+### **A V2 Submission would look like this:**
 ```json
 {
     "metadata": {
@@ -84,8 +67,6 @@ SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction
         "URL": "<Model URL>" or null,
         "Model Size(B)": 1000 or null,
         "Data Source": "Self-Reported",
-        "V1-Overall": 50.0,
-        "V2-Overall": 50.0
     },
     "metrics": {
         "image": {
@@ -121,7 +102,24 @@ SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction
     }
 }
 ```
-You may refer to the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for instructions about evaluating your model. \n
+
+### **TO SUBMIT V1 ONLY (Depreciated, but we still accept this format until 2025-06-30)**
+```json
+[
+    {
+        "Model": "<Model Name>",
+        "URL": "<Model URL>" or null,
+        "Model Size(B)": 1000 or null,
+        "Data Source": "Self-Reported",
+        "V1-Overall": 50.0,
+        "I-CLS": 50.0,
+        "I-QA": 50.0,
+        "I-RET": 50.0,
+        "I-VG": 50.0
+    }, 
+]
+```
+You may refer to the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for detailed instructions about evaluating your model. \n
 Please send us an email at [email protected], attaching the JSON file. We will review your submission and update the leaderboard accordingly. \n
 Please also share any feedback or suggestions you have for improving the leaderboard experience. We appreciate your contributions to the MMEB community!
 """

utils_v2.py

@@ -1,14 +1,13 @@
 import json
 import os
 import pandas as pd
-from utils import create_hyperlinked_names
-
-def sum_lst(lst):
-    assert isinstance(lst, list) and lst, f"Input should be a non-empty list, got {type(lst)}, size {len(lst)}"
-    total = lst[0]
-    for item in lst[1:]:
-        assert isinstance(item, (list, int, float)), f"Expected types are list and numbers, got {type(item)}"
-        total += item
+from utils import create_hyperlinked_names, process_model_size
+
+def sum_lol(lol):
+    assert isinstance(lol, list) and all(isinstance(i, list) for i in lol), f"Input should be a list of lists, got {type(lol)}"
+    total = []
+    for sublist in lol:
+        total.extend(sublist)
     return total
 
 SCORE_BASE_DIR = "scores"
@@ -21,7 +20,7 @@ DATASETS = {
         "I-VG": ['MSCOCO', 'RefCOCO', 'RefCOCO-Matching', 'Visual7W']
         }, 
     "visdoc": {
-        "VisDoc": ['ViDoRe_arxivqa', 'ViDoRe_docvqa', 'ViDoRe_infovqa', 'ViDoRe_tabfquad', 'ViDoRe_tatdqa', 'ViDoRe_shiftproject', 'ViDoRe_syntheticDocQA_artificial_intelligence', 'ViDoRe_syntheticDocQA_energy', 'ViDoRe_syntheticDocQA_government_reports', 'ViDoRe_syntheticDocQA_healthcare_industry', 'VisRAG_ArxivQA', 'VisRAG_ChartQA', 'VisRAG_MP-DocVQA', 'VisRAG_SlideVQA', 'VisRAG_InfoVQA', 'VisRAG_PlotQA', 'ViDoSeek-page', 'ViDoSeek-doc', 'MMLongBench-page', 'MMLongBench-doc']
+        "VisDoc": ['ViDoRe_arxivqa', 'ViDoRe_docvqa', 'ViDoRe_infovqa', 'ViDoRe_tabfquad', 'ViDoRe_tatdqa', 'ViDoRe_shiftproject', 'ViDoRe_syntheticDocQA_artificial_intelligence', 'ViDoRe_syntheticDocQA_energy', 'ViDoRe_syntheticDocQA_government_reports', 'ViDoRe_syntheticDocQA_healthcare_industry', 'VisRAG_ArxivQA', 'VisRAG_ChartQA', 'VisRAG_MP-DocVQA', 'VisRAG_SlideVQA', 'VisRAG_InfoVQA', 'VisRAG_PlotQA', 'ViDoSeek-page', 'ViDoSeek-doc', 'MMLongBench-page', 'MMLongBench-doc', "ViDoRe_esg_reports_human_labeled_v2", "ViDoRe_biomedical_lectures_v2", "ViDoRe_biomedical_lectures_v2_multilingual", "ViDoRe_economics_reports_v2", "ViDoRe_economics_reports_v2_multilingual", "ViDoRe_esg_reports_v2", "ViDoRe_esg_reports_v2_multilingual"]
         }, 
     "video": {
         "V-CLS": ['K700', 'UCF101', 'HMDB51', 'SmthSmthV2', 'Breakfast'], 
@@ -30,8 +29,8 @@ DATASETS = {
         "V-MRET": ['QVHighlight', 'Charades-STA', 'MomentSeeker', 'ActivityNetQA']
         }
 }
-ALL_DATASETS_SPLITS = {k: sum_lst(list(v.values())) for k, v in DATASETS.items()}
-ALL_DATASETS = sum_lst(list(ALL_DATASETS_SPLITS.values()))
+ALL_DATASETS_SPLITS = {k: sum_lol(list(v.values())) for k, v in DATASETS.items()}
+ALL_DATASETS = sum_lol(list(ALL_DATASETS_SPLITS.values()))
 MODALITIES = list(DATASETS.keys())
 SPECIAL_METRICS = {
     '__default__': 'hit@1',
@@ -45,24 +44,29 @@ COLUMN_NAMES = BASE_COLS + ["Overall", 'Image-Overall', 'Video-Overall', 'VisDoc
 DATA_TITLE_TYPE = BASE_DATA_TITLE_TYPE + \
                     ['number'] * 3
 
-TASKS_I = ['Image-Overall'] + ALL_DATASETS_SPLITS['image']
+TASKS_I = ['Image-Overall'] + TASKS[1:5] + ALL_DATASETS_SPLITS['image']
 COLUMN_NAMES_I = BASE_COLS + TASKS_I
 DATA_TITLE_TYPE_I = BASE_DATA_TITLE_TYPE + \
-                    ['number'] * len(TASKS_I)
+                    ['number'] * (len(TASKS_I) + 4)
 
-TASKS_V = ['Video-Overall'] + ALL_DATASETS_SPLITS['video']
+TASKS_V = ['Video-Overall'] + TASKS[6:10] + ALL_DATASETS_SPLITS['video']
 COLUMN_NAMES_V = BASE_COLS + TASKS_V
 DATA_TITLE_TYPE_V = BASE_DATA_TITLE_TYPE + \
-                    ['number'] * len(TASKS_V)
+                    ['number'] * (len(TASKS_V) + 4)
 
 TASKS_D = ['VisDoc'] + ALL_DATASETS_SPLITS['visdoc']
 COLUMN_NAMES_D = BASE_COLS + TASKS_D
 DATA_TITLE_TYPE_D = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(TASKS_D)
 
-TABLE_INTRODUCTION = """**I-CLS**: Image Classification, **I-QA**: (Image) Visual Question Answering, **I-RET**: Image Retrieval, **I-VG**: (Image) Visual Grounding \n
-**V-CLS**: Video Classification, **V-QA**: (Video) Visual Question Answering, **V-RET**: Video Retrieval, **V-MRET**: Video Moment Retrieval \n
-**VisDoc**: Visual Document Understanding \n"""
+TABLE_INTRODUCTION = """**MMEB**: Massive MultiModal Embedding Benchmark \n
+                        Models are ranked based on **Overall**"""
+TABLE_INTRODUCTION_I = """**I-CLS**: Image Classification, **I-QA**: (Image) Visual Question Answering, **I-RET**: Image Retrieval, **I-VG**: (Image) Visual Grounding \n
+                        Models are ranked based on **Image-Overall**"""
+TABLE_INTRODUCTION_V = """**V-CLS**: Video Classification, **V-QA**: (Video) Visual Question Answering, **V-RET**: Video Retrieval, **V-MRET**: Video Moment Retrieval \n
+                        Models are ranked based on **Video-Overall**"""
+TABLE_INTRODUCTION_D = """**VisDoc**: Visual Document Understanding \n
+                        Models are ranked based on **VisDoc**"""
 
 LEADERBOARD_INFO = """
 ## Dataset Summary
@@ -112,16 +116,16 @@ def calculate_score(raw_scores=None):
     avg_scores = {}
 
     # Calculate overall score for all datasets
-    avg_scores['Overall'] = None # get_avg(sum(all_scores.values()), len(ALL_DATASETS))
+    avg_scores['Overall'] = get_avg(sum(all_scores.values()), len(ALL_DATASETS))
 
     # Calculate scores for each modality
     for modality in MODALITIES:
-        datasets_for_each_modality = ALL_DATASETS_SPLITS.get(modality, [])
+        datasets_for_each_modality = ALL_DATASETS_SPLITS[modality]
         avg_scores[f"{modality.capitalize()}-Overall"] = get_avg(
             sum(all_scores.get(dataset, 0.0) for dataset in datasets_for_each_modality),
             len(datasets_for_each_modality)
         )
-
+    
     # Calculate scores for each sub-task
     for modality, datasets_list in DATASETS.items():
         for sub_task, datasets in datasets_list.items():
@@ -136,20 +140,27 @@ def generate_model_row(data):
     row = {
         'Models': metadata.get('model_name', None), 
         'Model Size(B)': metadata.get('model_size', None),
-        'URL': metadata.get('url', None)
+        'URL': metadata.get('url', None), 
+        'Data Source': metadata.get('data_source', 'Self-Reported'),
     }
     scores = calculate_score(data['metrics'])
     row.update(scores)
     return row
 
+def rank_models(df, column='Overall'):
+    """Ranks the models based on the specific score."""
+    df = df.sort_values(by=column, ascending=False).reset_index(drop=True)
+    df['Rank'] = range(1, len(df) + 1)
+    return df
+
 def get_df():
     """Generates a DataFrame from the loaded data."""
     all_data = load_data()
     rows = [generate_model_row(data) for data in all_data]
     df = pd.DataFrame(rows)
-    df = df.sort_values(by='Overall', ascending=False).reset_index(drop=True)
-    df['Rank'] = range(1, len(df) + 1)
+    df['Model Size(B)'] = df['Model Size(B)'].apply(process_model_size)
     df = create_hyperlinked_names(df)
+    df = rank_models(df, column='Overall')
     return df
 
 def refresh_data():