Remove unused files and configurations, including .gitignore, Makefile, requirements.txt, and various source files.

.gitattributes

@@ -25,6 +25,7 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
@@ -32,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,13 +0,0 @@
-auto_evals/
-venv/
-__pycache__/
-.env
-.ipynb_checkpoints
-*ipynb
-.vscode/
-
-eval-queue/
-eval-results/
-eval-queue-bk/
-eval-results-bk/
-logs/

.pre-commit-config.yaml

@@ -1,53 +0,0 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-default_language_version:
-  python: python3
-
-ci:
-  autofix_prs: true
-  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
-  autoupdate_schedule: quarterly
-
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
-    hooks:
-      - id: check-yaml
-      - id: check-case-conflict
-      - id: detect-private-key
-      - id: check-added-large-files
-        args: ['--maxkb=1000']
-      - id: requirements-txt-fixer
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        name: Format imports
-
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-      - id: black
-        name: Format code
-        additional_dependencies: ['click==8.0.2']
-
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    # Ruff version.
-    rev: 'v0.0.267'
-    hooks:
-      - id: ruff

Makefile

@@ -1,13 +0,0 @@
-.PHONY: style format
-
-
-style:
-	python -m black --line-length 119 .
-	python -m isort .
-	ruff check --fix .
-
-
-quality:
-	python -m black --check --line-length 119 .
-	python -m isort --check-only .
-	ruff check .

README.md

@@ -1,44 +1,13 @@
 ---
-title: The Arabic Rag Leaderboard
-emoji: 🥇
+title: The Arabic RAG Leaderboard
+emoji: 📊
 colorFrom: green
 colorTo: indigo
 sdk: gradio
+sdk_version: 5.7.1
 app_file: app.py
 pinned: true
-license: apache-2.0
+short_description: The only leaderboard you will require for your RAG needs 🏆
 ---
 
-# Start the configuration
-
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
-```
-
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
-# Code logic for more complex edits
-
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,204 +1,668 @@
-import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+import os
+import json
+import numpy as np
 import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
+import gradio as gr
+from huggingface_hub import HfApi, hf_hub_download
+
+
+OWNER = "Navid-AI"
+DATASET_REPO_ID = f"{OWNER}/requests-dataset"
+
+HEADER = """<div style="text-align: center; margin-bottom: 20px;">
+    <h1>The Arabic RAG Leaderboard</h1>
+    <p style="font-size: 14px; color: #888;">The only leaderboard you will require for your RAG needs 🏆</p>
+</div>
+
+"""
+
+ABOUT_SECTION = """
+## About
+
+The AraGen Leaderboard is designed to evaluate and compare the performance of Chat Arabic Large Language Models (LLMs) on a set of generative tasks. By leveraging the new **3C3H** evaluation measure which evaluate the model's output across six dimensions —Correctness, Completeness, Conciseness, Helpfulness, Honesty, and Harmlessness— the leaderboard provides a comprehensive and holistic evaluation of a model's performance in generating human-like and ethically responsible content.
+
+### Why Focus on Chat Models?
+
+AraGen Leaderboard —And 3C3H in general— is specifically designed to assess **chat models**, which interact in conversational settings, intended for end user interaction and require a blend of factual accuracy and user-centric dialogue capabilities. While it is technically possible to submit foundational models, we kindly ask users to refrain from doing so. For evaluations of foundational models using likelihood accuracy based benchmarks, please refer to the [Open Arabic LLM Leaderboard (OALL)](https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard).
+
+### How to Submit Your Model?
+
+Navigate to the submission section below to submit your open chat model from the HuggingFace Hub for evaluation. Ensure that your model is public and the submmited metadata (precision, revision, #params) is accurate.
+
+### Contact
+
+For any inquiries or assistance, feel free to reach out through the community tab at [Inception AraGen Community](https://huggingface.co/spaces/inceptionai/AraGen-Leaderboard/discussions) or via [email](mailto:[email protected]).
+"""
+
+CITATION_BUTTON_LABEL = """
+Copy the following snippet to cite these results
+"""
+
+CITATION_BUTTON_TEXT = """
+@misc{AraGen,
+  author = {El Filali, Ali and Sengupta, Neha and Abouelseoud, Arwa and Nakov, Preslav and Fourrier, Clémentine},
+  title = {Rethinking LLM Evaluation with 3C3H: AraGen Benchmark and Leaderboard},
+  year = {2024},
+  publisher = {Inception},
+  howpublished = "url{https://huggingface.co/spaces/inceptionai/AraGen-Leaderboard}"
+}
+"""
+
+
+def load_results():
+    # Get the current directory of the script and construct the path to results.json
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    results_file = os.path.join(current_dir, "assets", "results", "results.json")
+    
+    # Load the JSON data from the specified file
+    with open(results_file, 'r') as f:
+        data = json.load(f)
+    
+    # Filter out any entries that only contain '_last_sync_timestamp'
+    filtered_data = []
+    for entry in data:
+        # If '_last_sync_timestamp' is the only key, skip it
+        if len(entry.keys()) == 1 and "_last_sync_timestamp" in entry:
+            continue
+        filtered_data.append(entry)
+    
+    data = filtered_data
+    
+    # Lists to collect data
+    data_3c3h = []
+    data_tasks = []
+    
+    for model_data in data:
+        # Extract model meta data
+        meta = model_data.get('Meta', {})
+        model_name = meta.get('Model Name', 'UNK')
+        revision = meta.get('Revision', 'UNK')
+        precision = meta.get('Precision', 'UNK')
+        params = meta.get('Params', 'UNK')
+        license = meta.get('License', 'UNK')
+        
+        # Convert "Model Size" to numeric, treating "UNK" as infinity
+        try:
+            model_size_numeric = float(params)
+        except (ValueError, TypeError):
+            model_size_numeric = np.inf
+        
+        # 3C3H Scores
+        scores_data = model_data.get('claude-3.5-sonnet Scores', {})
+        scores_3c3h = scores_data.get('3C3H Scores', {})
+        scores_tasks = scores_data.get('Tasks Scores', {})
+        
+        # Multiply scores by 100 to get percentages (keep them as numeric values)
+        formatted_scores_3c3h = {k: v*100 for k, v in scores_3c3h.items()}
+        formatted_scores_tasks = {k: v*100 for k, v in scores_tasks.items()}
+        
+        # For 3C3H Scores DataFrame
+        data_entry_3c3h = {
+            'Model Name': model_name,
+            'Revision': revision,
+            'License': license,
+            'Precision': precision,
+            'Model Size': model_size_numeric,  # Numeric value for sorting
+            '3C3H Score': formatted_scores_3c3h.get("3C3H Score", np.nan),
+            'Correctness': formatted_scores_3c3h.get("Correctness", np.nan),
+            'Completeness': formatted_scores_3c3h.get("Completeness", np.nan),
+            'Conciseness': formatted_scores_3c3h.get("Conciseness", np.nan),
+            'Helpfulness': formatted_scores_3c3h.get("Helpfulness", np.nan),
+            'Honesty': formatted_scores_3c3h.get("Honesty", np.nan),
+            'Harmlessness': formatted_scores_3c3h.get("Harmlessness", np.nan),
+        }
+        data_3c3h.append(data_entry_3c3h)
+        
+        # For Tasks Scores DataFrame
+        data_entry_tasks = {
+            'Model Name': model_name,
+            'Revision': revision,
+            'License': license,
+            'Precision': precision,
+            'Model Size': model_size_numeric,  # Numeric value for sorting
+            **formatted_scores_tasks
+        }
+        data_tasks.append(data_entry_tasks)
+    
+    df_3c3h = pd.DataFrame(data_3c3h)
+    df_tasks = pd.DataFrame(data_tasks)
+    
+    # Round the numeric score columns to 4 decimal places
+    score_columns_3c3h = ['3C3H Score', 'Correctness', 'Completeness', 'Conciseness', 'Helpfulness', 'Honesty', 'Harmlessness']
+    df_3c3h[score_columns_3c3h] = df_3c3h[score_columns_3c3h].round(4)
+    
+    # Replace np.inf with a large number in 'Model Size Filter' for filtering
+    max_model_size_value = 1000  # Define a maximum value
+    df_3c3h['Model Size Filter'] = df_3c3h['Model Size'].replace(np.inf, max_model_size_value)
+    
+    # Sort df_3c3h by '3C3H Score' descending if column exists
+    if '3C3H Score' in df_3c3h.columns:
+        df_3c3h = df_3c3h.sort_values(by='3C3H Score', ascending=False)
+        df_3c3h.insert(0, 'Rank', range(1, len(df_3c3h) + 1))  # Add Rank column starting from 1
+    else:
+        df_3c3h.insert(0, 'Rank', range(1, len(df_3c3h) + 1))
+    
+    # Extract task columns
+    task_columns = [col for col in df_tasks.columns if col not in ['Model Name', 'Revision', 'License', 'Precision', 'Model Size', 'Model Size Filter']]
+    
+    # Round the task score columns to 4 decimal places
+    if task_columns:
+        df_tasks[task_columns] = df_tasks[task_columns].round(4)
+    
+    # Replace np.inf with a large number in 'Model Size Filter' for filtering
+    df_tasks['Model Size Filter'] = df_tasks['Model Size'].replace(np.inf, max_model_size_value)
+    
+    # Sort df_tasks by the first task column if it exists
+    if task_columns:
+        first_task = task_columns[0]
+        df_tasks = df_tasks.sort_values(by=first_task, ascending=False)
+        df_tasks.insert(0, 'Rank', range(1, len(df_tasks) + 1))  # Add Rank column starting from 1
+    else:
+        df_tasks = df_tasks.sort_values(by='Model Name', ascending=True)
+        df_tasks.insert(0, 'Rank', range(1, len(df_tasks) + 1))
+    
+    return df_3c3h, df_tasks, task_columns
+
+def load_requests(status_folder):
+    api = HfApi()
+    requests_data = []
+    folder_path_in_repo = status_folder  # 'pending', 'finished', or 'failed'
+
+    hf_api_token = os.environ.get('HF_API_TOKEN', None)
+
+    try:
+        # List files in the dataset repository
+        files_info = api.list_repo_files(
+            repo_id=DATASET_REPO_ID,
+            repo_type="dataset",
+            token=hf_api_token
+        )
+    except Exception as e:
+        print(f"Error accessing dataset repository: {e}")
+        return pd.DataFrame()  # Return empty DataFrame if repository not found or inaccessible
+
+    # Filter files in the desired folder
+    files_in_folder = [f for f in files_info if f.startswith(f"{folder_path_in_repo}/") and f.endswith('.json')]
+
+    for file_path in files_in_folder:
+        try:
+            # Download the JSON file
+            local_file_path = hf_hub_download(
+                repo_id=DATASET_REPO_ID,
+                filename=file_path,
+                repo_type="dataset",
+                token=hf_api_token
+            )
+            # Load JSON data
+            with open(local_file_path, 'r') as f:
+                request = json.load(f)
+            requests_data.append(request)
+        except Exception as e:
+            print(f"Error loading file {file_path}: {e}")
+            continue  # Skip files that can't be loaded
+
+    df = pd.DataFrame(requests_data)
+    return df
+
+def submit_model(model_name, revision, precision, params, license):
+    # Load existing evaluations
+    df_3c3h, df_tasks, _ = load_results()
+    existing_models_results = df_3c3h[['Model Name', 'Revision', 'Precision']]
+
+    # Handle 'Missing' precision
+    if precision == 'Missing':
+        precision = None
+    else:
+        precision = precision.strip().lower()
+
+    # Load pending and finished requests from the dataset repository
+    df_pending = load_requests('pending')
+    df_finished = load_requests('finished')
+
+    # Check if model is already evaluated
+    model_exists_in_results = ((existing_models_results['Model Name'] == model_name) &
+                               (existing_models_results['Revision'] == revision) &
+                               (existing_models_results['Precision'] == precision)).any()
+    if model_exists_in_results:
+        return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' has already been evaluated.**"
+
+    # Check if model is in pending requests
+    if not df_pending.empty:
+        existing_models_pending = df_pending[['model_name', 'revision', 'precision']]
+        model_exists_in_pending = ((existing_models_pending['model_name'] == model_name) &
+                                   (existing_models_pending['revision'] == revision) &
+                                   (existing_models_pending['precision'] == precision)).any()
+        if model_exists_in_pending:
+            return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' is already in the pending evaluations.**"
+
+    # Check if model is in finished requests
+    if not df_finished.empty:
+        existing_models_finished = df_finished[['model_name', 'revision', 'precision']]
+        model_exists_in_finished = ((existing_models_finished['model_name'] == model_name) &
+                                    (existing_models_finished['revision'] == revision) &
+                                    (existing_models_finished['precision'] == precision)).any()
+        if model_exists_in_finished:
+            return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' has already been evaluated.**"
+
+    # Check if model exists on HuggingFace Hub
+    api = HfApi()
+    try:
+        model_info = api.model_info(model_name)
+    except Exception as e:
+        return f"**Error: Could not find model '{model_name}' on HuggingFace Hub. Please ensure the model name is correct and the model is public.**"
+
+    # Proceed with submission
+    status = "PENDING"
+
+    # Prepare the submission data
+    submission = {
+        "model_name": model_name,
+        "license": license,
+        "revision": revision,
+        "precision": precision,
+        "status": status,
+        "params": params
+    }
+
+    # Serialize the submission to JSON
+    submission_json = json.dumps(submission, indent=2)
+
+    # Define the file path in the repository
+    org_model = model_name.split('/')
+    if len(org_model) != 2:
+        return "**Please enter the full model name including the organization or username, e.g., 'inceptionai/jais-family-30b-8k'**"
+    org, model_id = org_model
+    precision_str = precision if precision else 'Missing'
+    file_path_in_repo = f"pending/{org}/{model_id}_eval_request_{revision}_{precision_str}.json"
+
+    # Upload the submission to the dataset repository
+    try:
+        hf_api_token = os.environ.get('HF_API_TOKEN', None)
+        api.upload_file(
+            path_or_fileobj=submission_json.encode('utf-8'),
+            path_in_repo=file_path_in_repo,
+            repo_id=DATASET_REPO_ID,
+            repo_type="dataset",
+            token=hf_api_token
+        )
+    except Exception as e:
+        return f"**Error: Could not submit the model. {str(e)}**"
+
+    return f"**Model '{model_name}' has been submitted for evaluation.**"
+
+def main():
+    df_3c3h, df_tasks, task_columns = load_results()
+
+    # Extract unique Precision and License values for filters
+    precision_options_3c3h = sorted(df_3c3h['Precision'].dropna().unique().tolist())
+    precision_options_3c3h = [p for p in precision_options_3c3h if p != 'UNK']
+    precision_options_3c3h.append('Missing')
+
+    license_options_3c3h = sorted(df_3c3h['License'].dropna().unique().tolist())
+    license_options_3c3h = [l for l in license_options_3c3h if l != 'UNK']
+    license_options_3c3h.append('Missing')
+
+    precision_options_tasks = sorted(df_tasks['Precision'].dropna().unique().tolist())
+    precision_options_tasks = [p for p in precision_options_tasks if p != 'UNK']
+    precision_options_tasks.append('Missing')
+
+    license_options_tasks = sorted(df_tasks['License'].dropna().unique().tolist())
+    license_options_tasks = [l for l in license_options_tasks if l != 'UNK']
+    license_options_tasks.append('Missing')
+
+    # Get min and max model sizes for sliders, handling 'inf' values
+    min_model_size_3c3h = int(df_3c3h['Model Size Filter'].min())
+    max_model_size_3c3h = int(df_3c3h['Model Size Filter'].max())
+
+    min_model_size_tasks = int(df_tasks['Model Size Filter'].min())
+    max_model_size_tasks = int(df_tasks['Model Size Filter'].max())
+
+    # Exclude 'Model Size Filter' from column selectors
+    column_choices_3c3h = [col for col in df_3c3h.columns if col != 'Model Size Filter']
+    column_choices_tasks = [col for col in df_tasks.columns if col != 'Model Size Filter']
+
+    with gr.Blocks() as demo:
+        gr.Markdown(HEADER)
+        
+        with gr.Tabs():
+            with gr.Tab("Retrieval"):
+                with gr.Tabs():
+                    with gr.Tab("Leaderboard"):
                         with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
+                            search_box_retrieval = gr.Textbox(
+                                placeholder="Search for models...", 
+                                label="Search", 
+                                interactive=True
+                            )
+
+                        with gr.Row():
+                            license_filter_retrieval = gr.CheckboxGroup(
+                                choices=license_options_3c3h,
+                                value=license_options_3c3h.copy(),  # Default all selected
+                                label="Filter by License",
+                            )
+                            precision_filter_retrieval = gr.CheckboxGroup(
+                                choices=precision_options_3c3h,
+                                value=precision_options_3c3h.copy(),  # Default all selected
+                                label="Filter by Precision",
                             )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
                         with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
+                            model_size_min_filter_3c3h = gr.Slider(
+                                minimum=min_model_size_3c3h,
+                                maximum=max_model_size_3c3h,
+                                value=min_model_size_3c3h,
+                                step=1,
+                                label="Minimum Model Size",
+                                interactive=True
                             )
+                            model_size_max_filter_3c3h = gr.Slider(
+                                minimum=min_model_size_3c3h,
+                                maximum=max_model_size_3c3h,
+                                value=max_model_size_3c3h,
+                                step=1,
+                                label="Maximum Model Size",
+                                interactive=True
+                            )
+                        
+                        leaderboard_3c3h = gr.Dataframe(
+                            df_3c3h[['Rank', 'Model Name', '3C3H Score', 'Correctness', 'Completeness',
+                                   'Conciseness', 'Helpfulness', 'Honesty', 'Harmlessness']],
+                            interactive=False
+                        )
+                        
+                        def filter_df_3c3h(search_query, selected_cols, precision_filters, license_filters, min_size, max_size):
+                            filtered_df = df_3c3h.copy()
+                            
+                            # Ensure min_size <= max_size
+                            if min_size > max_size:
+                                min_size, max_size = max_size, min_size
+                            
+                            # Apply search filter
+                            if search_query:
+                                filtered_df = filtered_df[filtered_df['Model Name'].str.contains(search_query, case=False, na=False)]
+                            
+                            # Apply Precision filter
+                            if precision_filters:
+                                include_missing = 'Missing' in precision_filters
+                                selected_precisions = [p for p in precision_filters if p != 'Missing']
+                                if include_missing:
+                                    filtered_df = filtered_df[
+                                        (filtered_df['Precision'].isin(selected_precisions)) |
+                                        (filtered_df['Precision'] == 'UNK') |
+                                        (filtered_df['Precision'].isna())
+                                    ]
+                                else:
+                                    filtered_df = filtered_df[filtered_df['Precision'].isin(selected_precisions)]
+                            
+                            # Apply License filter
+                            if license_filters:
+                                include_missing = 'Missing' in license_filters
+                                selected_licenses = [l for l in license_filters if l != 'Missing']
+                                if include_missing:
+                                    filtered_df = filtered_df[
+                                        (filtered_df['License'].isin(selected_licenses)) |
+                                        (filtered_df['License'] == 'UNK') |
+                                        (filtered_df['License'].isna())
+                                    ]
+                                else:
+                                    filtered_df = filtered_df[filtered_df['License'].isin(selected_licenses)]
+                            
+                            # Apply Model Size filter
+                            filtered_df = filtered_df[
+                                (filtered_df['Model Size Filter'] >= min_size) &
+                                (filtered_df['Model Size Filter'] <= max_size)
+                            ]
+                            
+                            # Remove existing 'Rank' column if present
+                            if 'Rank' in filtered_df.columns:
+                                filtered_df = filtered_df.drop(columns=['Rank'])
+                            
+                            # Recalculate Rank after filtering
+                            filtered_df = filtered_df.reset_index(drop=True)
+                            filtered_df.insert(0, 'Rank', range(1, len(filtered_df) + 1))
+                            
+                            # Ensure selected columns are present
+                            selected_cols = [col for col in selected_cols if col in filtered_df.columns]
+                            
+                            return filtered_df[selected_cols]
+                        
+                        # Bind the filter function to the appropriate events
+                        filter_inputs_3c3h = [
+                            search_box_retrieval,
+                            precision_filter_retrieval,
+                            license_filter_retrieval,
+                            model_size_min_filter_3c3h,
+                            model_size_max_filter_3c3h
+                        ]
+                        search_box_retrieval.submit(
+                            filter_df_3c3h,
+                            inputs=filter_inputs_3c3h,
+                            outputs=leaderboard_3c3h
+                        )
+                        
+                        # Bind change events for CheckboxGroups and sliders
+                        for component in filter_inputs_3c3h:
+                            component.change(
+                                filter_df_3c3h,
+                                inputs=filter_inputs_3c3h,
+                                outputs=leaderboard_3c3h
+                            )
+                    
+                    with gr.Tab("Submit Retriever"): 
+
+                        model_name_input = gr.Textbox(
+                            label="Model", 
+                            placeholder="Enter the full model name from HuggingFace Hub (e.g., inceptionai/jais-family-30b-8k)"
+                        )
+                        revision_input = gr.Textbox(
+                            label="Revision", 
+                            placeholder="main", 
+                            value="main"
+                        )
+                        precision_input = gr.Dropdown(
+                            choices=["float16", "float32", "bfloat16", "8bit", "4bit"], 
+                            label="Precision",
+                            value="float16"
+                        )
+                        params_input = gr.Textbox(
+                            label="Params", 
+                            placeholder="Enter the approximate number of parameters as Integer (e.g., 7, 13, 30, 70 ...)"
+                        )
+                        # Changed from Dropdown to Textbox with default value "Open"
+                        license_input = gr.Textbox(
+                            label="License", 
+                            placeholder="Enter the license type (Generic one is 'Open' in case no License is provided)", 
+                            value="Open"
+                        )
+                        submit_button = gr.Button("Submit Model")
+                        submission_result = gr.Markdown()
+
+                        submit_button.click(
+                            submit_model,
+                            inputs=[model_name_input, revision_input, precision_input, params_input, license_input],
+                            outputs=submission_result
+                        )
+                        
+                        # Load pending, finished, and failed requests
+                        df_pending = load_requests('pending')
+                        df_finished = load_requests('finished')
+                        df_failed = load_requests('failed')
 
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
+                        # Display the tables
+                        gr.Markdown("## Evaluation Status")
+                        with gr.Accordion(f"Pending Evaluations ({len(df_pending)})", open=False):
+                            if not df_pending.empty:
+                                gr.Dataframe(df_pending)
+                            else:
+                                gr.Markdown("No pending evaluations.")
+                        with gr.Accordion(f"Finished Evaluations ({len(df_finished)})", open=False):
+                            if not df_finished.empty:
+                                gr.Dataframe(df_finished)
+                            else:
+                                gr.Markdown("No finished evaluations.")
+                        with gr.Accordion(f"Failed Evaluations ({len(df_failed)})", open=False):
+                            if not df_failed.empty:
+                                gr.Dataframe(df_failed)
+                            else:
+                                gr.Markdown("No failed evaluations.")
+
+            with gr.Tab("Reranking"):
+                with gr.Tabs():
+                    with gr.Tab("Leaderboard"):
+                        
                         with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
+                            search_box_tasks = gr.Textbox(
+                                placeholder="Search for models...", 
+                                label="Search", 
+                                interactive=True
+                            )
+                        with gr.Row():
+                            column_selector_tasks = gr.CheckboxGroup(
+                                choices=column_choices_tasks,
+                                value=['Rank', 'Model Name'] + task_columns,
+                                label="Select columns to display",
+                            )
+                        with gr.Row():
+                            license_filter_tasks = gr.CheckboxGroup(
+                                choices=license_options_tasks,
+                                value=license_options_tasks.copy(),  # Default all selected
+                                label="Filter by License",
+                            )
+                            precision_filter_tasks = gr.CheckboxGroup(
+                                choices=precision_options_tasks,
+                                value=precision_options_tasks.copy(),  # Default all selected
+                                label="Filter by Precision",
+                            )
+                        with gr.Row():
+                            model_size_min_filter_tasks = gr.Slider(
+                                minimum=min_model_size_tasks,
+                                maximum=max_model_size_tasks,
+                                value=min_model_size_tasks,
+                                step=1,
+                                label="Minimum Model Size",
+                                interactive=True
+                            )
+                            model_size_max_filter_tasks = gr.Slider(
+                                minimum=min_model_size_tasks,
+                                maximum=max_model_size_tasks,
+                                value=max_model_size_tasks,
+                                step=1,
+                                label="Maximum Model Size",
+                                interactive=True
+                            )
+                        
+                        leaderboard_tasks = gr.Dataframe(
+                            df_tasks[['Rank', 'Model Name'] + task_columns],
+                            interactive=False
+                        )
+                        
+                        def filter_df_tasks(search_query, selected_cols, precision_filters, license_filters, min_size, max_size):
+                            filtered_df = df_tasks.copy()
+                            
+                            # Ensure min_size <= max_size
+                            if min_size > max_size:
+                                min_size, max_size = max_size, min_size
+                            
+                            # Apply search filter
+                            if search_query:
+                                filtered_df = filtered_df[filtered_df['Model Name'].str.contains(search_query, case=False, na=False)]
+                            
+                            # Apply Precision filter
+                            if precision_filters:
+                                include_missing = 'Missing' in precision_filters
+                                selected_precisions = [p for p in precision_filters if p != 'Missing']
+                                if include_missing:
+                                    filtered_df = filtered_df[
+                                        (filtered_df['Precision'].isin(selected_precisions)) |
+                                        (filtered_df['Precision'] == 'UNK') |
+                                        (filtered_df['Precision'].isna())
+                                    ]
+                                else:
+                                    filtered_df = filtered_df[filtered_df['Precision'].isin(selected_precisions)]
+                            
+                            # Apply License filter
+                            if license_filters:
+                                include_missing = 'Missing' in license_filters
+                                selected_licenses = [l for l in license_filters if l != 'Missing']
+                                if include_missing:
+                                    filtered_df = filtered_df[
+                                        (filtered_df['License'].isin(selected_licenses)) |
+                                        (filtered_df['License'] == 'UNK') |
+                                        (filtered_df['License'].isna())
+                                    ]
+                                else:
+                                    filtered_df = filtered_df[filtered_df['License'].isin(selected_licenses)]
+                            
+                            # Apply Model Size filter
+                            filtered_df = filtered_df[
+                                (filtered_df['Model Size Filter'] >= min_size) &
+                                (filtered_df['Model Size Filter'] <= max_size)
+                            ]
+                            
+                            # Remove existing 'Rank' column if present
+                            if 'Rank' in filtered_df.columns:
+                                filtered_df = filtered_df.drop(columns=['Rank'])
+                            
+                            # Sort by the first task column if it exists
+                            if task_columns:
+                                first_task = task_columns[0]
+                                filtered_df = filtered_df.sort_values(by=first_task, ascending=False)
+                            else:
+                                filtered_df = filtered_df.sort_values(by='Model Name', ascending=True)
+                            
+                            # Recalculate Rank after filtering
+                            filtered_df = filtered_df.reset_index(drop=True)
+                            filtered_df.insert(0, 'Rank', range(1, len(filtered_df) + 1))
+                            
+                            # Ensure selected columns are present
+                            selected_cols = [col for col in selected_cols if col in filtered_df.columns]
+                            
+                            return filtered_df[selected_cols]
+                        
+                        # Bind the filter function to the appropriate events
+                        filter_inputs_tasks = [
+                            search_box_tasks,
+                            column_selector_tasks,
+                            precision_filter_tasks,
+                            license_filter_tasks,
+                            model_size_min_filter_tasks,
+                            model_size_max_filter_tasks
+                        ]
+                        search_box_tasks.submit(
+                            filter_df_tasks,
+                            inputs=filter_inputs_tasks,
+                            outputs=leaderboard_tasks
+                        )
+                        
+                        # Bind change events for CheckboxGroups and sliders
+                        for component in filter_inputs_tasks:
+                            component.change(
+                                filter_df_tasks,
+                                inputs=filter_inputs_tasks,
+                                outputs=leaderboard_tasks
                             )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
+                    with gr.Tab("Submit Reranker"):
+                        pass
 
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
+            with gr.Tab("LLM Context Answering"):
+                with gr.Tabs():
+                    with gr.Tab("Leaderboard"):
+                        pass
+                    with gr.Tab("Submit Here"):
+                        pass
+
+            with gr.Row():
+                with gr.Accordion("📙 Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        lines=20,
+                        elem_id="citation-button",
+                        show_copy_button=True,
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
 
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
+    demo.launch()
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -1,13 +0,0 @@
-[tool.ruff]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-
-[tool.isort]
-profile = "black"
-line_length = 119
-
-[tool.black]
-line-length = 119

requirements.txt

@@ -1,16 +0,0 @@
-APScheduler
-black
-datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.13
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece

src/about.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">The Arabic RAG Leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/display/css_html_js.py

@@ -1,105 +0,0 @@
-custom_css = """
-
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
- 
-#search-bar {
-    padding: 0px;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-#leaderboard-table td:nth-child(2),
-#leaderboard-table th:nth-child(2) {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
-#filter_type{
-    border: 0;
-    padding-left: 0;
-    padding-top: 0;
-}
-#filter_type label {
-    display: flex;
-}
-#filter_type label > span{
-    margin-top: var(--spacing-lg);
-    margin-right: 0.5em;
-}
-#filter_type label > .wrap{
-    width: 103px;
-}
-#filter_type label > .wrap .wrap-inner{  
-    padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input{
-    width: 1px
-}
-#filter-columns-type{
-    border:0;
-    padding:0.5;
-}
-#filter-columns-size{
-    border:0;
-    padding:0.5;
-}
-#box-filter > .form{
-    border: 0
-}
-"""
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,110 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/envs.py

@@ -1,25 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
-
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,196 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/populate.py

@@ -1,58 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )