Update requirements and refactor model submission logic to improve error handling and data loading

requirements.txt

@@ -1,3 +1,3 @@
 fuzzywuzzy
-Levenshtein
+python-Levenshtein
 python-dotenv

utils.py

@@ -3,7 +3,8 @@ import pandas as pd
 import json
 import os
 from pathlib import Path
-from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub import HfApi
+from datasets import load_dataset
 
 api = HfApi()
 
@@ -48,7 +49,7 @@ def get_model_info(model_id, verbose=False):
         return num_downloads, num_likes, license, num_parameters, supported_precisions
     except Exception as e:
         print(f"Error: Could not fetch model information. {str(e)}")
-        return 0, 0, "Unknown", 0, []
+        return 0, 0, "Unknown", 0, ["Missing"]
 
 def fetch_model_information(model_name):
     try:
@@ -60,61 +61,65 @@ def fetch_model_information(model_name):
         return
     return gr.update(choices=supported_precisions, value=supported_precisions[0]), license, num_parameters, num_downloads, num_likes
 
-def submit_model(model_name, revision, precision, params, license, task, pending_gradio_df):
+def load_requests(status_folder, task_type=None):
+    # Load the dataset from the HuggingFace Hub
+    ds = load_dataset(DATASET_REPO_ID, split="test")
+    df = ds.to_pandas()
+
+    # Filter the dataframe based on the status folder and task type
+    df = df[df['status'] == status_folder.upper()]
+    df = df[df['task'] == task_type] if task_type else df
+    df.drop(columns=['status', 'task'], inplace=True)
+
+    return df
 
-    if float(params) == 0:
-        return "Model size cannot be zero. If the Auto Fetch feature failed and your model is suitable for evaluation, please open a community discussion for help.", pending_gradio_df
+def submit_model(model_name, revision, precision, params, license, task):
+    # Load pending and finished requests from the dataset repository
+    df_pending = load_requests('pending', task_type=task)
+    df_finished = load_requests('finished', task_type=task)
+    df_failed = load_requests('failed', task_type=task)
+
+    # Check if Auto Fetch feature couldn't fetch model info
+    if float(params) == 0 and precision == 'Missing':
+        return "I think the auto-fetch feature couldn't fetch model info. If your model is not suitable for this task evaluation then this is expected, but if it's suitable and this behavior happened with you then please open a community discussion in the leaderboard discussion section and we will fix it ASAP.", df_pending
     
+    # Check if model size is in valid range
     if float(params) > 5000:
-        return "Model size should be less than 5000 million parameters (5 billion) 👀", pending_gradio_df
+        return "Model size should be less than 5000 million parameters (5 billion) 👀", df_pending
 
     # Handle 'Missing' precision
     if precision == 'Missing':
         precision = None
     else:
         precision = precision.strip().lower()
-
-    # Load pending and finished requests from the dataset repository
-    df_pending = load_requests('pending')
-    df_finished = load_requests('finished')
-    df_failed = load_requests('failed')
-
-    # Check if model is in pending requests
-    if not df_pending.empty:
-        existing_models_pending = df_pending[['model_name', 'revision', 'precision', 'task']]
-        model_exists_in_pending = ((existing_models_pending['model_name'] == model_name) &
-                                   (existing_models_pending['revision'] == revision) &
-                                   (existing_models_pending['precision'] == precision.capitalize()) &
-                                      (existing_models_pending['task'] == task)).any()
-        if model_exists_in_pending:
-            return f"Model {model_name} is already in the evaluation queue as a {task} 🚀", pending_gradio_df
+    
+    # Helper function to check if model exists in a dataframe
+    def model_exists_in_df(df):
+
+        if df.empty:
+            return False
+        return ((df['model_name'] == model_name) & 
+                (df['revision'] == revision) & 
+                (df['precision'] == precision)).any()
+    
+    # Check if model is already in pending requests
+    if model_exists_in_df(df_pending):
+        return f"Model {model_name} is already in the evaluation queue as a {task} 👍", df_pending
 
     # Check if model is in finished requests
-    if not df_finished.empty:
-        existing_models_finished = df_finished[['model_name', 'revision', 'precision', 'task']]
-        model_exists_in_finished = ((existing_models_finished['model_name'] == model_name) &
-                                    (existing_models_finished['revision'] == revision) &
-                                    (existing_models_finished['precision'] == precision.capitalize()) &
-                                    (existing_models_finished['task'] == task)).any()
-        if model_exists_in_finished:
-            return f"Model {model_name} has already been evaluated as a {task} 🎉", pending_gradio_df
+    if model_exists_in_df(df_finished):
+        return f"Model {model_name} has already been evaluated as a {task} 🎉", df_pending
         
     # Check if model is in failed requests
-    if not df_failed.empty:
-        existing_models_failed = df_failed[['model_name', 'revision', 'precision', 'task']]
-        model_exists_in_failed = ((existing_models_failed['model_name'] == model_name) &
-                                  (existing_models_failed['revision'] == revision) &
-                                  (existing_models_failed['precision'] == precision.capitalize()) &
-                                  (existing_models_failed['task'] == task)).any()
-        if model_exists_in_failed:
-            return f"Model {model_name} has previously failed evaluation as a {task} ❌", pending_gradio_df
+    if model_exists_in_df(df_failed):
+        return f"Model {model_name} has previously failed evaluation as a {task} ❌", df_pending
 
     # Check if model exists on HuggingFace Hub
     try:
         api.model_info(model_name)
     except Exception as e:
         print(f"Error fetching model info: {e}")
-        return f"Model {model_name} not found on HuggingFace Hub 🤷‍♂️", pending_gradio_df
+        return f"Model {model_name} not found on HuggingFace Hub 🤷‍♂️", df_pending
 
     # Proceed with submission
     status = "PENDING"
@@ -136,7 +141,7 @@ def submit_model(model_name, revision, precision, params, license, task, pending
     # Define the file path in the repository
     org_model = model_name.split('/')
     if len(org_model) != 2:
-        return "Please enter the full model name including the organization or username, e.g., 'intfloat/multilingual-e5-large-instruct' 🤷‍♂️", pending_gradio_df
+        return "Please enter the full model name including the organization or username, e.g., 'intfloat/multilingual-e5-large-instruct' 🤷‍♂️", df_pending
     org, model_id = org_model
     precision_str = precision if precision else 'Missing'
     file_path_in_repo = f"pending/{org}/{model_id}_eval_request_{revision}_{precision_str}_{task.lower()}.json"
@@ -152,71 +157,10 @@ def submit_model(model_name, revision, precision, params, license, task, pending
         )
     except Exception as e:
         print(f"Error uploading file: {e}")
-        return f"Error: Could not submit model '{model_name}' for evaluation.", pending_gradio_df
+        return f"Error: Could not submit model '{model_name}' for evaluation.", df_pending
 
-    # Update the pending requests DataFrame
-    if isinstance(pending_gradio_df, str):
-        pending_gradio_df = pd.DataFrame(columns=["model_name", "license", "revision", "precision", "status", "params", "task"])
-    elif isinstance(pending_gradio_df, dict):
-        pending_gradio_df = pd.DataFrame(pending_gradio_df)
-
-    pending_gradio_df = pending_gradio_df._append(
-        {
-            "model_name": model_name,
-            "license": license,
-            "revision": revision,
-            "precision": precision.capitalize(),
-            "status": status,
-            "params": params,
-            "task": task
-        },
-        ignore_index=True
-    )
-    return f"Model {model_name} has been submitted successfully as a {task} 🚀", pending_gradio_df
-
-def load_requests(status_folder, task_type=None):
-    api = HfApi()
-    requests_data = []
-    folder_path_in_repo = status_folder  # 'pending', 'finished', or 'failed'
-
-    try:
-        # Use the cached token
-        files_info = api.list_repo_files(
-            repo_id=DATASET_REPO_ID,
-            repo_type="dataset",
-            token=HF_TOKEN
-        )
-    except Exception as e:
-        print(f"Error accessing dataset repository: {e}")
-        return pd.DataFrame()  # Return empty DataFrame if repository not found or inaccessible
-
-    # Filter files in the desired folder
-    files_in_folder = [f for f in files_info if f.startswith(f"{folder_path_in_repo}/") and f.endswith('.json')]
-
-    for file_path in files_in_folder:
-        try:
-            # Download the JSON file
-            local_file_path = hf_hub_download(
-                repo_id=DATASET_REPO_ID,
-                filename=file_path,
-                repo_type="dataset",
-                token=HF_TOKEN
-            )
-            # Load JSON data
-            with open(local_file_path, 'r') as f:
-                request = json.load(f)
-            requests_data.append(request)
-        except Exception as e:
-            print(f"Error loading file {file_path}: {e}")
-            continue  # Skip files that can't be loaded
-
-    df = pd.DataFrame(requests_data)
-
-    # Filter by task type
-    if task_type and not df.empty:
-        df = df[df['task'] == task_type]
-
-    return df
+    df_pending = load_requests('pending', task_type=task)
+    return f"Model {model_name} has been submitted successfully as a {task} 🚀", df_pending
 
 
 def submit_gradio_module(task_type):
@@ -282,11 +226,8 @@ def submit_gradio_module(task_type):
 
     # Display the tables
     gr.Markdown("## Evaluation Status")
-    with gr.Accordion(f"Pending Evaluations ({len(df_pending)})", open=False):
-        if not df_pending.empty:
-            pending_gradio_df = gr.Dataframe(df_pending)
-        else:
-            pending_gradio_df = gr.Markdown("No pending evaluations.")
+    with gr.Accordion(f"Pending Evaluations ({len(df_pending)})", open=True):
+        pending_gradio_df = gr.Dataframe(df_pending)
     with gr.Accordion(f"Finished Evaluations ({len(df_finished)})", open=False):
         if not df_finished.empty:
             gr.Dataframe(df_finished)
@@ -300,6 +241,6 @@ def submit_gradio_module(task_type):
 
     submit_button.click(
         submit_model,
-        inputs=[model_name_input, revision_input, precision_input, params_input, license_input, var, pending_gradio_df],
+        inputs=[model_name_input, revision_input, precision_input, params_input, license_input, var],
         outputs=[submission_result, pending_gradio_df],
     )