Add handling for failed model evaluations in submission process

utils.py

@@ -72,6 +72,7 @@ def submit_model(model_name, revision, precision, params, license, task, pending
     # Load pending and finished requests from the dataset repository
     df_pending = load_requests('pending')
     df_finished = load_requests('finished')
+    df_failed = load_requests('failed')
 
     # Check if model is in pending requests
     if not df_pending.empty:
@@ -91,14 +92,24 @@ def submit_model(model_name, revision, precision, params, license, task, pending
                                     (existing_models_finished['precision'] == precision.capitalize()) &
                                     (existing_models_finished['task'] == task)).any()
         if model_exists_in_finished:
-            return f"Model {model_name} has already been evaluated as a {task} 🎉"
+            return f"Model {model_name} has already been evaluated as a {task} 🎉", pending_gradio_df
+        
+    # Check if model is in failed requests
+    if not df_failed.empty:
+        existing_models_failed = df_failed[['model_name', 'revision', 'precision', 'task']]
+        model_exists_in_failed = ((existing_models_failed['model_name'] == model_name) &
+                                  (existing_models_failed['revision'] == revision) &
+                                  (existing_models_failed['precision'] == precision.capitalize()) &
+                                  (existing_models_failed['task'] == task)).any()
+        if model_exists_in_failed:
+            return f"Model {model_name} has previously failed evaluation as a {task} ❌", pending_gradio_df
 
     # Check if model exists on HuggingFace Hub
     try:
         api.model_info(model_name)
     except Exception as e:
         print(f"Error fetching model info: {e}")
-        return f"Model {model_name} not found on HuggingFace Hub 🤷‍♂️"
+        return f"Model {model_name} not found on HuggingFace Hub 🤷‍♂️", pending_gradio_df
 
     # Proceed with submission
     status = "PENDING"
@@ -120,7 +131,7 @@ def submit_model(model_name, revision, precision, params, license, task, pending
     # Define the file path in the repository
     org_model = model_name.split('/')
     if len(org_model) != 2:
-        return "Please enter the full model name including the organization or username, e.g., 'intfloat/multilingual-e5-large-instruct' 🤷‍♂️"
+        return "Please enter the full model name including the organization or username, e.g., 'intfloat/multilingual-e5-large-instruct' 🤷‍♂️", pending_gradio_df
     org, model_id = org_model
     precision_str = precision if precision else 'Missing'
     file_path_in_repo = f"pending/{org}/{model_id}_eval_request_{revision}_{precision_str}_{task.lower()}.json"
@@ -136,9 +147,22 @@ def submit_model(model_name, revision, precision, params, license, task, pending
         )
     except Exception as e:
         print(f"Error uploading file: {e}")
-        return f"Error: Could not submit model '{model_name}' for evaluation."
+        return f"Error: Could not submit model '{model_name}' for evaluation.", pending_gradio_df
 
-    return f"Model {model_name} has been submitted successfully as a {task} 🚀"
+    # Update the pending requests DataFrame
+    pending_gradio_df = pending_gradio_df._append(
+        {
+            "model_name": model_name,
+            "license": license,
+            "revision": revision,
+            "precision": precision.capitalize(),
+            "status": status,
+            "params": params,
+            "task": task
+        },
+        ignore_index=True
+    )
+    return f"Model {model_name} has been submitted successfully as a {task} 🚀", pending_gradio_df
 
 def load_requests(status_folder, task_type=None):
     api = HfApi()
@@ -252,8 +276,7 @@ def submit_gradio_module(task_type):
         if not df_pending.empty:
             pending_gradio_df = gr.Dataframe(df_pending)
         else:
-            gr.Markdown("No pending evaluations.")
-            pending_gradio_df = None
+            pending_gradio_df = gr.Markdown("No pending evaluations.")
     with gr.Accordion(f"Finished Evaluations ({len(df_finished)})", open=False):
         if not df_finished.empty:
             gr.Dataframe(df_finished)