update scripts

app.py

@@ -1,9 +1,13 @@
+import logging
+
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 import src.envs as envs
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO
+from main_backend import PENDING_STATUS, RUNNING_STATUS, FINISHED_STATUS, FAILED_STATUS
+from src.backend import sort_queue
+from src.envs import EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, RESULTS_REPO
 import src.backend.manage_requests as manage_requests
 
 import socket
@@ -16,6 +20,9 @@ import src.submission.submit as submit
 import os
 import datetime
 import spacy_transformers
+import pprint
+
+pp = pprint.PrettyPrinter(width=80)
 
 TOKEN = os.environ.get("H4_TOKEN", None)
 print("TOKEN", TOKEN)
@@ -47,22 +54,70 @@ original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_d
 leaderboard_df = original_df.copy()
 
 def process_pending_evals():
-    if len(pending_eval_queue_df) == 0:
-        print("No pending evaluations found.")
+    # if len(pending_eval_queue_df) == 0:
+    #     print("No pending evaluations found.")
+    #     return
+    #
+    # for _, eval_request in pending_eval_queue_df.iterrows():
+    #     import re
+    #     model_link = eval_request['model']
+    #     match = re.search(r'>([^<]+)<', model_link)
+    #     if match:
+    #         eval_request['model'] = match.group(1)  # 赋值给 eval_request['model']
+    #     else:
+    #         eval_request['model'] = model_link  # 如果无法匹配，保留原始字符串
+    #
+    #     print(f"Evaluating model: {eval_request['model']}")
+    #
+    #     # 调用评估函数
+    #     run_eval_suite.run_evaluation(
+    #         eval_request=eval_request,
+    #         local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
+    #         results_repo=envs.RESULTS_REPO,
+    #         batch_size=1,
+    #         device=envs.DEVICE,
+    #         no_cache=True,
+    #         need_check=False,  # 根据需要设定是否需要检查
+    #         write_results=False  # 根据需要设定是否写入结果
+    #     )
+    #     print(f"Finished evaluation for model: {eval_request['model']}")
+    #     # Update the status to FINISHED
+    #     manage_requests.set_eval_request(
+    #         api=envs.API,
+    #         eval_request=eval_request,
+    #         new_status="FINISHED",
+    #         hf_repo=envs.QUEUE_REPO,
+    #         local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
+    #     )
+    current_pending_status = [PENDING_STATUS]
+    print('_________________')
+    manage_requests.check_completed_evals(
+        api=envs.API,
+        checked_status=RUNNING_STATUS,
+        completed_status=FINISHED_STATUS,
+        failed_status=FAILED_STATUS,
+        hf_repo=envs.QUEUE_REPO,
+        local_dir=envs.EVAL_REQUESTS_PATH_BACKEND,
+        hf_repo_results=envs.RESULTS_REPO,
+        local_dir_results=envs.EVAL_RESULTS_PATH_BACKEND
+    )
+    logging.info("Checked completed evals")
+    eval_requests = manage_requests.get_eval_requests(
+        job_status=current_pending_status,
+        hf_repo=envs.QUEUE_REPO,
+        local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
+    )
+    logging.info("Got eval requests")
+    eval_requests = sort_queue.sort_models_by_priority(api=envs.API, models=eval_requests)
+    logging.info("Sorted eval requests")
+
+    print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
+    if len(eval_requests) == 0:
+        print("No eval requests found. Exiting.")
         return
 
-    for _, eval_request in pending_eval_queue_df.iterrows():
-        import re
-        model_link = eval_request['model']
-        match = re.search(r'>([^<]+)<', model_link)
-        if match:
-            eval_request['model'] = match.group(1)  # 赋值给 eval_request['model']
-        else:
-            eval_request['model'] = model_link  # 如果无法匹配，保留原始字符串
-
-        print(f"Evaluating model: {eval_request['model']}")
-
-        # 调用评估函数
+    for eval_request in eval_requests:
+        pp.pprint(eval_request)
         run_eval_suite.run_evaluation(
             eval_request=eval_request,
             local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
@@ -70,21 +125,21 @@ def process_pending_evals():
             batch_size=1,
             device=envs.DEVICE,
             no_cache=True,
-            need_check=False,  # 根据需要设定是否需要检查
-            write_results=False  # 根据需要设定是否写入结果
+            need_check= False,
+            write_results= False
         )
-        print(f"Finished evaluation for model: {eval_request['model']}")
+        logging.info(f"Eval finished for model {eval_request.model}, now setting status to finished")
+
         # Update the status to FINISHED
         manage_requests.set_eval_request(
             api=envs.API,
             eval_request=eval_request,
-            new_status="FINISHED",
+            new_status=FINISHED_STATUS,
             hf_repo=envs.QUEUE_REPO,
             local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
         )
 
 
-
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame,