[ADD] Submission of private models

app.py

@@ -50,7 +50,7 @@ from src.display.utils import (
     fields,
     render_generation_templates
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN, PRIVATE_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval, PLACEHOLDER_DATASET_WISE_NORMALIZATION_CONFIG
 

src/about.py

@@ -101,7 +101,7 @@ NUM_FEWSHOT = 0  # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """""" #<h1 align="center" id="space-title"> NER Leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title" style="color: red;"> [DEV Mode] </h1>"""
 # LOGO = """<img src="https://equalengineers.com/wp-content/uploads/2024/04/dummy-logo-5b.png" alt="Clinical X HF" width="500" height="333">"""
 LOGO = """<img src="https://huggingface.co/spaces/m42-health/MEDIC-Benchmark/resolve/main/assets/logo_medic_4.png" alt="Clinical X HF" width="40%" style="display: block; margin-left: auto; margin-right: auto;">"""
 FIVE_PILLAR_DIAGRAM = """<img src="https://huggingface.co/spaces/m42-health/MEDIC-Benchmark/resolve/main/assets/MEDIC_Diagram.jpg" alt="MEDIC Diagram" width="52%" style="display: block; margin-left: auto; margin-right: auto;">"""

src/populate.py

@@ -46,7 +46,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             file_path = os.path.join(save_path, entry)
             with open(file_path) as fp:
                 data = json.load(fp)
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"])
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"]) if not data["private"] else data["model_name"]
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
             # changes to be made here
             data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
@@ -63,7 +63,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 with open(file_path) as fp:
                     data = json.load(fp)
                 # print(data)
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"])
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"]) if not data["private"] else data["model_name"]
                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
                 data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
                 data[EvalQueueColumn.open_ended_status.name] = data["status"]["open-ended"]

src/submission/submit.py

@@ -4,7 +4,7 @@ import ast
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO, PRIVATE_REPO
 from src.submission.check_validity import (
     already_submitted_models,
     check_model_card,
@@ -61,9 +61,11 @@ def add_new_eval(
     """
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
+    if not REQUESTED_MODELS and not PRIVATE_REPO:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
     if model.startswith("/"):
+        if not PRIVATE_REPO:
+            return styled_error("Private models are not allowed to be submitted to the public queue.")
         user_name = ""
         model_path = model
         private = True
@@ -88,12 +90,12 @@ def add_new_eval(
         revision = "main"
 
     # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
+    if weight_type in ["Delta", "Adapter"] and not private:
         base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
         if not base_model_on_hub:
             return styled_error(f'Base model "{base_model}" {error}')
 
-    if not weight_type == "Adapter":
+    if not weight_type == "Adapter" and not private:
         model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
@@ -101,7 +103,7 @@ def add_new_eval(
 
     # Is the model info correctly filled?
     try:
-        if model.startswith("/"):
+        if not private:
             model_info = API.model_info(repo_id=model, revision=revision)
             model_size = get_model_size(model_info=model_info)
             license = model_info.cardData["license"]
@@ -112,7 +114,7 @@ def add_new_eval(
         else:
             model_size = None
             license = None
-            likes = 0
+            likes = -1
     except Exception:
         return styled_error("Could not get your model information. Please fill it up properly.")
 
@@ -158,16 +160,19 @@ def add_new_eval(
 
     # Check for duplicate submission
 
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+    if not PRIVATE_REPO and f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted. Add the revision if the model has been updated.")
 
     print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    if not private:
+        OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+        
+    else:
+        OUT_DIR = f"{EVAL_REQUESTS_PATH}/offline"
+        model_path = model_path[1:] if model_path.startswith("/") else model_path
+        model_path = model_path.replace("/", "+-+")
+        out_path = f"{OUT_DIR}/{model_path}_{revision}_{precision}_{weight_type}_eval_request.json"
     os.makedirs(OUT_DIR, exist_ok=True)
-    if model_path.startswith("/"):
-        os.makedirs(f"{OUT_DIR}/{model_path}", exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_{revision}_{precision}_{weight_type}_eval_request.json"
-
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))