Submit error handling

src/submission/submit.py

@@ -29,176 +29,179 @@ def add_new_eval(
     hf_model_id: str,
     contact_email: str
 ):
+    try:
+        with open(upload, mode="r") as f:
+            data = json.load(f)    
+        
+        results = data['results']
+
+        acc_keys = ['exact_match,none', 'exact_match,flexible-extract', 'exact_match,strict-match']
+
+        ret = {
+            'eval_name': eval_name,
+            'precision': precision,
+            'hf_model_id': hf_model_id,
+            'contact_email': contact_email
+            }
+
+        for k, v in results.items():
+            for acc_k in acc_keys:
+                if acc_k in v and k in BENCHMARK_COLS:
+                    ret[k] = v[acc_k]    
+        #validation
+        for k,v in ret.items():
+            if k in ['eval_name', 'precision', 'hf_model_id', 'contact_email']:
+                continue
+            if k not in BENCHMARK_COLS:
+                print(f"Missing: {k}")
+                return styled_error(f'Missing: {k}')
+
+        if len(BENCHMARK_COLS) != len(ret) - 4:
+            print(f"Missing columns")
+            return styled_error(f'Missing columns')
+
+        # TODO add complex validation
+        #print(results.keys())
+        #print(BENCHMARK_COLS)
+        #for input_col in results.keys():
+        #    if input_col not in BENCHMARK_COLS:
+        #        print(input_col)
+        #        return styled_error(f'Missing: {input_col}')
+        #ret.update({i:j['acc,none'] for i,j in results.items()})
+        # fake data for testing...
+        #ret.update({i:round(np.random.normal(1, 0.5, 1)[0], 2) for i,j in results.items()})
+        
+        user_name = "czechbench_leaderboard"
+        OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+
+        existing_eval_names = []
+        for fname in glob(f"{OUT_DIR}/*.json"):
+            with open(fname, mode="r") as f:
+                existing_eval = json.load(f)
+            existing_eval_names.append(existing_eval['eval_name'])
+
+        if ret['eval_name'] in existing_eval_names:
+            print(f"Model name {ret['eval_name']} is used!")
+            return styled_error(f"Model name {ret['eval_name']} is used!")
+
+        out_path = f"{OUT_DIR}/{eval_name}_eval_request.json"
+
+        with open(out_path, "w") as f:
+            f.write(json.dumps(ret))
+
+
+        print("Uploading eval file")
+
+        print("path_or_fileobj: ", out_path)
+        print("path_in_repo: ",out_path.split("eval-queue/")[1])
+        print("repo_id: ", RESULTS_REPO)
+        print("repo_type: ", "dataset")
+
+        response = API.upload_file(
+            path_or_fileobj=out_path,
+            path_in_repo=out_path.split("eval-queue/")[1],
+            repo_id=RESULTS_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {eval_name} to eval queue",
+        )
 
-    with open(upload, mode="r") as f:
-        data = json.load(f)    
-    
-    results = data['results']
-
-    acc_keys = ['exact_match,none', 'exact_match,flexible-extract', 'exact_match,strict-match']
-
-    ret = {
-        'eval_name': eval_name,
-        'precision': precision,
-        'hf_model_id': hf_model_id,
-        'contact_email': contact_email
+        """
+        global REQUESTED_MODELS
+        global USERS_TO_SUBMISSION_DATES
+        if not REQUESTED_MODELS:
+            REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+        user_name = ""
+        model_path = model
+        if "/" in model:
+            user_name = model.split("/")[0]
+            model_path = model.split("/")[1]
+
+        precision = precision.split(" ")[0]
+        current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        if model_type is None or model_type == "":
+            return styled_error("Please select a model type.")
+
+        # Does the model actually exist?
+        if revision == "":
+            revision = "main"
+
+        # Is the model on the hub?
+        if weight_type in ["Delta", "Adapter"]:
+            base_model_on_hub, error, _ = is_model_on_hub(
+                model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True
+            )
+            if not base_model_on_hub:
+                return styled_error(f'Base model "{base_model}" {error}')
+
+        if not weight_type == "Adapter":
+            model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, test_tokenizer=True)
+            if not model_on_hub:
+                return styled_error(f'Model "{model}" {error}')
+
+        # Is the model info correctly filled?
+        try:
+            model_info = API.model_info(repo_id=model, revision=revision)
+        except Exception:
+            return styled_error("Could not get your model information. Please fill it up properly.")
+
+        model_size = get_model_size(model_info=model_info, precision=precision)
+
+        # Were the model card and license filled?
+        try:
+            license = model_info.cardData["license"]
+        except Exception:
+            return styled_error("Please select a license for your model")
+
+        modelcard_OK, error_msg = check_model_card(model)
+        if not modelcard_OK:
+            return styled_error(error_msg)
+
+        # Seems good, creating the eval
+        print("Adding new eval")
+
+        eval_entry = {
+            "model": model,
+            "base_model": base_model,
+            "revision": revision,
+            "precision": precision,
+            "weight_type": weight_type,
+            "status": "PENDING",
+            "submitted_time": current_time,
+            "model_type": model_type,
+            "likes": model_info.likes,
+            "params": model_size,
+            "license": license,
         }
 
-    for k, v in results.items():
-        for acc_k in acc_keys:
-            if acc_k in v and k in BENCHMARK_COLS:
-                ret[k] = v[acc_k]    
-    #validation
-    for k,v in ret.items():
-        if k in ['eval_name', 'precision', 'hf_model_id', 'contact_email']:
-            continue
-        if k not in BENCHMARK_COLS:
-            print(f"Missing: {k}")
-            return styled_error(f'Missing: {k}')
-
-    if len(BENCHMARK_COLS) != len(ret) - 4:
-        print(f"Missing columns")
-        return styled_error(f'Missing columns')
-
-    # TODO add complex validation
-    #print(results.keys())
-    #print(BENCHMARK_COLS)
-    #for input_col in results.keys():
-    #    if input_col not in BENCHMARK_COLS:
-    #        print(input_col)
-    #        return styled_error(f'Missing: {input_col}')
-    #ret.update({i:j['acc,none'] for i,j in results.items()})
-    # fake data for testing...
-    #ret.update({i:round(np.random.normal(1, 0.5, 1)[0], 2) for i,j in results.items()})
-    
-    user_name = "czechbench_leaderboard"
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-
-    existing_eval_names = []
-    for fname in glob(f"{OUT_DIR}/*.json"):
-        with open(fname, mode="r") as f:
-            existing_eval = json.load(f)
-        existing_eval_names.append(existing_eval['eval_name'])
-
-    if ret['eval_name'] in existing_eval_names:
-        print(f"Model name {ret['eval_name']} is used!")
-        return styled_error(f"Model name {ret['eval_name']} is used!")
-
-    out_path = f"{OUT_DIR}/{eval_name}_eval_request.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(ret))
-
-
-    print("Uploading eval file")
-
-    print("path_or_fileobj: ", out_path)
-    print("path_in_repo: ",out_path.split("eval-queue/")[1])
-    print("repo_id: ", RESULTS_REPO)
-    print("repo_type: ", "dataset")
-
-    response = API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=RESULTS_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {eval_name} to eval queue",
-    )
-
-    """
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(
-            model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True
+        # Check for duplicate submission
+        if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+            return styled_warning("This model has been already submitted.")
+
+        print("Creating eval file")
+        OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+        os.makedirs(OUT_DIR, exist_ok=True)
+        out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+
+        with open(out_path, "w") as f:
+            f.write(json.dumps(eval_entry))
+
+        print("Uploading eval file")
+        API.upload_file(
+            path_or_fileobj=out_path,
+            path_in_repo=out_path.split("eval-queue/")[1],
+            repo_id=QUEUE_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {model} to eval queue",
         )
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
 
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-    """
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    ), "", "", "", ""
+        # Remove the local file
+        os.remove(out_path)
+        """
+        return styled_message(
+            "Your results have been successfully submitted. They will be added to the leaderboard upon verification."
+        ), "", "", "", ""
+    
+    except Exception as e:
+        return styled_error(f"An error occurred: {e}"), "", "", "", ""