Clémentine
commited on
Commit
·
2edf30a
1
Parent(s):
2c89e68
removed need for tokens in the leaderboard + removed skull in flagged models
Browse files- src/auto_leaderboard/get_model_metadata.py +0 -3
- src/init.py +17 -24
src/auto_leaderboard/get_model_metadata.py
CHANGED
|
@@ -100,13 +100,10 @@ def get_model_type(leaderboard_data: List[dict]):
|
|
| 100 |
model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
|
| 101 |
|
| 102 |
def flag_models(leaderboard_data:List[dict]):
|
| 103 |
-
flag_symbol = "💀"
|
| 104 |
for model_data in leaderboard_data:
|
| 105 |
if model_data["model_name_for_query"] in FLAGGED_MODELS:
|
| 106 |
issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
|
| 107 |
issue_link = model_hyperlink(FLAGGED_MODELS[model_data["model_name_for_query"]], f"See discussion #{issue_num}")
|
| 108 |
-
|
| 109 |
-
model_data[AutoEvalColumn.model_type_symbol.name] = flag_symbol
|
| 110 |
model_data[AutoEvalColumn.model.name] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
|
| 111 |
|
| 112 |
def apply_metadata(leaderboard_data: List[dict]):
|
|
|
|
| 100 |
model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
|
| 101 |
|
| 102 |
def flag_models(leaderboard_data:List[dict]):
|
|
|
|
| 103 |
for model_data in leaderboard_data:
|
| 104 |
if model_data["model_name_for_query"] in FLAGGED_MODELS:
|
| 105 |
issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
|
| 106 |
issue_link = model_hyperlink(FLAGGED_MODELS[model_data["model_name_for_query"]], f"See discussion #{issue_num}")
|
|
|
|
|
|
|
| 107 |
model_data[AutoEvalColumn.model.name] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
|
| 108 |
|
| 109 |
def apply_metadata(leaderboard_data: List[dict]):
|
src/init.py
CHANGED
|
@@ -1,8 +1,6 @@
|
|
| 1 |
import os
|
| 2 |
from huggingface_hub import Repository
|
| 3 |
|
| 4 |
-
H4_TOKEN = os.environ.get("H4_TOKEN", None)
|
| 5 |
-
|
| 6 |
|
| 7 |
def get_all_requested_models(requested_models_dir):
|
| 8 |
depth = 1
|
|
@@ -20,28 +18,23 @@ def load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, QUEUE_PATH, RESULTS_PATH):
|
|
| 20 |
eval_results_repo = None
|
| 21 |
requested_models = None
|
| 22 |
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
eval_results_repo.git_pull()
|
| 41 |
-
|
| 42 |
-
requested_models = get_all_requested_models("eval-queue")
|
| 43 |
-
else:
|
| 44 |
-
print("No HuggingFace token provided. Skipping evaluation requests and results.")
|
| 45 |
|
| 46 |
return eval_queue_repo, requested_models, eval_results_repo
|
| 47 |
|
|
|
|
| 1 |
import os
|
| 2 |
from huggingface_hub import Repository
|
| 3 |
|
|
|
|
|
|
|
| 4 |
|
| 5 |
def get_all_requested_models(requested_models_dir):
|
| 6 |
depth = 1
|
|
|
|
| 18 |
eval_results_repo = None
|
| 19 |
requested_models = None
|
| 20 |
|
| 21 |
+
print("Pulling evaluation requests and results.")
|
| 22 |
+
|
| 23 |
+
eval_queue_repo = Repository(
|
| 24 |
+
local_dir=QUEUE_PATH,
|
| 25 |
+
clone_from=QUEUE_REPO,
|
| 26 |
+
repo_type="dataset",
|
| 27 |
+
)
|
| 28 |
+
eval_queue_repo.git_pull()
|
| 29 |
+
|
| 30 |
+
eval_results_repo = Repository(
|
| 31 |
+
local_dir=RESULTS_PATH,
|
| 32 |
+
clone_from=RESULTS_REPO,
|
| 33 |
+
repo_type="dataset",
|
| 34 |
+
)
|
| 35 |
+
eval_results_repo.git_pull()
|
| 36 |
+
|
| 37 |
+
requested_models = get_all_requested_models("eval-queue")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
return eval_queue_repo, requested_models, eval_results_repo
|
| 40 |
|