Caches model cards and dim_seq_size while first intiating the leaderboard

app.py

@@ -151,7 +151,10 @@ def add_rank(df):
     df.fillna("", inplace=True)
     return df
 
-def get_mteb_data(tasks=["Clustering"], langs=[], datasets=[], fillna=True, add_emb_dim=True, task_to_metric=TASK_TO_METRIC, rank=True):
+MODEL_CARD_METADATA = {}
+MODEL_EMB_DIM = {}
+def get_mteb_data(tasks=["Clustering"], langs=[], datasets=[], fillna=True, add_emb_dim=True, task_to_metric=TASK_TO_METRIC, rank=True, refresh=True):
+    global MODEL_CARD_METADATA, MODEL_EMB_DIM
     api = API
     models = api.list_models(filter="mteb")
     # Initialize list to models that we cannot fetch metadata from
@@ -174,12 +177,15 @@ def get_mteb_data(tasks=["Clustering"], langs=[], datasets=[], fillna=True, add_
                 res["Embedding Dimensions"] = EXTERNAL_MODEL_TO_DIM.get(model, "")
                 res["Max Tokens"] = EXTERNAL_MODEL_TO_SEQLEN.get(model, "")
             df_list.append(res)
-    
+
     for model in models:
         if model.modelId in MODELS_TO_SKIP: continue
-        print("MODEL", model)
-        readme_path = hf_hub_download(model.modelId, filename="README.md")
-        meta = metadata_load(readme_path)
+        print("MODEL", model.modelId)
+        if model.modelId not in MODEL_CARD_METADATA or refresh:
+            readme_path = hf_hub_download(model.modelId, filename="README.md")
+            meta = metadata_load(readme_path)
+            MODEL_CARD_METADATA[model.modelId] = meta
+        meta = MODEL_CARD_METADATA[model.modelId]
         if "model-index" not in meta:
             continue
         # meta['model-index'][0]["results"] is list of elements like:
@@ -211,9 +217,11 @@ def get_mteb_data(tasks=["Clustering"], langs=[], datasets=[], fillna=True, add_
             if add_emb_dim:
                 try:
                     # Fails on gated repos, so we only include scores for them
-                    out["Embedding Dimensions"], out["Max Tokens"], out["Model Size (Million Parameters)"], out["Memory Usage (GB, fp32)"] = get_dim_seq_size(model)
+                    if model.modelId not in MODEL_EMB_DIM or refresh:
+                        MODEL_EMB_DIM[model.modelId] = get_dim_seq_size(model)
+                    out["Embedding Dimensions"], out["Max Tokens"], out["Model Size (Million Parameters)"], out["Memory Usage (GB, fp32)"] = MODEL_EMB_DIM[model.modelId]
                 except:
-                    pass
+                    MODEL_EMB_DIM[model.modelId] = None, None, None, None
             df_list.append(out)
         if model.library_name == "sentence-transformers" or "sentence-transformers" in model.tags or "modules.json" in {file.rfilename for file in model.siblings}:
             SENTENCE_TRANSFORMERS_COMPATIBLE_MODELS.add(out["Model"])
@@ -231,7 +239,9 @@ def get_mteb_data(tasks=["Clustering"], langs=[], datasets=[], fillna=True, add_
         df.fillna("", inplace=True)
     return df
 
-def get_mteb_average(task_dict: dict):
+# Get dict with a task list for each task category
+# E.g. {"Classification": ["AmazonMassiveIntentClassification (en)", ...], "PairClassification": ["SprintDuplicateQuestions", ...]}
+def get_mteb_average(task_dict: dict, refresh=True):
     all_tasks = reduce(lambda x, y: x + y, task_dict.values())
     DATA_OVERALL = get_mteb_data(
         tasks=list(task_dict.keys()),
@@ -239,6 +249,7 @@ def get_mteb_average(task_dict: dict):
         fillna=False,
         add_emb_dim=True,
         rank=False,
+        refresh=refresh
     )
     # Debugging:
     # DATA_OVERALL.to_csv("overall.csv")
@@ -277,7 +288,7 @@ for board, board_config in BOARDS_CONFIG.items():
         "data_tasks": {}
     }
     if board_config["has_overall"]:
-        data_overall, data_tasks = get_mteb_average(board_config["tasks"])
+        data_overall, data_tasks = get_mteb_average(board_config["tasks"], refresh=False)
         boards_data[board]["data_overall"] = data_overall
         boards_data[board]["data_tasks"] = data_tasks
         all_data_tasks.extend(data_tasks.values())
@@ -287,7 +298,7 @@ for board, board_config in BOARDS_CONFIG.items():
             if len(task_category_list) > 1:
                 columns.append("Average")
             print(board, task_category, task_category_list)
-            data_task_category = get_mteb_data([task_category], [], task_category_list)[columns + task_category_list]
+            data_task_category = get_mteb_data([task_category], [], task_category_list, refresh=False)[columns + task_category_list]
             boards_data[board]["data_tasks"][task_category] = data_task_category
             all_data_tasks.append(data_task_category)