Spaces:
Running
Running
Enabled Overall with details, mainly for CSV format
Browse files
server.py
CHANGED
@@ -178,6 +178,7 @@ class LeaderboardServer:
|
|
178 |
self.TASKS_METADATA = json.load(open(TASKS_METADATA_PATH))
|
179 |
self.TASKS_CATEGORIES = {self.TASKS_METADATA[task]["category"] for task in self.TASKS_METADATA}
|
180 |
self.TASKS_CATEGORY_OVERALL = "Overall"
|
|
|
181 |
self.CATEGORY_TO_TASK_ABBREVIATION_TO_DETAILS = self._prepare_category_to_task_abbr_to_details()
|
182 |
self.MAX_LENGTH_OF_MODEL_TITLE = 28
|
183 |
self.DIR_DATAFRAMES_CSV = "./dataframes_csv"
|
@@ -224,7 +225,7 @@ class LeaderboardServer:
|
|
224 |
def update_leaderboard(self):
|
225 |
self._update_models_and_tournament_results()
|
226 |
|
227 |
-
categories = [self.TASKS_CATEGORY_OVERALL] + sorted(self.TASKS_CATEGORIES)
|
228 |
|
229 |
leaderboard_dataframes = {
|
230 |
category: self._get_leaderboard(category=category) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity'])
|
@@ -304,6 +305,7 @@ class LeaderboardServer:
|
|
304 |
sorted_abbreviation2name = dict.fromkeys(sorted(abbreviation2name.keys()))
|
305 |
sorted_abbreviation2name.update(abbreviation2name)
|
306 |
category2abbreviation2name[self.TASKS_CATEGORY_OVERALL] = sorted_abbreviation2name
|
|
|
307 |
|
308 |
return category2abbreviation2name
|
309 |
|
@@ -448,7 +450,7 @@ class LeaderboardServer:
|
|
448 |
match_results = {}
|
449 |
for task in self.tournament_results[submission_id][competitor_id]:
|
450 |
task_category = self.TASKS_METADATA[task]["category"]
|
451 |
-
if category in (task_category, self.TASKS_CATEGORY_OVERALL):
|
452 |
if to_csv:
|
453 |
match_results[task] = self.tournament_results[submission_id][competitor_id][task]["significant"]
|
454 |
else:
|
@@ -554,7 +556,7 @@ class LeaderboardServer:
|
|
554 |
for task in self.TASKS_METADATA.keys():
|
555 |
|
556 |
task_category = self.TASKS_METADATA[task]["category"]
|
557 |
-
if category not in (self.TASKS_CATEGORY_OVERALL, task_category):
|
558 |
continue
|
559 |
else:
|
560 |
# tournament_results
|
@@ -567,7 +569,7 @@ class LeaderboardServer:
|
|
567 |
task_score = num_of_wins / num_of_competitors * 100 if num_of_competitors > 0 else 100
|
568 |
win_score.setdefault(task_category, []).append(task_score)
|
569 |
|
570 |
-
if category
|
571 |
local_results[task] = task_score
|
572 |
for metric in VISIBLE_METRICS:
|
573 |
visible_metrics_map_word_to_header[task + "_" + metric] = self.TASKS_METADATA[task]["abbreviation"] + " " + metric
|
@@ -580,9 +582,10 @@ class LeaderboardServer:
|
|
580 |
for c in win_score:
|
581 |
win_score[c] = sum(win_score[c]) / len(win_score[c])
|
582 |
|
583 |
-
if category
|
584 |
-
|
585 |
-
|
|
|
586 |
local_results["average_score"] = sum(win_score.values()) / len(win_score)
|
587 |
else:
|
588 |
local_results["average_score"] = win_score[category]
|
|
|
178 |
self.TASKS_METADATA = json.load(open(TASKS_METADATA_PATH))
|
179 |
self.TASKS_CATEGORIES = {self.TASKS_METADATA[task]["category"] for task in self.TASKS_METADATA}
|
180 |
self.TASKS_CATEGORY_OVERALL = "Overall"
|
181 |
+
self.TASKS_CATEGORY_OVERALL_DETAILS = "Overall with details"
|
182 |
self.CATEGORY_TO_TASK_ABBREVIATION_TO_DETAILS = self._prepare_category_to_task_abbr_to_details()
|
183 |
self.MAX_LENGTH_OF_MODEL_TITLE = 28
|
184 |
self.DIR_DATAFRAMES_CSV = "./dataframes_csv"
|
|
|
225 |
def update_leaderboard(self):
|
226 |
self._update_models_and_tournament_results()
|
227 |
|
228 |
+
categories = [self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS] + sorted(self.TASKS_CATEGORIES)
|
229 |
|
230 |
leaderboard_dataframes = {
|
231 |
category: self._get_leaderboard(category=category) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity'])
|
|
|
305 |
sorted_abbreviation2name = dict.fromkeys(sorted(abbreviation2name.keys()))
|
306 |
sorted_abbreviation2name.update(abbreviation2name)
|
307 |
category2abbreviation2name[self.TASKS_CATEGORY_OVERALL] = sorted_abbreviation2name
|
308 |
+
category2abbreviation2name[self.TASKS_CATEGORY_OVERALL_DETAILS] = sorted_abbreviation2name
|
309 |
|
310 |
return category2abbreviation2name
|
311 |
|
|
|
450 |
match_results = {}
|
451 |
for task in self.tournament_results[submission_id][competitor_id]:
|
452 |
task_category = self.TASKS_METADATA[task]["category"]
|
453 |
+
if category in (task_category, self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS):
|
454 |
if to_csv:
|
455 |
match_results[task] = self.tournament_results[submission_id][competitor_id][task]["significant"]
|
456 |
else:
|
|
|
556 |
for task in self.TASKS_METADATA.keys():
|
557 |
|
558 |
task_category = self.TASKS_METADATA[task]["category"]
|
559 |
+
if category not in (self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS, task_category):
|
560 |
continue
|
561 |
else:
|
562 |
# tournament_results
|
|
|
569 |
task_score = num_of_wins / num_of_competitors * 100 if num_of_competitors > 0 else 100
|
570 |
win_score.setdefault(task_category, []).append(task_score)
|
571 |
|
572 |
+
if category in (task_category, self.TASKS_CATEGORY_OVERALL_DETAILS):
|
573 |
local_results[task] = task_score
|
574 |
for metric in VISIBLE_METRICS:
|
575 |
visible_metrics_map_word_to_header[task + "_" + metric] = self.TASKS_METADATA[task]["abbreviation"] + " " + metric
|
|
|
582 |
for c in win_score:
|
583 |
win_score[c] = sum(win_score[c]) / len(win_score[c])
|
584 |
|
585 |
+
if category in (self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS):
|
586 |
+
if category == self.TASKS_CATEGORY_OVERALL:
|
587 |
+
for c in win_score:
|
588 |
+
local_results[c] = win_score[c]
|
589 |
local_results["average_score"] = sum(win_score.values()) / len(win_score)
|
590 |
else:
|
591 |
local_results["average_score"] = win_score[category]
|