idolezal commited on
Commit
690c597
ยท
1 Parent(s): acca45a

Enabled Overall with details, mainly for CSV format

Browse files
Files changed (1) hide show
  1. server.py +10 -7
server.py CHANGED
@@ -178,6 +178,7 @@ class LeaderboardServer:
178
  self.TASKS_METADATA = json.load(open(TASKS_METADATA_PATH))
179
  self.TASKS_CATEGORIES = {self.TASKS_METADATA[task]["category"] for task in self.TASKS_METADATA}
180
  self.TASKS_CATEGORY_OVERALL = "Overall"
 
181
  self.CATEGORY_TO_TASK_ABBREVIATION_TO_DETAILS = self._prepare_category_to_task_abbr_to_details()
182
  self.MAX_LENGTH_OF_MODEL_TITLE = 28
183
  self.DIR_DATAFRAMES_CSV = "./dataframes_csv"
@@ -224,7 +225,7 @@ class LeaderboardServer:
224
  def update_leaderboard(self):
225
  self._update_models_and_tournament_results()
226
 
227
- categories = [self.TASKS_CATEGORY_OVERALL] + sorted(self.TASKS_CATEGORIES)
228
 
229
  leaderboard_dataframes = {
230
  category: self._get_leaderboard(category=category) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity'])
@@ -304,6 +305,7 @@ class LeaderboardServer:
304
  sorted_abbreviation2name = dict.fromkeys(sorted(abbreviation2name.keys()))
305
  sorted_abbreviation2name.update(abbreviation2name)
306
  category2abbreviation2name[self.TASKS_CATEGORY_OVERALL] = sorted_abbreviation2name
 
307
 
308
  return category2abbreviation2name
309
 
@@ -448,7 +450,7 @@ class LeaderboardServer:
448
  match_results = {}
449
  for task in self.tournament_results[submission_id][competitor_id]:
450
  task_category = self.TASKS_METADATA[task]["category"]
451
- if category in (task_category, self.TASKS_CATEGORY_OVERALL):
452
  if to_csv:
453
  match_results[task] = self.tournament_results[submission_id][competitor_id][task]["significant"]
454
  else:
@@ -554,7 +556,7 @@ class LeaderboardServer:
554
  for task in self.TASKS_METADATA.keys():
555
 
556
  task_category = self.TASKS_METADATA[task]["category"]
557
- if category not in (self.TASKS_CATEGORY_OVERALL, task_category):
558
  continue
559
  else:
560
  # tournament_results
@@ -567,7 +569,7 @@ class LeaderboardServer:
567
  task_score = num_of_wins / num_of_competitors * 100 if num_of_competitors > 0 else 100
568
  win_score.setdefault(task_category, []).append(task_score)
569
 
570
- if category == task_category:
571
  local_results[task] = task_score
572
  for metric in VISIBLE_METRICS:
573
  visible_metrics_map_word_to_header[task + "_" + metric] = self.TASKS_METADATA[task]["abbreviation"] + " " + metric
@@ -580,9 +582,10 @@ class LeaderboardServer:
580
  for c in win_score:
581
  win_score[c] = sum(win_score[c]) / len(win_score[c])
582
 
583
- if category == self.TASKS_CATEGORY_OVERALL:
584
- for c in win_score:
585
- local_results[c] = win_score[c]
 
586
  local_results["average_score"] = sum(win_score.values()) / len(win_score)
587
  else:
588
  local_results["average_score"] = win_score[category]
 
178
  self.TASKS_METADATA = json.load(open(TASKS_METADATA_PATH))
179
  self.TASKS_CATEGORIES = {self.TASKS_METADATA[task]["category"] for task in self.TASKS_METADATA}
180
  self.TASKS_CATEGORY_OVERALL = "Overall"
181
+ self.TASKS_CATEGORY_OVERALL_DETAILS = "Overall with details"
182
  self.CATEGORY_TO_TASK_ABBREVIATION_TO_DETAILS = self._prepare_category_to_task_abbr_to_details()
183
  self.MAX_LENGTH_OF_MODEL_TITLE = 28
184
  self.DIR_DATAFRAMES_CSV = "./dataframes_csv"
 
225
  def update_leaderboard(self):
226
  self._update_models_and_tournament_results()
227
 
228
+ categories = [self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS] + sorted(self.TASKS_CATEGORIES)
229
 
230
  leaderboard_dataframes = {
231
  category: self._get_leaderboard(category=category) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity'])
 
305
  sorted_abbreviation2name = dict.fromkeys(sorted(abbreviation2name.keys()))
306
  sorted_abbreviation2name.update(abbreviation2name)
307
  category2abbreviation2name[self.TASKS_CATEGORY_OVERALL] = sorted_abbreviation2name
308
+ category2abbreviation2name[self.TASKS_CATEGORY_OVERALL_DETAILS] = sorted_abbreviation2name
309
 
310
  return category2abbreviation2name
311
 
 
450
  match_results = {}
451
  for task in self.tournament_results[submission_id][competitor_id]:
452
  task_category = self.TASKS_METADATA[task]["category"]
453
+ if category in (task_category, self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS):
454
  if to_csv:
455
  match_results[task] = self.tournament_results[submission_id][competitor_id][task]["significant"]
456
  else:
 
556
  for task in self.TASKS_METADATA.keys():
557
 
558
  task_category = self.TASKS_METADATA[task]["category"]
559
+ if category not in (self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS, task_category):
560
  continue
561
  else:
562
  # tournament_results
 
569
  task_score = num_of_wins / num_of_competitors * 100 if num_of_competitors > 0 else 100
570
  win_score.setdefault(task_category, []).append(task_score)
571
 
572
+ if category in (task_category, self.TASKS_CATEGORY_OVERALL_DETAILS):
573
  local_results[task] = task_score
574
  for metric in VISIBLE_METRICS:
575
  visible_metrics_map_word_to_header[task + "_" + metric] = self.TASKS_METADATA[task]["abbreviation"] + " " + metric
 
582
  for c in win_score:
583
  win_score[c] = sum(win_score[c]) / len(win_score[c])
584
 
585
+ if category in (self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS):
586
+ if category == self.TASKS_CATEGORY_OVERALL:
587
+ for c in win_score:
588
+ local_results[c] = win_score[c]
589
  local_results["average_score"] = sum(win_score.values()) / len(win_score)
590
  else:
591
  local_results["average_score"] = win_score[category]