Add Solbench score

src/display/utils.py

@@ -38,7 +38,8 @@ auto_eval_column_dict = [
         "", "str", True, never_hidden=True)),
     ("model", ColumnContent, create_column_content(
         "Model", "markdown", True, never_hidden=True)),
-    ("average", ColumnContent, create_column_content("Average", "number", True)),
+    ("solbench", ColumnContent, create_column_content("Score", "number", True)),
+    # ("average", ColumnContent, create_column_content("Average", "number", True)),
 ]
 
 # Add task-specific columns

src/leaderboard/read_evals.py

@@ -114,7 +114,11 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        solbench = sum([
+            self.results.get('naive_judge', 0),
+            self.results.get('human_eval_solidity', 0)
+        ]) / 2
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -124,7 +128,8 @@ class EvalResult:
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.model_name),
             AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            # AutoEvalColumn.average.name: average,
+            AutoEvalColumn.solbench.name: solbench,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,

src/populate.py

@@ -19,7 +19,8 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    # df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df.sort_values(by=[AutoEvalColumn.solbench.name], ascending=False)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced