brunneis commited on
Commit
1eb7d16
·
unverified ·
1 Parent(s): 0f68941

Update score definition

Browse files
Files changed (1) hide show
  1. src/leaderboard/read_evals.py +4 -4
src/leaderboard/read_evals.py CHANGED
@@ -115,10 +115,10 @@ class EvalResult:
115
  def to_dict(self):
116
  """Converts the Eval Result to a dict compatible with our dataframe display"""
117
  # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
118
- solbench = sum([
119
- self.results.get('naive_judge', 0),
120
- self.results.get('human_eval_solidity', 0)
121
- ]) / 2
122
  data_dict = {
123
  "eval_name": self.eval_name, # not a column, just a save name,
124
  AutoEvalColumn.precision.name: self.precision.value.name,
 
115
  def to_dict(self):
116
  """Converts the Eval Result to a dict compatible with our dataframe display"""
117
  # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
118
+ solbench = (
119
+ self.results.get('naive_judge', 0) * 0.3 +
120
+ self.results.get('human_eval_solidity', 0) * 0.7
121
+ )
122
  data_dict = {
123
  "eval_name": self.eval_name, # not a column, just a save name,
124
  AutoEvalColumn.precision.name: self.precision.value.name,