Refactor Tasks to load by yaml configuration file

src/display/utils.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass, make_dataclass
 from enum import Enum
 from typing import List
 import pandas as pd
+from yaml import safe_load
 from src.envs import GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS
 
 def fields(raw_class):
@@ -21,61 +22,9 @@ class Task:
     link: str = None
     description: str = None
 
-class Tasks(Enum):
-    oab_exams = Task(
-        benchmark="oab_exams",
-        metric="exact_match",
-        col_name="OAB Exams",
-        baseline=25.0, 
-        human_baseline=50.0,
-        few_shot=5,
-        limit=None,
-        task_list=["oab_exams_generate"],
-        link="https://huggingface.co/datasets/eduagarcia/oab_exams",
-        description="OAB Exams is a dataset of 2,000 questions from the Brazilian Bar Association's exams."
-    )
-    brazilian_court_decisions_judgment = Task(
-        benchmark="brazilian_court_decisions_judgment",
-        metric="f1_macro",
-        col_name="BR Court Decisions",
-        baseline=33.33, 
-        human_baseline=100.0,
-        few_shot=5,
-        limit=None,
-        task_list=["brazilian_court_decisions_judgment_generate"],
-        link="https://huggingface.co/datasets/joelniklaus/brazilian_court_decisions",
-        description="A classification dataset of court decisions from the Tribunal de Justiça de Alagoas (TJAL, the State Supreme Court of Alagoas (Brazil)."
-    )
-    datalawyer_frases = Task(
-        benchmark="datalawyer_frases",
-        metric="f1_macro",
-        col_name="DL Frases",
-        baseline=10.0, 
-        human_baseline=100.0,
-        few_shot=15,
-        limit=2000,
-        task_list=["datalawyer_frases_generate"],
-        link="https://huggingface.co/datasets/eduagarcia/portuguese_benchmark",
-        description="A classification dataset"
-    )
-    rrip = Task(
-        benchmark="rrip",
-        metric="f1_macro",
-        col_name="RRIP",
-        baseline=12.5, 
-        human_baseline=100.0,
-        few_shot=15,
-        limit=None,
-        task_list=["rrip_generate"],
-        link="https://huggingface.co/datasets/eduagarcia/portuguese_benchmark",
-        description="A classification dataset"
-    )
-    #arc = Task("arc:challenge", "acc_norm", "ARC", 25.0, 80.0)
-    #hellaswag = Task("hellaswag", "acc_norm", "HellaSwag", 25.0, 95.0)
-    #mmlu = Task("hendrycksTest", "acc", "MMLU", 25.0, 89.8)
-    #truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA", 25.0, 94.0)
-    #winogrande = Task("winogrande", "acc", "Winogrande", 50.0, 94.0)
-    #gsm8k = Task("gsm8k", "acc", "GSM8K", 0.21, 100)
+with open('tasks_config.yaml', 'r', encoding='utf-8') as f:
+    tasks_config = safe_load(f)
+    Tasks = Enum('Tasks', {k: Task(**v) for k, v in tasks_config['tasks'].items()})
 
 # These classes are for user facing column names,
 # to avoid having to change them all around the code

tasks_config.yaml

@@ -0,0 +1,52 @@
+version: 0.0.4
+tasks:
+  oab_exams:
+    benchmark: oab_exams
+    col_name: OAB Exams
+    task_list:
+    - oab_exams_generate
+    metric: exact_match
+    few_shot: 5
+    limit: null
+    baseline: 25.0
+    human_baseline: 50.0
+    description: OAB Exams is a dataset of 2,000 questions from the Brazilian Bar
+      Association's exams.
+    link: https://huggingface.co/datasets/eduagarcia/oab_exams
+  brazilian_court_decisions_judgment:
+    benchmark: brazilian_court_decisions_judgment
+    col_name: BR Court Decisions
+    task_list:
+    - brazilian_court_decisions_judgment_generate
+    metric: f1_macro
+    few_shot: 5
+    limit: null
+    baseline: 33.33
+    human_baseline: 100.0
+    description: A classification dataset of court decisions from the Tribunal de
+      Justiça de Alagoas (TJAL, the State Supreme Court of Alagoas (Brazil).
+    link: https://huggingface.co/datasets/joelniklaus/brazilian_court_decisions
+  datalawyer_frases:
+    benchmark: datalawyer_frases
+    col_name: DL Frases
+    task_list:
+    - datalawyer_frases_generate
+    metric: f1_macro
+    few_shot: 15
+    limit: 2000
+    baseline: 10.0
+    human_baseline: 100.0
+    description: A classification dataset
+    link: https://huggingface.co/datasets/eduagarcia/portuguese_benchmark
+  rrip:
+    benchmark: rrip
+    col_name: RRIP
+    task_list:
+    - rrip_generate
+    metric: f1_macro
+    few_shot: 15
+    limit: null
+    baseline: 12.5
+    human_baseline: 100.0
+    description: A classification dataset
+    link: https://huggingface.co/datasets/eduagarcia/portuguese_benchmark