Add dataset field to code completion task
Browse files
src/leaderboard_formatting.py
CHANGED
|
@@ -20,6 +20,7 @@ COLUMNS_PRETTY = {
|
|
| 20 |
"EM non_informative": "EM non-informative",
|
| 21 |
"EM random": "EM random",
|
| 22 |
"EM all": "EM all",
|
|
|
|
| 23 |
}
|
| 24 |
|
| 25 |
|
|
@@ -47,16 +48,19 @@ METRICS_PER_TASK = {
|
|
| 47 |
|
| 48 |
SORT_COLUMN_PER_TASK = {
|
| 49 |
"commit_message_generation": "ROUGE-1",
|
| 50 |
-
"project_code_completion": "
|
| 51 |
}
|
| 52 |
|
| 53 |
|
| 54 |
def get_columns_per_task(task_id: str) -> List[str]:
|
| 55 |
metrics_per_task = METRICS_PER_TASK[task_id]
|
| 56 |
-
|
|
|
|
| 57 |
return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"]
|
| 58 |
|
| 59 |
|
| 60 |
def get_types_per_task(task_id: str) -> List[str]:
|
| 61 |
metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0))
|
|
|
|
|
|
|
| 62 |
return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]
|
|
|
|
| 20 |
"EM non_informative": "EM non-informative",
|
| 21 |
"EM random": "EM random",
|
| 22 |
"EM all": "EM all",
|
| 23 |
+
"dataset": "Dataset",
|
| 24 |
}
|
| 25 |
|
| 26 |
|
|
|
|
| 48 |
|
| 49 |
SORT_COLUMN_PER_TASK = {
|
| 50 |
"commit_message_generation": "ROUGE-1",
|
| 51 |
+
"project_code_completion": "Dataset"
|
| 52 |
}
|
| 53 |
|
| 54 |
|
| 55 |
def get_columns_per_task(task_id: str) -> List[str]:
|
| 56 |
metrics_per_task = METRICS_PER_TASK[task_id]
|
| 57 |
+
if task_id == 'project_code_completion':
|
| 58 |
+
return ["Model Name", "Availability", "Context Size", "Dataset"] + metrics_per_task + ["Submitted By", "Resources"]
|
| 59 |
return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"]
|
| 60 |
|
| 61 |
|
| 62 |
def get_types_per_task(task_id: str) -> List[str]:
|
| 63 |
metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0))
|
| 64 |
+
if task_id == 'project_code_completion':
|
| 65 |
+
return ["html", "markdown", "markdown", "html"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]
|
| 66 |
return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]
|