Add dataset metadata about human/machine translation

datasets.json

@@ -11,6 +11,7 @@
             "language_modeling"
         ],
         "parallel": true,
+        "translation": "human",
         "base": "FLORES",
         "implemented": true,
         "group": "Low-Resource Languages"
@@ -25,8 +26,9 @@
             "speech_recognition"
         ],
         "parallel": true,
+        "translation": "human",
         "base": "FLORES",
-        "implemented": true,
+        "implemented": false,
         "group": "Low-Resource Languages"
     },
     {
@@ -39,8 +41,68 @@
             "speech_recognition"
         ],
         "parallel": null,
+        "translation": "human",
         "group": "Low-Resource Languages"
     },
+    {
+        "name": "Global MMLU",
+        "author": "Cohere",
+        "author_url": "https://cohere.com",
+        "url": "https://huggingface.co/datasets/CohereForAI/Global-MMLU",
+        "n_languages": 42,
+        "languages": [
+            "am",
+            "ar",
+            "bn",
+            "cs",
+            "de",
+            "el",
+            "en",
+            "es",
+            "fa",
+            "fil",
+            "fr",
+            "ha",
+            "he",
+            "hi",
+            "id",
+            "ig",
+            "it",
+            "ja",
+            "ko",
+            "ky",
+            "lt",
+            "mg",
+            "ms",
+            "ne",
+            "nl",
+            "ny",
+            "pl",
+            "pt",
+            "ro",
+            "ru",
+            "si",
+            "sn",
+            "so",
+            "sr",
+            "sv",
+            "sw",
+            "te",
+            "tr",
+            "uk",
+            "vi",
+            "yo",
+            "zh"
+        ],
+        "tasks": [
+            "question_answering"
+        ],
+        "parallel": true,
+        "translation": "mixed",
+        "base": "MMLU",
+        "implemented": true,
+        "group": "Multitask Language Understanding"
+    },
     {
         "name": "MMMLU",
         "author": "OpenAI",
@@ -67,7 +129,9 @@
             "question_answering"
         ],
         "parallel": true,
+        "translation": "human",
         "base": "MMLU",
+        "implemented": true,
         "group": "Multitask Language Understanding"
     },
     {
@@ -100,7 +164,9 @@
             "question_answering"
         ],
         "parallel": true,
+        "translation": "human",
         "base": "MMLU",
+        "implemented": true,
         "group": "Multitask Language Understanding"
     },
     {
@@ -146,7 +212,9 @@
             "question_answering"
         ],
         "parallel": true,
+        "translation": "machine",
         "base": "MMLU",
+        "implemented": false,
         "group": "Multitask Language Understanding"
     },
     {
@@ -181,65 +249,25 @@
             "question_answering"
         ],
         "parallel": true,
+        "translation": "machine",
         "base": "MMLU",
+        "implemented": false,
         "group": "Multitask Language Understanding"
     },
     {
-        "name": "Global MMLU",
-        "author": "Cohere",
-        "author_url": "https://cohere.com",
-        "url": "https://huggingface.co/datasets/CohereForAI/Global-MMLU",
-        "n_languages": 42,
-        "languages": [
-            "am",
-            "ar",
-            "bn",
-            "cs",
-            "de",
-            "el",
-            "en",
-            "es",
-            "fa",
-            "fil",
-            "fr",
-            "ha",
-            "he",
-            "hi",
-            "id",
-            "ig",
-            "it",
-            "ja",
-            "ko",
-            "ky",
-            "lt",
-            "mg",
-            "ms",
-            "ne",
-            "nl",
-            "ny",
-            "pl",
-            "pt",
-            "ro",
-            "ru",
-            "si",
-            "sn",
-            "so",
-            "sr",
-            "sv",
-            "sw",
-            "te",
-            "tr",
-            "uk",
-            "vi",
-            "yo",
-            "zh"
-        ],
+        "name": "Uhuru ARC Easy",
+        "author": "Masakhane",
+        "author_url": "https://www.masakhane.io",
+        "url": "https://huggingface.co/datasets/masakhane/uhura-arc-easy",
+        "n_languages": 6,
         "tasks": [
             "question_answering"
         ],
         "parallel": true,
-        "base": "MMLU",
-        "group": "Multitask Language Understanding"
+        "translation": "human",
+        "base": "AI2 ARC",
+        "implemented": false,
+        "group": "Abstract Reasoning"
     },
     {
         "name": "Okapi ARC Challenge",
@@ -251,20 +279,9 @@
             "question_answering"
         ],
         "parallel": true,
+        "translation": "machine",
         "base": "AI2 ARC",
-        "group": "Abstract Reasoning"
-    },
-    {
-        "name": "Uhuru ARC Easy",
-        "author": "Masakhane",
-        "author_url": "https://www.masakhane.io",
-        "url": "https://huggingface.co/datasets/masakhane/uhura-arc-easy",
-        "n_languages": 6,
-        "tasks": [
-            "question_answering"
-        ],
-        "parallel": true,
-        "base": "AI2 ARC",
+        "implemented": false,
         "group": "Abstract Reasoning"
     },
     {
@@ -277,33 +294,39 @@
             "question_answering"
         ],
         "parallel": true,
+        "translation": "machine",
         "base": "AI2 ARC",
+        "implemented": false,
         "group": "Abstract Reasoning"
     },
     {
-        "name": "Okapi TruthfulQA",
-        "author": "Academic",
-        "author_url": null,
-        "url": "https://huggingface.co/datasets/jon-tow/okapi_truthfulqa/tree/main/data",
-        "n_languages": 31,
+        "name": "Uhura TruthfulQA",
+        "author": "Masakhane",
+        "author_url": "https://www.masakhane.io",
+        "url": "https://huggingface.co/datasets/masakhane/uhura-truthfulqa",
+        "n_languages": 6,
         "tasks": [
             "question_answering"
         ],
         "parallel": true,
+        "translation": "human",
         "base": "TruthfulQA",
+        "implemented": false,
         "group": "Truthfulness"
     },
     {
-        "name": "Uhura TruthfulQA",
-        "author": "Masakhane",
-        "author_url": "https://www.masakhane.io",
-        "url": "https://huggingface.co/datasets/masakhane/uhura-truthfulqa",
-        "n_languages": 6,
+        "name": "Okapi TruthfulQA",
+        "author": "Academic",
+        "author_url": null,
+        "url": "https://huggingface.co/datasets/jon-tow/okapi_truthfulqa/tree/main/data",
+        "n_languages": 31,
         "tasks": [
             "question_answering"
         ],
         "parallel": true,
+        "translation": "machine",
         "base": "TruthfulQA",
+        "implemented": false,
         "group": "Truthfulness"
     },
     {
@@ -316,7 +339,9 @@
             "question_answering"
         ],
         "parallel": true,
+        "translation": "machine",
         "base": "TruthfulQA",
+        "implemented": false,
         "group": "Truthfulness"
     },
     {
@@ -344,7 +369,9 @@
             "logic"
         ],
         "parallel": true,
+        "translation": "human",
         "base": "MNLI",
+        "implemented": false,
         "group": "Natural Language Inference"
     },
     {
@@ -383,7 +410,9 @@
             "question_answering"
         ],
         "parallel": true,
+        "translation": "machine",
         "base": "HellaSwag",
+        "implemented": false,
         "group": "Adversarial Language Modelling"
     },
     {
@@ -396,7 +425,9 @@
             "question_answering"
         ],
         "parallel": true,
+        "translation": "machine",
         "base": "HellaSwag",
+        "implemented": false,
         "group": "Adversarial Language Modelling"
     },
     {
@@ -422,7 +453,9 @@
             "math"
         ],
         "parallel": true,
+        "translation": "human",
         "base": "MGSM",
+        "implemented": false,
         "group": "Grade School Math"
     },
     {
@@ -435,7 +468,9 @@
             "math"
         ],
         "parallel": true,
+        "translation": "machine",
         "base": "MGSM",
+        "implemented": false,
         "group": "Grade School Math"
     },
     {

frontend/src/components/DatasetTable.js

@@ -7,8 +7,9 @@ import 'primeicons/primeicons.css'
 
 const DatasetTable = ({ data }) => {
   const [filters, setFilters] = useState({
-    n_languages: { value: null, matchMode: FilterMatchMode.BETWEEN },
     tasks: { value: null, matchMode: FilterMatchMode.IN },
+    translation: { value: null, matchMode: FilterMatchMode.IN },
+    n_languages: { value: null, matchMode: FilterMatchMode.BETWEEN },
     parallel: { value: null, matchMode: FilterMatchMode.EQUALS },
     base: { value: null, matchMode: FilterMatchMode.IN },
   })
@@ -43,6 +44,16 @@ const DatasetTable = ({ data }) => {
     return <a href={rowData.url} target='_blank' style={{ textDecoration: 'none', color: 'inherit' }}><i className='pi pi-external-link' style={{ fontSize: '0.8rem' }} /></a>
   }
 
+  const translationBodyTemplate = rowData => {
+    const translationIcons = {
+      human: <i className='pi pi-user' title='Human-translated' />,
+      machine: <i className='pi pi-cog' title='Machine-translated' />,
+      mixed: <><i className='pi pi-user' title='Partially human-translated' /> <i className='pi pi-cog' title='Partially machine-translated' /></>,
+    }
+    const icon = translationIcons[rowData.translation] ?? <></>
+    return <div style={{ textAlign: 'center' }}>{icon}</div>
+  }
+
   const nLanguagesBodyTemplate = rowData => {
     return <div style={{ textAlign: 'center' }}>
       {rowData.n_languages}
@@ -67,6 +78,23 @@ const DatasetTable = ({ data }) => {
     )
   }
 
+  const translationRowFilterTemplate = options => {
+    return (
+      <MultiSelect
+        value={options.value}
+        options={['human', 'mixed', 'machine']}
+        onChange={e => {
+          options.filterApplyCallback(e.value)
+          setFilters(prevFilters => ({
+            ...prevFilters,
+            translation: { value: e.value, matchMode: FilterMatchMode.IN }
+          }))
+        }}
+        placeholder='All translation modes'
+      />
+    )
+  }
+
   return (
     <DataTable
       value={table}
@@ -88,7 +116,7 @@ const DatasetTable = ({ data }) => {
       <Column
         field='implemented'
         header={null}
-        sortable
+        headerTooltip='Whether the dataset has been integrated into this benchmark'
         style={{ maxWidth: '5rem' }}
         body={implementedBodyTemplate}
       />
@@ -103,7 +131,7 @@ const DatasetTable = ({ data }) => {
         field='name'
         header='Name'
         body={nameBodyTemplate}
-        style={{ minWidth: '5rem' }}
+        style={{ minWidth: '10rem' }}
         frozen
       />
       <Column
@@ -120,9 +148,19 @@ const DatasetTable = ({ data }) => {
         style={{ minWidth: '10rem', maxWidth: '15rem' }}
         body={tasksBodyTemplate}
       />
+      <Column
+        field='translation'
+        header={<i className='pi pi-language' />}
+        headerTooltip='Whether the dataset has been translated by humans or machines'
+        filter
+        filterElement={translationRowFilterTemplate}
+        showFilterMatchModes={false}
+        body={translationBodyTemplate}
+      />
       <Column
         field='n_languages'
         header='Languages'
+        headerTooltip='Number of languages in the dataset'
         filter
         sortable
         style={{ minWidth: '5rem', maxWidth: '10rem' }}