Back to bert mini for leaderboard submission

Finetune BERT.ipynb

@@ -10,15 +10,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 17,
    "id": "73e72549-69f2-46b5-b0f5-655777139972",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-24T18:21:58.280871Z",
-     "iopub.status.busy": "2025-01-24T18:21:58.280785Z",
-     "iopub.status.idle": "2025-01-24T18:22:01.627392Z",
-     "shell.execute_reply": "2025-01-24T18:22:01.627134Z",
-     "shell.execute_reply.started": "2025-01-24T18:21:58.280861Z"
+     "iopub.execute_input": "2025-01-24T18:59:00.459773Z",
+     "iopub.status.busy": "2025-01-24T18:59:00.458472Z",
+     "iopub.status.idle": "2025-01-24T18:59:00.517418Z",
+     "shell.execute_reply": "2025-01-24T18:59:00.517026Z",
+     "shell.execute_reply.started": "2025-01-24T18:59:00.459726Z"
     }
    },
    "outputs": [],
@@ -36,7 +36,8 @@
     "    EvalResult,\n",
     ")\n",
     "from datasets import DatasetDict, load_dataset\n",
-    "from torch.utils.data import Dataset, DataLoader"
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "from statsmodels.stats.proportion import proportion_confint"
    ]
   },
   {
@@ -478,7 +479,42 @@
     "acc 0.645, energy 0.273 Wh\n",
     "\n",
     "bert-base\\\n",
-    "acc, energy"
+    "acc 0.691, energy 1.053 Wh"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "6c35f222-79d9-4166-8601-8a6240a49c91",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-24T19:03:41.276772Z",
+     "iopub.status.busy": "2025-01-24T19:03:41.276125Z",
+     "iopub.status.idle": "2025-01-24T19:03:41.284530Z",
+     "shell.execute_reply": "2025-01-24T19:03:41.283079Z",
+     "shell.execute_reply.started": "2025-01-24T19:03:41.276731Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(0.6284344081642794, 0.6817389605903139)"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "nobs = 1219\n",
+    "acc = 0.656\n",
+    "proportion_confint(\n",
+    "    count=int(nobs * acc),\n",
+    "    nobs=nobs,\n",
+    "    method=\"jeffreys\",\n",
+    ")"
    ]
   },
   {

tasks/text.py

@@ -13,7 +13,7 @@ from .utils.emissions import tracker, clean_emissions_data, get_space_info
 
 router = APIRouter()
 
-MODEL_TYPE = "bert-base"
+MODEL_TYPE = "bert-mini"
 DESCRIPTIONS = {
     "baseline": "baseline most common class",
     "bert-base": "bert base fine tuned on just training data, Nvidia T4 small",
@@ -77,7 +77,6 @@ def bert_model(test_dataset: dict, model_type: str):
         print("Starting model run.")
         predictions = np.array([])
         for batch in dataloader:
-            print("    Running a batch.")
             test_input_ids = batch["input_ids"].to(device)
             test_attention_mask = batch["attention_mask"].to(device)
             outputs = model(test_input_ids, test_attention_mask)