Point submission at bert tiny.

Finetune BERT.ipynb

@@ -14,11 +14,11 @@
    "id": "73e72549-69f2-46b5-b0f5-655777139972",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-21T19:25:48.302003Z",
-     "iopub.status.busy": "2025-01-21T19:25:48.301808Z",
-     "iopub.status.idle": "2025-01-21T19:25:50.698806Z",
-     "shell.execute_reply": "2025-01-21T19:25:50.698535Z",
-     "shell.execute_reply.started": "2025-01-21T19:25:48.301982Z"
+     "iopub.execute_input": "2025-01-22T00:15:28.938894Z",
+     "iopub.status.busy": "2025-01-22T00:15:28.938077Z",
+     "iopub.status.idle": "2025-01-22T00:15:34.317293Z",
+     "shell.execute_reply": "2025-01-22T00:15:34.316942Z",
+     "shell.execute_reply.started": "2025-01-22T00:15:28.938839Z"
     }
    },
    "outputs": [],
@@ -45,11 +45,11 @@
    "id": "07e0787e-c72b-41f3-baba-43cef3f8d6f8",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-21T19:25:50.699344Z",
-     "iopub.status.busy": "2025-01-21T19:25:50.699200Z",
-     "iopub.status.idle": "2025-01-21T19:25:50.701241Z",
-     "shell.execute_reply": "2025-01-21T19:25:50.700993Z",
-     "shell.execute_reply.started": "2025-01-21T19:25:50.699335Z"
+     "iopub.execute_input": "2025-01-22T00:15:34.318082Z",
+     "iopub.status.busy": "2025-01-22T00:15:34.317923Z",
+     "iopub.status.idle": "2025-01-22T00:15:34.320079Z",
+     "shell.execute_reply": "2025-01-22T00:15:34.319851Z",
+     "shell.execute_reply.started": "2025-01-22T00:15:34.318073Z"
     }
    },
    "outputs": [],
@@ -67,15 +67,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 15,
    "id": "d4b79fb9-5e70-4600-8885-94bc0a6e917c",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-21T19:25:50.701789Z",
-     "iopub.status.busy": "2025-01-21T19:25:50.701708Z",
-     "iopub.status.idle": "2025-01-21T19:25:50.707095Z",
-     "shell.execute_reply": "2025-01-21T19:25:50.706788Z",
-     "shell.execute_reply.started": "2025-01-21T19:25:50.701781Z"
+     "iopub.execute_input": "2025-01-22T00:18:10.466025Z",
+     "iopub.status.busy": "2025-01-22T00:18:10.465289Z",
+     "iopub.status.idle": "2025-01-22T00:18:10.482505Z",
+     "shell.execute_reply": "2025-01-22T00:18:10.481605Z",
+     "shell.execute_reply.started": "2025-01-22T00:18:10.465973Z"
     }
    },
    "outputs": [],
@@ -188,20 +188,20 @@
     "        metrics = print_model_status(\n",
     "            epoch, num_epochs, model, train_dataloader, test_dataloader\n",
     "        )\n",
-    "        return metrics"
+    "    return metrics"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 16,
    "id": "07131bce-23ad-4787-8622-cce401f3e5ce",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-21T19:25:50.707655Z",
-     "iopub.status.busy": "2025-01-21T19:25:50.707519Z",
-     "iopub.status.idle": "2025-01-21T19:25:50.718311Z",
-     "shell.execute_reply": "2025-01-21T19:25:50.718037Z",
-     "shell.execute_reply.started": "2025-01-21T19:25:50.707646Z"
+     "iopub.execute_input": "2025-01-22T00:18:10.964716Z",
+     "iopub.status.busy": "2025-01-22T00:18:10.963608Z",
+     "iopub.status.idle": "2025-01-22T00:18:10.971834Z",
+     "shell.execute_reply": "2025-01-22T00:18:10.970949Z",
+     "shell.execute_reply.started": "2025-01-22T00:18:10.964671Z"
     }
    },
    "outputs": [],
@@ -217,15 +217,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 17,
    "id": "695bc080-bbd7-4937-af5b-50db1c936500",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-21T19:25:50.718754Z",
-     "iopub.status.busy": "2025-01-21T19:25:50.718677Z",
-     "iopub.status.idle": "2025-01-21T19:25:50.721834Z",
-     "shell.execute_reply": "2025-01-21T19:25:50.721583Z",
-     "shell.execute_reply.started": "2025-01-21T19:25:50.718746Z"
+     "iopub.execute_input": "2025-01-22T00:18:11.117610Z",
+     "iopub.status.busy": "2025-01-22T00:18:11.117201Z",
+     "iopub.status.idle": "2025-01-22T00:18:11.128421Z",
+     "shell.execute_reply": "2025-01-22T00:18:11.127145Z",
+     "shell.execute_reply.started": "2025-01-22T00:18:11.117580Z"
     }
    },
    "outputs": [],
@@ -321,57 +321,66 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "id": "34a7c310-c486-4db1-b94d-4363c3d3df5b",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-21T19:25:50.724036Z",
-     "iopub.status.busy": "2025-01-21T19:25:50.723968Z"
+     "iopub.execute_input": "2025-01-22T00:18:31.584691Z",
+     "iopub.status.busy": "2025-01-22T00:18:31.584113Z",
+     "iopub.status.idle": "2025-01-22T00:18:38.462642Z",
+     "shell.execute_reply": "2025-01-22T00:18:38.462384Z",
+     "shell.execute_reply.started": "2025-01-22T00:18:31.584650Z"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-21 19:18:35 Epoch  0/3 done. Loss: Train 2.184, Test 2.190; and Acc: Train 0.131, Test 0.129\n",
+      "2025-01-21 19:18:36 Epoch  1/3 done. Loss: Train 1.979, Test 2.002; and Acc: Train 0.244, Test 0.222\n",
+      "2025-01-21 19:18:37 Epoch  2/3 done. Loss: Train 1.915, Test 1.949; and Acc: Train 0.277, Test 0.258\n",
+      "2025-01-21 19:18:38 Epoch  3/3 done. Loss: Train 1.873, Test 1.917; and Acc: Train 0.276, Test 0.259\n"
+     ]
+    }
+   ],
    "source": [
     "model, tokenizer, regime, metrics = run_training(\n",
-    "    max_dataset_size=16 * 10,\n",
+    "    max_dataset_size=16 * 100,\n",
     "    bert_variety=\"google/bert_uncased_L-2_H-128_A-2\",\n",
     "    max_length=128,\n",
-    "    num_epochs=4,\n",
+    "    num_epochs=3,\n",
     "    batch_size=32,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "32abaa1b-11f4-4793-97b8-36bb2dc29d56",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "regime"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "fe108690-bcc1-4667-9f8e-907a1a8ac2ec",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metrics"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 23,
    "id": "0aedfcca-843e-4f4c-8062-3e4625161bcc",
    "metadata": {
     "editable": true,
+    "execution": {
+     "iopub.execute_input": "2025-01-22T00:18:46.417009Z",
+     "iopub.status.busy": "2025-01-22T00:18:46.416419Z",
+     "iopub.status.idle": "2025-01-22T00:18:46.529320Z",
+     "shell.execute_reply": "2025-01-22T00:18:46.529078Z",
+     "shell.execute_reply.started": "2025-01-22T00:18:46.416962Z"
+    },
     "slideshow": {
      "slide_type": ""
     },
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-21 19:18:46 Predictions: tensor([0, 0, 0, 0, 0, 0, 0], device='mps:0')\n"
+     ]
+    }
+   ],
    "source": [
     "model.eval()\n",
     "test_text = [\n",
@@ -436,15 +445,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 24,
    "id": "37794952-703c-466c-9d26-ee6cb2834246",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-21T18:35:29.897653Z",
-     "iopub.status.busy": "2025-01-21T18:35:29.897020Z",
-     "iopub.status.idle": "2025-01-21T18:35:29.901748Z",
-     "shell.execute_reply": "2025-01-21T18:35:29.901032Z",
-     "shell.execute_reply.started": "2025-01-21T18:35:29.897609Z"
+     "iopub.execute_input": "2025-01-22T00:19:05.789872Z",
+     "iopub.status.busy": "2025-01-22T00:19:05.789108Z",
+     "iopub.status.idle": "2025-01-22T00:19:05.796074Z",
+     "shell.execute_reply": "2025-01-22T00:19:05.794974Z",
+     "shell.execute_reply.started": "2025-01-22T00:19:05.789815Z"
     }
    },
    "outputs": [],
@@ -459,15 +468,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 25,
    "id": "28354e8c-886a-4523-8968-8c688c13f6a3",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-21T18:42:35.614137Z",
-     "iopub.status.busy": "2025-01-21T18:42:35.613694Z",
-     "iopub.status.idle": "2025-01-21T18:45:35.341816Z",
-     "shell.execute_reply": "2025-01-21T18:45:35.341535Z",
-     "shell.execute_reply.started": "2025-01-21T18:42:35.614111Z"
+     "iopub.execute_input": "2025-01-22T00:19:06.183379Z",
+     "iopub.status.busy": "2025-01-22T00:19:06.182544Z",
+     "iopub.status.idle": "2025-01-22T00:21:02.201321Z",
+     "shell.execute_reply": "2025-01-22T00:21:02.201016Z",
+     "shell.execute_reply.started": "2025-01-22T00:19:06.183320Z"
     }
    },
    "outputs": [
@@ -475,22 +484,22 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2025-01-21 10:43:44 Epoch  0/15 done. Loss: Train 2.177, Test 2.172; and Acc: Train 0.063, Test 0.071\n",
-      "2025-01-21 10:43:52 Epoch  1/15 done. Loss: Train 1.786, Test 1.823; and Acc: Train 0.383, Test 0.354\n",
-      "2025-01-21 10:44:00 Epoch  2/15 done. Loss: Train 1.579, Test 1.628; and Acc: Train 0.465, Test 0.436\n",
-      "2025-01-21 10:44:07 Epoch  3/15 done. Loss: Train 1.431, Test 1.498; and Acc: Train 0.510, Test 0.484\n",
-      "2025-01-21 10:44:14 Epoch  4/15 done. Loss: Train 1.304, Test 1.402; and Acc: Train 0.555, Test 0.515\n",
-      "2025-01-21 10:44:22 Epoch  5/15 done. Loss: Train 1.212, Test 1.339; and Acc: Train 0.585, Test 0.535\n",
-      "2025-01-21 10:44:29 Epoch  6/15 done. Loss: Train 1.128, Test 1.288; and Acc: Train 0.611, Test 0.546\n",
-      "2025-01-21 10:44:36 Epoch  7/15 done. Loss: Train 1.039, Test 1.241; and Acc: Train 0.643, Test 0.559\n",
-      "2025-01-21 10:44:44 Epoch  8/15 done. Loss: Train 1.003, Test 1.236; and Acc: Train 0.665, Test 0.555\n",
-      "2025-01-21 10:44:51 Epoch  9/15 done. Loss: Train 0.897, Test 1.183; and Acc: Train 0.708, Test 0.568\n",
-      "2025-01-21 10:44:58 Epoch 10/15 done. Loss: Train 0.852, Test 1.187; and Acc: Train 0.724, Test 0.572\n",
-      "2025-01-21 10:45:06 Epoch 11/15 done. Loss: Train 0.769, Test 1.154; and Acc: Train 0.755, Test 0.581\n",
-      "2025-01-21 10:45:13 Epoch 12/15 done. Loss: Train 0.764, Test 1.197; and Acc: Train 0.752, Test 0.573\n",
-      "2025-01-21 10:45:20 Epoch 13/15 done. Loss: Train 0.660, Test 1.153; and Acc: Train 0.797, Test 0.590\n",
-      "2025-01-21 10:45:28 Epoch 14/15 done. Loss: Train 0.588, Test 1.143; and Acc: Train 0.820, Test 0.594\n",
-      "2025-01-21 10:45:35 Epoch 15/15 done. Loss: Train 0.579, Test 1.200; and Acc: Train 0.822, Test 0.575\n"
+      "2025-01-21 19:19:11 Epoch  0/15 done. Loss: Train 2.055, Test 2.058; and Acc: Train 0.189, Test 0.191\n",
+      "2025-01-21 19:19:19 Epoch  1/15 done. Loss: Train 1.772, Test 1.805; and Acc: Train 0.354, Test 0.321\n",
+      "2025-01-21 19:19:26 Epoch  2/15 done. Loss: Train 1.530, Test 1.578; and Acc: Train 0.468, Test 0.446\n",
+      "2025-01-21 19:19:33 Epoch  3/15 done. Loss: Train 1.373, Test 1.437; and Acc: Train 0.518, Test 0.500\n",
+      "2025-01-21 19:19:41 Epoch  4/15 done. Loss: Train 1.254, Test 1.353; and Acc: Train 0.572, Test 0.541\n",
+      "2025-01-21 19:19:48 Epoch  5/15 done. Loss: Train 1.159, Test 1.289; and Acc: Train 0.597, Test 0.568\n",
+      "2025-01-21 19:19:55 Epoch  6/15 done. Loss: Train 1.068, Test 1.241; and Acc: Train 0.634, Test 0.567\n",
+      "2025-01-21 19:20:03 Epoch  7/15 done. Loss: Train 0.988, Test 1.199; and Acc: Train 0.668, Test 0.589\n",
+      "2025-01-21 19:20:10 Epoch  8/15 done. Loss: Train 0.911, Test 1.176; and Acc: Train 0.700, Test 0.587\n",
+      "2025-01-21 19:20:18 Epoch  9/15 done. Loss: Train 0.858, Test 1.169; and Acc: Train 0.721, Test 0.587\n",
+      "2025-01-21 19:20:25 Epoch 10/15 done. Loss: Train 0.782, Test 1.151; and Acc: Train 0.747, Test 0.599\n",
+      "2025-01-21 19:20:32 Epoch 11/15 done. Loss: Train 0.717, Test 1.143; and Acc: Train 0.771, Test 0.604\n",
+      "2025-01-21 19:20:40 Epoch 12/15 done. Loss: Train 0.657, Test 1.135; and Acc: Train 0.794, Test 0.610\n",
+      "2025-01-21 19:20:47 Epoch 13/15 done. Loss: Train 0.612, Test 1.147; and Acc: Train 0.819, Test 0.597\n",
+      "2025-01-21 19:20:54 Epoch 14/15 done. Loss: Train 0.553, Test 1.152; and Acc: Train 0.835, Test 0.599\n",
+      "2025-01-21 19:21:02 Epoch 15/15 done. Loss: Train 0.509, Test 1.166; and Acc: Train 0.857, Test 0.597\n"
      ]
     }
    ],
@@ -511,25 +520,232 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 28,
    "id": "ec2516f9-79f2-4ae1-ab9a-9a51a7a50587",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-21T18:57:29.278360Z",
-     "iopub.status.busy": "2025-01-21T18:57:29.276985Z",
-     "iopub.status.idle": "2025-01-21T18:57:29.289810Z",
-     "shell.execute_reply": "2025-01-21T18:57:29.288574Z",
-     "shell.execute_reply.started": "2025-01-21T18:57:29.278315Z"
+     "iopub.execute_input": "2025-01-22T00:23:23.018234Z",
+     "iopub.status.busy": "2025-01-22T00:23:23.017592Z",
+     "iopub.status.idle": "2025-01-22T00:23:23.049365Z",
+     "shell.execute_reply": "2025-01-22T00:23:23.048870Z",
+     "shell.execute_reply.started": "2025-01-22T00:23:23.018186Z"
     },
     "scrolled": true
    },
    "outputs": [
     {
-     "ename": "SyntaxError",
-     "evalue": "invalid syntax. Perhaps you forgot a comma? (3495586751.py, line 4)",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;36m  Cell \u001b[0;32mIn[35], line 4\u001b[0;36m\u001b[0m\n\u001b[0;31m    base_model=static_hyperparams[],\u001b[0m\n\u001b[0m               ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax. Perhaps you forgot a comma?\n"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---\n",
+      "base_model: google/bert_uncased_L-2_H-128_A-2\n",
+      "datasets:\n",
+      "- QuotaClimat/frugalaichallenge-text-train\n",
+      "language:\n",
+      "- en\n",
+      "license: apache-2.0\n",
+      "model_name: frugal-ai-text-bert-tiny\n",
+      "pipeline_tag: text-classification\n",
+      "tags:\n",
+      "- model_hub_mixin\n",
+      "- pytorch_model_hub_mixin\n",
+      "- climate\n",
+      "---\n",
+      "\n",
+      "# Model Card for Model ID\n",
+      "\n",
+      "<!-- Provide a quick summary of what the model is/does. -->\n",
+      "\n",
+      "Classify text into 8 categories of climate misinformation.\n",
+      "\n",
+      "## Model Details\n",
+      "\n",
+      "### Model Description\n",
+      "\n",
+      "<!-- Provide a longer summary of what this model is. -->\n",
+      "\n",
+      "Fine trained BERT for classifying climate information as part of the Frugal AI Challenge, for submission to https://huggingface.co/frugal-ai-challenge and scoring on accuracy and efficiency. Trainied on only the non-evaluation 80% of the data, so it's (non-cheating) score will be lower.\n",
+      "\n",
+      "- **Developed by:** Andre Bach\n",
+      "- **Funded by [optional]:** N/A\n",
+      "- **Shared by [optional]:** Andre Bach\n",
+      "- **Model type:** Text classification\n",
+      "- **Language(s) (NLP):** ['en']\n",
+      "- **License:** apache-2.0\n",
+      "- **Finetuned from model [optional]:** google/bert_uncased_L-2_H-128_A-2\n",
+      "\n",
+      "### Model Sources [optional]\n",
+      "\n",
+      "<!-- Provide the basic links for the model. -->\n",
+      "\n",
+      "- **Repository:** frugal-ai-text-bert-tiny\n",
+      "- **Paper [optional]:** [More Information Needed]\n",
+      "- **Demo [optional]:** [More Information Needed]\n",
+      "\n",
+      "## Uses\n",
+      "\n",
+      "<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->\n",
+      "\n",
+      "### Direct Use\n",
+      "\n",
+      "<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "### Downstream Use [optional]\n",
+      "\n",
+      "<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "### Out-of-Scope Use\n",
+      "\n",
+      "<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "## Bias, Risks, and Limitations\n",
+      "\n",
+      "<!-- This section is meant to convey both technical and sociotechnical limitations. -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "### Recommendations\n",
+      "\n",
+      "<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->\n",
+      "\n",
+      "Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.\n",
+      "\n",
+      "## How to Get Started with the Model\n",
+      "\n",
+      "Use the code below to get started with the model.\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "## Training Details\n",
+      "\n",
+      "### Training Data\n",
+      "\n",
+      "<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "### Training Procedure\n",
+      "\n",
+      "<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->\n",
+      "\n",
+      "#### Preprocessing [optional]\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "\n",
+      "#### Training Hyperparameters\n",
+      "\n",
+      "- **Training regime:** {'max_dataset_size': 'full', 'bert_variety': 'google/bert_uncased_L-2_H-128_A-2', 'max_length': 256, 'num_epochs': 15, 'batch_size': 16} <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->\n",
+      "\n",
+      "#### Speeds, Sizes, Times [optional]\n",
+      "\n",
+      "<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "## Evaluation\n",
+      "\n",
+      "<!-- This section describes the evaluation protocols and provides the results. -->\n",
+      "\n",
+      "### Testing Data, Factors & Metrics\n",
+      "\n",
+      "#### Testing Data\n",
+      "\n",
+      "<!-- This should link to a Dataset Card if possible. -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "#### Factors\n",
+      "\n",
+      "<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "#### Metrics\n",
+      "\n",
+      "<!-- These are the evaluation metrics being used, ideally with a description of why. -->\n",
+      "\n",
+      "{'train_loss': 0.5085738757594687, 'train_acc': 0.8565270935960592, 'test_loss': 1.1659069603139705, 'test_acc': 0.5972108285479901}\n",
+      "\n",
+      "### Results\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "#### Summary\n",
+      "\n",
+      "\n",
+      "\n",
+      "## Model Examination [optional]\n",
+      "\n",
+      "<!-- Relevant interpretability work for the model goes here -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "## Environmental Impact\n",
+      "\n",
+      "<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->\n",
+      "\n",
+      "Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).\n",
+      "\n",
+      "- **Hardware Type:** [More Information Needed]\n",
+      "- **Hours used:** [More Information Needed]\n",
+      "- **Cloud Provider:** [More Information Needed]\n",
+      "- **Compute Region:** [More Information Needed]\n",
+      "- **Carbon Emitted:** [More Information Needed]\n",
+      "\n",
+      "## Technical Specifications [optional]\n",
+      "\n",
+      "### Model Architecture and Objective\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "### Compute Infrastructure\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "#### Hardware\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "#### Software\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "## Citation [optional]\n",
+      "\n",
+      "<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->\n",
+      "\n",
+      "**BibTeX:**\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "**APA:**\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "## Glossary [optional]\n",
+      "\n",
+      "<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "## More Information [optional]\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "## Model Card Authors [optional]\n",
+      "\n",
+      "[More Information Needed]\n",
+      "\n",
+      "## Model Card Contact\n",
+      "\n",
+      "[More Information Needed]\n"
      ]
     }
    ],
@@ -553,16 +769,8 @@
     "    shared_by=\"Andre Bach\",\n",
     "    model_type=\"Text classification\",\n",
     "    repo=model_and_repo_name,\n",
-    "    training_regime=dict(\n",
-    "        max_dataset_size=\"full\",\n",
-    "        bert_variety=\"bert-base-uncased\",\n",
-    "        max_length=256,\n",
-    "        num_epochs=3,\n",
-    "        batch_size=16,\n",
-    "    ),\n",
-    "    testing_metrics=dict(\n",
-    "        loss_train=0.154, loss_test=0.978, acc_train=0.959, acc_test=0.705\n",
-    "    ),\n",
+    "    training_regime=training_regime,\n",
+    "    testing_metrics=testing_metrics,\n",
     ")\n",
     "# print(card_data.to_yaml())\n",
     "print(card)"
@@ -570,15 +778,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 30,
    "id": "29d3bbf9-ab2a-48e2-a550-e16da5025720",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-20T22:11:59.827681Z",
-     "iopub.status.busy": "2025-01-20T22:11:59.827001Z",
-     "iopub.status.idle": "2025-01-20T22:11:59.831852Z",
-     "shell.execute_reply": "2025-01-20T22:11:59.831047Z",
-     "shell.execute_reply.started": "2025-01-20T22:11:59.827635Z"
+     "iopub.execute_input": "2025-01-22T00:23:51.131078Z",
+     "iopub.status.busy": "2025-01-22T00:23:51.130578Z",
+     "iopub.status.idle": "2025-01-22T00:23:51.135440Z",
+     "shell.execute_reply": "2025-01-22T00:23:51.134263Z",
+     "shell.execute_reply.started": "2025-01-22T00:23:51.131042Z"
     }
    },
    "outputs": [],
@@ -589,15 +797,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 34,
    "id": "e3b099c6-6b98-473b-8797-5032213b9fcb",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-20T22:12:00.576369Z",
-     "iopub.status.busy": "2025-01-20T22:12:00.575421Z",
-     "iopub.status.idle": "2025-01-20T22:12:01.065512Z",
-     "shell.execute_reply": "2025-01-20T22:12:01.065237Z",
-     "shell.execute_reply.started": "2025-01-20T22:12:00.576294Z"
+     "iopub.execute_input": "2025-01-22T00:24:18.616547Z",
+     "iopub.status.busy": "2025-01-22T00:24:18.615990Z",
+     "iopub.status.idle": "2025-01-22T00:24:18.669435Z",
+     "shell.execute_reply": "2025-01-22T00:24:18.669063Z",
+     "shell.execute_reply.started": "2025-01-22T00:24:18.616509Z"
     }
    },
    "outputs": [
@@ -605,7 +813,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2025-01-20 14:12:01 Predictions: tensor([0, 0, 3, 6, 2, 4, 6], device='mps:0')\n"
+      "2025-01-21 19:24:18 Predictions: tensor([0, 0, 3, 1, 2, 6, 6], device='mps:0')\n"
      ]
     }
    ],
@@ -638,27 +846,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 35,
    "id": "befb94b5-88bf-40fc-8b26-cf373d1256e0",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-20T22:12:15.099356Z",
-     "iopub.status.busy": "2025-01-20T22:12:15.098818Z",
-     "iopub.status.idle": "2025-01-20T22:12:33.175760Z",
-     "shell.execute_reply": "2025-01-20T22:12:33.174719Z",
-     "shell.execute_reply.started": "2025-01-20T22:12:15.099315Z"
+     "iopub.execute_input": "2025-01-22T00:24:41.153062Z",
+     "iopub.status.busy": "2025-01-22T00:24:41.152049Z",
+     "iopub.status.idle": "2025-01-22T00:24:43.436376Z",
+     "shell.execute_reply": "2025-01-22T00:24:43.435250Z",
+     "shell.execute_reply.started": "2025-01-22T00:24:41.153018Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "fbc09ae2c5614831a2fb02fa48a44fd1",
+       "model_id": "ef4fd0b071034f7d9cba6aa0ab69d148",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "model.safetensors:   0%|          | 0.00/438M [00:00<?, ?B/s]"
+       "model.safetensors:   0%|          | 0.00/17.6M [00:00<?, ?B/s]"
       ]
      },
      "metadata": {},
@@ -667,74 +875,88 @@
     {
      "data": {
       "text/plain": [
-       "CommitInfo(commit_url='https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-base/commit/bdc2daf80d9647566ef56297f2cdc32f898170df', commit_message='Push model using huggingface_hub.', commit_description='', oid='bdc2daf80d9647566ef56297f2cdc32f898170df', pr_url=None, repo_url=RepoUrl('https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-base', endpoint='https://huggingface.co', repo_type='model', repo_id='Nonnormalizable/frugal-ai-text-bert-base'), pr_revision=None, pr_num=None)"
+       "CommitInfo(commit_url='https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-tiny/commit/512fbc46e1cfc7456f4e9f2331a30d66a9052d88', commit_message='Push model using huggingface_hub.', commit_description='', oid='512fbc46e1cfc7456f4e9f2331a30d66a9052d88', pr_url=None, repo_url=RepoUrl('https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-tiny', endpoint='https://huggingface.co', repo_type='model', repo_id='Nonnormalizable/frugal-ai-text-bert-tiny'), pr_revision=None, pr_num=None)"
       ]
      },
-     "execution_count": 19,
+     "execution_count": 35,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "model_final.push_to_hub(\"frugal-ai-text-bert-base\")"
+    "model_final.push_to_hub(model_and_repo_name)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 36,
    "id": "251ef9ee-8ba3-495f-8fe6-a93aa63168ce",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-20T22:12:33.178424Z",
-     "iopub.status.busy": "2025-01-20T22:12:33.178028Z",
-     "iopub.status.idle": "2025-01-20T22:12:34.321979Z",
-     "shell.execute_reply": "2025-01-20T22:12:34.320974Z",
-     "shell.execute_reply.started": "2025-01-20T22:12:33.178397Z"
+     "iopub.execute_input": "2025-01-22T00:24:48.887758Z",
+     "iopub.status.busy": "2025-01-22T00:24:48.887178Z",
+     "iopub.status.idle": "2025-01-22T00:24:49.581460Z",
+     "shell.execute_reply": "2025-01-22T00:24:49.580127Z",
+     "shell.execute_reply.started": "2025-01-22T00:24:48.887716Z"
     }
    },
    "outputs": [
     {
      "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f3f6ee50ec314983a492ab7f4f5ef1bc",
+       "version_major": 2,
+       "version_minor": 0
+      },
       "text/plain": [
-       "CommitInfo(commit_url='https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-base/commit/9081285a20fa0d62c5c1580aa17884de2b3bc236', commit_message='Upload tokenizer', commit_description='', oid='9081285a20fa0d62c5c1580aa17884de2b3bc236', pr_url=None, repo_url=RepoUrl('https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-base', endpoint='https://huggingface.co', repo_type='model', repo_id='Nonnormalizable/frugal-ai-text-bert-base'), pr_revision=None, pr_num=None)"
+       "README.md:   0%|          | 0.00/320 [00:00<?, ?B/s]"
       ]
      },
-     "execution_count": 20,
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-tiny/commit/1870eebdb54a6a636d7dd32a5d923abc8d1baaec', commit_message='Upload tokenizer', commit_description='', oid='1870eebdb54a6a636d7dd32a5d923abc8d1baaec', pr_url=None, repo_url=RepoUrl('https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-tiny', endpoint='https://huggingface.co', repo_type='model', repo_id='Nonnormalizable/frugal-ai-text-bert-tiny'), pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 36,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "tokenizer_final.push_to_hub(\"frugal-ai-text-bert-base\")"
+    "tokenizer_final.push_to_hub(model_and_repo_name)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 38,
    "id": "863d3553-89a6-4188-a8d0-eaa0b6bccb6c",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-01-20T22:12:34.324003Z",
-     "iopub.status.busy": "2025-01-20T22:12:34.323725Z",
-     "iopub.status.idle": "2025-01-20T22:12:35.350962Z",
-     "shell.execute_reply": "2025-01-20T22:12:35.350482Z",
-     "shell.execute_reply.started": "2025-01-20T22:12:34.323976Z"
+     "iopub.execute_input": "2025-01-22T00:25:24.402856Z",
+     "iopub.status.busy": "2025-01-22T00:25:24.402275Z",
+     "iopub.status.idle": "2025-01-22T00:25:25.011133Z",
+     "shell.execute_reply": "2025-01-22T00:25:25.009553Z",
+     "shell.execute_reply.started": "2025-01-22T00:25:24.402817Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "CommitInfo(commit_url='https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-base/commit/b3078a95ea36d71c1d1bf0d153e069b83f74bddf', commit_message='Upload README.md with huggingface_hub', commit_description='', oid='b3078a95ea36d71c1d1bf0d153e069b83f74bddf', pr_url=None, repo_url=RepoUrl('https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-base', endpoint='https://huggingface.co', repo_type='model', repo_id='Nonnormalizable/frugal-ai-text-bert-base'), pr_revision=None, pr_num=None)"
+       "CommitInfo(commit_url='https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-tiny/commit/62d658978d97614f10c3d69b6595a0fb6b8a2d4c', commit_message='Upload README.md with huggingface_hub', commit_description='', oid='62d658978d97614f10c3d69b6595a0fb6b8a2d4c', pr_url=None, repo_url=RepoUrl('https://huggingface.co/Nonnormalizable/frugal-ai-text-bert-tiny', endpoint='https://huggingface.co', repo_type='model', repo_id='Nonnormalizable/frugal-ai-text-bert-tiny'), pr_revision=None, pr_num=None)"
       ]
      },
-     "execution_count": 21,
+     "execution_count": 38,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "card.push_to_hub(\"Nonnormalizable/frugal-ai-text-bert-base\")"
+    "card.push_to_hub(f\"Nonnormalizable/{model_and_repo_name}\")"
    ]
   },
   {
@@ -766,7 +988,202 @@
   },
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
-    "state": {},
+    "state": {
+     "05dc5be1c7754da082079d8c8fdd3a2a": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "07fc7cde058940a29aba80b9b1f18247": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "14a18f1cb1f244b28fa35c504023c27d": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "layout": "IPY_MODEL_07fc7cde058940a29aba80b9b1f18247",
+       "style": "IPY_MODEL_05dc5be1c7754da082079d8c8fdd3a2a",
+       "value": "model.safetensors: 100%"
+      }
+     },
+     "1aedcea94f5a43919f4b0dd14eb54ab0": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "bar_style": "success",
+       "layout": "IPY_MODEL_3a0167722aae49eb87c0cc1a0b381a7c",
+       "max": 320,
+       "style": "IPY_MODEL_75738a2d58574271b291cf982f86074a",
+       "value": 320
+      }
+     },
+     "350a27df5da248b3b06dbad6b28bc789": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "bar_style": "success",
+       "layout": "IPY_MODEL_eaf57ba3273f42d4900ce2ebae398892",
+       "max": 17552376,
+       "style": "IPY_MODEL_aa86a038e70b43368dd5ccb65b653f86",
+       "value": 17552376
+      }
+     },
+     "3a0167722aae49eb87c0cc1a0b381a7c": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "40d60079070e4bd2b81fbcfd695cd759": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "605845a520ee426fa2330ded19d8a617": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "7402f6bafb5f485e9f172cdf017572c7": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "75738a2d58574271b291cf982f86074a": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "7a56b64874594d50bb09ee06bb656e54": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "7bc93085c8f54642b6005f0f701772fa": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "8bdfbdb11197418d894a54ec6d487906": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "layout": "IPY_MODEL_ae882ab226794053a4f906ac3b5e49b9",
+       "style": "IPY_MODEL_605845a520ee426fa2330ded19d8a617",
+       "value": " 17.6M/17.6M [00:00&lt;00:00, 27.8MB/s]"
+      }
+     },
+     "9a5aac888e72447fa2f379cd3e25a11c": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "aa86a038e70b43368dd5ccb65b653f86": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "acfe7063ca0f4ea6acd6f308071f4418": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "layout": "IPY_MODEL_9a5aac888e72447fa2f379cd3e25a11c",
+       "style": "IPY_MODEL_7bc93085c8f54642b6005f0f701772fa",
+       "value": "README.md: 100%"
+      }
+     },
+     "ae882ab226794053a4f906ac3b5e49b9": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "d84082e3eb2e4ababf340e8262c5489c": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "eaf57ba3273f42d4900ce2ebae398892": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "ef4fd0b071034f7d9cba6aa0ab69d148": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "children": [
+        "IPY_MODEL_14a18f1cb1f244b28fa35c504023c27d",
+        "IPY_MODEL_350a27df5da248b3b06dbad6b28bc789",
+        "IPY_MODEL_8bdfbdb11197418d894a54ec6d487906"
+       ],
+       "layout": "IPY_MODEL_7402f6bafb5f485e9f172cdf017572c7"
+      }
+     },
+     "f3f6ee50ec314983a492ab7f4f5ef1bc": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "children": [
+        "IPY_MODEL_acfe7063ca0f4ea6acd6f308071f4418",
+        "IPY_MODEL_1aedcea94f5a43919f4b0dd14eb54ab0",
+        "IPY_MODEL_f8f939982d7542969718ad06c692c488"
+       ],
+       "layout": "IPY_MODEL_d84082e3eb2e4ababf340e8262c5489c"
+      }
+     },
+     "f8f939982d7542969718ad06c692c488": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "layout": "IPY_MODEL_40d60079070e4bd2b81fbcfd695cd759",
+       "style": "IPY_MODEL_7a56b64874594d50bb09ee06bb656e54",
+       "value": " 320/320 [00:00&lt;00:00, 25.4kB/s]"
+      }
+     }
+    },
     "version_major": 2,
     "version_minor": 0
    }

tasks/text.py

@@ -18,7 +18,7 @@ DESCRIPTIONS = {
     "bert-medium": "to be implemented",
     "bert-small": "to be implemented",
     "bert-mini": "to be implemented",
-    "bert-tiny": "to be implemented",
+    "bert-tiny": "bert tiny finetuned",
 }
 
 ROUTE = "/text"
@@ -73,7 +73,7 @@ def bert_model(test_dataset: dict, model_type: str):
 @router.post(ROUTE, tags=["Text Task"])
 async def evaluate_text(
     request: TextEvaluationRequest,
-    model_type: str = "bert-base",
+    model_type: str = "bert-tiny",
     # This should be an API query parameter, but it looks like the submission repo
     # https://huggingface.co/spaces/frugal-ai-challenge/submission-portal
     # is built in a way to not accept any other endpoints or parameters.