Feat(P3L1): Configure TrainingArguments and instantiate Trainer

01-FineTuning-QA.ipynb

@@ -800,9 +800,155 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "id": "1d26984c-395e-4b7b-a8cf-1c71fb61394b",
    "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Training Arguments defined:\n"
+     ]
+    }
+   ],
+   "source": [
+    "#Sub-Step 7.2: Define Training Arguments\n",
+    "\n",
+    "from transformers import TrainingArguments\n",
+    "\n",
+    "# Define the directory where model checkpoints will be saved during/after training\n",
+    "output_directory = \"malay-qa-model-finetuned\" # You can change this name if you like\n",
+    "\n",
+    "# Configure the training parameters\n",
+    "# These values are starting points; adjust based on performance and GPU memory.\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=output_directory,              # Directory to save model checkpoints and logs\n",
+    "\n",
+    "    # --- Logging & Saving ---\n",
+    "    logging_strategy=\"steps\",                 # Log training loss at specified intervals\n",
+    "    logging_steps=5,                         # Log training loss every 5 steps (adjust based on dataset size)\n",
+    "    save_strategy=\"epoch\",                    # Save a checkpoint at the end of each epoch\n",
+    "    # save_total_limit=2,                     # Optional: Only keep the latest 2 checkpoints\n",
+    "\n",
+    "    # --- Evaluation ---\n",
+    "    eval_strategy=\"epoch\",              # Run evaluation at the end of each epoch\n",
+    "\n",
+    "    # --- Training Hyperparameters ---\n",
+    "    learning_rate=2e-5,                       # Starting learning rate (AdamW optimizer default: 5e-5) - 2e-5 is common for fine-tuning\n",
+    "    num_train_epochs=1,                       # Number of times to iterate over the entire training dataset. START SMALL (1-3).\n",
+    "    per_device_train_batch_size=4,            # Number of training examples per batch per GPU/CPU. DECREASE if you get \"CUDA out of memory\" errors. Start small (4 or 8).\n",
+    "    per_device_eval_batch_size=4,             # Batch size for evaluation. DECREASE if memory errors during eval.\n",
+    "    weight_decay=0.01,                        # Adds a small penalty to large weights to prevent overfitting\n",
+    "\n",
+    "    # --- Other Settings ---\n",
+    "    push_to_hub=False,                        # Set to True later if you want to upload model to Hugging Face Hub\n",
+    "    report_to=\"none\",                         # Disable integrations like WandB/TensorBoard for simplicity now\n",
+    "    # fp16=torch.cuda.is_available(),         # Optional: Enable mixed-precision training if on GPU (can speed up training and save memory) - requires 'accelerate' library\n",
+    ")\n",
+    "\n",
+    "print(\"\\nTraining Arguments defined:\")\n",
+    "# Display the arguments to verify\n",
+    "# Note: It might show many more default arguments as well\n",
+    "# print(training_args)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "11dc845d-4f57-4931-b15b-0c7d948fc770",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Using DefaultDataCollator.\n"
+     ]
+    }
+   ],
+   "source": [
+    "#Sub-Step 7.3: Define Data Collator\n",
+    "\n",
+    "from transformers import DefaultDataCollator\n",
+    "\n",
+    "# DefaultDataCollator is suitable for standard token classification, sequence classification,\n",
+    "# and also QA tasks where inputs are already padded to max_length by the tokenizer.\n",
+    "# It converts lists of dictionaries into batches of tensors.\n",
+    "data_collator = DefaultDataCollator()\n",
+    "\n",
+    "print(\"\\nUsing DefaultDataCollator.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "33d37330-080e-4400-bd14-272d5d770780",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Instantiating Trainer...\n",
+      "Trainer instantiated successfully.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\amiru\\AppData\\Local\\Temp\\ipykernel_14448\\3333510307.py:21: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
+      "  trainer = Trainer(\n"
+     ]
+    }
+   ],
+   "source": [
+    "#Sub-Step 7.4: Instantiate the Trainer\n",
+    "\n",
+    "from transformers import Trainer\n",
+    "import torch # Ensure torch is imported if checking CUDA\n",
+    "\n",
+    "# --- Verify required variables exist ---\n",
+    "required_vars = ['model', 'training_args', 'tokenized_datasets', 'tokenizer', 'data_collator']\n",
+    "for var_name in required_vars:\n",
+    "    if not var_name in locals():\n",
+    "        print(f\"ERROR: Required variable '{var_name}' not found. Please run previous steps.\")\n",
+    "        raise NameError(f\"Variable '{var_name}' not defined.\")\n",
+    "\n",
+    "# Check dataset structure again (paranoid check)\n",
+    "if not isinstance(tokenized_datasets, DatasetDict) or 'train' not in tokenized_datasets or 'eval' not in tokenized_datasets:\n",
+    "    print(\"ERROR: 'tokenized_datasets' is not a DatasetDict or lacks 'train'/'eval' splits.\")\n",
+    "    raise TypeError(\"'tokenized_datasets' has incorrect type or structure.\")\n",
+    "\n",
+    "print(\"\\nInstantiating Trainer...\")\n",
+    "try:\n",
+    "    # Create the Trainer instance\n",
+    "    trainer = Trainer(\n",
+    "        model=model,                         # The pre-trained QA model loaded in Step 6\n",
+    "        args=training_args,                  # The configuration object defined above\n",
+    "        train_dataset=tokenized_datasets[\"train\"], # The tokenized training data split\n",
+    "        eval_dataset=tokenized_datasets[\"eval\"],   # The tokenized evaluation data split\n",
+    "        tokenizer=tokenizer,                 # The tokenizer (used for saving/padding consistency)\n",
+    "        data_collator=data_collator,         # How to create batches from dataset samples\n",
+    "        # compute_metrics=compute_metrics,   # We're skipping custom QA metrics (EM/F1) for Level 1 MVP\n",
+    "    )\n",
+    "    print(\"Trainer instantiated successfully.\")\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(f\"ERROR: Failed to instantiate Trainer. Check input arguments and configurations.\")\n",
+    "    print(f\"Error details: {e}\")\n",
+    "    raise"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "078ecf6d-b2d0-47f4-b20b-d57ab02ca498",
+   "metadata": {},
    "outputs": [],
    "source": []
   }