training pipeline version control bug fix

training_pipeline.ipynb

@@ -2,9 +2,17 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-12 10:07:12,956 INFO: PyTorch version 2.5.1 available.\n"
+     ]
+    }
+   ],
    "source": [
     "import hopsworks\n",
     "from sentence_transformers import SentenceTransformer, InputExample, losses\n",
@@ -16,18 +24,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2025-01-08 19:52:22,417 INFO: Closing external client and cleaning up certificates.\n",
-      "Connection closed.\n",
-      "2025-01-08 19:52:22,421 INFO: Initializing external client\n",
-      "2025-01-08 19:52:22,421 INFO: Base URL: https://c.app.hopsworks.ai:443\n",
-      "2025-01-08 19:52:23,548 INFO: Python Engine initialized.\n",
+      "2025-01-12 10:07:14,851 INFO: Initializing external client\n",
+      "2025-01-12 10:07:14,852 INFO: Base URL: https://c.app.hopsworks.ai:443\n",
+      "2025-01-12 10:07:15,245 WARNING: InsecureRequestWarning: Unverified HTTPS request is being made to host 'c.app.hopsworks.ai'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n",
+      "\n",
+      "2025-01-12 10:07:18,039 INFO: Python Engine initialized.\n",
       "\n",
       "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/1158296\n"
      ]
@@ -51,7 +59,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Finished: Reading data from Hopsworks, using Hopsworks Feature Query Service (0.84s) \n"
+      "Finished: Reading data from Hopsworks, using Hopsworks Feature Query Service (2.46s) \n"
      ]
     }
    ],
@@ -109,8 +117,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2025-01-08 19:25:05,476 INFO: Use pytorch device_name: cpu\n",
-      "2025-01-08 19:25:05,477 INFO: Load pretrained SentenceTransformer: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2\n"
+      "2025-01-12 10:07:23,794 INFO: Use pytorch device_name: cpu\n",
+      "2025-01-12 10:07:23,795 INFO: Load pretrained SentenceTransformer: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2\n"
      ]
     }
    ],
@@ -159,12 +167,12 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "65a11878fdad456a94ae2e4d44e403a3",
+       "model_id": "13a4c4779de349a4a93c26a2a952d713",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "  0%|          | 0/3 [00:00<?, ?it/s]"
+       "  0%|          | 0/6 [00:00<?, ?it/s]"
       ]
      },
      "metadata": {},
@@ -174,28 +182,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'train_runtime': 5.2094, 'train_samples_per_second': 2.879, 'train_steps_per_second': 0.576, 'train_loss': 0.27454523245493573, 'epoch': 3.0}\n",
-      "2025-01-08 19:25:14,162 INFO: Save model to ./finetuned_model\n"
+      "{'train_runtime': 16.1772, 'train_samples_per_second': 4.265, 'train_steps_per_second': 0.371, 'train_loss': 0.18365144729614258, 'epoch': 3.0}\n",
+      "2025-01-12 10:07:44,670 INFO: Save model to ./finetuned_model\n"
      ]
     },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "7bc7a5e2e56e4abe8bbf47e5ed251d6a",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Computing widget examples:   0%|          | 0/1 [00:00<?, ?example/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a4218c62846f43c7be217513f8fd86de",
+       "model_id": "a36ca79a9a5245c3931717a3c466bba9",
        "version_major": 2,
        "version_minor": 0
       },
@@ -265,12 +259,50 @@
    "cell_type": "code",
    "execution_count": 15,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-12 10:07:45,746 WARNING: InsecureRequestWarning: Unverified HTTPS request is being made to host 'c.app.hopsworks.ai'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n",
+      "\n"
+     ]
+    }
+   ],
    "source": [
     "# Get Model Registry\n",
     "mr = project.get_model_registry()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-12 10:08:24,657 WARNING: VersionWarning: No version provided for getting model `job_matching_sentence_transformer`, defaulting to `1`.\n",
+      "\n",
+      "Model already exists with version 1\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Check if the model already exists and get the latest version\n",
+    "try:\n",
+    "    existing_model = mr.get_model(name=\"job_matching_sentence_transformer\")\n",
+    "    latest_version = existing_model.version\n",
+    "    print(f\"Model already exists with version {latest_version}\")\n",
+    "except:\n",
+    "    # If the model doesn't exist, set version to 1\n",
+    "    latest_version = 0\n",
+    "\n",
+    "# Set the new version dynamically\n",
+    "new_version = latest_version + 1"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 19,
@@ -284,7 +316,7 @@
     "    model_schema=model_schema,\n",
     "    input_example=X_train_sample,\n",
     "    description=\"Finetuned SentenceTransformer for job matching\",\n",
-    "    version=1\n",
+    "    version=new_version,\n",
     ")"
    ]
   },