using data.dir for datasets

app.py

@@ -5,6 +5,7 @@ import faiss
 import numpy as np
 import os
 import time
+import json 
 
 # ✅ Ensure FAISS is installed
 os.system("pip install faiss-cpu")
@@ -12,15 +13,32 @@ os.system("pip install faiss-cpu")
 def log(message):
     print(f"✅ {message}")
 
-# ✅ Load the datasets
-log("📥 Loading datasets...")
+
+DATA_DIR = "data"
+
+def load_local_dataset(dataset_name):
+    """Load a dataset from a JSON file."""
+    file_path = os.path.join(DATA_DIR, f"{dataset_name}.json")
+    
+    if os.path.exists(file_path):
+        with open(file_path, "r") as f:
+            data = json.load(f)
+        print(f"✅ Loaded {dataset_name} from {file_path}")
+        return data
+    else:
+        print(f"❌ ERROR: {dataset_name} file not found!")
+        return None
+
+# ✅ Load all datasets from local storage
 datasets = {
-    "sales": load_dataset("goendalf666/sales-conversations", trust_remote_code=True),
-    "blended": load_dataset("blended_skill_talk", trust_remote_code=True),
-    "dialog": load_dataset("daily_dialog", trust_remote_code=True),
-    "multiwoz": load_dataset("multi_woz_v22", trust_remote_code=True),
+    "sales": load_local_dataset("sales"),
+    "blended": load_local_dataset("blended"),
+    "dialog": load_local_dataset("dialog"),
+    "multiwoz": load_local_dataset("multiwoz"),
 }
-log("✅ Datasets loaded.")
+
+print("✅ Datasets loaded from local storage!")
+
 
 # ✅ Step 1: Run Embedding Script (Import and Run)
 log("🚀 Running embeddings script...")