saving datasets in json

embeddings.py

@@ -3,11 +3,44 @@ from transformers import AutoTokenizer, AutoModel
 import faiss
 import torch
 import numpy as np
+import os
+import json
 
 def log(message):
     print(f"✅ {message}")
 
 
+
+# ✅ Ensure data folder exists
+DATA_DIR = "data"
+os.makedirs(DATA_DIR, exist_ok=True)
+
+# ✅ List of datasets
+datasets_list = {
+    "sales": "goendalf666/sales-conversations",
+    "blended": "blended_skill_talk",
+    "dialog": "daily_dialog",
+    "multiwoz": "multi_woz_v22",
+}
+
+def save_dataset_to_file(dataset_name, dataset):
+    """Save dataset to a local JSON file."""
+    file_path = os.path.join(DATA_DIR, f"{dataset_name}.json")
+    
+    with open(file_path, "w") as f:
+        json.dump(dataset["train"].to_dict(), f)
+    
+    print(f"✅ Saved {dataset_name} to {file_path}")
+
+# ✅ Load & Save all datasets
+for name, dataset_id in datasets_list.items():
+    dataset = load_dataset(dataset_id, split="train")
+    save_dataset_to_file(name, dataset)
+    
+print("✅ All datasets saved locally!")
+
+
+
 # ✅ Load datasets
 datasets = {
     "sales": load_dataset("goendalf666/sales-conversations"),