updating proper format to embed the datasets and print statements to get progress

embeddings.py

@@ -27,12 +27,52 @@ def embed_text(texts):
         embeddings = model(**inputs).last_hidden_state.mean(dim=1).cpu().numpy()
     return embeddings
 
+
 # ✅ Extract and embed the datasets
 def create_embeddings(dataset_name, dataset):
-    print(f"Creating embeddings for {dataset_name}...")
-    texts = [text for text in dataset["train"]['text']]  # Adjust the field depending on dataset structure
-    embeddings = embed_text(texts)
-    return embeddings
+    print(f"📥 Creating embeddings for {dataset_name}...")
+    
+    if dataset_name == "goendalf666/sales-conversations":
+        texts = [" ".join(row.values()) for row in dataset["train"]]
+
+    elif dataset_name == "AlekseyKorshuk/persona-chat":
+        texts = [" ".join(utterance["candidates"]) for utterance in dataset["train"]["utterances"]]
+
+    elif dataset_name == "blended_skill_talk":
+        texts = [" ".join(row["free_messages"] + row["guided_messages"]) for row in dataset["train"]]
+
+    elif dataset_name == "daily_dialog":
+        texts = [" ".join(row["dialog"]) for row in dataset["train"]]
+
+    elif dataset_name == "multi_woz_v22":
+        texts = [" ".join(row["turns"]["utterance"]) for row in dataset["train"]]
+
+    else:
+        print(f"⚠️ Warning: Dataset {dataset_name} not handled properly!")
+        texts = []
+
+    # ✅ Verify dataset extraction
+    if len(texts) == 0:
+        print(f"❌ ERROR: No text extracted from {dataset_name}! Check dataset structure.")
+    else:
+        print(f"✅ Extracted {len(texts)} texts from {dataset_name}. Sample:\n{texts[:3]}")
+
+    return texts
+
+# ✅ Embed and store in FAISS
+for name, dataset in datasets.items():
+    texts = create_embeddings(name, dataset)
+
+    if len(texts) > 0:  # ✅ Only embed if texts exist
+        embeddings = embed_text(texts)
+        print(f"✅ Generated embeddings shape: {embeddings.shape}")
+
+        index = save_embeddings_to_faiss(embeddings)
+        print(f"✅ Embeddings for {name} saved to FAISS.")
+    else:
+        print(f"⚠️ Skipping embedding for {name} (No valid texts).")
+
+
 
 # ✅ Save embeddings to a database
 def save_embeddings_to_faiss(embeddings, index_name="my_embeddings"):