fix: normalize faiss

faiss_index/index.py

@@ -61,20 +61,21 @@ def build_faiss_index(papers, dataset_dir=DATASET_DIR):
                 torch.cuda.empty_cache()
     
     # Convert to numpy array and build FAISS index
-    embeddings = np.array(embeddings)
+    embeddings = np.array(embeddings, dtype=np.float32)  # Ensure float32 type
     dimension = embeddings.shape[1]
     
-    # Use more efficient index type
-    index = faiss.IndexFlatIP(dimension)  # Simple but efficient dot-product index
+    # Normalize the vectors manually
+    norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+    embeddings = embeddings / norms
     
-    # Normalize vectors to use dot product as similarity
-    faiss.normalize_L2(embeddings)
+    # Create FAISS index
+    index = faiss.IndexFlatIP(dimension)
     index.add(embeddings)
     
     # Create and save the dataset
     dataset = Dataset.from_dict({
         "text": texts,
-        "embeddings": embeddings,
+        "embeddings": embeddings.tolist(),  # Convert to list for storage
         "title": [p["title"] for p in papers]
     })