Create create_faiss.py

create_faiss.py

@@ -0,0 +1,25 @@
+from datasets import load_dataset
+import faiss
+from sentence_transformers import SentenceTransformer
+import numpy as np
+
+# Load the Pile dataset (legal text)
+dataset = load_dataset("EleutherAI/the_pile", split="train")
+
+# Extract legal-related documents
+law_data = [item['text'] for item in dataset if item['meta']['pile_set_name'] == 'Pile-CC']
+
+# Load embedding model
+model = SentenceTransformer("all-MiniLM-L6-v2")
+
+# Generate embeddings
+embeddings = model.encode(law_data, convert_to_numpy=True)
+
+# Create FAISS index
+index = faiss.IndexFlatL2(embeddings.shape[1])
+index.add(embeddings)
+
+# Save FAISS index
+faiss.write_index(index, "faiss_index.bin")
+
+print("✅ FAISS index saved successfully!")