create_faiss.py

from datasets import load_dataset
import faiss
from sentence_transformers import SentenceTransformer
import numpy as np

# Load the US-LegalKit dataset
dataset = load_dataset("macadeliccc/US-LegalKit", split="train")

# Extract legal text documents
law_data = [item['text'] for item in dataset if 'text' in item]

# Load embedding model
model = SentenceTransformer("all-MiniLM-L6-v2")

# Generate embeddings
embeddings = model.encode(law_data, convert_to_numpy=True)

# Create FAISS index
dimension = embeddings.shape[1]
index = faiss.IndexFlatL2(dimension)  # L2 Distance Index
index.add(embeddings)  # Add vectors to FAISS index

# Save FAISS index
faiss.write_index(index, "faiss_index.bin")

print("✅ FAISS index saved successfully as 'faiss_index.bin'!")