empaly / script.py
pankajsingh3012's picture
Upload 7 files
1762587 verified
raw
history blame contribute delete
642 Bytes
# with open("local_data.txt","r") as f:
#
# data =f.read()
# # for x in data:
# # print(x)
# with open("data.txt","+a") as d:
# for x in data.split("."):
# d.write(str(x)+".")
import faiss
import numpy as np
import google.generativeai as genai
from sentence_transformers import SentenceTransformer
with open("data.txt", "r", encoding="utf-8") as f:
snippets = [line.strip() for line in f if line.strip()]
embed_model = SentenceTransformer("all-MiniLM-L6-v2")
embeddings = embed_model.encode(snippets)
index = faiss.IndexFlatL2(embeddings.shape[1])
index.add(np.array(embeddings))
print(index)