reranking added

.gitignore

@@ -0,0 +1,2 @@
+
+*.pyc

app.py

@@ -7,17 +7,25 @@ Created on Wed Mar 22 19:59:54 2023
 import gradio as gr
 from simiandb import Simiandb
 from langchain.embeddings import HuggingFaceEmbeddings
+from sentence_transformers import CrossEncoder
 
 
 
 
 model_name = "all-MiniLM-L6-v2"
 hf = HuggingFaceEmbeddings(model_name=model_name)
+cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
 
 documentdb = Simiandb("mystore", embedding_function=hf, mode="a")
 
 def search(query):
-    return documentdb.similarity_search(query)
+    hits = documentdb.similarity_search(query)
+    cross_inp = [[query, hit] for hit in hits]
+    cross_scores = cross_encoder.predict(cross_inp)
+    hits = [hit for _, hit in sorted(zip(cross_scores, hits), reverse=True)]
+    return hits[0]
 
 iface = gr.Interface(fn=search, inputs="text", outputs="text")
-iface.launch()
+iface.launch()
+
+#print(search("what is the balloon boy hoax"))

simiandb.py

@@ -178,7 +178,7 @@ class Simiandb():
         batch = self._vector_table.chunkshape[0]*25
         res = np.ascontiguousarray(np.empty(shape=(count,), dtype="float32"))
         end = 0
-        a = time()
+
         while end!=count:
             end += batch
             end = end if end <= count else count 
@@ -189,7 +189,7 @@ class Simiandb():
 
         indices = np.argpartition(res, -k)[-k:] #from https://stackoverflow.com/questions/6910641/how-do-i-get-indices-of-n-maximum-values-in-a-numpy-array
         indices = indices[np.argsort(res[indices])[::-1]]
-        print(time() -a)
+
         return indices