hybrid test

app.py

@@ -7,8 +7,9 @@ from qdrant_client import QdrantClient
 from langchain.retrievers import ContextualCompressionRetriever
 from langchain.retrievers.document_compressors import CrossEncoderReranker
 from langchain_community.cross_encoders import HuggingFaceCrossEncoder
-from langchain_qdrant import FastEmbedSparse, RetrievalMode
 from appStore.prep_data import process_giz_worldwide
+from appStore.prep_utils import create_documents
+from appStore.embed import hybrid_embed_chunks
 
 # get the device to be used eithe gpu or cpu
 device = 'cuda' if cuda.is_available() else 'cpu'
@@ -18,28 +19,6 @@ st.set_page_config(page_title="SEARCH IATI",layout='wide')
 st.title("SEARCH IATI Database")
 var=st.text_input("enter keyword")
 
-def embed_chunks(chunks):
-    """
-    takes the chunks and does the hybrid embedding for the list of chunks
-    """
-    embeddings = HuggingFaceEmbeddings(
-        model_kwargs = {'device': device},
-        encode_kwargs = {'normalize_embeddings': True},
-        model_name='BAAI/bge-m3'
-    )
-    #sparse_embeddings = FastEmbedSparse(model_name="Qdrant/bm25")
-    # placeholder for collection
-    print("starting embedding")
-    qdrant_collections = {}
-    qdrant_collections['all'] = Qdrant.from_documents(
-                chunks,
-                embeddings,
-                path="/data/local_qdrant",
-                collection_name='all',
-            )
-            
-    print(qdrant_collections)
-    print("vector embeddings done")
     
 @st.cache_resource    
 def get_local_qdrant(): 
@@ -77,7 +56,12 @@ def get_context(vectorstore,query):
 # first we create the chunks for iati documents
 chunks = process_giz_worldwide()
 for i in range(5):
-    print(chunks.loc[0,'chunks'])
+    print(i,"\n",chunks.loc[i,'chunks'])
+temp_df = chunks[:5]
+temp_doc = create_documents(temp_df,'chunks')
+hybrid_embed_chunks(temp_doc)
+
+
 #print("chunking done")
 
 # once the chunks are done, we perform hybrid emebddings
@@ -85,7 +69,7 @@ for i in range(5):
 
 # vectorstores = get_local_qdrant() 
 # vectorstore = vectorstores['all']
-# button=st.button("search")
+button=st.button("search")
 # results= get_context(vectorstore, f"find the relvant paragraphs for: {var}")
 if button:
      st.write(f"Found {len(results)} results for query:{var}")

appStore/embed.py

@@ -0,0 +1,31 @@
+from langchain_community.embeddings import HuggingFaceEmbeddings, HuggingFaceInferenceAPIEmbeddings
+from langchain_community.vectorstores import Qdrant
+from langchain_qdrant import FastEmbedSparse, RetrievalMode
+# get the device to be used eithe gpu or cpu
+device = 'cuda' if cuda.is_available() else 'cpu'
+
+
+def hybrid_embed_chunks(chunks):
+    """
+    takes the chunks and does the hybrid embedding for the list of chunks
+    """
+    embeddings = HuggingFaceEmbeddings(
+        model_kwargs = {'device': device},
+        encode_kwargs = {'normalize_embeddings': True},
+        model_name='BAAI/bge-m3'
+    )
+    sparse_embeddings = FastEmbedSparse(model_name="Qdrant/bm25")
+    # placeholder for collection
+    print("starting embedding")
+    #qdrant_collections = {}
+    Qdrant.from_documents(
+                chunks,
+                embeddings,
+                sparse_embeddings = sparse_embeddings,
+                path="/data/local_qdrant",
+                collection_name='giz_worldwide',
+                retrieval_mode=RetrievalMode.HYBRID,
+            )    
+            
+    print(qdrant_collections)
+    print("vector embeddings done")