appStore/embed.py

from langchain_qdrant import QdrantVectorStore
from langchain_qdrant import FastEmbedSparse, RetrievalMode
from torch import cuda
import streamlit as st
from langchain_huggingface import HuggingFaceEmbeddings
from appStore.prep_utils import get_client

# get the device to be used eithe gpu or cpu
device = 'cuda' if cuda.is_available() else 'cpu'


def hybrid_embed_chunks(docs, collection_name, del_if_exists = False):
    """
    takes the chunks and does the hybrid embedding for the list of chunks
    """

    # Dense Embeddings function
    embeddings = HuggingFaceEmbeddings(
        model_kwargs = {'device': device},
        encode_kwargs = {'normalize_embeddings': True},
        model_name='BAAI/bge-m3'
    )
    
    # Sparse Embedding Function
    sparse_embeddings = FastEmbedSparse(model_name="Qdrant/bm25")

    # get exisitng client
    client = get_client()

    # create collection
    if del_if_exists:
        client.delete_collection(collection_name=f"{collection_name}")
        
    client.create_collection(
    collection_name=collection_name,
    vectors_config={
        "text-dense": models.VectorParams(size=1024, distance=models.Distance.COSINE, on_disk = True)
    },
    sparse_vectors_config={
        "text-sparse": models.SparseVectorParams(index=models.SparseIndexParams(
                on_disk=True,
            )
        )
    },)

    # create Vector store
    vector_store = QdrantVectorStore(
            client=client,
            collection_name=collection_name,
            embedding=embeddings,
            vector_name="text-dense",
            sparse_embedding = sparse_embeddings,
            sparse_vector_name="text-sparse",
            retrieval_mode=RetrievalMode.HYBRID,
        )

    print("starting embedding")
    vector_store.add_documents(docs)
    print("vector embeddings done")

@st.cache_resource    
def get_local_qdrant(collection_name): 
    """once the local qdrant server is created this is used to make the connection to exisitng server"""

    qdrant_collections = {}
    embeddings = HuggingFaceEmbeddings(
        model_kwargs = {'device': device},
        encode_kwargs = {'normalize_embeddings': True},
        model_name='BAAI/bge-m3')
    client = QdrantClient(path="/data/local_qdrant") 
    sparse_embeddings = FastEmbedSparse(model_name="Qdrant/bm25")
    print("Collections in local Qdrant:",client.get_collections())
    qdrant_collections[collection_name] = Qdrant(client=client, collection_name=collection_name, 
                                       embeddings=embeddings,
                                      )
    return qdrant_collections