Update app.py

app.py

@@ -1,5 +1,6 @@
 from langchain.chat_models import ChatOpenAI
 from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.retrievers import BM25Retriever,EnsembleRetriever
 from langchain.vectorstores import Qdrant
 from langchain.text_splitter import CharacterTextSplitter
 from langchain.chains import ConversationalRetrievalChain
@@ -25,7 +26,7 @@ for filename in os.listdir(pdf_folder_path):
         documents.extend(loader.load())
 
 # Split the documents into manageable chunks
-text_splitter = CharacterTextSplitter(chunk_size=512, chunk_overlap=25)
+text_splitter = CharacterTextSplitter(chunk_size=128, chunk_overlap=5)
 docs = text_splitter.split_documents(documents)
 
 # Initialize embeddings and Qdrant client
@@ -35,7 +36,7 @@ qdrant_client = QdrantClient(":memory:")
 # Recreate Qdrant collection
 qdrant_client.create_collection(
     collection_name="langchain_collection",
-    vectors_config=qdrant_models.VectorParams(size=1536, distance=qdrant_models.Distance.COSINE)
+    vectors_config=qdrant_models.VectorParams(size=3000, distance=qdrant_models.Distance.COSINE)
 )
 
 # Set up the sparse embeddings for hybrid retrieval
@@ -55,15 +56,19 @@ vector_store = QdrantVectorStore.from_documents(
 memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
 
 # Set up the retriever
-retriever = vector_store.as_retriever( search_kwargs={"k": 3})
+retriever = vector_store.as_retriever(search_type="similarity", search_kwargs={"k": 3})
+
+keyword_retriever = BM25Retriever.from_documents(docs)
+keyword_retriever.k =  3
+ensemble_retriever = EnsembleRetriever(retrievers=[retriever,keyword_retriever],
+                                       weights=[0.6, 0.5])
 
-# Set up the language model
 llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0.3)
 
 # Set up the conversational retrieval chain with memory
 conversational_chain = ConversationalRetrievalChain.from_llm(
     llm=llm,
-    retriever=retriever,
+    retriever=ensemble_retriever,
     memory=memory,
     verbose=True 
 )