Update app.py

app.py

@@ -12,15 +12,11 @@ from langchain.chains.combine_documents import create_stuff_documents_chain
 from langchain_community.chat_message_histories.streamlit import StreamlitChatMessageHistory
 from langchain.prompts import PromptTemplate
 from langchain.vectorstores import Chroma
-from utills import load_txt_documents, split_docs, load_uploaded_documents, retriever_from_chroma
 from langchain.text_splitter import TokenTextSplitter, RecursiveCharacterTextSplitter
 from langchain_community.document_loaders.directory import DirectoryLoader
-from HTML_templates import css, bot_template, user_template
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.runnables import RunnablePassthrough
-from langchain import hub
-from langchain.retrievers import ContextualCompressionRetriever
-from langchain.retrievers.document_compressors import LLMChainExtractor
+
 
 lang_api_key = os.getenv("lang_api_key")
 
@@ -53,7 +49,6 @@ def create_retriever_from_chroma(vectorstore_path="./docs/chroma/", search_type=
         vectorstore = Chroma(persist_directory=vectorstore_path,embedding_function=embeddings)
         
     else:
-        # Load documents from the specified data path
         st.write("Vector store doesnt exist and will be created now")
         loader = DirectoryLoader('./data/', glob="./*.txt", loader_cls=TextLoader)
         docs = loader.load()
@@ -61,13 +56,12 @@ def create_retriever_from_chroma(vectorstore_path="./docs/chroma/", search_type=
         
         text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
         chunk_size=chunk_size, chunk_overlap=chunk_overlap,
-        separators=["\n \n \n", "\n \n", "\n1" , "(?<=\. )", " ", ""]
+        separators=["\n\n \n\n","\n\n\n", "\n\n", r"In \[[0-9]+\]", r"\n+", r"\s+"],
+        is_separator_regex = True
     )
         split_docs = text_splitter.split_documents(docs)
 
         
-
-        # Create the vectorstore
         vectorstore = Chroma.from_documents(
             documents=split_docs, embedding=embeddings, persist_directory=vectorstore_path
         )
@@ -97,7 +91,7 @@ def main():
     st.header("Chat with multiple Lithuanian Law Documents:" ":books:")
     
     st.markdown("Hi, I am Birute (Powered by qwen2-0_5b model), chat assistant, based on republic of Lithuania law documents. You can choose below information retrieval type and how many documents you want to be retrieved.")
-    st.markdown("Available Documents: LR_Civil_Code_2022, LR_Constitution_2022, LR_Criminal_Code_2018, LR_Criminal_Procedure_code_2022,LR_Labour_code_2010. P.S it's a shame that there are no newest documents translations... ")
+    st.markdown("Available Documents: LR_Civil_Code_2022, LR_Constitution_2022, LR_Criminal_Code_2018, LR_Criminal_Procedure_code_2022,LR_Labour_code_2010. P.S it's a shame that there are no newest documents translations into English... ")
 
     if "messages" not in st.session_state:
         st.session_state["messages"] = [
@@ -106,9 +100,9 @@ def main():
 
 
     search_type = st.selectbox(
-        "Choose search type. Options are [Max marginal relevance search (similarity) , Similarity search (similarity). Default value (mmr)]", 
+        "Choose search type. Options are [Max marginal relevance search (similarity) , Similarity search (similarity). Default value (similarity)]", 
         options=["mmr", "similarity"], 
-        index=1  # Default to "mmr"
+        index=1  
     )
 
     k = st.select_slider(
@@ -116,7 +110,7 @@ def main():
         options=list(range(2, 16)), 
         value=4  
     )
-    retriever = create_retriever_from_chroma(vectorstore_path="docs/chroma/", search_type=search_type, k=k, chunk_size=200, chunk_overlap=30)
+    retriever = create_retriever_from_chroma(vectorstore_path="docs/chroma/", search_type=search_type, k=k, chunk_size=350, chunk_overlap=30)