Update loaders/common.py

loaders/common.py

@@ -6,6 +6,24 @@ from langchain.schema import Document
 import streamlit as st
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from stats import add_usage
+import re
+
+def clean_chat_text(text):
+    """Clean chat export text to remove special characters and format consistently"""
+    # Remove non-printable characters
+    text = ''.join(char for char in text if char.isprintable())
+    
+    # Clean up WhatsApp-style timestamps and phone numbers
+    text = re.sub(r'\[\d{1,2}/\d{1,2}/\d{2,4},\s+\d{1,2}:\d{1,2}:\d{1,2}\s+[AP]M\]', '', text)
+    text = re.sub(r'‪\+\d{2,3}\s*\d{3,10}\s*\d{3,10}‬', '', text)
+    
+    # Remove joining messages
+    text = re.sub(r'joined using this group\'s invite link', '', text)
+    
+    # Remove extra whitespace
+    text = ' '.join(text.split())
+    
+    return text
 
 def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
     documents = []
@@ -30,17 +48,30 @@ def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
     text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
     
     documents = text_splitter.split_documents(documents)
-    # Add the document sha1 as metadata to each document
-    docs_with_metadata = [Document(page_content=doc.page_content, metadata={"file_sha1": file_sha1,"file_size":file_size ,"file_name": file_name, 
-                                                                            "chunk_size": chunk_size, "chunk_overlap": chunk_overlap, "date": dateshort,
-                                                                            "user" : st.session_state["username"]}) 
-                          for doc in documents]
+    
+    # Clean the text content before creating metadata
+    docs_with_metadata = [Document(page_content=clean_chat_text(doc.page_content),
+                                 metadata={"file_sha1": file_sha1,
+                                         "file_size": file_size,
+                                         "file_name": file_name, 
+                                         "chunk_size": chunk_size,
+                                         "chunk_overlap": chunk_overlap,
+                                         "date": dateshort,
+                                         "user": st.session_state["username"]}) 
+                         for doc in documents]
     
     try:
+        # Add debug logging before vector store addition
+        print(f"Attempting to add {len(docs_with_metadata)} documents")
+        print(f"Sample cleaned content: {docs_with_metadata[0].page_content[:200] if docs_with_metadata else 'No documents'}")
+        
         vector_store.add_documents(docs_with_metadata)
+        
         if stats_db:
-            add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,"file_type": file_suffix, 
-                                                             "chunk_size": chunk_size, "chunk_overlap": chunk_overlap})
+            add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,
+                                                             "file_type": file_suffix, 
+                                                             "chunk_size": chunk_size,
+                                                             "chunk_overlap": chunk_overlap})
     except Exception as e:
         print(f"Error adding documents to vector store:")
         print(f"Exception: {str(e)}")
@@ -54,4 +85,7 @@ def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
         print(f"First document preview (truncated):")
         if docs_with_metadata:
             print(docs_with_metadata[0].page_content[:500])
-        raise  # Re-raise the exception after logging
+            
+        # Additional debug info for vector store
+        print(f"Vector store type: {type(vector_store).__name__}")
+        raise