File size: 4,005 Bytes
4e00df7 f65663c 4e00df7 8056ec2 4e00df7 f65663c 4e00df7 8056ec2 f65663c 8056ec2 f65663c 8056ec2 f65663c 8056ec2 f65663c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 |
import tempfile
import time
import os
from utils import compute_sha1_from_file
from langchain.schema import Document
import streamlit as st
from langchain.text_splitter import RecursiveCharacterTextSplitter
from stats import add_usage
import re
def clean_chat_text(text):
"""Clean chat export text to remove special characters and format consistently"""
# Remove non-printable characters
text = ''.join(char for char in text if char.isprintable())
# Clean up WhatsApp-style timestamps and phone numbers
text = re.sub(r'\[\d{1,2}/\d{1,2}/\d{2,4},\s+\d{1,2}:\d{1,2}:\d{1,2}\s+[AP]M\]', '', text)
text = re.sub(r'\+\d{2,3}\s*\d{3,10}\s*\d{3,10}', '', text)
# Remove joining messages
text = re.sub(r'joined using this group\'s invite link', '', text)
# Remove extra whitespace
text = ' '.join(text.split())
return text
def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
documents = []
file_name = file.name
file_size = file.size
if st.secrets.self_hosted == "false":
if file_size > 1000000:
st.error("File size is too large. Please upload a file smaller than 1MB or self host.")
return
dateshort = time.strftime("%Y%m%d")
with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as tmp_file:
tmp_file.write(file.getvalue())
tmp_file.flush()
loader = loader_class(tmp_file.name)
documents = loader.load()
file_sha1 = compute_sha1_from_file(tmp_file.name)
os.remove(tmp_file.name)
chunk_size = st.session_state['chunk_size']
chunk_overlap = st.session_state['chunk_overlap']
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
documents = text_splitter.split_documents(documents)
# Clean the text content before creating metadata
docs_with_metadata = [Document(page_content=clean_chat_text(doc.page_content),
metadata={"file_sha1": file_sha1,
"file_size": file_size,
"file_name": file_name,
"chunk_size": chunk_size,
"chunk_overlap": chunk_overlap,
"date": dateshort,
"user": st.session_state["username"]})
for doc in documents]
try:
# Add debug logging before vector store addition
print(f"Attempting to add {len(docs_with_metadata)} documents")
print(f"Sample cleaned content: {docs_with_metadata[0].page_content[:200] if docs_with_metadata else 'No documents'}")
vector_store.add_documents(docs_with_metadata)
if stats_db:
add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,
"file_type": file_suffix,
"chunk_size": chunk_size,
"chunk_overlap": chunk_overlap})
except Exception as e:
print(f"Error adding documents to vector store:")
print(f"Exception: {str(e)}")
print(f"Input details:")
print(f"File name: {file_name}")
print(f"File size: {file_size}")
print(f"File SHA1: {file_sha1}")
print(f"Number of documents: {len(docs_with_metadata)}")
print(f"Chunk size: {chunk_size}")
print(f"Chunk overlap: {chunk_overlap}")
print(f"First document preview (truncated):")
if docs_with_metadata:
print(docs_with_metadata[0].page_content[:500])
# Additional debug info for vector store
print(f"Vector store type: {type(vector_store).__name__}")
raise |