Update app.py

app.py

@@ -47,26 +47,19 @@ def get_url_content(url):
         file.write(pdf.read())
         file.close()
         doc = fitz.open('pdf.pdf')
-        return (url, ''.join([text for page in doc for text in page.get_text()]))
+        return (url, ''.join([text for page in doc for text in page.get_text("text")]))
     else:
         soup = BeautifulSoup(response.content, 'html.parser')
         content = soup.find_all('div', class_='wpb_content_element')
-        text = [c.get_text().strip() for c in content if c.get_text().strip() != '']
-        text = [line for item in text for line in item.split('\n') if line.strip() != '']
+        text = ' '.join([c.get_text().strip() for c in content if c.get_text().strip() != ''])
+        return (url, text)
 
-        # Check if 'ARTS ON:' exists in the list
-        try:
-            arts_on = text.index('ARTS ON:')
-            return (url, '\n'.join(text[:arts_on]))
-        except ValueError:
-            # If 'ARTS ON:' is not found, return what was found without truncating
-            return (url, '\n'.join(text))
 
 
 @st.cache_resource
 def get_retriever(urls):
     all_content = [get_url_content(url) for url in urls]
-    documents = [Document(page_content=doc, metadata={'url': url}) for (url, doc) in all_content]
+    documents = [Document(text=doc, metadata={'url': url}) for (url, doc) in all_content]
 
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=200)
     docs = text_splitter.split_documents(documents)