Update app.py

app.py

@@ -47,24 +47,19 @@ def get_url_content(url):
     response = requests.get(url)
     if url.endswith('.pdf'):
         pdf = io.BytesIO(response.content)
-        file = open('pdf.pdf', 'wb')
-        file.write(pdf.read())
-        file.close()
         doc = fitz.open(stream=pdf, filetype="pdf")
-        return (url, ''.join([text for page in doc for text in page.get_text()]))
+        return (url, ''.join(page.get_text() for page in doc))
     else:
         soup = BeautifulSoup(response.content, 'html.parser')
         content = soup.find_all('div', class_='wpb_content_element')
         text = [c.get_text().strip() for c in content if c.get_text().strip() != '']
         text = [line for item in text for line in item.split('\n') if line.strip() != '']
-
-        # Post processing to exclude footer content, only if 'ARTS ON:' is present.
+        # Exclude footer content
         try:
             arts_on_index = text.index('ARTS ON:')
             return (url, '\n'.join(text[:arts_on_index]))
         except ValueError:
-            return (url, '\n'.join(text))  # If 'ARTS ON:' not found, return full text
-
+            return (url, '\n'.join(text))  # Return full text if specific marker not found
 
 @st.cache_resource
 def get_retriever(urls):