Update app.py

app.py

@@ -50,17 +50,17 @@ def get_url_content(url):
         return (url, ''.join([text for page in doc for text in page.get_text()]))
     else:
         soup = BeautifulSoup(response.content, 'html.parser')
-
-        # Content containers. Here wordpress specific container css class name
-        # used. This will be different for each website.
         content = soup.find_all('div', class_='wpb_content_element')
         text = [c.get_text().strip() for c in content if c.get_text().strip() != '']
         text = [line for item in text for line in item.split('\n') if line.strip() != '']
 
-        # Post processing to exclude footer content.
-        # This will be different for each website.
-        arts_on = text.index('ARTS ON:')
-        return (url, '\n'.join(text[:arts_on]))
+        # Check if 'ARTS ON:' exists in the list
+        try:
+            arts_on = text.index('ARTS ON:')
+            return (url, '\n'.join(text[:arts_on]))
+        except ValueError:
+            # If 'ARTS ON:' is not found, return what was found without truncating
+            return (url, '\n'.join(text))
 
 
 @st.cache_resource