Update app.py

app.py

@@ -8,6 +8,8 @@ from transformers import pipeline
 from gtts import gTTS
 import tempfile
 import os
+from bs4 import BeautifulSoup
+import requests
 
 # CPU-friendly summarization LLM
 summary_pipe = pipeline("text2text-generation", model="google/flan-t5-base", device=-1)
@@ -15,7 +17,7 @@ llm = HuggingFacePipeline(pipeline=summary_pipe)
 
 # Summarization prompt
 summary_prompt = PromptTemplate.from_template("""
-Summarize the following webpage content in a clear, concise way:
+Summarize the following article content in a clear, concise way:
 
 {text}
 
@@ -24,17 +26,31 @@ Summary:
 
 summary_chain = LLMChain(llm=llm, prompt=summary_prompt)
 
+def extract_main_content(url):
+    try:
+        response = requests.get(url, timeout=10)
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        # Remove navigation, header, footer, sidebars, and scripts
+        for tag in soup(["nav", "header", "footer", "aside", "script", "style", "noscript"]):
+            tag.decompose()
+
+        # Extract main content using tags with significant paragraph text
+        paragraphs = soup.find_all("p")
+        content = "\n".join([p.get_text() for p in paragraphs if len(p.get_text()) > 60])
+        return content.strip()
+    except Exception as e:
+        return f"Error extracting article content: {str(e)}"
+
 def url_to_audio_summary(url):
     try:
-        loader = WebBaseLoader(url)
-        docs = loader.load()
-        splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=100)
-        splits = splitter.split_documents(docs)
+        article_text = extract_main_content(url)
+        if article_text.startswith("Error"):
+            return article_text, None
 
-        full_text = "\n".join([s.page_content for s in splits])
-        summary = summary_chain.run(text=full_text)
+        summary = summary_chain.run(text=article_text)
 
-        # Use gTTS for TTS since Hugging Face TTS model failed
+        # Use gTTS for TTS
         tts = gTTS(text=summary)
         temp_path = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
         tts.save(temp_path.name)
@@ -52,7 +68,7 @@ iface = gr.Interface(
         gr.Audio(label="Audio Summary")
     ],
     title="URL to Audio Summary Agent",
-    description="Summarizes article from a URL and gives an audio summary. CPU-only using gTTS."
+    description="Summarizes only the article content from a URL and gives an audio summary. CPU-only using gTTS."
 )
 
 if __name__ == "__main__":