Update rss_processor.py

rss_processor.py

@@ -12,13 +12,13 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 # Hugging Face setup
-HF_API_TOKEN = os.getenv("DEMO_HF_API_TOKEN", "DEMO_HF_API_TOKEN")
+HF_API_TOKEN = os.getenv("DEMO_HF_API_TOKEN", "YOUR_HF_API_TOKEN")
 HF_MODEL = "Qwen/Qwen-72B-Instruct"
-REPO_ID = "broadfield-dev/news-rag-db"  # Ensure this is your repo
+REPO_ID = "broadfield-dev/news-rag-db"
 LOCAL_DB_DIR = "chroma_db"
 
 # Explicitly login to Hugging Face Hub
-#login(token=HF_API_TOKEN)
+login(token=HF_API_TOKEN)
 client = InferenceClient(model=HF_MODEL, token=HF_API_TOKEN)
 
 # RSS feeds
@@ -78,17 +78,25 @@ hf_api = HfApi()
 def fetch_rss_feeds():
     articles = []
     for feed_url in RSS_FEEDS:
-        feed = feedparser.parse(feed_url)
-        for entry in feed.entries[:5]:
-            image = entry.get("media_content", [{}])[0].get("url") or entry.get("media_thumbnail", [{}])[0].get("url") or None
-            articles.append({
-                "title": entry.get("title", "No Title"),
-                "link": entry.get("link", ""),
-                "description": entry.get("summary", entry.get("description", "No Description")),
-                "published": entry.get("published", "Unknown Date"),
-                "category": categorize_feed(feed_url),
-                "image": image,
-            })
+        try:
+            logger.info(f"Fetching feed: {feed_url}")
+            feed = feedparser.parse(feed_url)
+            if feed.bozo:
+                logger.warning(f"Failed to parse {feed_url}: {feed.bozo_exception}")
+                continue
+            for entry in feed.entries[:5]:
+                image = entry.get("media_content", [{}])[0].get("url") or entry.get("media_thumbnail", [{}])[0].get("url") or None
+                articles.append({
+                    "title": entry.get("title", "No Title"),
+                    "link": entry.get("link", ""),
+                    "description": entry.get("summary", entry.get("description", "No Description")),
+                    "published": entry.get("published", "Unknown Date"),
+                    "category": categorize_feed(feed_url),
+                    "image": image,
+                })
+            logger.info(f"Processed {len(feed.entries[:5])} entries from {feed_url}")
+        except Exception as e:
+            logger.error(f"Error fetching {feed_url}: {e}")
     return articles
 
 def categorize_feed(url):
@@ -126,23 +134,27 @@ def categorize_article(text):
 def process_and_store_articles(articles):
     documents = []
     for article in articles:
-        summary = summarize_article(article["description"])
-        sentiment = categorize_article(article["description"])
-        doc = Document(
-            page_content=summary,
-            metadata={
-                "title": article["title"],
-                "link": article["link"],
-                "original_description": article["description"],
-                "published": article["published"],
-                "category": article["category"],
-                "sentiment": sentiment,
-                "image": article["image"] if article["image"] else "https://via.placeholder.com/150",
-            }
-        )
-        documents.append(doc)
+        try:
+            summary = summarize_article(article["description"])
+            sentiment = categorize_article(article["description"])
+            doc = Document(
+                page_content=summary,
+                metadata={
+                    "title": article["title"],
+                    "link": article["link"],
+                    "original_description": article["description"],
+                    "published": article["published"],
+                    "category": article["category"],
+                    "sentiment": sentiment,
+                    "image": article["image"],
+                }
+            )
+            documents.append(doc)
+        except Exception as e:
+            logger.error(f"Error processing article {article['title']}: {e}")
     vector_db.add_documents(documents)
     vector_db.persist()
+    logger.info("Vector DB persisted")
     upload_to_hf_hub()
 
 def upload_to_hf_hub():