Update rss_processor.py

rss_processor.py

@@ -69,6 +69,7 @@ vector_db = Chroma(persist_directory=LOCAL_DB_DIR, embedding_function=embedding_
 
 def fetch_rss_feeds():
     articles = []
+    seen_articles = set()  # Track unique articles by title and link
     for feed_url in RSS_FEEDS:
         try:
             logger.info(f"Fetching feed: {feed_url}")
@@ -77,16 +78,22 @@ def fetch_rss_feeds():
                 logger.warning(f"Failed to parse {feed_url}: {feed.bozo_exception}")
                 continue
             for entry in feed.entries[:5]:
-                image = entry.get("media_content", [{}])[0].get("url") or entry.get("media_thumbnail", [{}])[0].get("url") or ""
-                articles.append({
-                    "title": entry.get("title", "No Title"),
-                    "link": entry.get("link", ""),
-                    "description": entry.get("summary", entry.get("description", "No Description")),
-                    "published": entry.get("published", "Unknown Date"),
-                    "category": categorize_feed(feed_url),
-                    "image": image if image else "",
-                })
-            logger.info(f"Processed {len(feed.entries[:5])} entries from {feed_url}")
+                title = entry.get("title", "No Title")
+                link = entry.get("link", "")
+                # Create a unique key for deduplication
+                article_key = f"{title}|{link}"
+                if article_key not in seen_articles:
+                    seen_articles.add(article_key)
+                    image = entry.get("media_content", [{}])[0].get("url") or entry.get("media_thumbnail", [{}])[0].get("url") or ""
+                    articles.append({
+                        "title": title,
+                        "link": link,
+                        "description": entry.get("summary", entry.get("description", "No Description")),
+                        "published": entry.get("published", "Unknown Date"),
+                        "category": categorize_feed(feed_url),
+                        "image": image if image else "",
+                    })
+            logger.info(f"Processed {len(feed.entries[:5])} unique entries from {feed_url}")
         except Exception as e:
             logger.error(f"Error fetching {feed_url}: {e}")
     return articles
@@ -109,7 +116,6 @@ def process_and_store_articles(articles):
     documents = []
     for article in articles:
         try:
-            # Ensure no None values in metadata
             metadata = {
                 "title": article["title"] or "No Title",
                 "link": article["link"] or "",