Update app.py

app.py

@@ -1,155 +1,9 @@
 import os
-import feedparser
 from flask import Flask, render_template, request
-from huggingface_hub import HfApi, InferenceClient
-from langchain.vectorstores import Chroma
-from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.docstore.document import Document
-import requests
-import shutil
+from rss_processor import fetch_rss_feeds, process_and_store_articles, vector_db
 
-# Flask app setup
 app = Flask(__name__)
 
-# Hugging Face setup
-HF_API_TOKEN = os.getenv("HF_API_TOKEN", "YOUR_HF_API_TOKEN")
-HF_MODEL = "Qwen/Qwen-72B-Instruct"
-REPO_ID = "broadfield-dev/news-rag-db"
-LOCAL_DB_DIR = "chroma_db"
-client = InferenceClient(model=HF_MODEL, token=HF_API_TOKEN)
-
-# Comprehensive RSS feeds
-RSS_FEEDS = [
-    "https://www.sciencedaily.com/rss/top/science.xml",
-    "https://www.horoscope.com/us/horoscopes/general/rss/horoscope-rss.aspx",
-    "http://rss.cnn.com/rss/cnn_allpolitics.rss",
-    "https://phys.org/rss-feed/physics-news/",
-    "https://www.spaceweatherlive.com/en/news/rss",
-    "https://weather.com/feeds/rss",
-    "https://www.wired.com/feed/rss",
-    "https://www.nasa.gov/rss/dyn/breaking_news.rss",
-    "https://www.nationalgeographic.com/feed/",
-    "https://www.nature.com/nature.rss",
-    "https://www.scientificamerican.com/rss/",
-    "https://www.newscientist.com/feed/home/",
-    "https://www.livescience.com/feeds/all",
-    "https://www.hindustantimes.com/feed/horoscope/rss",
-    "https://www.washingtonpost.com/wp-srv/style/horoscopes/rss.xml",
-    "https://astrostyle.com/feed/",
-    "https://www.vogue.com/feed/rss",
-    "https://feeds.bbci.co.uk/news/politics/rss.xml",
-    "https://www.reuters.com/arc/outboundfeeds/newsletter-politics/?outputType=xml",
-    "https://www.politico.com/rss/politics.xml",
-    "https://thehill.com/feed/",
-    "https://www.aps.org/publications/apsnews/updates/rss.cfm",
-    "https://www.quantamagazine.org/feed/",
-    "https://www.sciencedaily.com/rss/matter_energy/physics.xml",
-    "https://physicsworld.com/feed/",
-    "https://www.swpc.noaa.gov/rss.xml",
-    "https://www.nasa.gov/rss/dyn/solar_system.rss",
-    "https://weather.com/science/space/rss",
-    "https://www.space.com/feeds/space-weather",
-    "https://www.accuweather.com/en/rss",
-    "https://feeds.bbci.co.uk/weather/feeds/rss/5day/world/",
-    "https://www.weather.gov/rss",
-    "https://www.foxweather.com/rss",
-    "https://techcrunch.com/feed/",
-    "https://arstechnica.com/feed/",
-    "https://gizmodo.com/rss",
-    "https://www.theverge.com/rss/index.xml",
-    "https://www.space.com/feeds/all",
-    "https://www.universetoday.com/feed/",
-    "https://skyandtelescope.org/feed/",
-    "https://www.esa.int/rss",
-    "https://www.smithsonianmag.com/rss/",
-    "https://www.popsci.com/rss.xml",
-    "https://www.discovermagazine.com/rss",
-    "https://www.atlasobscura.com/feeds/latest"
-]
-
-# Embedding model
-embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-vector_db = Chroma(persist_directory=LOCAL_DB_DIR, embedding_function=embedding_model)
-hf_api = HfApi()
-
-def fetch_rss_feeds():
-    articles = []
-    for feed_url in RSS_FEEDS:
-        feed = feedparser.parse(feed_url)
-        for entry in feed.entries[:5]:  # Limit to 5 per feed
-            articles.append({
-                "title": entry.get("title", "No Title"),
-                "link": entry.get("link", ""),
-                "description": entry.get("summary", entry.get("description", "No Description")),
-                "published": entry.get("published", "Unknown Date"),
-                "category": categorize_feed(feed_url),
-            })
-    return articles
-
-def categorize_feed(url):
-    if "sciencedaily" in url or "phys.org" in url:
-        return "Science & Physics"
-    elif "horoscope" in url:
-        return "Astrology"
-    elif "politics" in url:
-        return "Politics"
-    elif "spaceweather" in url or "nasa" in url:
-        return "Solar & Space"
-    elif "weather" in url:
-        return "Earth Weather"
-    else:
-        return "Cool Stuff"
-
-def summarize_article(text):
-    prompt = f"Summarize the following text concisely:\n\n{text}"
-    response = client.text_generation(prompt, max_new_tokens=100, temperature=0.7)
-    return response.strip()
-
-def categorize_article(text):
-    prompt = f"Classify the sentiment as positive, negative, or neutral:\n\n{text}"
-    response = client.text_generation(prompt, max_new_tokens=10, temperature=0.7)
-    return response.strip()
-
-def process_and_store_articles(articles):
-    documents = []
-    for article in articles:
-        summary = summarize_article(article["description"])
-        sentiment = categorize_article(article["description"])
-        doc = Document(
-            page_content=summary,
-            metadata={
-                "title": article["title"],
-                "link": article["link"],
-                "original_description": article["description"],
-                "published": article["published"],
-                "category": article["category"],
-                "sentiment": sentiment,
-            }
-        )
-        documents.append(doc)
-    vector_db.add_documents(documents)
-    vector_db.persist()
-    upload_to_hf_hub()
-
-def upload_to_hf_hub():
-    if os.path.exists(LOCAL_DB_DIR):
-        try:
-            hf_api.create_repo(repo_id=REPO_ID, repo_type="dataset", exist_ok=True)
-        except Exception as e:
-            print(f"Error creating repo: {e}")
-        for root, _, files in os.walk(LOCAL_DB_DIR):
-            for file in files:
-                local_path = os.path.join(root, file)
-                remote_path = os.path.relpath(local_path, LOCAL_DB_DIR)
-                hf_api.upload_file(
-                    path_or_fileobj=local_path,
-                    path_in_repo=remote_path,
-                    repo_id=REPO_ID,
-                    repo_type="dataset",
-                    token=HF_API_TOKEN
-                )
-        print(f"Database uploaded to: {REPO_ID}")
-
 @app.route('/', methods=['GET', 'POST'])
 def index():
     articles = fetch_rss_feeds()
@@ -192,113 +46,5 @@ def index():
 
     return render_template("index.html", categorized_articles=categorized_articles)
 
-HTML_TEMPLATE = """
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>News Feed Hub</title>
-    <style>
-        body {
-            font-family: 'Arial', sans-serif;
-            margin: 0;
-            padding: 20px;
-            background-color: #f4f4f9;
-            color: #333;
-        }
-        h1 {
-            text-align: center;
-            color: #2c3e50;
-        }
-        .search-container {
-            text-align: center;
-            margin: 20px 0;
-        }
-        .search-bar {
-            width: 50%;
-            padding: 12px;
-            font-size: 16px;
-            border: 2px solid #3498db;
-            border-radius: 25px;
-            box-shadow: 0 2px 5px rgba(0,0,0,0.1);
-            outline: none;
-            transition: border-color 0.3s;
-        }
-        .search-bar:focus {
-            border-color: #2980b9;
-        }
-        .category-section {
-            margin: 30px 0;
-        }
-        .category-title {
-            background-color: #3498db;
-            color: white;
-            padding: 10px;
-            border-radius: 5px;
-            font-size: 1.4em;
-        }
-        .article {
-            background-color: white;
-            padding: 15px;
-            margin: 10px 0;
-            border-radius: 8px;
-            box-shadow: 0 2px 5px rgba(0,0,0,0.1);
-            transition: transform 0.2s;
-        }
-        .article:hover {
-            transform: translateY(-3px);
-        }
-        .title a {
-            font-size: 1.2em;
-            color: #2c3e50;
-            text-decoration: none;
-        }
-        .title a:hover {
-            color: #3498db;
-        }
-        .summary {
-            color: #555;
-            margin: 5px 0;
-        }
-        .sentiment {
-            font-style: italic;
-            color: #7f8c8d;
-        }
-        .published {
-            font-size: 0.9em;
-            color: #95a5a6;
-        }
-    </style>
-</head>
-<body>
-    <h1>News Feed Hub</h1>
-    <div class="search-container">
-        <form method="POST">
-            <input type="text" name="search" class="search-bar" placeholder="Search news semantically...">
-        </form>
-    </div>
-    {% for category, articles in categorized_articles.items() %}
-    <div class="category-section">
-        <div class="category-title">{{ category }}</div>
-        {% for article in articles %}
-        <div class="article">
-            <div class="title"><a href="{{ article.link }}" target="_blank">{{ article.title }}</a></div>
-            <div class="summary">{{ article.summary }}</div>
-            <div class="sentiment">Sentiment: {{ article.sentiment }}</div>
-            <div class="published">Published: {{ article.published }}</div>
-        </div>
-        {% endfor %}
-    </div>
-    {% endfor %}
-</body>
-</html>
-"""
-
 if __name__ == "__main__":
-    os.makedirs("templates", exist_ok=True)
-    with open("templates/index.html", "w") as f:
-        f.write(HTML_TEMPLATE)
-    if os.path.exists(LOCAL_DB_DIR):
-        shutil.rmtree(LOCAL_DB_DIR)
-    app.run(debug=True, host="0.0.0.0", port=7860)
+    app.run(host="0.0.0.0", port=7860)