app.py

import gradio as gr
import requests
from bs4 import BeautifulSoup
from openai import OpenAI
import json
import re
from urllib.parse import urljoin, urlparse
import time

class WebScrapingTool:
    def __init__(self):
        self.client = None
        self.system_prompt = """You are a specialized web data extraction assistant. Your core purpose is to browse and analyze the content of web pages based on user instructions, and return structured or unstructured information from the provided URL. Your capabilities include:

1. Navigating and reading web page content from a given URL.
2. Extracting textual content including headings, paragraphs, lists, and metadata.
3. Identifying and extracting HTML tables and presenting them in a clean, structured format.
4. Creating new, custom tables based on user queries by processing, reorganizing, or filtering the content found on the source page.

You must always follow these guidelines:
- Accurately extract and summarize both structured (tables, lists) and unstructured (paragraphs, articles) content.
- Clearly separate different types of data (e.g., summaries, tables, bullet points).
- When extracting textual content:
  - Maintain original meaning, structure, and tone.
  - Capture all relevant sections based on user instructions (e.g., only the "Overview" or "Methodology" sections).
- When extracting tables:
  - Preserve headers and align row data correctly.
  - Identify and differentiate multiple tables, if present.
- When creating custom tables:
  - Include only the relevant columns as per the user request.
  - Sort, filter, and reorganize data accordingly.
  - Use clear and consistent headers.

You must not hallucinate or infer data not present on the page. If content is missing, unclear, or restricted, say so explicitly.

Always respond based on the actual content from the provided link. If the page fails to load or cannot be accessed, inform the user immediately.

Your role is to act as an intelligent browser and data interpreter — able to read and reshape any web content to meet user needs."""

    def setup_client(self, api_key):
        """Initialize OpenAI client with OpenRouter"""
        try:
            self.client = OpenAI(
                base_url="https://openrouter.ai/api/v1",
                api_key=api_key,
            )
            return True, "API client initialized successfully!"
        except Exception as e:
            return False, f"Failed to initialize API client: {str(e)}"

    def scrape_webpage(self, url):
        """Scrape webpage content"""
        try:
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
            }
            
            response = requests.get(url, headers=headers, timeout=30)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Remove script and style elements
            for script in soup(["script", "style", "nav", "footer", "header"]):
                script.decompose()
            
            # Extract text content
            text_content = soup.get_text()
            
            # Clean up text
            lines = (line.strip() for line in text_content.splitlines())
            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
            text_content = ' '.join(chunk for chunk in chunks if chunk)
            
            # Extract tables
            tables = []
            for table in soup.find_all('table'):
                table_data = []
                headers = []
                
                # Extract headers
                header_row = table.find('tr')
                if header_row:
                    headers = [th.get_text().strip() for th in header_row.find_all(['th', 'td'])]
                
                # Extract rows
                for row in table.find_all('tr')[1:]:  # Skip header row
                    row_data = [td.get_text().strip() for td in row.find_all(['td', 'th'])]
                    if row_data:
                        table_data.append(row_data)
                
                if headers and table_data:
                    tables.append({
                        'headers': headers,
                        'data': table_data
                    })
            
            return {
                'success': True,
                'text': text_content[:15000],  # Limit text length
                'tables': tables,
                'title': soup.title.string if soup.title else "No title found"
            }
            
        except requests.RequestException as e:
            return {
                'success': False,
                'error': f"Failed to fetch webpage: {str(e)}"
            }
        except Exception as e:
            return {
                'success': False,
                'error': f"Error processing webpage: {str(e)}"
            }

    def analyze_content(self, scraped_data, user_query, api_key):
        """Analyze scraped content using DeepSeek V3"""
        if not self.client:
            success, message = self.setup_client(api_key)
            if not success:
                return f"Error: {message}"
        
        if not scraped_data['success']:
            return f"Error scraping webpage: {scraped_data['error']}"
        
        # Prepare content for AI analysis
        content_text = f"""
WEBPAGE CONTENT:
Title: {scraped_data['title']}

Main Text Content:
{scraped_data['text']}

Tables Found: {len(scraped_data['tables'])}
"""
        
        if scraped_data['tables']:
            content_text += "\n\nTABLES:\n"
            for i, table in enumerate(scraped_data['tables']):
                content_text += f"\nTable {i+1}:\n"
                content_text += f"Headers: {', '.join(table['headers'])}\n"
                content_text += "Data:\n"
                for row in table['data'][:10]:  # Limit rows
                    content_text += f"  {' | '.join(row)}\n"
        
        try:
            completion = self.client.chat.completions.create(
                extra_headers={
                    "HTTP-Referer": "https://gradio-web-scraper.com",
                    "X-Title": "AI Web Scraping Tool",
                },
                model="deepseek/deepseek-chat-v3-0324:free",
                messages=[
                    {"role": "system", "content": self.system_prompt},
                    {"role": "user", "content": f"Here is the webpage content:\n\n{content_text}\n\nUser Query: {user_query}"}
                ],
                temperature=0.1,
                max_tokens=4000
            )
            
            return completion.choices[0].message.content
            
        except Exception as e:
            return f"Error analyzing content: {str(e)}"

def create_interface():
    tool = WebScrapingTool()
    
    def process_request(api_key, url, user_query):
        if not api_key.strip():
            return "❌ Please enter your OpenRouter API key"
        
        if not url.strip():
            return "❌ Please enter a valid URL"
        
        if not user_query.strip():
            return "❌ Please enter your analysis query"
        
        # Add progress updates
        yield "🔄 Scraping webpage content..."
        
        # Scrape webpage
        scraped_data = tool.scrape_webpage(url)
        
        if not scraped_data['success']:
            yield f"❌ {scraped_data['error']}"
            return
        
        yield f"✅ Successfully scraped webpage!\n📄 Title: {scraped_data['title']}\n📊 Found {len(scraped_data['tables'])} tables\n\n🤖 Analyzing content with DeepSeek V3..."
        
        # Analyze content
        result = tool.analyze_content(scraped_data, user_query, api_key)
        
        yield f"✅ Analysis Complete!\n\n{result}"
    
    # Create Gradio interface
    with gr.Blocks(title="AI Web Scraping Tool", theme=gr.themes.Soft()) as app:
        gr.Markdown("""
        # 🤖 AI Web Scraping Tool
        ### Powered by DeepSeek V3 & OpenRouter
        
        Extract and analyze web content using advanced AI. Simply provide your OpenRouter API key, a URL, and describe what you want to extract.
        """)
        
        with gr.Row():
            with gr.Column(scale=2):
                api_key_input = gr.Textbox(
                    label="🔑 OpenRouter API Key",
                    placeholder="Enter your OpenRouter API key here...",
                    type="password",
                    info="Get your free API key from openrouter.ai"
                )
                
                url_input = gr.Textbox(
                    label="🌐 Website URL",
                    placeholder="https://example.com",
                    info="Enter the URL you want to scrape and analyze"
                )
                
                query_input = gr.Textbox(
                    label="📝 Analysis Query",
                    placeholder="What do you want to extract? (e.g., 'Extract main points and create a summary table')",
                    lines=3,
                    info="Describe what information you want to extract from the webpage"
                )
                
                with gr.Row():
                    analyze_btn = gr.Button("🚀 Analyze Website", variant="primary", size="lg")
                    clear_btn = gr.Button("🗑️ Clear", variant="secondary")
            
            with gr.Column(scale=3):
                output = gr.Textbox(
                    label="📊 Analysis Results",
                    lines=20,
                    max_lines=30,
                    show_copy_button=True,
                    interactive=False
                )
        
        # Example queries
        gr.Markdown("""
        ### 💡 Example Queries:
        - *"Extract the main summary and any data tables"*
        - *"Create a table of key statistics mentioned in the article"*
        - *"Summarize the main points in bullet format"*
        - *"Extract all numerical data and organize it in a table"*
        - *"Find and extract contact information and company details"*
        """)
        
        # Example websites
        with gr.Accordion("📋 Try These Example URLs", open=False):
            examples = [
                ["https://www.imf.org/en/Publications/WEO", "Extract economic outlook summary and GDP projections"],
                ["https://en.wikipedia.org/wiki/List_of_countries_by_GDP_(nominal)", "Create a table of top 10 countries by GDP"],
                ["https://www.who.int/news", "Summarize the latest health news"],
                ["https://www.nasdaq.com/market-activity/stocks", "Extract stock market data and trends"]
            ]
            
            for url, query in examples:
                gr.Markdown(f"**URL:** `{url}`  \n**Query:** *{query}*")
        
        # Event handlers
        analyze_btn.click(
            fn=process_request,
            inputs=[api_key_input, url_input, query_input],
            outputs=output,
            show_progress=True
        )
        
        clear_btn.click(
            fn=lambda: ("", "", "", ""),
            outputs=[api_key_input, url_input, query_input, output]
        )
        
        # Auto-fill example
        def fill_example():
            return (
                "",  # API key remains empty
                "https://www.imf.org/en/Publications/WEO/Issues/2024/04/16/world-economic-outlook-april-2024",
                """1. Extract a summary of the main economic outlook from this page.
2. Extract any available tables or figures with global GDP growth projections.
3. Create a new table showing:
   - Country/Region
   - Projected GDP Growth (2024)
   - Change from Previous Forecast (if available)
4. Highlight the top 3 fastest-growing economies in a separate mini-table."""
            )
        
        example_btn = gr.Button("📋 Load IMF Example", variant="secondary")
        example_btn.click(
            fn=fill_example,
            outputs=[url_input, query_input]
        )
    
    return app

if __name__ == "__main__":
    # Create and launch the app
    app = create_interface()
    
    # Launch with public sharing enabled
    app.launch(
        share=True,
    )