Spaces:
Running
Running
| import gradio as gr | |
| import requests | |
| from bs4 import BeautifulSoup | |
| from openai import OpenAI | |
| import json | |
| import re | |
| from urllib.parse import urljoin, urlparse | |
| import time | |
| class WebScrapingTool: | |
| def __init__(self): | |
| self.client = None | |
| self.system_prompt = """You are a specialized web data extraction assistant. Your core purpose is to browse and analyze the content of web pages based on user instructions, and return structured or unstructured information from the provided URL. Your capabilities include: | |
| 1. Navigating and reading web page content from a given URL. | |
| 2. Extracting textual content including headings, paragraphs, lists, and metadata. | |
| 3. Identifying and extracting HTML tables and presenting them in a clean, structured format. | |
| 4. Creating new, custom tables based on user queries by processing, reorganizing, or filtering the content found on the source page. | |
| You must always follow these guidelines: | |
| - Accurately extract and summarize both structured (tables, lists) and unstructured (paragraphs, articles) content. | |
| - Clearly separate different types of data (e.g., summaries, tables, bullet points). | |
| - When extracting textual content: | |
| - Maintain original meaning, structure, and tone. | |
| - Capture all relevant sections based on user instructions (e.g., only the "Overview" or "Methodology" sections). | |
| - When extracting tables: | |
| - Preserve headers and align row data correctly. | |
| - Identify and differentiate multiple tables, if present. | |
| - When creating custom tables: | |
| - Include only the relevant columns as per the user request. | |
| - Sort, filter, and reorganize data accordingly. | |
| - Use clear and consistent headers. | |
| You must not hallucinate or infer data not present on the page. If content is missing, unclear, or restricted, say so explicitly. | |
| Always respond based on the actual content from the provided link. If the page fails to load or cannot be accessed, inform the user immediately. | |
| Your role is to act as an intelligent browser and data interpreter — able to read and reshape any web content to meet user needs.""" | |
| def setup_client(self, api_key): | |
| """Initialize OpenAI client with OpenRouter""" | |
| try: | |
| self.client = OpenAI( | |
| base_url="https://openrouter.ai/api/v1", | |
| api_key=api_key, | |
| ) | |
| return True, "API client initialized successfully!" | |
| except Exception as e: | |
| return False, f"Failed to initialize API client: {str(e)}" | |
| def scrape_webpage(self, url): | |
| """Scrape webpage content""" | |
| try: | |
| headers = { | |
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' | |
| } | |
| response = requests.get(url, headers=headers, timeout=30) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| # Remove script and style elements | |
| for script in soup(["script", "style", "nav", "footer", "header"]): | |
| script.decompose() | |
| # Extract text content | |
| text_content = soup.get_text() | |
| # Clean up text | |
| lines = (line.strip() for line in text_content.splitlines()) | |
| chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) | |
| text_content = ' '.join(chunk for chunk in chunks if chunk) | |
| # Extract tables | |
| tables = [] | |
| for table in soup.find_all('table'): | |
| table_data = [] | |
| headers = [] | |
| # Extract headers | |
| header_row = table.find('tr') | |
| if header_row: | |
| headers = [th.get_text().strip() for th in header_row.find_all(['th', 'td'])] | |
| # Extract rows | |
| for row in table.find_all('tr')[1:]: # Skip header row | |
| row_data = [td.get_text().strip() for td in row.find_all(['td', 'th'])] | |
| if row_data: | |
| table_data.append(row_data) | |
| if headers and table_data: | |
| tables.append({ | |
| 'headers': headers, | |
| 'data': table_data | |
| }) | |
| return { | |
| 'success': True, | |
| 'text': text_content[:15000], # Limit text length | |
| 'tables': tables, | |
| 'title': soup.title.string if soup.title else "No title found" | |
| } | |
| except requests.RequestException as e: | |
| return { | |
| 'success': False, | |
| 'error': f"Failed to fetch webpage: {str(e)}" | |
| } | |
| except Exception as e: | |
| return { | |
| 'success': False, | |
| 'error': f"Error processing webpage: {str(e)}" | |
| } | |
| def analyze_content(self, scraped_data, user_query, api_key): | |
| """Analyze scraped content using DeepSeek V3""" | |
| if not self.client: | |
| success, message = self.setup_client(api_key) | |
| if not success: | |
| return f"Error: {message}" | |
| if not scraped_data['success']: | |
| return f"Error scraping webpage: {scraped_data['error']}" | |
| # Prepare content for AI analysis | |
| content_text = f""" | |
| WEBPAGE CONTENT: | |
| Title: {scraped_data['title']} | |
| Main Text Content: | |
| {scraped_data['text']} | |
| Tables Found: {len(scraped_data['tables'])} | |
| """ | |
| if scraped_data['tables']: | |
| content_text += "\n\nTABLES:\n" | |
| for i, table in enumerate(scraped_data['tables']): | |
| content_text += f"\nTable {i+1}:\n" | |
| content_text += f"Headers: {', '.join(table['headers'])}\n" | |
| content_text += "Data:\n" | |
| for row in table['data'][:10]: # Limit rows | |
| content_text += f" {' | '.join(row)}\n" | |
| try: | |
| completion = self.client.chat.completions.create( | |
| extra_headers={ | |
| "HTTP-Referer": "https://gradio-web-scraper.com", | |
| "X-Title": "AI Web Scraping Tool", | |
| }, | |
| model="deepseek/deepseek-chat-v3-0324:free", | |
| messages=[ | |
| {"role": "system", "content": self.system_prompt}, | |
| {"role": "user", "content": f"Here is the webpage content:\n\n{content_text}\n\nUser Query: {user_query}"} | |
| ], | |
| temperature=0.1, | |
| max_tokens=4000 | |
| ) | |
| return completion.choices[0].message.content | |
| except Exception as e: | |
| return f"Error analyzing content: {str(e)}" | |
| def create_interface(): | |
| tool = WebScrapingTool() | |
| def process_request(api_key, url, user_query): | |
| if not api_key.strip(): | |
| return "❌ Please enter your OpenRouter API key" | |
| if not url.strip(): | |
| return "❌ Please enter a valid URL" | |
| if not user_query.strip(): | |
| return "❌ Please enter your analysis query" | |
| # Add progress updates | |
| yield "🔄 Scraping webpage content..." | |
| # Scrape webpage | |
| scraped_data = tool.scrape_webpage(url) | |
| if not scraped_data['success']: | |
| yield f"❌ {scraped_data['error']}" | |
| return | |
| yield f"✅ Successfully scraped webpage!\n📄 Title: {scraped_data['title']}\n📊 Found {len(scraped_data['tables'])} tables\n\n🤖 Analyzing content with DeepSeek V3..." | |
| # Analyze content | |
| result = tool.analyze_content(scraped_data, user_query, api_key) | |
| yield f"✅ Analysis Complete!\n\n{result}" | |
| # Create Gradio interface | |
| with gr.Blocks(title="AI Web Scraping Tool", theme=gr.themes.Soft()) as app: | |
| gr.Markdown(""" | |
| # 🤖 AI Web Scraping Tool | |
| ### Powered by DeepSeek V3 & OpenRouter | |
| Extract and analyze web content using advanced AI. Simply provide your OpenRouter API key, a URL, and describe what you want to extract. | |
| """) | |
| with gr.Row(): | |
| with gr.Column(scale=2): | |
| api_key_input = gr.Textbox( | |
| label="🔑 OpenRouter API Key", | |
| placeholder="Enter your OpenRouter API key here...", | |
| type="password", | |
| info="Get your free API key from openrouter.ai" | |
| ) | |
| url_input = gr.Textbox( | |
| label="🌐 Website URL", | |
| placeholder="https://example.com", | |
| info="Enter the URL you want to scrape and analyze" | |
| ) | |
| query_input = gr.Textbox( | |
| label="📝 Analysis Query", | |
| placeholder="What do you want to extract? (e.g., 'Extract main points and create a summary table')", | |
| lines=3, | |
| info="Describe what information you want to extract from the webpage" | |
| ) | |
| with gr.Row(): | |
| analyze_btn = gr.Button("🚀 Analyze Website", variant="primary", size="lg") | |
| clear_btn = gr.Button("🗑️ Clear", variant="secondary") | |
| with gr.Column(scale=3): | |
| output = gr.Textbox( | |
| label="📊 Analysis Results", | |
| lines=20, | |
| max_lines=30, | |
| show_copy_button=True, | |
| interactive=False | |
| ) | |
| # Example queries | |
| gr.Markdown(""" | |
| ### 💡 Example Queries: | |
| - *"Extract the main summary and any data tables"* | |
| - *"Create a table of key statistics mentioned in the article"* | |
| - *"Summarize the main points in bullet format"* | |
| - *"Extract all numerical data and organize it in a table"* | |
| - *"Find and extract contact information and company details"* | |
| """) | |
| # Example websites | |
| with gr.Accordion("📋 Try These Example URLs", open=False): | |
| examples = [ | |
| ["https://www.imf.org/en/Publications/WEO", "Extract economic outlook summary and GDP projections"], | |
| ["https://en.wikipedia.org/wiki/List_of_countries_by_GDP_(nominal)", "Create a table of top 10 countries by GDP"], | |
| ["https://www.who.int/news", "Summarize the latest health news"], | |
| ["https://www.nasdaq.com/market-activity/stocks", "Extract stock market data and trends"] | |
| ] | |
| for url, query in examples: | |
| gr.Markdown(f"**URL:** `{url}` \n**Query:** *{query}*") | |
| # Event handlers | |
| analyze_btn.click( | |
| fn=process_request, | |
| inputs=[api_key_input, url_input, query_input], | |
| outputs=output, | |
| show_progress=True | |
| ) | |
| clear_btn.click( | |
| fn=lambda: ("", "", "", ""), | |
| outputs=[api_key_input, url_input, query_input, output] | |
| ) | |
| # Auto-fill example | |
| def fill_example(): | |
| return ( | |
| "", # API key remains empty | |
| "https://www.imf.org/en/Publications/WEO/Issues/2024/04/16/world-economic-outlook-april-2024", | |
| """1. Extract a summary of the main economic outlook from this page. | |
| 2. Extract any available tables or figures with global GDP growth projections. | |
| 3. Create a new table showing: | |
| - Country/Region | |
| - Projected GDP Growth (2024) | |
| - Change from Previous Forecast (if available) | |
| 4. Highlight the top 3 fastest-growing economies in a separate mini-table.""" | |
| ) | |
| example_btn = gr.Button("📋 Load IMF Example", variant="secondary") | |
| example_btn.click( | |
| fn=fill_example, | |
| outputs=[url_input, query_input] | |
| ) | |
| return app | |
| if __name__ == "__main__": | |
| # Create and launch the app | |
| app = create_interface() | |
| # Launch with public sharing enabled | |
| app.launch( | |
| share=True, | |
| ) |