Upload 5 files

app (7).py

@@ -0,0 +1,65 @@
+
+import streamlit as st
+import os
+from scraper import fetch_html_selenium, format_data, save_raw_data, save_formatted_data
+from pagination_detector import detect_pagination_elements
+from assets import PRICING
+import google.generativeai as genai
+
+# Access API keys from Hugging Face Secrets
+openai_api_key = os.getenv('OPENAI_API_KEY')
+google_api_key = os.getenv('GOOGLE_API_KEY')
+groq_api_key = os.getenv('GROQ_API_KEY')
+
+# Check if the keys are available
+if not openai_api_key or not google_api_key or not groq_api_key:
+    st.error("API keys are missing! Please add them as secrets in Hugging Face.")
+
+# Initialize Streamlit app
+st.set_page_config(page_title="Universal Web Scraper", page_icon="🦑")
+st.title("Universal Web Scraper 🦑")
+
+# Initialize session state variables if they don't exist
+if 'results' not in st.session_state:
+    st.session_state['results'] = None
+if 'perform_scrape' not in st.session_state:
+    st.session_state['perform_scrape'] = False
+
+# Sidebar components
+st.sidebar.title("Web Scraper Settings")
+model_selection = st.sidebar.selectbox("Select Model", options=list(PRICING.keys()), index=0)
+url_input = st.sidebar.text_input("Enter URL(s) separated by whitespace")
+
+# Add toggle to show/hide tags field
+show_tags = st.sidebar.checkbox("Enable Scraping", value=False)
+
+# Conditionally show tags input based on the toggle
+tags = []
+if show_tags:
+    tags = st.sidebar.text_input("Enter Fields to Extract (comma-separated)").split(",")
+
+st.sidebar.markdown("---")
+# Add pagination toggle and input
+use_pagination = st.sidebar.checkbox("Enable Pagination", value=False)
+pagination_details = None
+if use_pagination:
+    pagination_details = st.sidebar.text_input("Enter Pagination Details (optional)", help="Describe how to navigate through pages")
+
+st.sidebar.markdown("---")
+
+# Define the scraping function
+def perform_scrape():
+    raw_html = fetch_html_selenium(url_input)
+    markdown = format_data(raw_html)
+    save_raw_data(markdown, "scraped_data")
+
+    if use_pagination:
+        pagination_data, _, _ = detect_pagination_elements(url_input, pagination_details, model_selection, markdown)
+        return pagination_data
+
+    return markdown
+
+if st.sidebar.button("Scrape"):
+    with st.spinner("Scraping data..."):
+        result = perform_scrape()
+        st.write(result)

assets.py.txt

@@ -0,0 +1,145 @@
+assets.py
+
+"""
+This module contains configuration variables and constants
+that are used across different parts of the application.
+"""
+
+# List of user agents to mimic different users
+USER_AGENTS  = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 13_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.2 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 13_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:91.0) Gecko/20100101 Firefox/91.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 14_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 14_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 13_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
+]
+
+
+# Define the pricing for models without Batch API
+PRICING = {
+    "gpt-4o-mini": {
+        "input": 0.150 / 1_000_000,  # $0.150 per 1M input tokens
+        "output": 0.600 / 1_000_000, # $0.600 per 1M output tokens
+    },
+    "gpt-4o-2024-08-06": {
+        "input": 2.5 / 1_000_000,  # $2.5 per 1M input tokens
+        "output": 10 / 1_000_000, # $10 per 1M output tokens
+    },
+    "gemini-1.5-flash": {
+        "input": 0.075 / 1_000_000,  # $0.075 per 1M input tokens
+        "output": 0.30 / 1_000_000, # $0.30 per 1M output tokens
+    },
+    "Llama3.1 8B": {
+        "input": 0 ,  # Free
+        "output": 0 , # Free
+    },
+    "Groq Llama3.1 70b": {
+        "input": 0 ,  # Free
+        "output": 0 , # Free
+    },
+    # Add other models and their prices here if needed
+}
+
+# Timeout settings for web scraping
+TIMEOUT_SETTINGS = {
+    "page_load": 30,
+    "script": 10
+}
+            
+# Other reusable constants or configuration settings
+HEADLESS_OPTIONS = ["--disable-gpu", "--disable-dev-shm-usage","--window-size=1920,1080","--disable-search-engine-choice-screen"]
+
+#in case you don't need to open the website
+##HEADLESS_OPTIONS=HEADLESS_OPTIONS+[ "--headless=new"]
+
+#number of scrolls
+NUMBER_SCROLL=2
+
+
+LLAMA_MODEL_FULLNAME="lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF"
+GROQ_LLAMA_MODEL_FULLNAME="llama-3.1-70b-versatile"
+
+SYSTEM_MESSAGE = """You are an intelligent text extraction and conversion assistant. Your task is to extract structured information 
+                        from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, 
+                        with no additional commentary, explanations, or extraneous information. 
+                        You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language.
+                        Please process the following text and provide the output in pure JSON format with no words before or after the JSON:"""
+
+USER_MESSAGE = f"Extract the following information from the provided text:\nPage content:\n\n"
+        
+
+
+
+
+PROMPT_PAGINATION = """
+You are an assistant that extracts pagination elements from markdown content of websites your goal as a universal pagination scrapper of urls from all websites no matter how different they are.
+
+Please extract the following:
+
+- The url of the 'Next', 'More', 'See more', 'load more' or any other button indicating how to access the next page, if any, it should be 1 url and no more, if there are multiple urls with the same structure leave this empty.
+
+- A list of page URLs for pagination it should be a pattern of similar urls with pages that are numbered, if you detect this pattern and the numbers starts from a certain low number until a large number generate the rest of the urls even if they're not included, 
+your goal here is to give as many urls for the user to choose from in order for them to do further scraping, you will have to deal with very different websites that can potientially have so many urls of images and other elements, 
+detect only the urls that are clearly defining a pattern to show data on multiple pages, sometimes there is only a part of these urls and you have to combine it with the initial url, that will be provided for you at the end of this prompt.
+
+- The user can give you indications on how the pagination works for the specific website at the end of this prompt, if those indications are not empty pay special attention to them as they will directly help you understand the structure and the number of pages to generate.
+
+Provide the output as a JSON object with the following structure:
+
+{
+    "page_urls": ["url1", "url2", "url3",...,"urlN"]
+}
+
+Do not include any additional text or explanations.
+"""
+    
+        

pagination_detector.py.txt

@@ -0,0 +1,206 @@
+pagination_detector.py
+
+
+# pagination_detector.py
+
+import os
+import json
+from typing import List, Dict, Tuple, Union
+from pydantic import BaseModel, Field, ValidationError
+
+import tiktoken
+from dotenv import load_dotenv
+
+from openai import OpenAI
+import google.generativeai as genai
+from groq import Groq
+
+from assets import PROMPT_PAGINATION, PRICING, LLAMA_MODEL_FULLNAME, GROQ_LLAMA_MODEL_FULLNAME
+
+load_dotenv()
+import logging
+
+class PaginationData(BaseModel):
+    page_urls: List[str] = Field(default_factory=list, description="List of pagination URLs, including 'Next' button URL if present")
+
+def calculate_pagination_price(token_counts: Dict[str, int], model: str) -> float:
+    """
+    Calculate the price for pagination based on token counts and the selected model.
+    
+    Args:
+    token_counts (Dict[str, int]): A dictionary containing 'input_tokens' and 'output_tokens'.
+    model (str): The name of the selected model.
+
+    Returns:
+    float: The total price for the pagination operation.
+    """
+    input_tokens = token_counts['input_tokens']
+    output_tokens = token_counts['output_tokens']
+    
+    input_price = input_tokens * PRICING[model]['input']
+    output_price = output_tokens * PRICING[model]['output']
+    
+    return input_price + output_price
+
+def detect_pagination_elements(url: str, indications: str, selected_model: str, markdown_content: str) -> Tuple[Union[PaginationData, Dict, str], Dict, float]:
+    try:
+        """
+        Uses AI models to analyze markdown content and extract pagination elements.
+
+        Args:
+            selected_model (str): The name of the OpenAI model to use.
+            markdown_content (str): The markdown content to analyze.
+
+        Returns:
+            Tuple[PaginationData, Dict, float]: Parsed pagination data, token counts, and pagination price.
+        """ 
+        prompt_pagination = PROMPT_PAGINATION+"\n The url of the page to extract pagination from   "+url+"if the urls that you find are not complete combine them intelligently in a way that fit the pattern **ALWAYS GIVE A FULL URL**"
+        if indications != "":
+            prompt_pagination +=PROMPT_PAGINATION+"\n\n these are the users indications that, pay special attention to them: "+indications+"\n\n below are the markdowns of the website: \n\n"
+        else:
+            prompt_pagination +=PROMPT_PAGINATION+"\n There are no user indications in this case just apply the logic described. \n\n below are the markdowns of the website: \n\n"
+
+        if selected_model in ["gpt-4o-mini", "gpt-4o-2024-08-06"]:
+            # Use OpenAI API
+            client = OpenAI(api_key=os.getenv('OPENAI_API_KEY'))
+            completion = client.beta.chat.completions.parse(
+                model=selected_model,
+                messages=[
+                    {"role": "system", "content": prompt_pagination},
+                    {"role": "user", "content": markdown_content},
+                ],
+                response_format=PaginationData
+            )
+
+            # Extract the parsed response
+            parsed_response = completion.choices[0].message.parsed
+
+            # Calculate tokens using tiktoken
+            encoder = tiktoken.encoding_for_model(selected_model)
+            input_token_count = len(encoder.encode(markdown_content))
+            output_token_count = len(encoder.encode(json.dumps(parsed_response.dict())))
+            token_counts = {
+                "input_tokens": input_token_count,
+                "output_tokens": output_token_count
+            }
+
+            # Calculate the price
+            pagination_price = calculate_pagination_price(token_counts, selected_model)
+
+            return parsed_response, token_counts, pagination_price
+
+        elif selected_model == "gemini-1.5-flash":
+            # Use Google Gemini API
+            genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
+            model = genai.GenerativeModel(
+                'gemini-1.5-flash',
+                generation_config={
+                    "response_mime_type": "application/json",
+                    "response_schema": PaginationData
+                }
+            )
+            prompt = f"{prompt_pagination}\n{markdown_content}"
+            # Count input tokens using Gemini's method
+            input_tokens = model.count_tokens(prompt)
+            completion = model.generate_content(prompt)
+            # Extract token counts from usage_metadata
+            usage_metadata = completion.usage_metadata
+            token_counts = {
+                "input_tokens": usage_metadata.prompt_token_count,
+                "output_tokens": usage_metadata.candidates_token_count
+            }
+            # Get the result
+            response_content = completion.text
+            
+            # Log the response content and its type
+            logging.info(f"Gemini Flash response type: {type(response_content)}")
+            logging.info(f"Gemini Flash response content: {response_content}")
+            
+            # Try to parse the response as JSON
+            try:
+                parsed_data = json.loads(response_content)
+                if isinstance(parsed_data, dict) and 'page_urls' in parsed_data:
+                    pagination_data = PaginationData(**parsed_data)
+                else:
+                    pagination_data = PaginationData(page_urls=[])
+            except json.JSONDecodeError:
+                logging.error("Failed to parse Gemini Flash response as JSON")
+                pagination_data = PaginationData(page_urls=[])
+
+            # Calculate the price
+            pagination_price = calculate_pagination_price(token_counts, selected_model)
+
+            return pagination_data, token_counts, pagination_price
+
+        elif selected_model == "Llama3.1 8B":
+            # Use Llama model via OpenAI API pointing to local server
+            openai.api_key = "lm-studio"
+            openai.api_base = "http://localhost:1234/v1"
+            response = openai.ChatCompletion.create(
+                model=LLAMA_MODEL_FULLNAME,
+                messages=[
+                    {"role": "system", "content": prompt_pagination},
+                    {"role": "user", "content": markdown_content},
+                ],
+                temperature=0.7,
+            )
+            response_content = response['choices'][0]['message']['content'].strip()
+            # Try to parse the JSON
+            try:
+                pagination_data = json.loads(response_content)
+            except json.JSONDecodeError:
+                pagination_data = {"next_buttons": [], "page_urls": []}
+            # Token counts
+            token_counts = {
+                "input_tokens": response['usage']['prompt_tokens'],
+                "output_tokens": response['usage']['completion_tokens']
+            }
+            # Calculate the price
+            pagination_price = calculate_pagination_price(token_counts, selected_model)
+
+            return pagination_data, token_counts, pagination_price
+
+        elif selected_model == "Groq Llama3.1 70b":
+            # Use Groq client
+            client = Groq(api_key=os.environ.get("GROQ_API_KEY"))
+            response = client.chat.completions.create(
+                model=GROQ_LLAMA_MODEL_FULLNAME,
+                messages=[
+                    {"role": "system", "content": prompt_pagination},
+                    {"role": "user", "content": markdown_content},
+                ],
+            )
+            response_content = response.choices[0].message.content.strip()
+            # Try to parse the JSON
+            try:
+                pagination_data = json.loads(response_content)
+            except json.JSONDecodeError:
+                pagination_data = {"page_urls": []}
+            # Token counts
+            token_counts = {
+                "input_tokens": response.usage.prompt_tokens,
+                "output_tokens": response.usage.completion_tokens
+            }
+            # Calculate the price
+            pagination_price = calculate_pagination_price(token_counts, selected_model)
+
+            # Ensure the pagination_data is a dictionary
+            if isinstance(pagination_data, PaginationData):
+                pagination_data = pagination_data.dict()
+            elif not isinstance(pagination_data, dict):
+                pagination_data = {"page_urls": []}
+
+            return pagination_data, token_counts, pagination_price
+
+        else:
+            raise ValueError(f"Unsupported model: {selected_model}")
+
+    except Exception as e:
+        logging.error(f"An error occurred in detect_pagination_elements: {e}")
+        # Return default values if an error occurs
+        return PaginationData(page_urls=[]), {"input_tokens": 0, "output_tokens": 0}, 0.0
+
+
+            
+            
+        

requirements (28).txt

@@ -0,0 +1,15 @@
+openai
+python-dotenv
+pandas
+pydantic
+requests
+beautifulsoup4
+html2text
+tiktoken
+selenium
+readability-lxml
+streamlit
+streamlit-tags
+openpyxl
+groq
+google-generativeai

scraper.py.txt

@@ -0,0 +1,458 @@
+scraper.py
+
+import os
+import random
+import time
+import re
+import json
+from datetime import datetime
+from typing import List, Dict, Type
+
+import pandas as pd
+from bs4 import BeautifulSoup
+from pydantic import BaseModel, Field, create_model
+import html2text
+import tiktoken
+
+from dotenv import load_dotenv
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.action_chains import ActionChains
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+
+from openai import OpenAI
+import google.generativeai as genai
+from groq import Groq
+
+
+from assets import USER_AGENTS,PRICING,HEADLESS_OPTIONS,SYSTEM_MESSAGE,USER_MESSAGE,LLAMA_MODEL_FULLNAME,GROQ_LLAMA_MODEL_FULLNAME
+load_dotenv()
+
+# Set up the Chrome WebDriver options
+
+def setup_selenium():
+    options = Options()
+
+    # Randomly select a user agent from the imported list
+    user_agent = random.choice(USER_AGENTS)
+    options.add_argument(f"user-agent={user_agent}")
+
+    # Add other options
+    for option in HEADLESS_OPTIONS:
+        options.add_argument(option)
+
+    # Specify the path to the ChromeDriver
+    service = Service(r"./chromedriver-win64/chromedriver.exe")  
+
+    # Initialize the WebDriver
+    driver = webdriver.Chrome(service=service, options=options)
+    return driver
+
+def click_accept_cookies(driver):
+    """
+    Tries to find and click on a cookie consent button. It looks for several common patterns.
+    """
+    try:
+        # Wait for cookie popup to load
+        WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located((By.XPATH, "//button | //a | //div"))
+        )
+        
+        # Common text variations for cookie buttons
+        accept_text_variations = [
+            "accept", "agree", "allow", "consent", "continue", "ok", "I agree", "got it"
+        ]
+        
+        # Iterate through different element types and common text variations
+        for tag in ["button", "a", "div"]:
+            for text in accept_text_variations:
+                try:
+                    # Create an XPath to find the button by text
+                    element = driver.find_element(By.XPATH, f"//{tag}[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text}')]")
+                    if element:
+                        element.click()
+                        print(f"Clicked the '{text}' button.")
+                        return
+                except:
+                    continue
+
+        print("No 'Accept Cookies' button found.")
+    
+    except Exception as e:
+        print(f"Error finding 'Accept Cookies' button: {e}")
+
+def fetch_html_selenium(url):
+    driver = setup_selenium()
+    try:
+        driver.get(url)
+        
+        # Add random delays to mimic human behavior
+        time.sleep(1)  # Adjust this to simulate time for user to read or interact
+        driver.maximize_window()
+        
+
+        # Try to find and click the 'Accept Cookies' button
+        # click_accept_cookies(driver)
+
+        # Add more realistic actions like scrolling
+        driver.execute_script("window.scrollTo(0, document.body.scrollHeight/2);")
+        time.sleep(random.uniform(1.1, 1.8))  # Simulate time taken to scroll and read
+        driver.execute_script("window.scrollTo(0, document.body.scrollHeight/1.2);")
+        time.sleep(random.uniform(1.1, 1.8))
+        driver.execute_script("window.scrollTo(0, document.body.scrollHeight/1);")
+        time.sleep(random.uniform(1.1, 2.1))
+        html = driver.page_source
+        return html
+    finally:
+        driver.quit()
+
+def clean_html(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    
+    # Remove headers and footers based on common HTML tags or classes
+    for element in soup.find_all(['header', 'footer']):
+        element.decompose()  # Remove these tags and their content
+
+    return str(soup)
+
+
+def html_to_markdown_with_readability(html_content):
+
+    
+    cleaned_html = clean_html(html_content)  
+    
+    # Convert to markdown
+    markdown_converter = html2text.HTML2Text()
+    markdown_converter.ignore_links = False
+    markdown_content = markdown_converter.handle(cleaned_html)
+    
+    return markdown_content
+
+
+    
+def save_raw_data(raw_data: str, output_folder: str, file_name: str):
+    """Save raw markdown data to the specified output folder."""
+    os.makedirs(output_folder, exist_ok=True)
+    raw_output_path = os.path.join(output_folder, file_name)
+    with open(raw_output_path, 'w', encoding='utf-8') as f:
+        f.write(raw_data)
+    print(f"Raw data saved to {raw_output_path}")
+    return raw_output_path
+
+
+def remove_urls_from_file(file_path):
+    # Regex pattern to find URLs
+    url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
+
+    # Construct the new file name
+    base, ext = os.path.splitext(file_path)
+    new_file_path = f"{base}_cleaned{ext}"
+
+    # Read the original markdown content
+    with open(file_path, 'r', encoding='utf-8') as file:
+        markdown_content = file.read()
+
+    # Replace all found URLs with an empty string
+    cleaned_content = re.sub(url_pattern, '', markdown_content)
+
+    # Write the cleaned content to a new file
+    with open(new_file_path, 'w', encoding='utf-8') as file:
+        file.write(cleaned_content)
+    print(f"Cleaned file saved as: {new_file_path}")
+    return cleaned_content
+
+
+def create_dynamic_listing_model(field_names: List[str]) -> Type[BaseModel]:
+    """
+    Dynamically creates a Pydantic model based on provided fields.
+    field_name is a list of names of the fields to extract from the markdown.
+    """
+    # Create field definitions using aliases for Field parameters
+    field_definitions = {field: (str, ...) for field in field_names}
+    # Dynamically create the model with all field
+    return create_model('DynamicListingModel', **field_definitions)
+
+
+def create_listings_container_model(listing_model: Type[BaseModel]) -> Type[BaseModel]:
+    """
+    Create a container model that holds a list of the given listing model.
+    """
+    return create_model('DynamicListingsContainer', listings=(List[listing_model], ...))
+
+
+
+
+def trim_to_token_limit(text, model, max_tokens=120000):
+    encoder = tiktoken.encoding_for_model(model)
+    tokens = encoder.encode(text)
+    if len(tokens) > max_tokens:
+        trimmed_text = encoder.decode(tokens[:max_tokens])
+        return trimmed_text
+    return text
+
+def generate_system_message(listing_model: BaseModel) -> str:
+    """
+    Dynamically generate a system message based on the fields in the provided listing model.
+    """
+    # Use the model_json_schema() method to introspect the Pydantic model
+    schema_info = listing_model.model_json_schema()
+
+    # Extract field descriptions from the schema
+    field_descriptions = []
+    for field_name, field_info in schema_info["properties"].items():
+        # Get the field type from the schema info
+        field_type = field_info["type"]
+        field_descriptions.append(f'"{field_name}": "{field_type}"')
+
+    # Create the JSON schema structure for the listings
+    schema_structure = ",\n".join(field_descriptions)
+
+    # Generate the system message dynamically
+    system_message = f"""
+    You are an intelligent text extraction and conversion assistant. Your task is to extract structured information 
+                        from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, 
+                        with no additional commentary, explanations, or extraneous information. 
+                        You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language.
+                        Please process the following text and provide the output in pure JSON format with no words before or after the JSON:
+    Please ensure the output strictly follows this schema:
+
+    {{
+        "listings": [
+            {{
+                {schema_structure}
+            }}
+        ]
+    }} """
+
+    return system_message
+
+
+
+def format_data(data, DynamicListingsContainer, DynamicListingModel, selected_model):
+    token_counts = {}
+    
+    if selected_model in ["gpt-4o-mini", "gpt-4o-2024-08-06"]:
+        # Use OpenAI API
+        client = OpenAI(api_key=os.getenv('OPENAI_API_KEY'))
+        completion = client.beta.chat.completions.parse(
+            model=selected_model,
+            messages=[
+                {"role": "system", "content": SYSTEM_MESSAGE},
+                {"role": "user", "content": USER_MESSAGE + data},
+            ],
+            response_format=DynamicListingsContainer
+        )
+        # Calculate tokens using tiktoken
+        encoder = tiktoken.encoding_for_model(selected_model)
+        input_token_count = len(encoder.encode(USER_MESSAGE + data))
+        output_token_count = len(encoder.encode(json.dumps(completion.choices[0].message.parsed.dict())))
+        token_counts = {
+            "input_tokens": input_token_count,
+            "output_tokens": output_token_count
+        }
+        return completion.choices[0].message.parsed, token_counts
+
+    elif selected_model == "gemini-1.5-flash":
+        # Use Google Gemini API
+        genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
+        model = genai.GenerativeModel('gemini-1.5-flash',
+                generation_config={
+                    "response_mime_type": "application/json",
+                    "response_schema": DynamicListingsContainer
+                })
+        prompt = SYSTEM_MESSAGE + "\n" + USER_MESSAGE + data
+        # Count input tokens using Gemini's method
+        input_tokens = model.count_tokens(prompt)
+        completion = model.generate_content(prompt)
+        # Extract token counts from usage_metadata
+        usage_metadata = completion.usage_metadata
+        token_counts = {
+            "input_tokens": usage_metadata.prompt_token_count,
+            "output_tokens": usage_metadata.candidates_token_count
+        }
+        return completion.text, token_counts
+    
+    elif selected_model == "Llama3.1 8B":
+
+        # Dynamically generate the system message based on the schema
+        sys_message = generate_system_message(DynamicListingModel)
+        # print(SYSTEM_MESSAGE)
+        # Point to the local server
+        client = OpenAI(base_url="http://localhost:1234/v1", api_key="lm-studio")
+
+        completion = client.chat.completions.create(
+            model=LLAMA_MODEL_FULLNAME, #change this if needed (use a better model)
+            messages=[
+                {"role": "system", "content": sys_message},
+                {"role": "user", "content": USER_MESSAGE + data}
+            ],
+            temperature=0.7,
+            
+        )
+
+        # Extract the content from the response
+        response_content = completion.choices[0].message.content
+        print(response_content)
+        # Convert the content from JSON string to a Python dictionary
+        parsed_response = json.loads(response_content)
+        
+        # Extract token usage
+        token_counts = {
+            "input_tokens": completion.usage.prompt_tokens,
+            "output_tokens": completion.usage.completion_tokens
+        }
+
+        return parsed_response, token_counts
+    elif selected_model== "Groq Llama3.1 70b":
+        
+        # Dynamically generate the system message based on the schema
+        sys_message = generate_system_message(DynamicListingModel)
+        # print(SYSTEM_MESSAGE)
+        # Point to the local server
+        client = Groq(api_key=os.environ.get("GROQ_API_KEY"),)
+
+        completion = client.chat.completions.create(
+        messages=[
+            {"role": "system","content": sys_message},
+            {"role": "user","content": USER_MESSAGE + data}
+        ],
+        model=GROQ_LLAMA_MODEL_FULLNAME,
+    )
+
+        # Extract the content from the response
+        response_content = completion.choices[0].message.content
+        
+        # Convert the content from JSON string to a Python dictionary
+        parsed_response = json.loads(response_content)
+        
+        # completion.usage
+        token_counts = {
+            "input_tokens": completion.usage.prompt_tokens,
+            "output_tokens": completion.usage.completion_tokens
+        }
+
+        return parsed_response, token_counts
+    else:
+        raise ValueError(f"Unsupported model: {selected_model}")
+
+
+
+def save_formatted_data(formatted_data, output_folder: str, json_file_name: str, excel_file_name: str):
+    """Save formatted data as JSON and Excel in the specified output folder."""
+    os.makedirs(output_folder, exist_ok=True)
+    
+    # Parse the formatted data if it's a JSON string (from Gemini API)
+    if isinstance(formatted_data, str):
+        try:
+            formatted_data_dict = json.loads(formatted_data)
+        except json.JSONDecodeError:
+            raise ValueError("The provided formatted data is a string but not valid JSON.")
+    else:
+        # Handle data from OpenAI or other sources
+        formatted_data_dict = formatted_data.dict() if hasattr(formatted_data, 'dict') else formatted_data
+
+    # Save the formatted data as JSON
+    json_output_path = os.path.join(output_folder, json_file_name)
+    with open(json_output_path, 'w', encoding='utf-8') as f:
+        json.dump(formatted_data_dict, f, indent=4)
+    print(f"Formatted data saved to JSON at {json_output_path}")
+
+    # Prepare data for DataFrame
+    if isinstance(formatted_data_dict, dict):
+        # If the data is a dictionary containing lists, assume these lists are records
+        data_for_df = next(iter(formatted_data_dict.values())) if len(formatted_data_dict) == 1 else formatted_data_dict
+    elif isinstance(formatted_data_dict, list):
+        data_for_df = formatted_data_dict
+    else:
+        raise ValueError("Formatted data is neither a dictionary nor a list, cannot convert to DataFrame")
+
+    # Create DataFrame
+    try:
+        df = pd.DataFrame(data_for_df)
+        print("DataFrame created successfully.")
+
+        # Save the DataFrame to an Excel file
+        excel_output_path = os.path.join(output_folder, excel_file_name)
+        df.to_excel(excel_output_path, index=False)
+        print(f"Formatted data saved to Excel at {excel_output_path}")
+        
+        return df
+    except Exception as e:
+        print(f"Error creating DataFrame or saving Excel: {str(e)}")
+        return None
+
+def calculate_price(token_counts, model):
+    input_token_count = token_counts.get("input_tokens", 0)
+    output_token_count = token_counts.get("output_tokens", 0)
+    
+    # Calculate the costs
+    input_cost = input_token_count * PRICING[model]["input"]
+    output_cost = output_token_count * PRICING[model]["output"]
+    total_cost = input_cost + output_cost
+    
+    return input_token_count, output_token_count, total_cost
+
+
+def generate_unique_folder_name(url):
+    timestamp = datetime.now().strftime('%Y_%m_%d__%H_%M_%S')
+    url_name = re.sub(r'\W+', '_', url.split('//')[1].split('/')[0])  # Extract domain name and replace non-alphanumeric characters
+    return f"{url_name}_{timestamp}"
+
+
+def scrape_multiple_urls(urls, fields, selected_model):
+    output_folder = os.path.join('output', generate_unique_folder_name(urls[0]))
+    os.makedirs(output_folder, exist_ok=True)
+    
+    total_input_tokens = 0
+    total_output_tokens = 0
+    total_cost = 0
+    all_data = []
+    markdown = None  # We'll store the markdown for the first (or only) URL
+    
+    for i, url in enumerate(urls, start=1):
+        raw_html = fetch_html_selenium(url)
+        current_markdown = html_to_markdown_with_readability(raw_html)
+        if i == 1:
+            markdown = current_markdown  # Store markdown for the first URL
+        
+        input_tokens, output_tokens, cost, formatted_data = scrape_url(url, fields, selected_model, output_folder, i, current_markdown)
+        total_input_tokens += input_tokens
+        total_output_tokens += output_tokens
+        total_cost += cost
+        all_data.append(formatted_data)
+    
+    return output_folder, total_input_tokens, total_output_tokens, total_cost, all_data, markdown
+
+def scrape_url(url: str, fields: List[str], selected_model: str, output_folder: str, file_number: int, markdown: str):
+    """Scrape a single URL and save the results."""
+    try:
+        # Save raw data
+        save_raw_data(markdown, output_folder, f'rawData_{file_number}.md')
+
+        # Create the dynamic listing model
+        DynamicListingModel = create_dynamic_listing_model(fields)
+
+        # Create the container model that holds a list of the dynamic listing models
+        DynamicListingsContainer = create_listings_container_model(DynamicListingModel)
+        
+        # Format data
+        formatted_data, token_counts = format_data(markdown, DynamicListingsContainer, DynamicListingModel, selected_model)
+        
+        # Save formatted data
+        save_formatted_data(formatted_data, output_folder, f'sorted_data_{file_number}.json', f'sorted_data_{file_number}.xlsx')
+
+        # Calculate and return token usage and cost
+        input_tokens, output_tokens, total_cost = calculate_price(token_counts, selected_model)
+        return input_tokens, output_tokens, total_cost, formatted_data
+
+    except Exception as e:
+        print(f"An error occurred while processing {url}: {e}")
+        return 0, 0, 0, None
+
+
+                    
+