scrape-content.ipynb

{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-26T16:28:29.519384Z",
     "start_time": "2024-10-26T16:28:29.506673Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from fake_headers import Headers\n",
    "\n",
    "headers = Headers(headers=True).generate()\n",
    "headers"
   ],
   "id": "c60b4d771c2e0a21",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Accept': '*/*',\n",
       " 'Connection': 'keep-alive',\n",
       " 'User-Agent': 'Mozilla/5.0 (X11; Linux i686 on x86_64; rv:60.3.0) Gecko/20100101 Firefox/60.3.0',\n",
       " 'Accept-Language': 'en-US;q=0.5,en;q=0.3',\n",
       " 'DNT': '1',\n",
       " 'Referer': 'https://google.com'}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-26T16:28:29.719882Z",
     "start_time": "2024-10-26T16:28:29.531148Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "\n",
    "\n",
    "def scroll_and_wait(driver, scroll_pause_time=2):\n",
    "    \"\"\"\n",
    "    Scroll the page gradually and wait for images to load\n",
    "    \"\"\"\n",
    "    # Get scroll height\n",
    "    last_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "\n",
    "    while True:\n",
    "        # Scroll down gradually\n",
    "        for i in range(10):\n",
    "            driver.execute_script(f\"window.scrollTo(0, {(i + 1) * (last_height / 10)});\")\n",
    "            time.sleep(0.5)  # Short pause between each scroll step\n",
    "\n",
    "        # Wait for new images to load\n",
    "        time.sleep(scroll_pause_time)\n",
    "\n",
    "        # Calculate new scroll height and compare with last scroll height\n",
    "        new_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "        if new_height == last_height:\n",
    "            break\n",
    "        last_height = new_height\n",
    "\n",
    "\n",
    "def wait_for_images(driver, timeout=10):\n",
    "    \"\"\"\n",
    "    Wait for images to load and become visible\n",
    "    \"\"\"\n",
    "    try:\n",
    "        # Wait for all image elements to be present\n",
    "        WebDriverWait(driver, timeout).until(\n",
    "            EC.presence_of_all_elements_located((By.TAG_NAME, \"img\"))\n",
    "        )\n",
    "\n",
    "        # Get all image elements\n",
    "        images = driver.find_elements(By.TAG_NAME, \"img\")\n",
    "\n",
    "        # Wait for images to load\n",
    "        for img in images:\n",
    "            try:\n",
    "                WebDriverWait(driver, 2).until(\n",
    "                    lambda d: img.get_attribute('complete') == 'true' and\n",
    "                              img.get_attribute('naturalHeight') != '0'\n",
    "                )\n",
    "            except:\n",
    "                continue  # Skip images that don't load within timeout\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"Warning: Not all images could be loaded: {e}\")"
   ],
   "id": "11933d956e20b6b8",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-26T16:29:00.452959Z",
     "start_time": "2024-10-26T16:28:29.721884Z"
    }
   },
   "cell_type": "code",
   "source": [
    "chrome_options = Options()\n",
    "# chrome_options.add_argument(\"--headless\")\n",
    "# chrome_options.add_argument(\"--disable-gpu\")\n",
    "# chrome_options.add_argument(\"--no-sandbox\")\n",
    "# chrome_options.add_argument(\"--disable-dev-shm-usage\")\n",
    "\n",
    "# Add fake headers\n",
    "for key, value in headers.items():\n",
    "    chrome_options.add_argument(f'--{key.lower()}={value}')\n",
    "\n",
    "# Additional configurations to appear more human-like\n",
    "chrome_options.add_argument(\"--disable-blink-features=AutomationControlled\")\n",
    "chrome_options.add_argument(\"--window-size=1920,1080\")\n",
    "\n",
    "# Enable images in headless mode\n",
    "chrome_options.add_argument(\"--force-device-scale-factor=1\")\n",
    "chrome_options.add_argument(\"--high-dpi-support=1\")\n",
    "\n",
    "# Privacy and fingerprinting prevention\n",
    "chrome_options.add_argument(\"--disable-blink-features\")\n",
    "chrome_options.add_argument(\"--disable-infobars\")\n",
    "chrome_options.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n",
    "chrome_options.add_experimental_option(\"useAutomationExtension\", False)\n",
    "\n",
    "# Enable JavaScript\n",
    "chrome_options.add_argument(\"--enable-javascript\")\n",
    "\n",
    "driver = webdriver.Chrome(options=chrome_options)\n",
    "\n",
    "driver.execute_cdp_cmd(\"Page.addScriptToEvaluateOnNewDocument\", {\n",
    "    \"source\": \"\"\"\n",
    "        Object.defineProperty(navigator, 'webdriver', {\n",
    "            get: () => undefined\n",
    "        })\n",
    "    \"\"\"\n",
    "})\n",
    "\n",
    "products_url = \"https://www.target.com/s?searchTerm=Peach&tref=typeahead%7Cterm%7CPeach%7C%7C%7Chistory\"\n",
    "driver.get(products_url)\n",
    "\n",
    "time.sleep(3)\n",
    "\n",
    "# Scroll and wait for content\n",
    "scroll_and_wait(driver)\n",
    "\n",
    "# Wait for images to load\n",
    "wait_for_images(driver)\n",
    "\n",
    "time.sleep(2)\n",
    "\n",
    "soup = BeautifulSoup(driver.page_source, \"html.parser\")\n",
    "driver.quit()"
   ],
   "id": "ac14cff825f0887f",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-26T16:54:58.190620Z",
     "start_time": "2024-10-26T16:54:58.165031Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from urllib.parse import urljoin\n",
    "import json\n",
    "from collections import Counter\n",
    "\n",
    "\n",
    "def get_element_signature(element):\n",
    "    \"\"\"\n",
    "    Create a signature for an element based on its structure.\n",
    "    \"\"\"\n",
    "    signature = {\n",
    "        'tag': element.name,\n",
    "        'classes': tuple(sorted(element.get('class', []))),\n",
    "        'child_tags': tuple(sorted(child.name for child in element.find_all(recursive=False) if child.name)),\n",
    "        'has_image': bool(element.find('img')),\n",
    "        'has_price': bool(any(c in element.get_text() for c in '$€£¥')),\n",
    "        'has_link': bool(element.find('a')),\n",
    "    }\n",
    "    return str(signature)\n",
    "\n",
    "\n",
    "def analyze_children_similarity(element):\n",
    "    \"\"\"\n",
    "    Analyze how similar the direct children of an element are.\n",
    "    \"\"\"\n",
    "    if not element.contents:\n",
    "        return 0, 0\n",
    "\n",
    "    # Get signatures for all direct children that are elements (have a tag name)\n",
    "    child_signatures = [\n",
    "        get_element_signature(child)\n",
    "        for child in element.find_all(recursive=False)\n",
    "        if child.name\n",
    "    ]\n",
    "\n",
    "    if not child_signatures:\n",
    "        return 0, 0\n",
    "\n",
    "    # Count how many times each signature appears and get the most common one\n",
    "    signature_counts = Counter(child_signatures)\n",
    "    most_common_sig, most_common_count = signature_counts.most_common(1)[0]\n",
    "    similarity_score = most_common_count / len(child_signatures)\n",
    "\n",
    "    return similarity_score, most_common_count\n",
    "\n",
    "\n",
    "def count_images_in_element(element):\n",
    "    \"\"\"\n",
    "    Count all images within an element, including nested ones.\n",
    "    \"\"\"\n",
    "    return len(element.find_all('img', recursive=True))\n",
    "\n",
    "\n",
    "def get_element_identifier(element):\n",
    "    \"\"\"\n",
    "    Create a unique identifier for an element including tag and classes.\n",
    "    \"\"\"\n",
    "    identifier = element.name\n",
    "    if element.get('class'):\n",
    "        identifier += f\" .{' .'.join(element['class'])}\"\n",
    "    if element.get('id'):\n",
    "        identifier += f\" #{element['id']}\"\n",
    "    return identifier\n",
    "\n",
    "\n",
    "def convert_relative_urls(soup, base_url):\n",
    "    \"\"\"\n",
    "    Convert all relative URLs in the soup object to absolute URLs.\n",
    "    \"\"\"\n",
    "    for tag in soup.find_all(href=True):\n",
    "        tag['href'] = urljoin(base_url, tag['href'])\n",
    "    for tag in soup.find_all(src=True):\n",
    "        tag['src'] = urljoin(base_url, tag['src'])\n",
    "    for tag in soup.find_all(attrs={'data-src': True}):\n",
    "        tag['data-src'] = urljoin(base_url, tag['data-src'])\n",
    "    return soup\n",
    "\n",
    "\n",
    "def find_image_rich_parents(soup, base_url, min_children=4, min_similarity=0.7):\n",
    "    \"\"\"\n",
    "    Find elements containing images and return both sorted list and detailed top element info.\n",
    "    \"\"\"\n",
    "    # Convert relative URLs to absolute\n",
    "    soup = convert_relative_urls(soup, base_url)\n",
    "\n",
    "    # Collect potential container elements with their scores\n",
    "    elements_with_scores = []\n",
    "    for element in soup.find_all():\n",
    "        if element.name in ['div', 'ul', 'section', 'main']:\n",
    "            similarity_score, similar_children_count = analyze_children_similarity(element)\n",
    "            image_count = count_images_in_element(element)\n",
    "\n",
    "            if similar_children_count >= min_children and similarity_score >= min_similarity and image_count > 0:\n",
    "                # Calculate combined score based on similarity and image count\n",
    "                combined_score = (similarity_score * similar_children_count * image_count)\n",
    "                elements_with_scores.append((element, image_count, combined_score))\n",
    "\n",
    "    if not elements_with_scores:\n",
    "        return [], {\"error\": \"No elements with images found\"}, \"\"\n",
    "\n",
    "    # Sort by combined score\n",
    "    elements_with_scores.sort(key=lambda x: x[2], reverse=True)\n",
    "\n",
    "    # Process elements for sorted list output\n",
    "    sorted_elements = []\n",
    "    for element, image_count, _ in elements_with_scores:\n",
    "        sorted_elements.append((get_element_identifier(element), image_count))\n",
    "\n",
    "    # Get top element (one with highest combined score)\n",
    "    top_element = elements_with_scores[0][0]\n",
    "\n",
    "    # Separate child elements with images\n",
    "    products = []\n",
    "    for child in top_element.find_all(recursive=False):\n",
    "        if child.name:  # Skip text nodes\n",
    "            product_info = {\n",
    "                \"html_content\": str(child),\n",
    "                \"images\": []\n",
    "            }\n",
    "\n",
    "            # Get all images within this product\n",
    "            for img in child.find_all('img', recursive=True):\n",
    "                image_info = {\n",
    "                    \"src\": img.get('src', 'No source'),\n",
    "                    \"alt\": img.get('alt', 'No alt text')\n",
    "                }\n",
    "                product_info[\"images\"].append(image_info)\n",
    "\n",
    "            products.append(product_info)\n",
    "\n",
    "    print(len(products))\n",
    "\n",
    "    # Create result dictionary for top element   \n",
    "    top_element_info = {\n",
    "        \"parent\": {\n",
    "            \"tag\": top_element.name,\n",
    "            \"identifier\": get_element_identifier(top_element),\n",
    "            \"classes\": top_element.get('class', []),\n",
    "            \"id\": top_element.get('id', None)\n",
    "        },\n",
    "        \"products_count\": len(products),\n",
    "        \"products\": products\n",
    "    }\n",
    "\n",
    "    # Create styled HTML output\n",
    "    style_tag = \"\"\"\n",
    "    <style>\n",
    "        div {\n",
    "            width: auto !important;\n",
    "            height: auto !important;\n",
    "        }\n",
    "    \n",
    "        img {\n",
    "            width: 300px;\n",
    "            height: 300px;\n",
    "            object-fit: contain;\n",
    "        }\n",
    "    \n",
    "        svg {\n",
    "            max-height: 10px;\n",
    "            max-width: 10px;\n",
    "        }\n",
    "    </style>\n",
    "    \"\"\"\n",
    "    html_output = style_tag + str(top_element)\n",
    "\n",
    "    return sorted_elements, json.dumps(top_element_info, indent=2), html_output\n",
    "\n",
    "\n",
    "def print_results(element_list):\n",
    "    \"\"\"\n",
    "    Print formatted results.\n",
    "    \"\"\"\n",
    "    print(\"\\nElements Containing Most Images (Lowest Level for Each Count):\")\n",
    "    print(\"-\" * 70)\n",
    "    print(\"Rank  Element Tag & Classes                           Image Count\")\n",
    "    print(\"-\" * 70)\n",
    "\n",
    "    for rank, element in enumerate(element_list, 1):\n",
    "        tag_info, count = element\n",
    "        rank_str = f\"{rank}.\"\n",
    "        rank_str = rank_str.ljust(5)\n",
    "        tag_info_padded = tag_info.ljust(45)\n",
    "        print(f\"{rank_str} {tag_info_padded} {count}\")"
   ],
   "id": "3830f2e224e84798",
   "outputs": [],
   "execution_count": 11
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "",
   "id": "80fa7f140d4da0a2"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-26T16:55:03.174631Z",
     "start_time": "2024-10-26T16:55:02.976453Z"
    }
   },
   "cell_type": "code",
   "source": [
    "base_url = products_url.rsplit('/', 1)[0]\n",
    "sorted_elements, top_element_info, html_output = find_image_rich_parents(soup, base_url)\n",
    "\n",
    "# Print sorted list\n",
    "print_results(sorted_elements)\n",
    "\n",
    "with open(\"output1.json\", \"w\") as file:\n",
    "    file.write(top_element_info)\n",
    "\n",
    "with open(\"output1.html\", \"w\") as file:\n",
    "    file.write(html_output)"
   ],
   "id": "20b0b8cd238de02d",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\n",
      "\n",
      "Elements Containing Most Images (Lowest Level for Each Count):\n",
      "----------------------------------------------------------------------\n",
      "Rank  Element Tag & Classes                           Image Count\n",
      "----------------------------------------------------------------------\n",
      "1.    div .sc-5da3fdcc-0 .cqdDWw                    51\n",
      "2.    div                                           1\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "",
   "id": "1465ddb6bce2981c"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}