include output

scrape-content.ipynb

@@ -174,8 +174,8 @@
   {
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2024-10-25T10:52:34.470225Z",
-     "start_time": "2024-10-25T10:52:34.458243Z"
+     "end_time": "2024-10-25T12:57:47.980010Z",
+     "start_time": "2024-10-25T12:57:47.962944Z"
     }
    },
    "cell_type": "code",
@@ -234,51 +234,14 @@
     "    return False\n",
     "\n",
     "\n",
-    "def print_results_with_content(element_list):\n",
+    "def find_image_rich_parents(soup, base_url):\n",
     "    \"\"\"\n",
-    "    Print formatted results including the inner content of elements.\n",
+    "    Find elements containing images and return both sorted list and detailed top element info.\n",
     "    \"\"\"\n",
-    "    print(\"\\nElements Containing Most Images (Lowest Level for Each Count):\")\n",
-    "    print(\"=\" * 100)\n",
-    "\n",
-    "    for rank, (tag_info, count, element) in enumerate(element_list, 1):\n",
-    "        print(f\"\\nRank {rank}:\")\n",
-    "        print(\"-\" * 100)\n",
-    "        print(f\"Element: {tag_info}\")\n",
-    "        print(f\"Image Count: {count}\")\n",
-    "        print(\"\\nContent Preview:\")\n",
-    "        print(\"-\" * 100)\n",
-    "\n",
-    "        # Get all immediate img tags\n",
-    "        immediate_images = element.find_all('img', recursive=False)\n",
-    "        nested_images = element.find_all('img', recursive=True)\n",
-    "\n",
-    "        print(f\"Direct images: {len(immediate_images)}\")\n",
-    "        print(f\"Total images (including nested): {len(nested_images)}\")\n",
-    "        print(\"\\nImage sources:\")\n",
-    "\n",
-    "        # Print image sources and alt text\n",
-    "        for img in nested_images:\n",
-    "            src = img.get('src', 'No source')\n",
-    "            alt = img.get('alt', 'No alt text')\n",
-    "            print(f\"- Source: {src}\")\n",
-    "            print(f\"  Alt text: {alt}\")\n",
-    "\n",
-    "        print(\"\\nFull HTML structure:\")\n",
-    "        print(\"-\" * 100)\n",
-    "        # Print formatted HTML structure\n",
-    "        html_content = element.prettify()\n",
-    "        print(html_content)\n",
-    "        print(\"=\" * 100)\n",
-    "\n",
-    "\n",
-    "def find_top_image_parent(soup, base_url):\n",
-    "    \"\"\"\n",
-    "    Find the element containing the most images at the lowest level and return its details as JSON.\n",
-    "    \"\"\"\n",
-    "    # Collect all elements with their image counts\n",
+    "    # Convert relative URLs to absolute\n",
     "    soup = convert_relative_urls(soup, base_url)\n",
     "\n",
+    "    # Collect all elements with their image counts\n",
     "    elements_with_counts = []\n",
     "    for element in soup.find_all():\n",
     "        if element.name != 'img':  # Skip img tags themselves\n",
@@ -290,67 +253,95 @@
     "    elements_with_counts.sort(key=lambda x: x[1], reverse=True)\n",
     "\n",
     "    if not elements_with_counts:\n",
-    "        return json.dumps({\"error\": \"No elements with images found\"}, indent=2)\n",
+    "        return [], {\"error\": \"No elements with images found\"}, \"\"\n",
     "\n",
+    "    # Process elements for sorted list\n",
+    "    sorted_elements = []\n",
     "    max_count = elements_with_counts[0][1]\n",
+    "    current_count = max_count\n",
+    "\n",
+    "    while current_count > 0 and len(sorted_elements) < 100:\n",
+    "        # Get all elements with current count\n",
+    "        current_elements = [(elem, count) for elem, count in elements_with_counts if count == current_count]\n",
+    "\n",
+    "        # Filter out elements that have children with the same count\n",
+    "        for element, count in current_elements:\n",
+    "            if not has_child_with_same_count(element, count, elements_with_counts):\n",
+    "                sorted_elements.append((get_element_identifier(element), count))\n",
+    "\n",
+    "        # Move to next highest count\n",
+    "        remaining_counts = [count for _, count in elements_with_counts if count < current_count]\n",
+    "        current_count = max(remaining_counts) if remaining_counts else 0\n",
+    "\n",
+    "    # Get detailed info for top element\n",
+    "    top_element = elements_with_counts[0][0]\n",
+    "    max_count = elements_with_counts[0][1]\n",
+    "\n",
+    "    # Find the lowest-level element among those with max count\n",
+    "    for element, count in elements_with_counts:\n",
+    "        if count == max_count and not has_child_with_same_count(element, count, elements_with_counts):\n",
+    "            top_element = element\n",
+    "            break\n",
+    "\n",
+    "    # Collect all images within the top element\n",
+    "    images = []\n",
+    "    for img in top_element.find_all('img', recursive=True):\n",
+    "        image_data = {\n",
+    "            \"src\": img.get('src', 'No source'),\n",
+    "            \"alt\": img.get('alt', 'No alt text')\n",
+    "        }\n",
+    "        for attr in ['title', 'width', 'height', 'class']:\n",
+    "            if img.get(attr):\n",
+    "                image_data[attr] = img[attr]\n",
+    "        images.append(image_data)\n",
+    "\n",
+    "    # Create result dictionary for top element\n",
+    "    top_element_info = {\n",
+    "        \"element\": {\n",
+    "            \"tag\": top_element.name,\n",
+    "            \"identifier\": get_element_identifier(top_element),\n",
+    "            \"classes\": top_element.get('class', []),\n",
+    "            \"id\": top_element.get('id', None)\n",
+    "        },\n",
+    "        \"image_count\": max_count,\n",
+    "        \"images\": images,\n",
+    "        \"html_content\": str(top_element)\n",
+    "    }\n",
+    "\n",
+    "    # Create styled HTML output\n",
+    "    style_tag = \"\"\"\n",
+    "    <style>\n",
+    "        img {\n",
+    "            width: 300px;\n",
+    "            height: 300px;\n",
+    "            object-fit: contain;\n",
+    "        }\n",
+    "    </style>\n",
+    "    \"\"\"\n",
+    "    html_output = style_tag + str(top_element)\n",
     "\n",
-    "    # Get all elements with max count\n",
-    "    top_elements = [(elem, count) for elem, count in elements_with_counts if count == max_count]\n",
-    "    print(len(elements_with_counts))\n",
-    "    # \n",
-    "    # # Find the lowest-level element among those with max count\n",
-    "    # top_element = None\n",
-    "    # for element, count in top_elements:\n",
-    "    #     if not has_child_with_same_count(element, count, elements_with_counts):\n",
-    "    #         top_element = element\n",
-    "    #         break\n",
-    "    # \n",
-    "    # if not top_element:\n",
-    "    #     return json.dumps({\"error\": \"No suitable element found\"}, indent=2)\n",
-    "    # \n",
-    "    # # Collect all images within the element\n",
-    "    # images = []\n",
-    "    # for img in top_element.find_all('img', recursive=True):\n",
-    "    #     image_data = {\n",
-    "    #         \"src\": img.get('src', 'No source'),\n",
-    "    #         \"alt\": img.get('alt', 'No alt text')\n",
-    "    #     }\n",
-    "    #     # Add any other attributes that exist\n",
-    "    #     for attr in ['title', 'width', 'height', 'class']:\n",
-    "    #         if img.get(attr):\n",
-    "    #             image_data[attr] = img[attr]\n",
-    "    #     images.append(image_data)\n",
-    "    # \n",
-    "    # # Create result dictionary\n",
-    "    # result = {\n",
-    "    #     \"element\": {\n",
-    "    #         \"tag\": top_element.name,\n",
-    "    #         \"identifier\": get_element_identifier(top_element),\n",
-    "    #         \"classes\": top_element.get('class', []),\n",
-    "    #         \"id\": top_element.get('id', None)\n",
-    "    #     },\n",
-    "    #     \"image_count\": max_count,\n",
-    "    #     \"images\": images,\n",
-    "    #     \"html_content\": str(top_element)\n",
-    "    # }\n",
-    "    # \n",
-    "    # # Create styled HTML output\n",
-    "    # style_tag = f\"\"\"\n",
-    "    # <style>\n",
-    "    #     img {{\n",
-    "    #         width: 300px;\n",
-    "    #         height: 300px;\n",
-    "    #         object-fit: contain;\n",
-    "    #     }}\n",
-    "    # </style>\n",
-    "    # \"\"\"\n",
-    "    # html_output = style_tag + str(top_element)\n",
-    "    # \n",
-    "    # return json.dumps(result, indent=2), html_output\n"
+    "    return sorted_elements, top_element_info, html_output\n",
+    "\n",
+    "\n",
+    "def print_results(element_list):\n",
+    "    \"\"\"\n",
+    "    Print formatted results.\n",
+    "    \"\"\"\n",
+    "    print(\"\\nElements Containing Most Images (Lowest Level for Each Count):\")\n",
+    "    print(\"-\" * 70)\n",
+    "    print(\"Rank  Element Tag & Classes                           Image Count\")\n",
+    "    print(\"-\" * 70)\n",
+    "\n",
+    "    for rank, element in enumerate(element_list, 1):\n",
+    "        tag_info, count = element\n",
+    "        rank_str = f\"{rank}.\"\n",
+    "        rank_str = rank_str.ljust(5)\n",
+    "        tag_info_padded = tag_info.ljust(45)\n",
+    "        print(f\"{rank_str} {tag_info_padded} {count}\")"
    ],
    "id": "3830f2e224e84798",
    "outputs": [],
-   "execution_count": 33
+   "execution_count": 46
   },
   {
    "metadata": {},
@@ -361,20 +352,23 @@
   {
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2024-10-25T10:52:36.684418Z",
-     "start_time": "2024-10-25T10:52:36.614623Z"
+     "end_time": "2024-10-25T13:00:02.064761Z",
+     "start_time": "2024-10-25T13:00:01.277970Z"
     }
    },
    "cell_type": "code",
    "source": [
     "base_url = products_url.rsplit('/', 1)[0]\n",
-    "find_top_image_parent(soup, base_url)\n",
-    "#\n",
-    "# with open(\"output.json\", \"w\") as file:\n",
-    "#     file.write(json_data)\n",
-    "# \n",
-    "# with open(\"output.html\", \"w\") as file:\n",
-    "#     file.write(html_content)"
+    "sorted_elements, top_element_info, html_output = find_image_rich_parents(soup, base_url)\n",
+    "\n",
+    "# Print sorted list\n",
+    "print_results(sorted_elements)\n",
+    "\n",
+    "with open(\"output1.json\", \"w\") as file:\n",
+    "    file.write(json.dumps(top_element_info, indent=2))\n",
+    "\n",
+    "with open(\"output1.html\", \"w\") as file:\n",
+    "    file.write(html_output)"
    ],
    "id": "20b0b8cd238de02d",
    "outputs": [
@@ -382,11 +376,112 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "411\n"
+      "\n",
+      "Elements Containing Most Images (Lowest Level for Each Count):\n",
+      "----------------------------------------------------------------------\n",
+      "Rank  Element Tag & Classes                           Image Count\n",
+      "----------------------------------------------------------------------\n",
+      "1.    div                                           63\n",
+      "2.    div .sc-5da3fdcc-0 .cqdDWw                    58\n",
+      "3.    div                                           5\n",
+      "4.    div .l-container-fixed .h-padding-h-x6 .h-padding-t-x6 .h-display-flex .h-flex-direction-row .h-flex-justify-space-between 4\n",
+      "5.    div                                           3\n",
+      "6.    a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "7.    a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "8.    a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "9.    a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "10.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "11.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "12.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "13.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "14.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "15.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "16.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "17.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "18.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "19.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "20.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "21.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "22.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "23.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "24.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "25.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "26.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "27.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "28.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "29.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "30.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "31.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "32.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "33.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "34.   a .sc-e851bd29-0 .dmfVmE .h-display-block     2\n",
+      "35.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "36.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "37.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "38.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "39.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "40.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "41.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "42.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "43.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "44.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "45.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "46.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "47.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "48.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "49.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "50.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "51.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "52.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "53.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "54.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "55.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "56.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "57.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "58.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "59.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "60.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "61.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "62.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "63.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "64.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "65.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "66.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "67.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "68.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "69.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "70.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "71.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "72.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "73.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "74.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "75.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "76.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "77.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "78.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "79.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "80.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "81.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "82.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "83.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "84.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "85.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "86.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "87.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "88.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "89.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "90.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "91.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "92.   picture .sc-68a8cd0e-0 .ldZWSf                1\n",
+      "93.   span                                          1\n",
+      "94.   span                                          1\n",
+      "95.   span                                          1\n",
+      "96.   span                                          1\n",
+      "97.   div .Illustration_IllustrationWrapper__xJP5g  1\n"
      ]
     }
    ],
-   "execution_count": 34
+   "execution_count": 49
   }
  ],
  "metadata": {