add more euristics to find logos, might be too much

Signed-off-by: Niv Sardi <[email protected]>

crawler/common/selectors.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 
-logo = "img[src*=logo]"
+img_logo = "img[src*=logo]"
+id_logo = "*[id*=logo]"
+cls_logo = "*[class*=logo]"
+
 logosbancos = "img[src*=logosbancos]"
 
 entity_http = "p.post-pagina-interior a[target=_blank][href*=http]"

crawler/imtool.py

@@ -9,7 +9,7 @@ from typing import NamedTuple
 from entity import Entity
 
 TILE_SIZE = 800
-TILE_OVERLAP = 0.2
+TILE_OVERLAP = 0.8
 
 class BoundingBox(NamedTuple):
     x: float = 0.0

crawler/screenshot.py

@@ -27,7 +27,9 @@ def sc_entity(e: Entity):
     driver.save_screenshot(f"{e.DATA_PATH}/{e.bco}.png")
     driver.save_full_page_screenshot(f"{e.DATA_PATH}/{e.bco}.full.png")
 
-    logos = driver.find_elements(By.CSS_SELECTOR, selectors.logo)
+    logos = driver.find_elements(By.CSS_SELECTOR, selectors.img_logo) or []
+    logos.extend(driver.find_elements(By.CSS_SELECTOR, selectors.id_logo) or [])
+    logos.extend(driver.find_elements(By.CSS_SELECTOR, selectors.cls_logo) or [])
     with open(f"{e.DATA_PATH}/{e.bco}.full.txt", 'w') as f:
         for i in logos:
             f.write(f"{e.bco} {coord_to_point(i.rect)}\n")

src/index.ts

@@ -22,6 +22,21 @@ queue.addEventListener("idle", async () => {
   console.log("all done")
 })
 
+async function get_logos(page, selector): {}[] {
+  const logos = await page.$$(selector) || [];
+  for (const i in logos) {
+    const bb = await page.evaluate(e => {
+      const { x, y, width, height } = e.getBoundingClientRect();
+      return {
+        x, y, width, height, top: window.screen.top, left: window.screen.left
+      }
+    }, logos[i])
+    logos[i].box = bb;
+  }
+  return logos;
+}
+
+
 function process(o: { url: string, bco: string, name: string }): Promise<void> {
   const promises: Promise<void>[] = [];
 
@@ -30,11 +45,22 @@ function process(o: { url: string, bco: string, name: string }): Promise<void> {
     promises.push(new Promise<void>((accept, _reject) => {
       page.once('load', async () => {
         try {
-          const logos = await page.$$(selectors.logo);
+          const imgs = await get_logos(page, selectors.img_logo);
+          const ids = await get_logos(page, selectors.id_logo);
+          const cls = await get_logos(page, selectors.class_logo);
+          const logos = [
+            ...imgs, ...ids, ...cls
+          ]
+
           let annotations = '';
           for (const i in logos) {
-            const bb = await logos[i].boundingBox();
-            if (!bb) continue;
+            const bb = logos[i].box
+            if (!bb
+              || (bb.width < 10)
+              || (bb.height < 10)
+              || (bb.x + bb.width < 0)
+              || (bb.y + bb.height < 0)) continue;
+            console.log('got bb', o.bco, bb)
 
             try {
               await logos[i].screenshot({ path: `./data/logos/${o.bco}.logo${i}.png` })

src/selectors.ts

@@ -1,5 +1,7 @@
 export default {
-  "logo": "img[src*=logo]",
+  "img_logo": "img[src*=logo]",
+  "id_logo": "*[id*=logo]",
+  "class_logo": "*[class*=logo]",
   "logosbancos": "img[src*=logosbancos]",
   "entity_http": "p.post-pagina-interior a[target=_blank][href*=http]",
   "entity_mailto": "p.post-pagina-interior a[target=_blank][href*=mailto]"