Upload trendyol_scraper_origin.py

scrape/trendyol_scraper_origin.py

@@ -32,7 +32,6 @@ def scrape_comments(url):
 
             last_height = new_height
 
-    driver = None
     try:
         chrome_options = webdriver.ChromeOptions()
         chrome_options.add_argument("--disable-notifications")
@@ -45,84 +44,104 @@ def scrape_comments(url):
         chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
 
         # HuggingFace Spaces için özel ayarlar
-        if os.getenv('SPACE_ID'):
-            chrome_options.binary_location = "/usr/bin/google-chrome"
-            service = ChromeService("/usr/local/bin/chromedriver")
-        else:
-            service = ChromeService(ChromeDriverManager().install())
-
-        driver = webdriver.Chrome(service=service, options=chrome_options)
-        driver.maximize_window()
-
-        driver.get(url)
+        chrome_options.add_argument("--disable-setuid-sandbox")
+        chrome_options.add_argument("--remote-debugging-port=9222")
+        chrome_options.add_argument("--disable-extensions")
+        chrome_options.add_argument("--disable-software-rasterizer")
 
         try:
-            WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable((By.ID, "onetrust-accept-btn-handler"))
-            ).click()
-        except:
-            pass  # Bazen cookie popup görünmeyebilir
-
-        comprehensive_scroll(driver)
+            # Önce ChromeDriverManager ile deneyin
+            service = ChromeService(ChromeDriverManager().install())
+            driver = webdriver.Chrome(service=service, options=chrome_options)
+        except Exception as e:
+            print(f"ChromeDriverManager failed: {str(e)}")
+            try:
+                # Eğer ChromeDriverManager başarısız olursa, doğrudan Chrome'u başlatın
+                driver = webdriver.Chrome(options=chrome_options)
+            except Exception as e:
+                print(f"Direct Chrome initialization failed: {str(e)}")
+                return None
 
-        comment_elements = driver.find_elements(
-            By.XPATH,
-            "/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div",
-        )
-        total_comments = len(comment_elements)
+        try:
+            driver.maximize_window()
+            driver.get(url)
 
-        data = []
-        for i in range(1, total_comments + 1):
-            kullanıcı_id = i
+            # Cookie popup'ı kapatmayı dene, başarısız olursa devam et
             try:
-                username_xpath = f"/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div[{i}]/div[1]/div[2]/div[1]"
-                username = driver.find_element(By.XPATH, username_xpath).text
+                WebDriverWait(driver, 10).until(
+                    EC.element_to_be_clickable((By.ID, "onetrust-accept-btn-handler"))
+                ).click()
             except:
-                username = "N/A"
+                print("Cookie popup not found or couldn't be closed")
+                pass
 
-            try:
-                comment_xpath = f"/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div[{i}]/div[2]/p"
-                comment = driver.find_element(By.XPATH, comment_xpath).text
-            except:
-                comment = "N/A"
+            comprehensive_scroll(driver)
 
-            try:
-                date_xpath = f"/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div[{i}]/div[1]/div[2]/div[2]"
-                date = driver.find_element(By.XPATH, date_xpath).text
-            except:
-                date = "N/A"
+            comment_elements = driver.find_elements(
+                By.XPATH,
+                "/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div",
+            )
+            total_comments = len(comment_elements)
+
+            data = []
+            for i in range(1, total_comments + 1):
+                kullanıcı_id = i
+                try:
+                    username_xpath = f"/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div[{i}]/div[1]/div[2]/div[1]"
+                    username = driver.find_element(By.XPATH, username_xpath).text
+                except:
+                    username = "N/A"
+
+                try:
+                    comment_xpath = f"/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div[{i}]/div[2]/p"
+                    comment = driver.find_element(By.XPATH, comment_xpath).text
+                except:
+                    comment = "N/A"
+
+                try:
+                    date_xpath = f"/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div[{i}]/div[1]/div[2]/div[2]"
+                    date = driver.find_element(By.XPATH, date_xpath).text
+                except:
+                    date = "N/A"
+
+                star_xpath_base = f"/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div[{i}]/div[1]/div[1]/div"
+                try:
+                    full_stars = driver.find_elements(
+                        By.XPATH,
+                        f"{star_xpath_base}/div[@class='star-w']/div[@class='full'][@style='width: 100%; max-width: 100%;']",
+                    )
+                    star_count = len(full_stars)
+                except:
+                    star_count = 0
+
+                data.append(
+                    {
+                        "Kullanıcı_id": kullanıcı_id,
+                        "Kullanıcı Adı": username,
+                        "Yorum": comment,
+                        "Tarih": date,
+                        "Yıldız Sayısı": star_count,
+                    }
+                )
 
-            star_xpath_base = f"/html/body/div[1]/div[4]/div/div/div/div/div[3]/div/div/div[3]/div[2]/div[{i}]/div[1]/div[1]/div"
+            df = pd.DataFrame(data)
+            return df
+
+        finally:
             try:
-                full_stars = driver.find_elements(
-                    By.XPATH,
-                    f"{star_xpath_base}/div[@class='star-w']/div[@class='full'][@style='width: 100%; max-width: 100%;']",
-                )
-                star_count = len(full_stars)
+                driver.quit()
             except:
-                star_count = 0
-
-            data.append(
-                {
-                    "Kullanıcı_id": kullanıcı_id,
-                    "Kullanıcı Adı": username,
-                    "Yorum": comment,
-                    "Tarih": date,
-                    "Yıldız Sayısı": star_count,
-                }
-            )
-
-        df = pd.DataFrame(data)
-        return df
+                pass
 
     except Exception as e:
         print(f"Hata oluştu: {str(e)}")
+        try:
+            if 'driver' in locals():
+                driver.quit()
+        except:
+            pass
         return None
 
-    finally:
-        if driver:
-            driver.quit()
-
 if __name__ == "__main__":
     # Test URL
     url = "https://www.trendyol.com/apple/macbook-air-m1-cip-8gb-256gb-ssd-macos-13-qhd-tasinabilir-bilgisayar-uzay-grisi-p-68042136/yorumlar"