fix linux driver

scrape/trendyol_scraper.py

@@ -1,129 +1,125 @@
-from selenium import webdriver
-from selenium.webdriver.chrome.service import Service
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.chrome.options import Options
-import time
-import pandas as pd
-import os
-import random
-
-def scrape_reviews(url):
-    # Create data directory if it doesn't exist
-    data_directory = "data"
-    if not os.path.exists(data_directory):
-        os.makedirs(data_directory)
-
-    def comprehensive_scroll(driver):
-        last_height = driver.execute_script("return document.body.scrollHeight")
-        while True:
-            # Rastgele scroll
-            driver.execute_script(f"window.scrollTo(0, {last_height * random.uniform(0.7, 1.0)});")
-            time.sleep(random.uniform(2, 4))  # Rastgele bekleme
-            new_height = driver.execute_script("return document.body.scrollHeight")
-            if new_height == last_height:
-                break
-            last_height = new_height
-
-    # Chrome ayarları
-    chrome_options = Options()
-    chrome_options.add_argument('--headless')
-    chrome_options.add_argument('--no-sandbox')
-    chrome_options.add_argument('--disable-dev-shm-usage')
-    chrome_options.add_argument('--disable-blink-features=AutomationControlled')
-    chrome_options.add_argument('--disable-extensions')
-    chrome_options.add_argument('--ignore-certificate-errors')
-    chrome_options.add_argument('--disable-gpu')
-    chrome_options.add_argument("--window-size=1920,1080")
-    
-    # User agent ekleme
-    user_agents = [
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
-    ]
-    chrome_options.add_argument(f'user-agent={random.choice(user_agents)}')
-    
-    # Bot tespitini zorlaştırmak için ek ayarlar
-    chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
-    chrome_options.add_experimental_option('useAutomationExtension', False)
-    
-    try:
-        driver = webdriver.Chrome(options=chrome_options)
-        
-        # WebDriver özelliğini gizle
-        driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
-        
-        # Sayfayı yükle
-        print("Sayfa yükleniyor...")
-        driver.get(url)
-        time.sleep(random.uniform(3, 5))  # Rastgele bekleme
-        
-        try:
-            # Çerez popup'ını bekle ve tıkla
-            WebDriverWait(driver, 15).until(
-                EC.element_to_be_clickable((By.ID, 'onetrust-accept-btn-handler'))
-            ).click()
-            time.sleep(random.uniform(1, 2))
-        except Exception as e:
-            print(f"Çerez popup'ı hatası: {str(e)}")
-        
-        print("Yorumlar yükleniyor...")
-        comprehensive_scroll(driver)
-        
-        # Yorumları bulmadan önce kısa bir bekleme
-        time.sleep(random.uniform(2, 3))
-        
-        comment_elements = driver.find_elements(By.XPATH, '//div[contains(@class, "comment-cards-item")]')
-        total_comments = len(comment_elements)
-        
-        if total_comments == 0:
-            print("Yorum elementi bulunamadı!")
-            return pd.DataFrame()
-            
-        print(f"Toplam {total_comments} yorum bulundu.")
-        
-        data = []
-        for i, element in enumerate(comment_elements, 1):
-            try:
-                username = element.find_element(By.CLASS_NAME, "user-name").text
-            except:
-                username = "N/A"
-
-            try:
-                comment = element.find_element(By.CLASS_NAME, "comment-text").text
-            except:
-                comment = "N/A"
-
-            try:
-                date = element.find_element(By.CLASS_NAME, "comment-date").text
-            except:
-                date = "N/A"
-
-            try:
-                stars = len(element.find_elements(By.CSS_SELECTOR, "div.full[style='width: 100%; max-width: 100%;']"))
-            except:
-                stars = 0
-
-            data.append({
-                "Kullanıcı_id": i,
-                "Kullanıcı Adı": username,
-                "Yorum": comment,
-                "Tarih": date,
-                "Yıldız Sayısı": stars
-            })
-            
-            # Her 5 yorumda bir rastgele bekleme
-            if i % 5 == 0:
-                time.sleep(random.uniform(0.5, 1))
-
-        return pd.DataFrame(data)
-
-    except Exception as e:
-        print(f"Hata detayı: {str(e)}")
-        return pd.DataFrame()
-
-    finally:
-        if 'driver' in locals():
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.chrome.options import Options
+from webdriver_manager.chrome import ChromeDriverManager
+from webdriver_manager.core.os_manager import ChromeType
+import time
+import pandas as pd
+import os
+import random
+
+def scrape_reviews(url):
+    # Create data directory if it doesn't exist
+    data_directory = "data"
+    if not os.path.exists(data_directory):
+        os.makedirs(data_directory)
+
+    def comprehensive_scroll(driver):
+        last_height = driver.execute_script("return document.body.scrollHeight")
+        while True:
+            # Rastgele scroll
+            driver.execute_script(f"window.scrollTo(0, {last_height * random.uniform(0.7, 1.0)});")
+            time.sleep(random.uniform(2, 4))
+            new_height = driver.execute_script("return document.body.scrollHeight")
+            if new_height == last_height:
+                break
+            last_height = new_height
+
+    # Chrome ayarları
+    chrome_options = Options()
+    chrome_options.add_argument('--headless')
+    chrome_options.add_argument('--no-sandbox')
+    chrome_options.add_argument('--disable-dev-shm-usage')
+    chrome_options.add_argument('--disable-blink-features=AutomationControlled')
+    chrome_options.add_argument('--disable-extensions')
+    chrome_options.add_argument('--ignore-certificate-errors')
+    chrome_options.add_argument('--disable-gpu')
+    chrome_options.add_argument("--window-size=1920,1080")
+    
+    # User agent ekleme
+    user_agents = [
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
+    ]
+    chrome_options.add_argument(f'user-agent={random.choice(user_agents)}')
+    
+    try:
+        # ChromeDriver'ı otomatik yönet
+        service = Service(ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install())
+        driver = webdriver.Chrome(service=service, options=chrome_options)
+        
+        # WebDriver özelliğini gizle
+        driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
+        
+        print("Sayfa yükleniyor...")
+        driver.get(url)
+        time.sleep(random.uniform(3, 5))
+        
+        try:
+            WebDriverWait(driver, 15).until(
+                EC.element_to_be_clickable((By.ID, 'onetrust-accept-btn-handler'))
+            ).click()
+            time.sleep(random.uniform(1, 2))
+        except Exception as e:
+            print(f"Çerez popup'ı hatası: {str(e)}")
+        
+        print("Yorumlar yükleniyor...")
+        comprehensive_scroll(driver)
+        time.sleep(random.uniform(2, 3))
+        
+        # Yorum elementlerini bul
+        comment_elements = driver.find_elements(By.CSS_SELECTOR, "div.comment-cards-item")
+        total_comments = len(comment_elements)
+        
+        if total_comments == 0:
+            print("Yorum elementi bulunamadı!")
+            return pd.DataFrame()
+            
+        print(f"Toplam {total_comments} yorum bulundu.")
+        
+        data = []
+        for i, element in enumerate(comment_elements, 1):
+            try:
+                username = element.find_element(By.CLASS_NAME, "user-name").text
+            except:
+                username = "N/A"
+
+            try:
+                comment = element.find_element(By.CLASS_NAME, "comment-text").text
+            except:
+                comment = "N/A"
+
+            try:
+                date = element.find_element(By.CLASS_NAME, "comment-date").text
+            except:
+                date = "N/A"
+
+            try:
+                stars = len(element.find_elements(By.CSS_SELECTOR, "div.full[style='width: 100%; max-width: 100%;']"))
+            except:
+                stars = 0
+
+            data.append({
+                "Kullanıcı_id": i,
+                "Kullanıcı Adı": username,
+                "Yorum": comment,
+                "Tarih": date,
+                "Yıldız Sayısı": stars
+            })
+            
+            if i % 5 == 0:
+                time.sleep(random.uniform(0.5, 1))
+
+        return pd.DataFrame(data)
+
+    except Exception as e:
+        print(f"Hata detayı: {str(e)}")
+        return pd.DataFrame()
+
+    finally:
+        if 'driver' in locals():
             driver.quit()