title: Selenium Web Scrape - Why is this script returning 500k rows?
body: <p>I made a script to scrape through a website to grab all product information from certain categories, but my code is returning 500 000+ rows when there are only 3000 items in that certain category.</p>
  
  <p>I'm also really new to Python so any help is appreciated.</p>
  
  <p>The code is attached below:</p>
  
  <pre><code>    # -*- coding: utf-8 -*-
  """
  Created on Mon Feb  4 20:31:23 2019
  
  @author: 
  """
  import requests
  from selenium import webdriver
  from selenium.webdriver.common.by import By
  import selenium.webdriver.support.ui as ui
  import selenium.webdriver.support.expected_conditions as EC
  from bs4 import BeautifulSoup
  import os, sys
  import time
  from urllib.parse import urljoin
  import pandas as pd
  import re
  import numpy as np
  
  # base set up
  
  options = webdriver.ChromeOptions()
  options.add_argument('--ignore-certificate-errors')
  options.add_argument('--ignore-ssl-errors')
  os.chdir("C:/Users/user/desktop/scripts/python")
  cwd = os.getcwd()
  main_dir = os.path.abspath(os.path.join(cwd, os.pardir))
  print('Main Directory:', main_dir)
  
  chromedriver = ("C:/Users/user/desktop/scripts/python/chromedriver.exe")
  os.environ["webdriver.chrome.driver"] = chromedriver
  # browser = webdriver.Chrome(options=options, executable_path=chromedriver)
  
  mainurl = "https://www.bunnings.com.au/our-range"
  
  headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}
  page = requests.get(mainurl, headers=headers)
  soup = BeautifulSoup(page.content, 'html.parser')
  
  # script start
  
  subcat = []
  for item in soup.findAll('ul', attrs={'class': 'chalkboard-menu'}):
      links = item.find_all('a')
      for link in links:
          subcat.append(urljoin(mainurl, link.get("href")))
  subcat
  
  result = pd.DataFrame()
  for adrs in subcat[0:1]:
  #    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}
  #    page = requests.get(adrs, headers=headers)
  #    soup = BeautifulSoup(page.content, 'html.parser')
  #    pagelink = adrs
  #    adrs="https://www.bunnings.com.au/our-range/storage-cleaning/cleaning/brushware-mops/indoor-brooms"
      catProd = pd.DataFrame()
      url = adrs
      browser = webdriver.Chrome(options=options, executable_path=chromedriver)
      browser.get(url)
  
      lenOfPage = browser.execute_script("window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;")
      match = False
      while (match == False):
          lastCount = lenOfPage
          time.sleep(3)
          lenOfPage = browser.execute_script("window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;")
          if lastCount == lenOfPage:
              match = True
      reached= False
      while (reached==False):
          try:
              browser.find_element_by_css_selector('#MoreProductsButton &gt; span').click()
              lenOfPage = browser.execute_script("window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;")
              match = True
              while (match == True):
                  lastCount = lenOfPage
                  time.sleep(3)
                  lenOfPage = browser.execute_script("window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;")
                  if lastCount == lenOfPage:
                      match = True
                      browser.find_element_by_css_selector('#content-layout_inside-anchor &gt; div.search-result__content &gt; div &gt; div &gt; section &gt; div:nth-child(4) &gt; div &gt; div:nth-child(2) &gt; div &gt; button &gt; div.view-more_btn_text').click()
          except:
              reached=True
  # grab the items
              page = browser.page_source
              soup = BeautifulSoup(page, 'html.parser')
              browser.close()
  
          for article in soup.findAll('article', attrs={'class':'product-list__item hproduct special-order-product'}):
              for product in article.findAll('img', attrs={'class': 'photo'}):
                  pName = product['alt']
                  pCat = adrs
                  pID = article['data-product-id']
                  temp= pd.DataFrame({'proID':[pID],'Product':[pName],'Category':[pCat]})
                  catProd=catProd.append(temp)
                  result = result.append(catProd)
          time.sleep(3)
          result.head()
  
  #writes to CSV
  writer = pd.ExcelWriter('test123123.xlsx')
  result.to_excel(writer,'Sheet1')
  writer.save()
  </code></pre>
  
  <p>The code takes something like 20 minutes to iterate through 3000~ items which is insane in my opinion but the main problem still lies that I'm getting way too many duplicates and 500 000 rows when there are only 3500 rows that I need for that certain category.  </p>