Scraping content from a javascript enabled website with load more button The script I've written is able to scrape name, address, phone and web address from a webpage using python and selenium. The main barrier I had to face was to exhaust the load more button to get the more content until all are displayed. I know the way I have written xpaths in the script is fragile but they serve the purpose for now. I ran my script and found the results as I expected. I hope there will be any better way to improve the design and performance of my script. Thanks in advance for taking care of it. Here is the full code:

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)

driver.get("https://www.zebra.com/us/en/partners/partner-application-locator.html")
driver.find_element_by_xpath('//div[@class="ft-dropdown"]/input[@placeholder="City, State/Province, Country"]').clear()
driver.find_element_by_xpath('//div[@class="ft-dropdown"]/input[@placeholder="City, State/Province, Country"]').send_keys("Colorado, USA")

wait.until(EC.visibility_of_element_located((By.XPATH, '//a[contains(@class,"ng-binding")]')))
driver.find_element_by_xpath('//a[contains(@class,"ng-binding")]').click()

while True:
    try:
        wait.until(EC.visibility_of_element_located((By.CLASS_NAME, 'showmore-bg')))
        driver.find_element_by_class_name('showmore-bg').click()
    except Exception:
        break

wait.until(EC.visibility_of_element_located((By.XPATH, '//div[contains(@class,"padding0")]')))
for items in driver.find_elements_by_xpath('//div[contains(@class,"padding0")]'):
    try:
        name = items.find_element_by_xpath('.//a[@class="ng-binding"]').text
    except Exception:
        name = ""
    try:
        address = items.find_element_by_xpath('.//div[contains(@class,"fullDetail-cmpAdres")]//p[@class="ng-binding"]').text
    except Exception:
        address = ""    
    try:
        phone = items.find_element_by_xpath('.//div[contains(@class,"fullDetail-cmpAdres")]//p[contains(@class,"ng-scope")]').text
    except Exception:
        phone = ""      
    try:
        website = items.find_element_by_xpath('.//a[contains(@class,"ng-scope")]').get_attribute("href")
    except Exception:
        website = ""

    print(name ,address, phone, website)

driver.quit()
There are a few major things that immediately caught my eye:


bare except clauses should be avoided in favor of catching more specific exceptions
element locators quality - you should NOT be relying on layout-oriented classes like padding0 and pure-technical Angular classes like ng-binding and ng-scope
wait.until() calls in your case return a WebElement instance - you don't have to re-find an element after the wait
I think the code may benefit from comments explaining the logic behind the browser element interactions


Improved code:

from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


def get_field_text_if_exists(item, selector):
    """Extracts a field by a CSS selector if exists."""
    try:
        return item.find_element_by_css_selector(selector).text
    except NoSuchElementException:
        return ""


def get_link_if_exists(item, selector):
    """Extracts an href attribute value by a CSS selector if exists."""
    try:
        return item.find_element_by_css_selector(selector).get_attribute("href")
    except NoSuchElementException:
        return ""


driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)

driver.get("https://www.zebra.com/us/en/partners/partner-application-locator.html")

location = driver.find_element_by_css_selector('.partnerLocation input')
location.clear()
location.send_keys("Colorado, USA")

# select the first suggestion from a suggestion dropdown
dropdown_suggestion = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, 'ul[id^=typeahead] li a')))
dropdown_suggestion.click()

# click more until no more results to load
while True:
    try:
        more_button = wait.until(EC.visibility_of_element_located((By.CLASS_NAME, 'showmore-bg'))).click()
    except TimeoutException:
        break

# wait for results to load
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, '.seclection-result .partners-detail')))

# parse results
for result in driver.find_elements_by_css_selector('.seclection-result .partners-detail'):
    name = get_field_text_if_exists(result, 'a')
    address = get_field_text_if_exists(result, '.fullDetail-cmpAdres')
    phone = get_field_text_if_exists(result, '.fullDetail-cmpAdres p[ng-if*=phone]')
    website = get_link_if_exists(result, 'a[ng-if*=website]')

    print(name, address, phone, website)

driver.quit()