I have this script working pretty well but I know that there must be many things that I could do better to make it more efficient.from selenium import webdriverimport refrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.common.exceptions import NoSuchElementException, TimeoutExceptionimport csvimport xlsxwriterimport sysreload(sys)sys.setdefaultencoding('utf-8')new_driver = webdriver.Firefox()driver = webdriver.Firefox()new_driver_handle = new_driver.current_window_handledriver_handle = driver.current_window_handledriver.get('http://www.kijiji.com')try:    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.ID, SignInLink)))finally:    linkElem = driver.find_element_by_id('SignInLink')    type(linkElem)    linkElem.click()  # follows the Read It Online linktry:    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.ID, LoginEmailOrNickname)))finally:    emailElem = driver.find_element_by_id('LoginEmailOrNickname')    emailElem.send_keys('l_gaudet133@mail.com')    passwordElem = driver.find_element_by_id('login-password')    passwordElem.send_keys('123456789')    passwordElem.submit()try:    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, houses for sale))                                              )finally:    linkElem = driver.find_element_by_partial_link_text(houses for sale)    type(linkElem)    linkElem.click()  # follows the Read It Online linktry:    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, Propri)))finally:    linkElem = driver.find_element_by_partial_link_text(Propri)    type(linkElem)    linkElem.click()  # follows the Read It Online linktry:    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CLASS_NAME, titlecount)))finally:    linkcount_label = driver.find_element_by_class_name(titlecount)    titlecount_text = linkcount_label.get_attribute('textContent')adnumber = 0title_count = re.sub([^0-9.], , titlecount_text)title_count = int(title_count)s = title_count / 20# helloFile = open('houses.txt', 'a')xbook = xlsxwriter.Workbook('Test.xlsx')xsheet = xbook.add_worksheet('Test')outputFile = open('output.csv', 'a')outputWriter = csv.writer(outputFile)outputWriter.writerow(    ['label1', 'label2', 'label3', 'label4', 'label5', 'label6', 'label7', 'label8', 'label9', 'label10'])for a in range(s):    driver.get(driver.current_url)    list_links = driver.find_elements_by_css_selector(*[class^='title enable-search-navigation-flag'])    list_links1 = []    for i1 in list_links:        list_links1.append(i1.get_attribute('href'))    b = 0    for i in list_links:        # Open a new tab and open individual kijiji ad        print list_links1[b]        if b > 0:            new_driver.get(list_links1[b])        if adnumber == 0:            new_driver.switch_to.window(new_driver_handle)            new_driver.get(list_links1[b])            try:                element = WebDriverWait(new_driver, 30).until(                    EC.presence_of_element_located((By.ID, SignInLink))                )            finally:                linkElem = new_driver.find_element_by_id('SignInLink')                type(linkElem)                linkElem.click()  # follows the Read It Online link            try:                element = WebDriverWait(new_driver, 30).until(                    EC.presence_of_element_located((By.ID, LoginEmailOrNickname)))            finally:                emailElem = new_driver.find_element_by_id('LoginEmailOrNickname')                emailElem.send_keys('williamleonardjohnson@gmail.com')                passwordElem = new_driver.find_element_by_id('login-password')                passwordElem.send_keys('WJ1029vc1')                passwordElem.submit()        new_driver.save_screenshot('out1.png')        texts = []        try:            element = WebDriverWait(new_driver, 30).until(                EC.element_to_be_clickable((By.CSS_SELECTOR, *[class*='phoneShowNumberButton']))            )        except (NoSuchElementException, TimeoutException):            pass        else:            linkElem = new_driver.find_element_by_css_selector(*[class*='phoneShowNumberButton'])            type(linkElem)            new_driver.implicitly_wait(2)            linkElem.click()            phone_num = linkElem.text            print linkElem.text        # if len(new_driver.find_elements_by_css_selector(*[class*='phoneShowNumberButton'])) > 0:        #     linkElem = new_driver.find_element_by_css_selector(*[class*='phoneShowNumberButton'])        #     type(linkElem)        #     linkElem.click()        #     new_driver.implicitly_wait(5)        #     phone_num = linkElem.text        #     print linkElem.text        if 'phone_num' in locals():            texts.append(phone_num)        else:            texts.append(',')        try:            element = WebDriverWait(new_driver, 30).until(EC.presence_of_element_located(                (By.CSS_SELECTOR, [class='ad-attributes']))            )        except (NoSuchElementException, TimeoutException):            continue        else:            info = new_driver.find_element_by_css_selector([class='ad-attributes'])        # info = driver.find_element_by_css_selector([class='ad-attributes']) was replaced by 161 to 165        description = new_driver.find_element_by_id(AdDescriptionTabs)        getinfo = info.find_elements_by_tag_name(td)        for a in getinfo:            texts.append(a.get_attribute('textContent'))        # print texts        # print description.get_attribute('textContent')        home_desc = description.get_attribute('textContent')        texts.append(home_desc)        texts1 = texts        str1 = '|'.join(texts)        str1 = re.sub(r'\s+', ' ', str1.replace(' |', ' |'))        # str1 = str1.strip('\n')        texts = str1.split('|')        outputWriter.writerow([str1])        if adnumber <= title_count:            xsheet.write_row(adnumber, 0, texts)            adnumber += 1        if 'phone_num' in locals():            del phone_num        b += 1    driver.switch_to.window(driver_handle)    try:        element = WebDriverWait(driver, 60).until(            EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, Suivante))        )    finally:        linkElem = driver.find_element_by_partial_link_text(Suivante)        type(linkElem)        linkElem.click()xbook.close()driver.quit()
Selenium Kijiji web scraper
python;python 2.7;web scraping;selenium