NoSuchElement exception with Selenium even after using Wait and checking page_soure
Question: I have this simple scraper that I am running. I am trying to scrape the search
results for letter q from sam.gov:

    
    
    from selenium import webdriver
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    from bs4 import BeautifulSoup
    import re
    import sys  
    
    reload(sys)  
    sys.setdefaultencoding('utf8')
    letter = 'q'
    
    driver = webdriver.PhantomJS()
    driver.set_window_size(1120, 550)
    
    driver.get("http://sam.gov")
    
    #element = WebDriverWait(driver, 10).until(
    #                EC.presence_of_element_located((By.ID, "pbG220e071f_2de75f_2d417d_2d9c61_2d027d324c8fec:_viewRoot:j_id12:search1"))
    #            )
    #element.click()
    driver.find_element_by_id('pbG220e071f_2de75f_2d417d_2d9c61_2d027d324c8fec:_viewRoot:j_id12:search1').click()
    
    driver.find_element_by_id(letter).send_keys(letter)
    driver.find_element_by_id('RegSearchButton').click()
    
    
    def crawl():
        bsObj = BeautifulSoup(driver.page_source, "html.parser")
        tableList = bsObj.find_all("table", {"class":"width100 menu_header_top_emr"}) 
        tdList = bsObj.find_all("td", {"class":"menu_header width100"})
    
        for table in tableList:
            item = table.find_all("span", {"class":"results_body_text"})
            print item[0].get_text().strip() + ', ' + item[1].get_text().strip() 
    
    if driver.find_element_by_id('anch_16'):
        crawl()
        driver.find_element_by_id('anch_16').click()
        print "Going to next page"
    else:
        crawl()
        print "Done with last page" 
    
    driver.quit()
    

When i run it gives a weird error which is bothering me:

Traceback (most recent call last):

    
    
      File "save.py", line 22, in <module>
        driver.find_element_by_id('pbG220e071f_2de75f_2d417d_2d9c61_2d027d324c8fec:_viewRoot:j_id12:search1').click()
      File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 269, in find_element_by_id
        return self.find_element(by=By.ID, value=id_)
      File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 752, in find_element
        'value': value})['value']
      File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 236, in execute
        self.error_handler.check_response(response)
      File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/errorhandler.py", line 192, in check_response
        raise exception_class(message, screen, stacktrace)
    selenium.common.exceptions.NoSuchElementException: Message: {"errorMessage":"Unable to find element with id 'pbG220e071f_2de75f_2d417d_2d9c61_2d027d324c8fec:_viewRoot:j_id12:search1'","request":{"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"153","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:40423","User-Agent":"Python-urllib/2.7"},"httpVersion":"1.1","method":"POST","post":"{\"using\": \"id\", \"sessionId\": \"eb7dfa50-70a7-11e6-b125-9ff4e2dbd485\", \"value\": \"pbG220e071f_2de75f_2d417d_2d9c61_2d027d324c8fec:_viewRoot:j_id12:search1\"}","url":"/element","urlParsed":{"anchor":"","query":"","file":"element","directory":"/","path":"/element","relative":"/element","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/element","queryKey":{},"chunks":["element"]},"urlOriginal":"/session/eb7dfa50-70a7-11e6-b125-9ff4e2dbd485/element"}}
    Screenshot: available via screen
    

I have since tried using an implicit wait of 60 right after i initialize the
browser. No luck

I have also tried webdriverwait (commented out in the code right below
`driver.get("http://sam.gov")`, and it gave me at TimeOutException.

The weird thing is if i do a `print driver.page_source` right after the get
call, the source is fine and it contains the following code which actually
contains the element with the id that I am searching for. There is no frame or
iframe either.

    
    
    <a id="pbG220e071f_2de75f_2d417d_2d9c61_2d027d324c8fec:_viewRoot:j_id12:search1" href="#" title="Search Records" onclick="if(typeof jsfcljs == 'function'){jsfcljs(document.getElementById('pbG220e071f_2de75f_2d417d_2d9c61_2d027d324c8fec:_viewRoot:j_id12'),{'pbG220e071f_2de75f_2d417d_2d9c61_2d027d324c8fec:_viewRoot:j_id12:search1':'pbG220e071f_2de75f_2d417d_2d9c61_2d027d324c8fec:_viewRoot:j_id12:search1'},'');}return false" class="button">
    


Answer: Id locator of the element looks like dynamically generated, you should try
some different locator.

You can try using `css_selector` as below:-

    
    
    driver.find_element_by_css_selector("a.button[title='Search Records']").click()
    

Or using `WebDriverWait` as:-

    
    
    element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "a.button[title='Search Records']")))
    element.click()
    

**Note** :- Before finding element make sure it's not inside any
`frame/iframe`. If it's inside any `frame/iframe` you need to switch that
`frame/iframe` before finding element as `driver.switch_to_frame("frame/iframe
id or name")`

