Reading a web page with Python When I am scraping from a web page I need a way to open and read it. Previously, I used two functions, one for requests and one for Selenium

def read_page_w_requests(url):
    try:
        headers = {'User-agent': 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) \
                    Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1'}
        r = requests.get(url, headers=headers)
        text = r.text
    except Exception, e:
        print "Read page with requests error:", e
        text = None
    return text


def read_page_w_selenium(driver, url, element_be_located=""):
    try:
        driver.get(url)
        wait = WebDriverWait(driver, 10)
        if element_be_located:
            wait.until(EC.visibility_of_element_located((By.ID, element_be_located)))
            text = driver.page_source
        else:
            text = driver.page_source

    except Exception, e:
        print "Read page error:", e
        text = None

    return text


I am thinking of using just one, that combines both:

import requests
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC


def get_driver():
    try:
        dcap = dict(DesiredCapabilities.PHANTOMJS)
        dcap["phantomjs.page.settings.userAgent"] = (
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 "
            "(KHTML, like Gecko) Chrome/15.0.87"
        )
        driver = webdriver.PhantomJS(desired_capabilities=dcap)    
        return driver
    except Exception, e:
        print "Driver error:", e
        return None

def get_page_source(url, driver="", element=""):
    if driver:
        try:
            driver.get(url)
            wait = WebDriverWait(driver, 10)
            if element:
                wait.until(EC.visibility_of_element_located((By.ID, element)))
            text = driver.page_source
        except Exception, e:
            print "Read page error:", e
            text = None

    else:    
        try:
            headers = {'User-agent': 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) \
                        Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1'}
            r = requests.get(url, headers=headers)
            text = r.text
        except Exception, e:
            print "Read page error:", e
            text = None

    return text


Is it a good idea to combine two functions that do the same thing with a different way in a single function? One of my concerns is the name of each function.
I don't think it's worth having as one function. Why? Because you don't get to share any of the functionality. Your singular wrapped function just tests if there's a driver value and will execute one of two completely separate code blocks as a result. Instead, consider having the two functions separate, and then a single wrapper function that you can call which will determine which one to call.

In other words:

def get_page_source(url, driver="", element=""):
    if driver:
        return read_page_w_selenium(driver, url, element)
    else:    
        return read_page_w_requests(url)


This way you only need to call get_page_source but you're not cramming two unrelated blocks into the same function.

Other notes, you shouldn't set driver as an empty string if you expect an object. Set the default value as None and then test for that. 

def get_page_source(url, driver=None, element=""):
    if driver is None:
        return read_page_w_selenium(driver, url, element)


Also it's confusing to change the order of arguments. Make url first in both functions so that the order is consistent.

Instead of using text = None then return text. You could use a premature return and return None, though in Python that's equivalent to return. But I think even better would be the opposite, just use a premature return when you successfully get the text:

def read_page_w_requests(url):
    try:
        headers = {'User-agent': 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) \
                    Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1'}
        r = requests.get(url, headers=headers)
        return r.text
    except Exception, e:
        print "Read page with requests error:", e


Now your exception doesn't even need to return None, because by default a Python function without an explicit return will just return None anyway.