scripts/scraper.py
def fetch_page(self, url):
        "Used for fetching all the remote pages."

        self.message("Fetching: " + url)

        try:
            response = requests.get(url, timeout=10)
        except requests.exceptions.ConnectionError:
            self.message("Can't connect to domain.")
        except requests.exceptions.ConnectTimeout:
            self.message("Connection timed out.")
        except requests.exceptions.ReadTimeout:
            self.message("Read timed out.")

        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            self.message("HTTP Error: %s" % response.status_code)

        return response.text
Used for fetching all the remote pages.
Used for fetching all the remote pages.
Used for fetching all the remote pages .
def fetch_page(self, url):
        self.message("Fetching: " + url)
        try:
            response = requests.get(url, timeout=10)
        except requests.exceptions.ConnectionError:
            self.message("Can't connect to domain.")
        except requests.exceptions.ConnectTimeout:
            self.message("Connection timed out.")
        except requests.exceptions.ReadTimeout:
            self.message("Read timed out.")
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            self.message("HTTP Error: %s" % response.status_code)
        return response.text
def fetch_page ( self , url ) : self . message ( "Fetching: " + url ) try : response = requests . get ( url , timeout = 10 ) except requests . exceptions . ConnectionError : self . message ( "Can't connect to domain." ) except requests . exceptions . ConnectTimeout : self . message ( "Connection timed out." ) except requests . exceptions . ReadTimeout : self . message ( "Read timed out." ) try : response . raise_for_status ( ) except requests . exceptions . HTTPError : self . message ( "HTTP Error: %s" % response . status_code ) return response . text
Used for fetching all the remote pages.
Used for fetching all the remote pages .
"Used for fetching all the remote pages."
self
url