'Get the HTML for an URL, possibly from an in-memory cache.
XXX TODO Note: this cache is never actually cleared. It\'s assumed that
the data won\'t get stale over the lifetime of a locator instance (not
necessarily true for the default_locator).'

def get_page(self, url):

    (scheme, netloc, path, _, _, _) = urlparse(url)
    if ((scheme == 'file') and os.path.isdir(url2pathname(path))):
        url = urljoin(ensure_slash(url), 'index.html')
    if (url in self._page_cache):
        result = self._page_cache[url]
        logger.debug('Returning    %s    from    cache:    %s', url, result)
    else:
        host = netloc.split(':', 1)[0]
        result = None
        if (host in self._bad_hosts):
            logger.debug('Skipping    %s    due    to    bad    host    %s', url, host)
        else:
            req = Request(url, headers={'Accept-encoding': 'identity'})
            try:
                logger.debug('Fetching    %s', url)
                resp = self.opener.open(req, timeout=self.timeout)
                logger.debug('Fetched    %s', url)
                headers = resp.info()
                content_type = headers.get('Content-Type', '')
                if HTML_CONTENT_TYPE.match(content_type):
                    final_url = resp.geturl()
                    data = resp.read()
                    encoding = headers.get('Content-Encoding')
                    if encoding:
                        decoder = self.decoders[encoding]
                        data = decoder(data)
                    encoding = 'utf-8'
                    m = CHARSET.search(content_type)
                    if m:
                        encoding = m.group(1)
                    try:
                        data = data.decode(encoding)
                    except UnicodeError:
                        data = data.decode('latin-1')
                    result = Page(data, final_url)
                    self._page_cache[final_url] = result
            except HTTPError as e:
                if (e.code != 404):
                    logger.exception('Fetch    failed:    %s:    %s', url, e)
            except URLError as e:
                logger.exception('Fetch    failed:    %s:    %s', url, e)
                with self._lock:
                    self._bad_hosts.add(host)
            except Exception as e:
                logger.exception('Fetch    failed:    %s:    %s', url, e)
            finally:
                self._page_cache[url] = result
    return result
