def get_page(self, url):
<EOL>scheme, netloc, path, _, _, _ = urlparse(url)<EOL>if scheme == '<STR_LIT:file>' and os.path.isdir(url2pathname(path)):<EOL><INDENT>url = urljoin(ensure_slash(url), '<STR_LIT>')<EOL><DEDENT>if url in self._page_cache:<EOL><INDENT>result = self._page_cache[url]<EOL>logger.debug('<STR_LIT>', url, result)<EOL><DEDENT>else:<EOL><INDENT>host = netloc.split('<STR_LIT::>', <NUM_LIT:1>)[<NUM_LIT:0>]<EOL>result = None<EOL>if host in self._bad_hosts:<EOL><INDENT>logger.debug('<STR_LIT>', url, host)<EOL><DEDENT>else:<EOL><INDENT>req = Request(url, headers={'<STR_LIT>': '<STR_LIT>'})<EOL>try:<EOL><INDENT>logger.debug('<STR_LIT>', url)<EOL>resp = self.opener.open(req, timeout=self.timeout)<EOL>logger.debug('<STR_LIT>', url)<EOL>headers = resp.info()<EOL>content_type = headers.get('<STR_LIT:Content-Type>', '<STR_LIT>')<EOL>if HTML_CONTENT_TYPE.match(content_type):<EOL><INDENT>final_url = resp.geturl()<EOL>data = resp.read()<EOL>encoding = headers.get('<STR_LIT>')<EOL>if encoding:<EOL><INDENT>decoder = self.decoders[encoding]   <EOL>data = decoder(data)<EOL><DEDENT>encoding = '<STR_LIT:utf-8>'<EOL>m = CHARSET.search(content_type)<EOL>if m:<EOL><INDENT>encoding = m.group(<NUM_LIT:1>)<EOL><DEDENT>try:<EOL><INDENT>data = data.decode(encoding)<EOL><DEDENT>except UnicodeError:<EOL><INDENT>data = data.decode('<STR_LIT>')    <EOL><DEDENT>result = Page(data, final_url)<EOL>self._page_cache[final_url] = result<EOL><DEDENT><DEDENT>except HTTPError as e:<EOL><INDENT>if e.code != <NUM_LIT>:<EOL><INDENT>logger.exception('<STR_LIT>', url, e)<EOL><DEDENT><DEDENT>except URLError as e:<EOL><INDENT>logger.exception('<STR_LIT>', url, e)<EOL>with self._lock:<EOL><INDENT>self._bad_hosts.add(host)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>logger.exception('<STR_LIT>', url, e)<EOL><DEDENT>finally:<EOL><INDENT>self._page_cache[url] = result   <EOL><DEDENT><DEDENT><DEDENT>return result<EOL>
Get the HTML for an URL, possibly from an in-memory cache.

XXX TODO Note: this cache is never actually cleared. It's assumed that
the data won't get stale over the lifetime of a locator instance (not
necessarily true for the default_locator).