def process_url(self, url, retrieve=False):
if url in self.scanned_urls and not retrieve:<EOL><INDENT>return<EOL><DEDENT>self.scanned_urls[url] = True<EOL>if not URL_SCHEME(url):<EOL><INDENT>self.process_filename(url)<EOL>return<EOL><DEDENT>else:<EOL><INDENT>dists = list(distros_for_url(url))<EOL>if dists:<EOL><INDENT>if not self.url_ok(url):<EOL><INDENT>return<EOL><DEDENT>self.debug("<STR_LIT>", url)<EOL><DEDENT><DEDENT>if dists or not retrieve or url in self.fetched_urls:<EOL><INDENT>list(map(self.add, dists))<EOL>return  <EOL><DEDENT>if not self.url_ok(url):<EOL><INDENT>self.fetched_urls[url] = True<EOL>return<EOL><DEDENT>self.info("<STR_LIT>", url)<EOL>f = self.open_url(url, "<STR_LIT>" % url)<EOL>if f is None: return<EOL>self.fetched_urls[url] = self.fetched_urls[f.url] = True<EOL>if '<STR_LIT:html>' not in f.headers.get('<STR_LIT>', '<STR_LIT>').lower():<EOL><INDENT>f.close()   <EOL>return<EOL><DEDENT>base = f.url     <EOL>page = f.read()<EOL>if not isinstance(page, str): <EOL><INDENT>if isinstance(f, urllib.error.HTTPError):<EOL><INDENT>charset = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>charset = f.headers.get_param('<STR_LIT>') or '<STR_LIT>'<EOL><DEDENT>page = page.decode(charset, "<STR_LIT:ignore>")<EOL><DEDENT>f.close()<EOL>for match in HREF.finditer(page):<EOL><INDENT>link = urllib.parse.urljoin(base, htmldecode(match.group(<NUM_LIT:1>)))<EOL>self.process_url(link)<EOL><DEDENT>if url.startswith(self.index_url) and getattr(f,'<STR_LIT:code>',None)!=<NUM_LIT>:<EOL><INDENT>page = self.process_index(url, page)<EOL><DEDENT>
Evaluate a URL as a possible download, and maybe retrieve it