'Evaluate a URL as a possible download, and maybe retrieve it'

def process_url(self, url, retrieve=False):

    if ((url in self.scanned_urls) and (not retrieve)):
        return
    self.scanned_urls[url] = True
    if (not URL_SCHEME(url)):
        self.process_filename(url)
        return
    else:
        dists = list(distros_for_url(url))
        if dists:
            if (not self.url_ok(url)):
                return
            self.debug('Found    link:    %s', url)
    if (dists or (not retrieve) or (url in self.fetched_urls)):
        list(map(self.add, dists))
        return
    if (not self.url_ok(url)):
        self.fetched_urls[url] = True
        return
    self.info('Reading    %s', url)
    self.fetched_urls[url] = True
    f = self.open_url(url, ('Download    error    on    %s:    %%s    --    Some    packages    may    not    be    found!' % url))
    if (f is None):
        return
    self.fetched_urls[f.url] = True
    if ('html' not in f.headers.get('content-type', '').lower()):
        f.close()
        return
    base = f.url
    page = f.read()
    if (not isinstance(page, str)):
        if isinstance(f, HTTPError):
            charset = 'latin-1'
        else:
            charset = (f.headers.get_param('charset') or 'latin-1')
        page = page.decode(charset, 'ignore')
    f.close()
    for match in HREF.finditer(page):
        link = urljoin(base, htmldecode(match.group(1)))
        self.process_url(link)
    if (url.startswith(self.index_url) and (getattr(f, 'code', None) != 404)):
        page = self.process_index(url, page)
