'Yields (page, page_url) from the given locations, skipping
locations that have errors, and adding download/homepage links'

def _get_pages(self, locations, req):

    all_locations = list(locations)
    seen = set()
    while all_locations:
        location = all_locations.pop(0)
        if (location in seen):
            continue
        seen.add(location)
        page = self._get_page(location, req)
        if (page is None):
            continue
        (yield page)
        for link in page.rel_links():
            normalized = normalize_name(req.name).lower()
            if ((not (normalized in self.allow_external)) and (not self.allow_all_external)):
                self.need_warn_external = True
                logger.debug(('Not    searching    %s    for    files    because    external    urls    are    disallowed.' % link))
                continue
            if ((link.trusted is not None) and (not link.trusted) and (not (normalized in self.allow_unverified))):
                logger.debug(('Not    searching    %s    for    urls,    it    is    an    untrusted    link    and    cannot    produce    safe    or    verifiable    files.' % link))
                self.need_warn_unverified = True
                continue
            all_locations.append(link)
