'Find every link on a HTML document verify if the domain is reachable or
not; after that, verify if the web found a different name for the target
site or if we found a new site that is linked. If the link points to a
dead site then report it (it could be pointing to some private address
or something...)'

def _get_dead_links(self, fuzzable_request):

    base_url = fuzzable_request.get_url().base_url()
    original_response = self._uri_opener.GET(fuzzable_request.get_uri(), cache=True)
    base_response = self._uri_opener.GET(base_url, cache=True)
    base_resp_body = base_response.get_body()
    try:
        dp = parser_cache.dpc.get_document_parser_for(original_response)
    except BaseFrameworkException:
        return []
    non_existent_response = self._get_non_exist(fuzzable_request)
    nonexist_resp_body = non_existent_response.get_body()
    (parsed_references, re_references) = dp.get_references()
    parsed_references.extend(re_references)
    res = []
    vhosts = self._verify_link_domain(parsed_references)
    for (domain, vhost_response) in self._send_in_threads(base_url, vhosts):
        vhost_resp_body = vhost_response.get_body()
        if (fuzzy_not_equal(vhost_resp_body, base_resp_body, 0.35) and fuzzy_not_equal(vhost_resp_body, nonexist_resp_body, 0.35)):
            res.append((domain, vhost_response.id))
        else:
            desc = u'The    content    of    "%s"    references    a    non    existent    domain:    "%s".    This    can    be    a    broken    link,    or    an    internal    domain    name.'
            desc %= (fuzzable_request.get_url(), domain)
            i = Info(u'Internal    hostname    in    HTML    link', desc, original_response.id, self.get_name())
            i.set_url(fuzzable_request.get_url())
            kb.kb.append(self, 'find_vhosts', i)
            om.out.information(i.get_desc())
    return res
