'All of my previous versions of is_404 were very complex and tried to
struggle with all possible cases. The truth is that in most "strange"
cases I was failing miserably, so now I changed my 404 detection once
again, but keeping it as simple as possible.
Also, and because I was trying to cover ALL CASES, I was performing a
lot of requests in order to cover them, which in most situations was
unnecessary.
So now I go for a much simple approach:
1- Cover the simplest case of all using only 1 HTTP request
2- Give the users the power to configure the 404 detection by
setting a string that identifies the 404 response (in case we
are missing it for some reason in case #1)
:param http_response: The HTTP response which we want to know if it
is a 404 or not.'

@lru_404_cache
def is_404(self, http_response):

    domain_path = http_response.get_url().get_domain_path()
    extension = http_response.get_url().get_extension()
    if (domain_path in cf.cf.get('always_404')):
        return True
    elif (domain_path in cf.cf.get('never_404')):
        return False
    if (cf.cf.get('string_match_404') and (cf.cf.get('string_match_404') in http_response)):
        return True
    if (http_response.get_code() == 404):
        return True
    path_extension = (domain_path, extension)
    if ((path_extension in self._directory_uses_404_codes) and (http_response.get_code() != 404)):
        return False
    with self._lock:
        if (not self._already_analyzed):
            self.generate_404_knowledge(http_response.get_url())
            self._already_analyzed = True
    resp_body = get_clean_body(http_response)
    resp_content_type = http_response.doc_type
    resp_path = http_response.get_url().get_domain_path().url_string
    max_similarity_with_404 = 0.0
    resp_path_in_db = False
    with self._lock:
        for resp_404 in self._404_responses:
            if (resp_content_type != resp_404.doc_type):
                continue
            if fuzzy_equal(resp_404.body, resp_body, IS_EQUAL_RATIO):
                msg = '"%s"    (id:%s)    is    a    404    [similarity_index    >    %s]'
                fmt = (http_response.get_url(), http_response.id, IS_EQUAL_RATIO)
                om.out.debug((msg % fmt))
                return True
            else:
                current_ratio = relative_distance(resp_404.body, resp_body)
                max_similarity_with_404 = max(max_similarity_with_404, current_ratio)
            if ((not resp_path_in_db) and (resp_path == resp_404.path)):
                resp_path_in_db = True
        if (resp_path_in_db and (max_similarity_with_404 < MUST_VERIFY_RATIO)):
            msg = '"%s"    (id:%s)    is    NOT    a    404    [similarity_index    <    %s    with    sample    path    in    404    DB].'
            args = (http_response.get_url(), http_response.id, MUST_VERIFY_RATIO)
            om.out.debug((msg % args))
            return False
        if self._is_404_with_extra_request(http_response, resp_body):
            four_oh_data = FourOhFourResponseFactory(http_response)
            self._404_responses.append(four_oh_data)
            msg = '"%s"    (id:%s)    is    a    404    (similarity_index    >    %s).    Adding    new    knowledge    to    the    404_responses    database    (length=%s).'
            fmt = (http_response.get_url(), http_response.id, IS_EQUAL_RATIO, len(self._404_responses))
            om.out.debug((msg % fmt))
            return True
        msg = '"%s"    (id:%s)    is    NOT    a    404    [similarity_index    <    %s].'
        args = (http_response.get_url(), http_response.id, IS_EQUAL_RATIO)
        om.out.debug((msg % args))
        return False
