'Scrape the html response.'

def _scrape_response(self, headers, response):

    server_type = None
    server_string = headers.get(u'server', u'')
    if (server_string and (u'jetty' in server_string.lower())):
        server_type = u'jetty'
    if (server_string and (u'coyote' in server_string.lower())):
        server_type = u'tomcat'
    reason = None
    full_html = u''
    dom_tree = None
    if (IS_PY3 and hasattr(response, u'decode')):
        response = response.decode()
    if response.startswith(u'<?xml'):
        try:
            soup = ElementTree.fromstring(response)
            reason_node = soup.find(u'lst[@name="error"]/str[@name="msg"]')
            tb_node = soup.find(u'lst[@name="error"]/str[@name="trace"]')
            if (reason_node is not None):
                full_html = reason = reason_node.text.strip()
            if (tb_node is not None):
                full_html = tb_node.text.strip()
                if (reason is None):
                    reason = full_html
            if (reason and full_html):
                return (reason, full_html)
        except ElementTree.ParseError:
            pass
    if (server_type == u'tomcat'):
        m = re.search(u'<(h1)[^>]*>\\s*(.+?)\\s*</\\1>', response, re.IGNORECASE)
        if m:
            reason = m.group(2)
        else:
            full_html = (u'%s' % response)
    else:
        try:
            dom_tree = ElementTree.fromstring(response)
            reason_node = None
            if (server_type == u'jetty'):
                reason_node = dom_tree.find(u'body/pre')
            else:
                reason_node = dom_tree.find(u'head/title')
            if (reason_node is not None):
                reason = reason_node.text
            if (reason is None):
                full_html = ElementTree.tostring(dom_tree)
        except SyntaxError as err:
            LOG.warning(u'Unable    to    extract    error    message    from    invalid    XML:    %s', err, extra={u'data': {u'response': response}})
            full_html = (u'%s' % response)
    full_html = force_unicode(full_html)
    full_html = full_html.replace(u'\n', u'')
    full_html = full_html.replace(u'\r', u'')
    full_html = full_html.replace(u'<br/>', u'')
    full_html = full_html.replace(u'<br    />', u'')
    full_html = full_html.strip()
    return (reason, full_html)
