'This method connects to the target URL or proxy and returns
the target URL page content'

@staticmethod
def getPage(**kwargs):

    if (isinstance(conf.delay, (int, float)) and (conf.delay > 0)):
        time.sleep(conf.delay)
    if conf.offline:
        return (None, None, None)
    elif (conf.dummy or (conf.murphyRate and ((randomInt() % conf.murphyRate) == 0))):
        if conf.murphyRate:
            time.sleep((randomInt() % (MAX_MURPHY_SLEEP_TIME + 1)))
        return (getUnicode(randomStr(int(randomInt()), alphabet=[chr(_) for _ in xrange(256)]), {}, int(randomInt())), None, (None if (not conf.murphyRate) else randomInt(3)))
    threadData = getCurrentThreadData()
    with kb.locks.request:
        kb.requestCounter += 1
        threadData.lastRequestUID = kb.requestCounter
    url = (kwargs.get('url', None) or conf.url)
    get = kwargs.get('get', None)
    post = kwargs.get('post', None)
    method = kwargs.get('method', None)
    cookie = kwargs.get('cookie', None)
    ua = (kwargs.get('ua', None) or conf.agent)
    referer = (kwargs.get('referer', None) or conf.referer)
    host = (kwargs.get('host', None) or conf.host)
    direct_ = kwargs.get('direct', False)
    multipart = kwargs.get('multipart', None)
    silent = kwargs.get('silent', False)
    raise404 = kwargs.get('raise404', True)
    timeout = (kwargs.get('timeout', None) or conf.timeout)
    auxHeaders = kwargs.get('auxHeaders', None)
    response = kwargs.get('response', False)
    ignoreTimeout = (kwargs.get('ignoreTimeout', False) or kb.ignoreTimeout or conf.ignoreTimeouts)
    refreshing = kwargs.get('refreshing', False)
    retrying = kwargs.get('retrying', False)
    crawling = kwargs.get('crawling', False)
    skipRead = kwargs.get('skipRead', False)
    if multipart:
        post = multipart
    websocket_ = url.lower().startswith('ws')
    if (not urlparse.urlsplit(url).netloc):
        url = urlparse.urljoin(conf.url, url)
    target = checkSameHost(url, conf.url)
    if (not retrying):
        threadData.retriesCount = 0
    url = url.replace('    ', '%20')
    conn = None
    code = None
    page = None
    _ = urlparse.urlsplit(url)
    requestMsg = (u'HTTP    request    [#%d]:\n%s    ' % (threadData.lastRequestUID, (method or (HTTPMETHOD.POST if (post is not None) else HTTPMETHOD.GET))))
    requestMsg += (('%s%s' % ((_.path or '/'), (('?%s' % _.query) if _.query else ''))) if (not any((refreshing, crawling))) else url)
    responseMsg = u'HTTP    response    '
    requestHeaders = u''
    responseHeaders = None
    logHeaders = u''
    skipLogTraffic = False
    raise404 = (raise404 and (not kb.ignoreNotFound))
    url = asciifyUrl(url)
    try:
        socket.setdefaulttimeout(timeout)
        if direct_:
            if ('?' in url):
                (url, params) = url.split('?', 1)
                params = urlencode(params)
                url = ('%s?%s' % (url, params))
        elif any((refreshing, crawling)):
            pass
        elif target:
            if (conf.forceSSL and (urlparse.urlparse(url).scheme != 'https')):
                url = re.sub('\\Ahttp:', 'https:', url, re.I)
                url = re.sub(':80/', ':443/', url, re.I)
            if ((PLACE.GET in conf.parameters) and (not get)):
                get = conf.parameters[PLACE.GET]
                if (not conf.skipUrlEncode):
                    get = urlencode(get, limit=True)
            if get:
                if ('?' in url):
                    url = ('%s%s%s' % (url, DEFAULT_GET_POST_DELIMITER, get))
                    requestMsg += ('%s%s' % (DEFAULT_GET_POST_DELIMITER, get))
                else:
                    url = ('%s?%s' % (url, get))
                    requestMsg += ('?%s' % get)
            if ((PLACE.POST in conf.parameters) and (not post) and (method != HTTPMETHOD.GET)):
                post = conf.parameters[PLACE.POST]
        elif get:
            url = ('%s?%s' % (url, get))
            requestMsg += ('?%s' % get)
        requestMsg += ('    %s' % httplib.HTTPConnection._http_vsn_str)
        headers = forgeHeaders({HTTP_HEADER.COOKIE: cookie, HTTP_HEADER.USER_AGENT: ua, HTTP_HEADER.REFERER: referer, HTTP_HEADER.HOST: host})
        if (HTTP_HEADER.COOKIE in headers):
            cookie = headers[HTTP_HEADER.COOKIE]
        if kb.authHeader:
            headers[HTTP_HEADER.AUTHORIZATION] = kb.authHeader
        if kb.proxyAuthHeader:
            headers[HTTP_HEADER.PROXY_AUTHORIZATION] = kb.proxyAuthHeader
        if (not getHeader(headers, HTTP_HEADER.ACCEPT)):
            headers[HTTP_HEADER.ACCEPT] = HTTP_ACCEPT_HEADER_VALUE
        if ((not getHeader(headers, HTTP_HEADER.HOST)) or (not target)):
            headers[HTTP_HEADER.HOST] = getHostHeader(url)
        if (not getHeader(headers, HTTP_HEADER.ACCEPT_ENCODING)):
            headers[HTTP_HEADER.ACCEPT_ENCODING] = (HTTP_ACCEPT_ENCODING_HEADER_VALUE if kb.pageCompress else 'identity')
        if ((post is not None) and (not multipart) and (not getHeader(headers, HTTP_HEADER.CONTENT_TYPE))):
            headers[HTTP_HEADER.CONTENT_TYPE] = POST_HINT_CONTENT_TYPES.get(kb.postHint, DEFAULT_CONTENT_TYPE)
        if (headers.get(HTTP_HEADER.CONTENT_TYPE) == POST_HINT_CONTENT_TYPES[POST_HINT.MULTIPART]):
            warnMsg = ("missing    'boundary    parameter'    in    '%s'    header.    " % HTTP_HEADER.CONTENT_TYPE)
            warnMsg += 'Will    try    to    reconstruct'
            singleTimeWarnMessage(warnMsg)
            boundary = findMultipartPostBoundary(conf.data)
            if boundary:
                headers[HTTP_HEADER.CONTENT_TYPE] = ('%s;    boundary=%s' % (headers[HTTP_HEADER.CONTENT_TYPE], boundary))
        if conf.keepAlive:
            headers[HTTP_HEADER.CONNECTION] = 'keep-alive'
        if (target and conf.requestFile):
            headers = OrderedDict(conf.httpHeaders)
            if cookie:
                headers[HTTP_HEADER.COOKIE] = cookie
        if auxHeaders:
            for (key, value) in auxHeaders.items():
                for _ in headers.keys():
                    if (_.upper() == key.upper()):
                        del headers[_]
                headers[key] = value
        for (key, value) in headers.items():
            del headers[key]
            value = unicodeencode(value, kb.pageEncoding)
            for char in ('\\r', '\\n'):
                value = re.sub(('(%s)([^    \\t])' % char), '\\g<1>\\t\\g<2>', value)
            headers[unicodeencode(key, kb.pageEncoding)] = value.strip('\r\n')
        url = unicodeencode(url)
        post = unicodeencode(post)
        if websocket_:
            ws = websocket.WebSocket()
            ws.settimeout(timeout)
            ws.connect(url, header=(('%s:    %s' % _) for _ in headers.items() if (_[0] not in ('Host',))), cookie=cookie)
            ws.send(urldecode((post or '')))
            page = ws.recv()
            ws.close()
            code = ws.status
            status = httplib.responses[code]
            class _(dict, ):
                pass
            responseHeaders = _(ws.getheaders())
            responseHeaders.headers = [('%s:    %s\r\n' % (_[0].capitalize(), _[1])) for _ in responseHeaders.items()]
            requestHeaders += '\n'.join([('%s:    %s' % (getUnicode((key.capitalize() if isinstance(key, basestring) else key)), getUnicode(value))) for (key, value) in responseHeaders.items()])
            requestMsg += ('\n%s' % requestHeaders)
            if (post is not None):
                requestMsg += ('\n\n%s' % getUnicode(post))
            requestMsg += '\n'
            threadData.lastRequestMsg = requestMsg
            logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg)
        else:
            if (method and (method not in (HTTPMETHOD.GET, HTTPMETHOD.POST))):
                method = unicodeencode(method)
                req = MethodRequest(url, post, headers)
                req.set_method(method)
            else:
                req = urllib2.Request(url, post, headers)
            requestHeaders += '\n'.join([('%s:    %s' % (getUnicode((key.capitalize() if isinstance(key, basestring) else key)), getUnicode(value))) for (key, value) in req.header_items()])
            if ((not getRequestHeader(req, HTTP_HEADER.COOKIE)) and conf.cj):
                conf.cj._policy._now = conf.cj._now = int(time.time())
                cookies = conf.cj._cookies_for_request(req)
                requestHeaders += ('\n%s' % ('Cookie:    %s' % ';'.join((('%s=%s' % (getUnicode(cookie.name), getUnicode(cookie.value))) for cookie in cookies))))
            if (post is not None):
                if (not getRequestHeader(req, HTTP_HEADER.CONTENT_LENGTH)):
                    requestHeaders += ('\n%s:    %d' % (string.capwords(HTTP_HEADER.CONTENT_LENGTH), len(post)))
            if (not getRequestHeader(req, HTTP_HEADER.CONNECTION)):
                requestHeaders += ('\n%s:    %s' % (HTTP_HEADER.CONNECTION, ('close' if (not conf.keepAlive) else 'keep-alive')))
            requestMsg += ('\n%s' % requestHeaders)
            if (post is not None):
                requestMsg += ('\n\n%s' % getUnicode(post))
            requestMsg += '\n'
            if (not multipart):
                threadData.lastRequestMsg = requestMsg
                logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg)
            if conf.cj:
                for cookie in conf.cj:
                    if (cookie.value is None):
                        cookie.value = ''
                    else:
                        for char in ('\\r', '\\n'):
                            cookie.value = re.sub(('(%s)([^    \\t])' % char), '\\g<1>\\t\\g<2>', cookie.value)
            conn = urllib2.urlopen(req)
            if ((not kb.authHeader) and getRequestHeader(req, HTTP_HEADER.AUTHORIZATION) and ((conf.authType or '').lower() == AUTH_TYPE.BASIC.lower())):
                kb.authHeader = getRequestHeader(req, HTTP_HEADER.AUTHORIZATION)
            if ((not kb.proxyAuthHeader) and getRequestHeader(req, HTTP_HEADER.PROXY_AUTHORIZATION)):
                kb.proxyAuthHeader = getRequestHeader(req, HTTP_HEADER.PROXY_AUTHORIZATION)
            if response:
                return (conn, None, None)
            if hasattr(conn, 'redurl'):
                page = ((threadData.lastRedirectMsg[1] if (kb.redirectChoice == REDIRECTION.NO) else Connect._connReadProxy(conn)) if (not skipRead) else None)
                skipLogTraffic = (kb.redirectChoice == REDIRECTION.NO)
                code = conn.redcode
            else:
                page = (Connect._connReadProxy(conn) if (not skipRead) else None)
            code = (code or (conn.code if conn else None))
            responseHeaders = conn.info()
            responseHeaders[URI_HTTP_HEADER] = conn.geturl()
            page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))
            status = getUnicode(conn.msg)
        kb.connErrorCounter = 0
        if (not refreshing):
            refresh = responseHeaders.get(HTTP_HEADER.REFRESH, '').split('url=')[(-1)].strip()
            if extractRegexResult(META_REFRESH_REGEX, page):
                refresh = extractRegexResult(META_REFRESH_REGEX, page)
                debugMsg = 'got    HTML    meta    refresh    header'
                logger.debug(debugMsg)
            if refresh:
                if (kb.alwaysRefresh is None):
                    msg = 'sqlmap    got    a    refresh    request    '
                    msg += '(redirect    like    response    common    to    login    pages).    '
                    msg += 'Do    you    want    to    apply    the    refresh    '
                    msg += 'from    now    on    (or    stay    on    the    original    page)?    [Y/n]'
                    choice = readInput(msg, default='Y')
                    kb.alwaysRefresh = (choice not in ('n', 'N'))
                if kb.alwaysRefresh:
                    if re.search('\\Ahttps?://', refresh, re.I):
                        url = refresh
                    else:
                        url = urlparse.urljoin(url, refresh)
                    threadData.lastRedirectMsg = (threadData.lastRequestUID, page)
                    kwargs['refreshing'] = True
                    kwargs['url'] = url
                    kwargs['get'] = None
                    kwargs['post'] = None
                    try:
                        return Connect._getPageProxy(**kwargs)
                    except SqlmapSyntaxException:
                        pass
        if (conn and (not conf.keepAlive)):
            try:
                if hasattr(conn.fp, '_sock'):
                    conn.fp._sock.close()
                conn.close()
            except Exception as ex:
                warnMsg = ("problem    occurred    during    connection    closing    ('%s')" % getSafeExString(ex))
                logger.warn(warnMsg)
    except urllib2.HTTPError as ex:
        page = None
        responseHeaders = None
        try:
            page = (ex.read() if (not skipRead) else None)
            responseHeaders = ex.info()
            responseHeaders[URI_HTTP_HEADER] = ex.geturl()
            page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))
        except socket.timeout:
            warnMsg = 'connection    timed    out    while    trying    '
            warnMsg += ('to    get    error    page    information    (%d)' % ex.code)
            logger.warn(warnMsg)
            return (None, None, None)
        except KeyboardInterrupt:
            raise
        except:
            pass
        finally:
            page = (page if isinstance(page, unicode) else getUnicode(page))
        code = ex.code
        kb.originalCode = (kb.originalCode or code)
        threadData.lastHTTPError = (threadData.lastRequestUID, code)
        kb.httpErrorCodes[code] = (kb.httpErrorCodes.get(code, 0) + 1)
        status = getUnicode(ex.msg)
        responseMsg += ('[#%d]    (%d    %s):\n' % (threadData.lastRequestUID, code, status))
        if responseHeaders:
            logHeaders = '\n'.join([('%s:    %s' % (getUnicode((key.capitalize() if isinstance(key, basestring) else key)), getUnicode(value))) for (key, value) in responseHeaders.items()])
        logHTTPTraffic(requestMsg, ('%s%s\n\n%s' % (responseMsg, logHeaders, (page or '')[:MAX_CONNECTION_CHUNK_SIZE])))
        skipLogTraffic = True
        if (conf.verbose <= 5):
            responseMsg += getUnicode(logHeaders)
        elif (conf.verbose > 5):
            responseMsg += ('%s\n\n%s' % (logHeaders, (page or '')[:MAX_CONNECTION_CHUNK_SIZE]))
        if (not multipart):
            logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
        if ((ex.code == httplib.UNAUTHORIZED) and (not conf.ignore401)):
            errMsg = 'not    authorized,    try    to    provide    right    HTTP    '
            errMsg += ('authentication    type    and    valid    credentials    (%d)' % code)
            raise SqlmapConnectionException(errMsg)
        elif (ex.code == httplib.NOT_FOUND):
            if raise404:
                errMsg = ('page    not    found    (%d)' % code)
                raise SqlmapConnectionException(errMsg)
            else:
                debugMsg = ('page    not    found    (%d)' % code)
                singleTimeLogMessage(debugMsg, logging.DEBUG)
                processResponse(page, responseHeaders)
        elif (ex.code == httplib.GATEWAY_TIMEOUT):
            if ignoreTimeout:
                return ((None if (not conf.ignoreTimeouts) else ''), None, None)
            else:
                warnMsg = ('unable    to    connect    to    the    target    URL    (%d    -    %s)' % (ex.code, httplib.responses[ex.code]))
                if ((threadData.retriesCount < conf.retries) and (not kb.threadException)):
                    warnMsg += '.    sqlmap    is    going    to    retry    the    request'
                    logger.critical(warnMsg)
                    return Connect._retryProxy(**kwargs)
                elif kb.testMode:
                    logger.critical(warnMsg)
                    return (None, None, None)
                else:
                    raise SqlmapConnectionException(warnMsg)
        else:
            debugMsg = ('got    HTTP    error    code:    %d    (%s)' % (code, status))
            logger.debug(debugMsg)
    except (urllib2.URLError, socket.error, socket.timeout, httplib.HTTPException, struct.error, binascii.Error, ProxyError, SqlmapCompressionException, WebSocketException, TypeError):
        tbMsg = traceback.format_exc()
        if ('no    host    given' in tbMsg):
            warnMsg = ('invalid    URL    address    used    (%s)' % repr(url))
            raise SqlmapSyntaxException(warnMsg)
        elif (('forcibly    closed' in tbMsg) or ('Connection    is    already    closed' in tbMsg)):
            warnMsg = 'connection    was    forcibly    closed    by    the    target    URL'
        elif ('timed    out' in tbMsg):
            if (not conf.disablePrecon):
                singleTimeWarnMessage('turning    off    pre-connect    mechanism    because    of    connection    time    out(s)')
                conf.disablePrecon = True
                if (kb.testMode and (kb.testType not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED))):
                    kb.responseTimes.clear()
            if (kb.testMode and (kb.testType not in (None, PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED))):
                singleTimeWarnMessage("there    is    a    possibility    that    the    target    (or    WAF/IPS/IDS)    is    dropping    'suspicious'    requests")
            warnMsg = 'connection    timed    out    to    the    target    URL'
        elif (('URLError' in tbMsg) or ('error' in tbMsg)):
            warnMsg = 'unable    to    connect    to    the    target    URL'
            match = re.search('Errno    \\d+\\]    ([^>]+)', tbMsg)
            if match:
                warnMsg += ("    ('%s')" % match.group(1).strip())
        elif ('NTLM' in tbMsg):
            warnMsg = 'there    has    been    a    problem    with    NTLM    authentication'
        elif ('BadStatusLine' in tbMsg):
            warnMsg = 'connection    dropped    or    unknown    HTTP    '
            warnMsg += 'status    code    received'
            if ((not conf.agent) and (not conf.randomAgent)):
                warnMsg += '.    Try    to    force    the    HTTP    User-Agent    '
                warnMsg += "header    with    option    '--user-agent'    or    switch    '--random-agent'"
        elif ('IncompleteRead' in tbMsg):
            warnMsg = 'there    was    an    incomplete    read    error    while    retrieving    data    '
            warnMsg += 'from    the    target    URL'
        elif ('Handshake    status' in tbMsg):
            status = re.search('Handshake    status    ([\\d]{3})', tbMsg)
            errMsg = (('websocket    handshake    status    %s' % status.group(1)) if status else 'unknown')
            raise SqlmapConnectionException(errMsg)
        else:
            warnMsg = 'unable    to    connect    to    the    target    URL'
        if (('BadStatusLine' not in tbMsg) and any((conf.proxy, conf.tor))):
            warnMsg += '    or    proxy'
        with kb.locks.connError:
            kb.connErrorCounter += 1
            if ((kb.connErrorCounter >= MAX_CONSECUTIVE_CONNECTION_ERRORS) and (kb.connErrorChoice is None)):
                message = 'there    seems    to    be    a    continuous    problem    with    connection    to    the    target.    '
                message += 'Are    you    sure    that    you    want    to    continue    '
                message += 'with    further    target    testing?    [y/N]    '
                kb.connErrorChoice = (readInput(message, default='N') in ('Y', 'y'))
            if (kb.connErrorChoice is False):
                raise SqlmapConnectionException(warnMsg)
        if silent:
            return (None, None, None)
        elif ('forcibly    closed' in tbMsg):
            logger.critical(warnMsg)
            return (None, None, None)
        elif (ignoreTimeout and any(((_ in tbMsg) for _ in ('timed    out', 'IncompleteRead')))):
            return ((None if (not conf.ignoreTimeouts) else ''), None, None)
        elif ((threadData.retriesCount < conf.retries) and (not kb.threadException)):
            warnMsg += '.    sqlmap    is    going    to    retry    the    request'
            if (not retrying):
                warnMsg += '(s)'
                logger.critical(warnMsg)
            else:
                logger.debug(warnMsg)
            return Connect._retryProxy(**kwargs)
        elif kb.testMode:
            logger.critical(warnMsg)
            return (None, None, None)
        else:
            raise SqlmapConnectionException(warnMsg)
    finally:
        if (isinstance(page, basestring) and (not isinstance(page, unicode))):
            if ((HTTP_HEADER.CONTENT_TYPE in (responseHeaders or {})) and (not re.search(TEXT_CONTENT_TYPE_REGEX, responseHeaders[HTTP_HEADER.CONTENT_TYPE]))):
                page = unicode(page, errors='ignore')
            else:
                page = getUnicode(page)
        socket.setdefaulttimeout(conf.timeout)
    processResponse(page, responseHeaders)
    if (conn and getattr(conn, 'redurl', None)):
        _ = urlparse.urlsplit(conn.redurl)
        _ = ('%s%s' % ((_.path or '/'), (('?%s' % _.query) if _.query else '')))
        requestMsg = re.sub('(\n[A-Z]+    ).+?(    HTTP/\\d)', ('\\g<1>%s\\g<2>' % getUnicode(_).replace('\\', '\\\\')), requestMsg, 1)
        if (kb.resendPostOnRedirect is False):
            requestMsg = re.sub('(\\[#\\d+\\]:\n)POST    ', '\\g<1>GET    ', requestMsg)
            requestMsg = re.sub('(?i)Content-length:    \\d+\n', '', requestMsg)
            requestMsg = re.sub('(?s)\n\n.+', '\n', requestMsg)
        responseMsg += ('[#%d]    (%d    %s):\n' % (threadData.lastRequestUID, conn.code, status))
    else:
        responseMsg += ('[#%d]    (%d    %s):\n' % (threadData.lastRequestUID, code, status))
    if responseHeaders:
        logHeaders = '\n'.join([('%s:    %s' % (getUnicode((key.capitalize() if isinstance(key, basestring) else key)), getUnicode(value))) for (key, value) in responseHeaders.items()])
    if (not skipLogTraffic):
        logHTTPTraffic(requestMsg, ('%s%s\n\n%s' % (responseMsg, logHeaders, (page or '')[:MAX_CONNECTION_CHUNK_SIZE])))
    if (conf.verbose <= 5):
        responseMsg += getUnicode(logHeaders)
    elif (conf.verbose > 5):
        responseMsg += ('%s\n\n%s' % (logHeaders, (page or '')[:MAX_CONNECTION_CHUNK_SIZE]))
    if (not multipart):
        logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
    return (page, responseHeaders, code)
