'This method calls a function to get the target URL page content
and returns its page MD5 hash or a boolean value in case of
string match check (\'--string\' command line parameter)'

@staticmethod
def queryPage(value=None, place=None, content=False, getRatioValue=False, silent=False, method=None, timeBasedCompare=False, noteResponseTime=True, auxHeaders=None, response=False, raise404=None, removeReflection=True):

    if conf.direct:
        return direct(value, content)
    get = None
    post = None
    cookie = None
    ua = None
    referer = None
    host = None
    page = None
    pageLength = None
    uri = None
    code = None
    if (not place):
        place = (kb.injection.place or PLACE.GET)
    if (not auxHeaders):
        auxHeaders = {}
    raise404 = ((place != PLACE.URI) if (raise404 is None) else raise404)
    method = (method or conf.method)
    value = agent.adjustLateValues(value)
    payload = agent.extractPayload(value)
    threadData = getCurrentThreadData()
    if conf.httpHeaders:
        headers = OrderedDict(conf.httpHeaders)
        contentType = max(((headers[_] if (_.upper() == HTTP_HEADER.CONTENT_TYPE.upper()) else None) for _ in headers.keys()))
        if ((kb.postHint or conf.skipUrlEncode) and kb.postUrlEncode):
            kb.postUrlEncode = False
            conf.httpHeaders = [_ for _ in conf.httpHeaders if (_[1] != contentType)]
            contentType = POST_HINT_CONTENT_TYPES.get(kb.postHint, PLAIN_TEXT_CONTENT_TYPE)
            conf.httpHeaders.append((HTTP_HEADER.CONTENT_TYPE, contentType))
    if payload:
        if kb.tamperFunctions:
            for function in kb.tamperFunctions:
                try:
                    payload = function(payload=payload, headers=auxHeaders)
                except Exception as ex:
                    errMsg = 'error    occurred    while    running    tamper    '
                    errMsg += ("function    '%s'    ('%s')" % (function.func_name, getSafeExString(ex)))
                    raise SqlmapGenericException(errMsg)
                if (not isinstance(payload, basestring)):
                    errMsg = ("tamper    function    '%s'    returns    " % function.func_name)
                    errMsg += ("invalid    payload    type    ('%s')" % type(payload))
                    raise SqlmapValueException(errMsg)
            value = agent.replacePayload(value, payload)
        logger.log(CUSTOM_LOGGING.PAYLOAD, safecharencode(payload.replace('\\', BOUNDARY_BACKSLASH_MARKER)).replace(BOUNDARY_BACKSLASH_MARKER, '\\'))
        if ((place == PLACE.CUSTOM_POST) and kb.postHint):
            if (kb.postHint in (POST_HINT.SOAP, POST_HINT.XML)):
                payload = payload.replace('>', '&gt;').replace('<', '&lt;')
            elif (kb.postHint == POST_HINT.JSON):
                if (payload.startswith('"') and payload.endswith('"')):
                    payload = json.dumps(payload[1:(-1)])
                else:
                    payload = json.dumps(payload)[1:(-1)]
            elif (kb.postHint == POST_HINT.JSON_LIKE):
                payload = payload.replace("'", REPLACEMENT_MARKER).replace('"', "'").replace(REPLACEMENT_MARKER, '"')
                if (payload.startswith('"') and payload.endswith('"')):
                    payload = json.dumps(payload[1:(-1)])
                else:
                    payload = json.dumps(payload)[1:(-1)]
                payload = payload.replace("'", REPLACEMENT_MARKER).replace('"', "'").replace(REPLACEMENT_MARKER, '"')
            value = agent.replacePayload(value, payload)
        elif ((((place in (PLACE.GET, PLACE.URI, PLACE.COOKIE)) or ((place == PLACE.CUSTOM_HEADER) and (value.split(',')[0] == HTTP_HEADER.COOKIE))) and (not conf.skipUrlEncode)) or ((place in (PLACE.POST, PLACE.CUSTOM_POST)) and kb.postUrlEncode)):
            skip = False
            if ((place == PLACE.COOKIE) or ((place == PLACE.CUSTOM_HEADER) and (value.split(',')[0] == HTTP_HEADER.COOKIE))):
                if (kb.cookieEncodeChoice is None):
                    msg = ('do    you    want    to    URL    encode    cookie    values    (implementation    specific)?    %s' % ('[Y/n]' if (not conf.url.endswith('.aspx')) else '[y/N]'))
                    choice = readInput(msg, default=('Y' if (not conf.url.endswith('.aspx')) else 'N'))
                    kb.cookieEncodeChoice = (choice.upper().strip() == 'Y')
                if (not kb.cookieEncodeChoice):
                    skip = True
            if (not skip):
                payload = urlencode(payload, '%', False, (place != PLACE.URI))
                value = agent.replacePayload(value, payload)
        if conf.hpp:
            if (not any((conf.url.lower().endswith(_.lower()) for _ in (WEB_API.ASP, WEB_API.ASPX)))):
                warnMsg = 'HTTP    parameter    pollution    should    work    only    against    '
                warnMsg += 'ASP(.NET)    targets'
                singleTimeWarnMessage(warnMsg)
            if (place in (PLACE.GET, PLACE.POST)):
                _ = re.escape(PAYLOAD_DELIMITER)
                match = re.search(('(?P<name>\\w+)=%s(?P<value>.+?)%s' % (_, _)), value)
                if match:
                    payload = match.group('value')
                    for splitter in (urlencode('    '), '    '):
                        if (splitter in payload):
                            (prefix, suffix) = (('*/', '/*') if (splitter == '    ') else (urlencode(_) for _ in ('*/', '/*')))
                            parts = payload.split(splitter)
                            parts[0] = ('%s%s' % (parts[0], suffix))
                            parts[(-1)] = ('%s%s=%s%s' % (DEFAULT_GET_POST_DELIMITER, match.group('name'), prefix, parts[(-1)]))
                            for i in xrange(1, (len(parts) - 1)):
                                parts[i] = ('%s%s=%s%s%s' % (DEFAULT_GET_POST_DELIMITER, match.group('name'), prefix, parts[i], suffix))
                            payload = ''.join(parts)
                    for splitter in (urlencode(','), ','):
                        payload = payload.replace(splitter, ('%s%s=' % (DEFAULT_GET_POST_DELIMITER, match.group('name'))))
                    value = agent.replacePayload(value, payload)
            else:
                warnMsg = 'HTTP    parameter    pollution    works    only    with    regular    '
                warnMsg += 'GET    and    POST    parameters'
                singleTimeWarnMessage(warnMsg)
    if place:
        value = agent.removePayloadDelimiters(value)
    if (PLACE.GET in conf.parameters):
        get = (conf.parameters[PLACE.GET] if ((place != PLACE.GET) or (not value)) else value)
    elif (place == PLACE.GET):
        get = value
    if (PLACE.POST in conf.parameters):
        post = (conf.parameters[PLACE.POST] if ((place != PLACE.POST) or (not value)) else value)
    elif (place == PLACE.POST):
        post = value
    if (PLACE.CUSTOM_POST in conf.parameters):
        post = (conf.parameters[PLACE.CUSTOM_POST].replace(CUSTOM_INJECTION_MARK_CHAR, '') if ((place != PLACE.CUSTOM_POST) or (not value)) else value)
        post = (post.replace(ASTERISK_MARKER, '*') if post else post)
    if (PLACE.COOKIE in conf.parameters):
        cookie = (conf.parameters[PLACE.COOKIE] if ((place != PLACE.COOKIE) or (not value)) else value)
    if (PLACE.USER_AGENT in conf.parameters):
        ua = (conf.parameters[PLACE.USER_AGENT] if ((place != PLACE.USER_AGENT) or (not value)) else value)
    if (PLACE.REFERER in conf.parameters):
        referer = (conf.parameters[PLACE.REFERER] if ((place != PLACE.REFERER) or (not value)) else value)
    if (PLACE.HOST in conf.parameters):
        host = (conf.parameters[PLACE.HOST] if ((place != PLACE.HOST) or (not value)) else value)
    if (PLACE.URI in conf.parameters):
        uri = (conf.url if ((place != PLACE.URI) or (not value)) else value)
    else:
        uri = conf.url
    if (value and (place == PLACE.CUSTOM_HEADER)):
        if (value.split(',')[0].capitalize() == PLACE.COOKIE):
            cookie = value.split(',', 1)[1]
        else:
            auxHeaders[value.split(',')[0]] = value.split(',', 1)[1]
    if conf.csrfToken:
        def _adjustParameter(paramString, parameter, newValue):
            retVal = paramString
            match = re.search(('%s=[^&]*' % re.escape(parameter)), paramString)
            if match:
                retVal = re.sub(re.escape(match.group(0)), ('%s=%s' % (parameter, newValue)), paramString)
            else:
                match = re.search(('(%s["\']:["\'])([^"\']+)' % re.escape(parameter)), paramString)
                if match:
                    retVal = re.sub(re.escape(match.group(0)), ('%s%s' % (match.group(1), newValue)), paramString)
            return retVal
        (page, headers, code) = Connect.getPage(url=(conf.csrfUrl or conf.url), data=(conf.data if (conf.csrfUrl == conf.url) else None), method=(conf.method if (conf.csrfUrl == conf.url) else None), cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST))
        match = re.search(('<input[^>]+name=[\\"\']?%s[\\"\']?\\s[^>]*value=(\\"([^\\"]+)|\'([^\']+)|([^    >]+))' % re.escape(conf.csrfToken)), (page or ''))
        token = ((match.group(2) or match.group(3) or match.group(4)) if match else None)
        if (not token):
            match = re.search(('%s[\\"\']:[\\"\']([^\\"\']+)' % re.escape(conf.csrfToken)), (page or ''))
            token = (match.group(1) if match else None)
        if (not token):
            if ((conf.csrfUrl != conf.url) and (code == httplib.OK)):
                if (headers and ('text/plain' in headers.get(HTTP_HEADER.CONTENT_TYPE, ''))):
                    token = page
            if ((not token) and conf.cj and any(((_.name == conf.csrfToken) for _ in conf.cj))):
                for _ in conf.cj:
                    if (_.name == conf.csrfToken):
                        token = _.value
                        if (not any(((conf.csrfToken in _) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}))))):
                            if post:
                                post = ('%s%s%s=%s' % (post, (conf.paramDel or DEFAULT_GET_POST_DELIMITER), conf.csrfToken, token))
                            elif get:
                                get = ('%s%s%s=%s' % (get, (conf.paramDel or DEFAULT_GET_POST_DELIMITER), conf.csrfToken, token))
                            else:
                                get = ('%s=%s' % (conf.csrfToken, token))
                        break
            if (not token):
                errMsg = ("anti-CSRF    token    '%s'    can't    be    found    at    '%s'" % (conf.csrfToken, (conf.csrfUrl or conf.url)))
                if (not conf.csrfUrl):
                    errMsg += '.    You    can    try    to    rerun    by    providing    '
                    errMsg += "a    valid    value    for    option    '--csrf-url'"
                raise SqlmapTokenException, errMsg
        if token:
            for place in (PLACE.GET, PLACE.POST):
                if (place in conf.parameters):
                    if ((place == PLACE.GET) and get):
                        get = _adjustParameter(get, conf.csrfToken, token)
                    elif ((place == PLACE.POST) and post):
                        post = _adjustParameter(post, conf.csrfToken, token)
            for i in xrange(len(conf.httpHeaders)):
                if (conf.httpHeaders[i][0].lower() == conf.csrfToken.lower()):
                    conf.httpHeaders[i] = (conf.httpHeaders[i][0], token)
    if conf.rParam:
        def _randomizeParameter(paramString, randomParameter):
            retVal = paramString
            match = re.search(('(\\A|\\b)%s=(?P<value>[^&;]+)' % re.escape(randomParameter)), paramString)
            if match:
                origValue = match.group('value')
                retVal = re.sub(('(\\A|\\b)%s=[^&;]+' % re.escape(randomParameter)), ('%s=%s' % (randomParameter, randomizeParameterValue(origValue))), paramString)
            return retVal
        for randomParameter in conf.rParam:
            for item in (PLACE.GET, PLACE.POST, PLACE.COOKIE, PLACE.URI, PLACE.CUSTOM_POST):
                if (item in conf.parameters):
                    if ((item == PLACE.GET) and get):
                        get = _randomizeParameter(get, randomParameter)
                    elif ((item in (PLACE.POST, PLACE.CUSTOM_POST)) and post):
                        post = _randomizeParameter(post, randomParameter)
                    elif ((item == PLACE.COOKIE) and cookie):
                        cookie = _randomizeParameter(cookie, randomParameter)
                    elif ((item == PLACE.URI) and uri):
                        uri = _randomizeParameter(uri, randomParameter)
    if conf.evalCode:
        delimiter = (conf.paramDel or DEFAULT_GET_POST_DELIMITER)
        variables = {'uri': uri, 'lastPage': threadData.lastPage, '_locals': locals()}
        originals = {}
        keywords = keyword.kwlist
        if ((not get) and (PLACE.URI in conf.parameters)):
            query = (urlparse.urlsplit(uri).query or '')
        else:
            query = None
        for item in filter(None, (get, (post if (not kb.postHint) else None), query)):
            for part in item.split(delimiter):
                if ('=' in part):
                    (name, value) = part.split('=', 1)
                    name = re.sub('[^\\w]', '', name.strip())
                    if (name in keywords):
                        name = ('%s%s' % (name, EVALCODE_KEYWORD_SUFFIX))
                    value = urldecode(value, convall=True, plusspace=((item == post) and kb.postSpaceToPlus))
                    variables[name] = value
        if cookie:
            for part in cookie.split((conf.cookieDel or DEFAULT_COOKIE_DELIMITER)):
                if ('=' in part):
                    (name, value) = part.split('=', 1)
                    name = re.sub('[^\\w]', '', name.strip())
                    if (name in keywords):
                        name = ('%s%s' % (name, EVALCODE_KEYWORD_SUFFIX))
                    value = urldecode(value, convall=True)
                    variables[name] = value
        while True:
            try:
                compiler.parse(unicodeencode(conf.evalCode.replace(';', '\n')))
            except SyntaxError as ex:
                original = replacement = ex.text.strip()
                for _ in re.findall('[A-Za-z_]+', original)[::(-1)]:
                    if (_ in keywords):
                        replacement = replacement.replace(_, ('%s%s' % (_, EVALCODE_KEYWORD_SUFFIX)))
                        break
                if (original == replacement):
                    conf.evalCode = conf.evalCode.replace(EVALCODE_KEYWORD_SUFFIX, '')
                    break
                else:
                    conf.evalCode = conf.evalCode.replace(ex.text.strip(), replacement)
            else:
                break
        originals.update(variables)
        evaluateCode(conf.evalCode, variables)
        for variable in variables.keys():
            if variable.endswith(EVALCODE_KEYWORD_SUFFIX):
                value = variables[variable]
                del variables[variable]
                variables[variable.replace(EVALCODE_KEYWORD_SUFFIX, '')] = value
        uri = variables['uri']
        for (name, value) in variables.items():
            if ((name != '__builtins__') and (originals.get(name, '') != value)):
                if isinstance(value, (basestring, int)):
                    found = False
                    value = getUnicode(value)
                    regex = ('((\\A|%s)%s=).+?(%s|\\Z)' % (re.escape(delimiter), re.escape(name), re.escape(delimiter)))
                    if re.search(regex, (get or '')):
                        found = True
                        get = re.sub(regex, ('\\g<1>%s\\g<3>' % value), get)
                    if re.search(regex, (post or '')):
                        found = True
                        post = re.sub(regex, ('\\g<1>%s\\g<3>' % value), post)
                    if re.search(regex, (query or '')):
                        found = True
                        uri = re.sub(regex.replace('\\A', '\\?'), ('\\g<1>%s\\g<3>' % value), uri)
                    regex = ('((\\A|%s)%s=).+?(%s|\\Z)' % (re.escape((conf.cookieDel or DEFAULT_COOKIE_DELIMITER)), name, re.escape((conf.cookieDel or DEFAULT_COOKIE_DELIMITER))))
                    if re.search(regex, (cookie or '')):
                        found = True
                        cookie = re.sub(regex, ('\\g<1>%s\\g<3>' % value), cookie)
                    if (not found):
                        if (post is not None):
                            post += ('%s%s=%s' % (delimiter, name, value))
                        elif (get is not None):
                            get += ('%s%s=%s' % (delimiter, name, value))
                        elif (cookie is not None):
                            cookie += ('%s%s=%s' % ((conf.cookieDel or DEFAULT_COOKIE_DELIMITER), name, value))
    if (not conf.skipUrlEncode):
        get = urlencode(get, limit=True)
    if (post is not None):
        if ((place not in (PLACE.POST, PLACE.CUSTOM_POST)) and hasattr(post, UNENCODED_ORIGINAL_VALUE)):
            post = getattr(post, UNENCODED_ORIGINAL_VALUE)
        elif kb.postUrlEncode:
            post = urlencode(post, spaceplus=kb.postSpaceToPlus)
    if timeBasedCompare:
        if (len(kb.responseTimes.get(kb.responseTimeMode, [])) < MIN_TIME_RESPONSES):
            clearConsoleLine()
            kb.responseTimes.setdefault(kb.responseTimeMode, [])
            if conf.tor:
                warnMsg = "it's    highly    recommended    to    avoid    usage    of    switch    '--tor'    for    "
                warnMsg += 'time-based    injections    because    of    its    high    latency    time'
                singleTimeWarnMessage(warnMsg)
            warnMsg = ('[%s]    [WARNING]    %stime-based    comparison    requires    ' % (time.strftime('%X'), ('(case)    ' if kb.responseTimeMode else '')))
            warnMsg += 'larger    statistical    model,    please    wait'
            dataToStdout(warnMsg)
            while (len(kb.responseTimes[kb.responseTimeMode]) < MIN_TIME_RESPONSES):
                value = (kb.responseTimePayload.replace(RANDOM_INTEGER_MARKER, str(randomInt(6))).replace(RANDOM_STRING_MARKER, randomStr()) if kb.responseTimePayload else kb.responseTimePayload)
                Connect.queryPage(value=value, content=True, raise404=False)
                dataToStdout('.')
            dataToStdout('    (done)\n')
        elif (not kb.testMode):
            warnMsg = 'it    is    very    important    to    not    stress    the    network    connection    '
            warnMsg += 'during    usage    of    time-based    payloads    to    prevent    potential    '
            warnMsg += 'disruptions    '
            singleTimeWarnMessage(warnMsg)
        if (not kb.laggingChecked):
            kb.laggingChecked = True
            deviation = stdev(kb.responseTimes[kb.responseTimeMode])
            if (deviation > WARN_TIME_STDEV):
                kb.adjustTimeDelay = ADJUST_TIME_DELAY.DISABLE
                warnMsg = 'considerable    lagging    has    been    detected    '
                warnMsg += 'in    connection    response(s).    Please    use    as    high    '
                warnMsg += "value    for    option    '--time-sec'    as    possible    (e.g.    "
                warnMsg += '10    or    more)'
                logger.critical(warnMsg)
    if (conf.safeFreq > 0):
        kb.queryCounter += 1
        if ((kb.queryCounter % conf.safeFreq) == 0):
            if conf.safeUrl:
                Connect.getPage(url=conf.safeUrl, post=conf.safePost, cookie=cookie, direct=True, silent=True, ua=ua, referer=referer, host=host)
            elif kb.safeReq:
                Connect.getPage(url=kb.safeReq.url, post=kb.safeReq.post, method=kb.safeReq.method, auxHeaders=kb.safeReq.headers)
    start = time.time()
    if (kb.nullConnection and (not content) and (not response) and (not timeBasedCompare)):
        noteResponseTime = False
        try:
            pushValue(kb.pageCompress)
            kb.pageCompress = False
            if (kb.nullConnection == NULLCONNECTION.HEAD):
                method = HTTPMETHOD.HEAD
            elif (kb.nullConnection == NULLCONNECTION.RANGE):
                auxHeaders[HTTP_HEADER.RANGE] = 'bytes=-1'
            (_, headers, code) = Connect.getPage(url=uri, get=get, post=post, method=method, cookie=cookie, ua=ua, referer=referer, host=host, silent=silent, auxHeaders=auxHeaders, raise404=raise404, skipRead=(kb.nullConnection == NULLCONNECTION.SKIP_READ))
            if headers:
                if ((kb.nullConnection in (NULLCONNECTION.HEAD, NULLCONNECTION.SKIP_READ)) and headers.get(HTTP_HEADER.CONTENT_LENGTH)):
                    pageLength = int(headers[HTTP_HEADER.CONTENT_LENGTH])
                elif ((kb.nullConnection == NULLCONNECTION.RANGE) and headers.get(HTTP_HEADER.CONTENT_RANGE)):
                    pageLength = int(headers[HTTP_HEADER.CONTENT_RANGE][(headers[HTTP_HEADER.CONTENT_RANGE].find('/') + 1):])
        finally:
            kb.pageCompress = popValue()
    if (not pageLength):
        try:
            (page, headers, code) = Connect.getPage(url=uri, get=get, post=post, method=method, cookie=cookie, ua=ua, referer=referer, host=host, silent=silent, auxHeaders=auxHeaders, response=response, raise404=raise404, ignoreTimeout=timeBasedCompare)
        except MemoryError:
            (page, headers, code) = (None, None, None)
            warnMsg = 'site    returned    insanely    large    response'
            if kb.testMode:
                warnMsg += '    in    testing    phase.    This    is    a    common    '
                warnMsg += 'behavior    in    custom    WAF/IPS/IDS    solutions'
            singleTimeWarnMessage(warnMsg)
    if conf.secondOrder:
        (page, headers, code) = Connect.getPage(url=conf.secondOrder, cookie=cookie, ua=ua, silent=silent, auxHeaders=auxHeaders, response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
    threadData.lastQueryDuration = calculateDeltaSeconds(start)
    threadData.lastPage = page
    threadData.lastCode = code
    kb.originalCode = (kb.originalCode or code)
    if kb.testMode:
        kb.testQueryCount += 1
    if timeBasedCompare:
        return wasLastResponseDelayed()
    elif noteResponseTime:
        kb.responseTimes.setdefault(kb.responseTimeMode, [])
        kb.responseTimes[kb.responseTimeMode].append(threadData.lastQueryDuration)
    if ((not response) and removeReflection):
        page = removeReflectiveValues(page, payload)
    kb.maxConnectionsFlag = (re.search(MAX_CONNECTIONS_REGEX, (page or ''), re.I) is not None)
    kb.permissionFlag = (re.search(PERMISSION_DENIED_REGEX, (page or ''), re.I) is not None)
    if (content or response):
        return (page, headers)
    if getRatioValue:
        return (comparison(page, headers, code, getRatioValue=False, pageLength=pageLength), comparison(page, headers, code, getRatioValue=True, pageLength=pageLength))
    else:
        return comparison(page, headers, code, getRatioValue, pageLength)
