'HTTP fetcher'

@gen.coroutine
def http_fetch(self, url, task):

    start_time = time.time()
    self.on_fetch(u'http', task)
    handle_error = (lambda x: self.handle_error(u'http', url, task, start_time, x))
    fetch = self.pack_tornado_request_parameters(url, task)
    task_fetch = task.get(u'fetch', {})
    session = cookies.RequestsCookieJar()
    if (u'Cookie' in fetch[u'headers']):
        c = http_cookies.SimpleCookie()
        try:
            c.load(fetch[u'headers'][u'Cookie'])
        except AttributeError:
            c.load(utils.utf8(fetch[u'headers'][u'Cookie']))
        for key in c:
            session.set(key, c[key])
        del fetch[u'headers'][u'Cookie']
    if (u'cookies' in fetch):
        session.update(fetch[u'cookies'])
        del fetch[u'cookies']
    max_redirects = task_fetch.get(u'max_redirects', 5)
    fetch[u'follow_redirects'] = False
    while True:
        if task_fetch.get(u'robots_txt', False):
            can_fetch = (yield self.can_fetch(fetch[u'headers'][u'User-Agent'], fetch[u'url']))
            if (not can_fetch):
                error = tornado.httpclient.HTTPError(403, u'Disallowed    by    robots.txt')
                raise gen.Return(handle_error(error))
        try:
            request = tornado.httpclient.HTTPRequest(**fetch)
            old_cookie_header = request.headers.get(u'Cookie')
            if old_cookie_header:
                del request.headers[u'Cookie']
            cookie_header = cookies.get_cookie_header(session, request)
            if cookie_header:
                request.headers[u'Cookie'] = cookie_header
            elif old_cookie_header:
                request.headers[u'Cookie'] = old_cookie_header
        except Exception as e:
            logger.exception(fetch)
            raise gen.Return(handle_error(e))
        try:
            response = (yield gen.maybe_future(self.http_client.fetch(request)))
        except tornado.httpclient.HTTPError as e:
            if e.response:
                response = e.response
            else:
                raise gen.Return(handle_error(e))
        extract_cookies_to_jar(session, response.request, response.headers)
        if ((response.code in (301, 302, 303, 307)) and response.headers.get(u'Location') and task_fetch.get(u'allow_redirects', True)):
            if (max_redirects <= 0):
                error = tornado.httpclient.HTTPError(599, (u'Maximum    (%d)    redirects    followed' % task_fetch.get(u'max_redirects', 5)), response)
                raise gen.Return(handle_error(error))
            if (response.code in (302, 303)):
                fetch[u'method'] = u'GET'
                if (u'body' in fetch):
                    del fetch[u'body']
            fetch[u'url'] = quote_chinese(urljoin(fetch[u'url'], response.headers[u'Location']))
            fetch[u'request_timeout'] -= (time.time() - start_time)
            if (fetch[u'request_timeout'] < 0):
                fetch[u'request_timeout'] = 0.1
            max_redirects -= 1
            continue
        result = {}
        result[u'orig_url'] = url
        result[u'content'] = (response.body or u'')
        result[u'headers'] = dict(response.headers)
        result[u'status_code'] = response.code
        result[u'url'] = (response.effective_url or url)
        result[u'time'] = (time.time() - start_time)
        result[u'cookies'] = session.get_dict()
        result[u'save'] = task_fetch.get(u'save')
        if response.error:
            result[u'error'] = utils.text(response.error)
        if (200 <= response.code < 300):
            logger.info(u'[%d]    %s:%s    %s    %.2fs', response.code, task.get(u'project'), task.get(u'taskid'), url, result[u'time'])
        else:
            logger.warning(u'[%d]    %s:%s    %s    %.2fs', response.code, task.get(u'project'), task.get(u'taskid'), url, result[u'time'])
        raise gen.Return(result)
