'Uses requests to download the feed, parsing it in feedparser. Will be storified later.'

@timelimit(30)
def fetch(self):

    start = time.time()
    identity = self.get_identity()
    log_msg = (u'%2s    --->    [%-30s]    ~FYFetching    feed    (~FB%d~FY),    last    update:    %s' % (identity, self.feed.log_title[:30], self.feed.id, (datetime.datetime.now() - self.feed.last_update)))
    logging.debug(log_msg)
    etag = self.feed.etag
    modified = (self.feed.last_modified.utctimetuple()[:7] if self.feed.last_modified else None)
    address = self.feed.feed_address
    if (self.options.get('force') or (random.random() <= 0.01)):
        self.options['force'] = True
        modified = None
        etag = None
        address = qurl(address, add={'_': random.randint(0, 10000)})
        logging.debug((u'            --->    [%-30s]    ~FBForcing    fetch:    %s' % (self.feed.log_title[:30], address)))
    elif ((not self.feed.fetched_once) or (not self.feed.known_good)):
        modified = None
        etag = None
    if self.options.get('feed_xml'):
        logging.debug((u'            --->    [%-30s]    ~FM~BKFeed    has    been    fat    pinged.    Ignoring    fat:    %s' % (self.feed.log_title[:30], len(self.options.get('feed_xml')))))
    if self.options.get('fpf'):
        self.fpf = self.options.get('fpf')
        logging.debug((u'            --->    [%-30s]    ~FM~BKFeed    fetched    in    real-time    with    fat    ping.' % self.feed.log_title[:30]))
        return (FEED_OK, self.fpf)
    if ('youtube.com' in address):
        try:
            youtube_feed = self.fetch_youtube(address)
        except requests.adapters.ConnectionError:
            youtube_feed = None
        if (not youtube_feed):
            logging.debug((u'            ***>    [%-30s]    ~FRYouTube    fetch    failed:    %s.' % (self.feed.log_title[:30], address)))
            return (FEED_ERRHTTP, None)
        self.fpf = feedparser.parse(youtube_feed)
    elif re.match('(https?)?://twitter.com/\\w+/?$', qurl(address, remove=['_'])):
        twitter_feed = self.fetch_twitter(address)
        if (not twitter_feed):
            logging.debug((u'            ***>    [%-30s]    ~FRTwitter    fetch    failed:    %s' % (self.feed.log_title[:30], address)))
            return (FEED_ERRHTTP, None)
        self.fpf = feedparser.parse(twitter_feed)
    if (not self.fpf):
        try:
            headers = self.feed.fetch_headers()
            if etag:
                headers['If-None-Match'] = etag
            if modified:
                short_weekdays = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
                months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
                modified_header = ('%s,    %02d    %s    %04d    %02d:%02d:%02d    GMT' % (short_weekdays[modified[6]], modified[2], months[(modified[1] - 1)], modified[0], modified[3], modified[4], modified[5]))
                headers['If-Modified-Since'] = modified_header
            if (etag or modified):
                headers['A-IM'] = 'feed'
            raw_feed = requests.get(address, headers=headers)
            if (raw_feed.status_code >= 400):
                logging.debug(('            ***>    [%-30s]    ~FRFeed    fetch    was    %s    status    code,    trying    fake    user    agent:    %s' % (self.feed.log_title[:30], raw_feed.status_code, raw_feed.headers)))
                raw_feed = requests.get(address, headers=self.feed.fetch_headers(fake=True))
            if (raw_feed.content and ('application/json' in raw_feed.headers.get('Content-Type', ''))):
                json_feed = self.fetch_json_feed(address, raw_feed)
                if (not json_feed):
                    logging.debug((u'            ***>    [%-30s]    ~FRJSON    fetch    failed:    %s' % (self.feed.log_title[:30], address)))
                    return (FEED_ERRHTTP, None)
                self.fpf = feedparser.parse(json_feed)
            elif (raw_feed.content and (raw_feed.status_code < 400)):
                response_headers = raw_feed.headers
                response_headers['Content-Location'] = raw_feed.url
                self.raw_feed = smart_unicode(raw_feed.content)
                self.fpf = feedparser.parse(self.raw_feed, response_headers=response_headers)
                if self.options.get('debug', False):
                    logging.debug(('    --->    [%-30s]    ~FBFeed    fetch    status    %s:    %s    length    /    %s' % (self.feed.log_title[:30], raw_feed.status_code, len(smart_unicode(raw_feed.content)), raw_feed.headers)))
        except Exception as e:
            logging.debug(('            ***>    [%-30s]    ~FRFeed    failed    to    fetch    with    request,    trying    feedparser:    %s' % (self.feed.log_title[:30], unicode(e)[:100])))
        if ((not self.fpf) or self.options.get('force_fp', False)):
            try:
                self.fpf = feedparser.parse(address, agent=self.feed.user_agent, etag=etag, modified=modified)
            except (TypeError, ValueError, KeyError, EOFError, MemoryError) as e:
                logging.debug((u'            ***>    [%-30s]    ~FRFeed    fetch    error:    %s' % (self.feed.log_title[:30], e)))
                pass
    if (not self.fpf):
        try:
            logging.debug((u'            ***>    [%-30s]    ~FRTurning    off    headers...' % self.feed.log_title[:30]))
            self.fpf = feedparser.parse(address, agent=self.feed.user_agent)
        except (TypeError, ValueError, KeyError, EOFError, MemoryError) as e:
            logging.debug((u'            ***>    [%-30s]    ~FRFetch    failed:    %s.' % (self.feed.log_title[:30], e)))
            return (FEED_ERRHTTP, None)
    logging.debug((u'            --->    [%-30s]    ~FYFeed    fetch    in    ~FM%.4ss' % (self.feed.log_title[:30], (time.time() - start))))
    return (FEED_OK, self.fpf)
