'Scrape data from MDN sources.'

def scrape(self):

    if (not self.sources):
        logger.warn(u'No    sources    to    scrape.')
        return self.sources
    first = True
    repeat = False
    cycle = 0
    state_counts = OrderedDict(((state, 0) for state in Source.STATES))
    state_counts[Source.STATE_INIT] = len(self.sources)
    while (first or repeat):
        first = False
        repeat = False
        source_total = ((len(self.sources) - state_counts[Source.STATE_DONE]) - state_counts[Source.STATE_ERROR])
        last_counts = state_counts
        state_counts = OrderedDict(((state, 0) for state in Source.STATES))
        new_sources = []
        cycle += 1
        source_num = 0
        for (source_key, source) in reversed(self.sources.items()):
            if (source.state in (Source.STATE_DONE, Source.STATE_ERROR)):
                state_counts[source.state] += 1
                continue
            source_num += 1
            dependencies = source.gather(self.requester, self.storage)
            new_sources.extend(dependencies)
            dep_count = len(dependencies)
            state_counts[source.state] += 1
            for dep in dependencies:
                if (u'%' in dep[1]):
                    logger.warn(u'Source    "%s"    has    a    percent    in    deps', source_key)
            log_func = logger.debug
            err_msg = u''
            if (source.state == Source.STATE_DONE):
                log_fmt = self._report_done
            elif (source.state == Source.STATE_ERROR):
                log_func = logger.warn
                err_msg = (u'"%s"' % source.error)
                log_fmt = self._report_error
            else:
                repeat = True
                log_fmt = self._report_progress
            log_func(log_fmt, {u'cycle': cycle, u'source_num': source_num, u'source_total': source_total, u'source_key': source_key, u'err_msg': err_msg, u'state': source.state, u'freshness': source.freshness, u'dep_count': dep_count, u'dep_s': (u'' if (dep_count == 1) else u's')})
            if (source.state not in (Source.STATE_DONE, Source.STATE_ERROR)):
                for (num, dep) in enumerate(dependencies):
                    logger.debug(u'*    Dep    %d:    %s', (num + 1), dep)
        repeat = (repeat or bool(new_sources))
        for (source_type, source_param, options) in new_sources:
            if self.add_source(source_type, source_param, **options):
                state_counts[Source.STATE_INIT] += 1
        logger.info(u'Scrape    progress,    cycle    %d:    %s', cycle, u',    '.join(((u'%d    %s' % (v, k)) for (k, v) in state_counts.items() if (v > 0))))
        if (last_counts == state_counts):
            logger.warn(u'Dependency    block    detected.    Aborting.')
            return self.sources
    logger.info(u'Scrape    complete.')
    return self.sources
