'Perform a classic web spidering process.
:param url_list: The list of URL strings
:param max_depth: The max link depth that we have to follow.
:param domain: The domain name we are checking'

def _spider_archive(self, url_list, max_depth, domain):

    res = []
    def spider_worker(url, max_depth, domain):
        if (url in self._already_crawled):
            return []
        self._already_crawled.add(url)
        try:
            http_response = self._uri_opener.GET(url, cache=True)
        except:
            return []
        url_regex_str = (self.INTERESTING_URLS_RE % domain)
        matched_urls = re.findall(url_regex_str, http_response.body)
        new_urls = [URL(u) for u in matched_urls]
        new_urls = [u.remove_fragment() for u in new_urls]
        new_urls = set(new_urls)
        if ((max_depth - 1) > 0):
            if new_urls:
                res.extend(new_urls)
                res.extend(self._spider_archive(new_urls, (max_depth - 1), domain))
        else:
            msg = 'Some    sections    of    the    archive.org    site    were    not    analyzed'
            msg += '    because    of    the    configured    max_depth.'
            om.out.debug(msg)
            return new_urls
    args = izip(url_list, repeat(max_depth), repeat(domain))
    self.worker_pool.map_multi_args(spider_worker, args)
    return list(set(res))
