'Get the urllist.txt file and parse it.
:param fuzzable_request: A fuzzable_request instance that contains
(among other things) the URL to test.'

@runonce(exc_class=RunOnce)
def crawl(self, fuzzable_request):

    base_url = fuzzable_request.get_url().base_url()
    urllist_url = base_url.url_join('urllist.txt')
    http_response = self._uri_opener.GET(urllist_url, cache=True)
    if (not is_404(http_response)):
        if self._is_urllist_txt(base_url, http_response.get_body()):
            desc = 'A    urllist.txt    file    was    found    at:    "%s",    this    file    might    expose    private    URLs    and    requires    a    manual    review.    The    scanner    will    add    all    URLs    listed    in    this    files    to    the    analysis    queue.'
            desc = (desc % urllist_url)
            i = Info('urllist.txt    file', desc, http_response.id, self.get_name())
            i.set_url(urllist_url)
            kb.kb.append(self, 'urllist.txt', i)
            om.out.information(i.get_desc())
        url_generator = self._extract_urls_generator(base_url, http_response.get_body())
        self.worker_pool.map(self.http_get_and_parse, url_generator)
