'Get the robots.txt file and parse it.
:param fuzzable_request: A fuzzable_request instance that contains
(among other things) the URL to test.'

@runonce(exc_class=RunOnce)
def crawl(self, fuzzable_request):

    dirs = []
    base_url = fuzzable_request.get_url().base_url()
    robots_url = base_url.url_join('robots.txt')
    http_response = self._uri_opener.GET(robots_url, cache=True)
    if (not is_404(http_response)):
        desc = 'A    robots.txt    file    was    found    at:    "%s",    this    file    might    expose    private    URLs    and    requires    a    manual    review.    The    scanner    will    add    all    URLs    listed    in    this    files    to    the    analysis    queue.'
        desc = (desc % robots_url)
        i = Info('robots.txt    file', desc, http_response.id, self.get_name())
        i.set_url(robots_url)
        kb.kb.append(self, 'robots.txt', i)
        om.out.information(i.get_desc())
        dirs.append(robots_url)
        for line in http_response.get_body().split('\n'):
            line = line.strip()
            if ((len(line) > 0) and (line[0] != '#') and ((line.upper().find('ALLOW') == 0) or (line.upper().find('DISALLOW') == 0))):
                url = line[(line.find(':') + 1):]
                url = url.strip()
                try:
                    url = base_url.url_join(url)
                except:
                    pass
                else:
                    dirs.append(url)
    self.worker_pool.map(self.http_get_and_parse, dirs)
