'For every directory, fetch a list of shell files and analyze the
response.
:param fuzzable_request: A fuzzable_request instance that contains
(among other things) the URL to test.'

def crawl(self, fuzzable_request):

    domain_path = fuzzable_request.get_url().get_domain_path()
    if (domain_path not in self._analyzed_dirs):
        self._analyzed_dirs.add(domain_path)
        self.setup()
        web_shells = self._iter_web_shells()
        args_iter = (domain_path.url_join(fname) for fname in web_shells)
        self.worker_pool.map(self._check_if_exists, args_iter)
