'For every directory, fetch a list of files and analyze the response.
:param fuzzable_request: A fuzzable_request instance that contains
(among other things) the URL to test.'

def crawl(self, fuzzable_request):

    domain_path = fuzzable_request.get_url().get_domain_path()
    if (domain_path not in self._analyzed_dirs):
        self._analyzed_dirs.add(domain_path)
        test_generator = self._url_generator(domain_path)
        self.worker_pool.map_multi_args(self._send_and_check, test_generator)
