def querycrawler(self, path=None, curdepth=0, maxdepth=1):
        self.log.debug('Crawler is visiting %s' % path)
        localcrawlpaths = list()
        if curdepth > maxdepth:
            self.log.info('maximum depth %s reached' % maxdepth)
            return
        r = self.request(path=path)
        if r is None:
            return
        response, responsebody = r
        try:
            soup = BeautifulSoup(responsebody)
        except:
            self.log.warn('could not parse the response body')
            return
        tags = soup('a')
        for tag in tags:
            try:
                href = tag["href"]
                if href is not None:
                    tmpu = urlparse(href)
                    if (tmpu[1] != '') and (self.target != tmpu[1]):
                        # not on the same domain name .. ignore
                        self.log.debug('Ignoring link because it is not on the same site %s' % href)
                        continue
                    if tmpu[0] not in ['http', 'https', '']:
                        self.log.debug('Ignoring link because it is not an http uri %s' % href)
                        continue
                    path = tmpu[2]
                    if not path.startswith('/'):
                        path = '/' + path
                    if len(tmpu[4]) > 0:
                        # found a query .. thats all we need                                                
                        location = urlunparse(('', '', path, tmpu[3], tmpu[4], ''))
                        self.log.info('Found query %s' % location)
                        return href
                    if path not in self.crawlpaths:
                        href = unquote(path)
                        self.log.debug('adding %s for crawling' % href)
                        self.crawlpaths.append(href)
                        localcrawlpaths.append(href)
            except __HOLE__:
                pass
        for nextpath in localcrawlpaths:
            r = self.querycrawler(path=nextpath, curdepth=curdepth + 1, maxdepth=maxdepth)
            if r:
                return r
dataset/ETHPy150Open sandrogauci/wafw00f/wafw00f/lib/evillib.py/waftoolsengine.querycrawler