'Visits the next link in Crawler._queue.
If the link is on a domain recently visited (< Crawler.delay) it is skipped.
Parses the content at the link for new links and adds them to the queue,
according to their Crawler.priority().
Visited links (and content) are passed to Crawler.visit().'

def crawl(self, method=DEPTH, **kwargs):

    link = self.pop()
    if (link is None):
        return False
    if (link.url not in self.visited):
        t = time.time()
        url = URL(link.url)
        if (url.mimetype == 'text/html'):
            try:
                kwargs.setdefault('unicode', True)
                html = url.download(**kwargs)
                for new in self.parse(html, url=link.url):
                    new.url = abs(new.url, base=(url.redirect or link.url))
                    new.url = self.normalize(new.url)
                    if (new.url == link.url):
                        continue
                    if (new.url in self.visited):
                        continue
                    if (new.url in self._queued):
                        continue
                    if (self.follow(new) is False):
                        continue
                    if (self.domains and (not base(new.url).endswith(tuple(self.domains)))):
                        continue
                    if ((self.QUEUE is not None) and ((self.QUEUE * 1.25) < len(self._queue))):
                        self._queue = self._queue[:self.QUEUE]
                        self._queued.clear()
                        self._queued.update(dict(((q[2].url, True) for q in self._queue)))
                    self.push(new, priority=self.priority(new, method=method), sort=self.sort)
                self.visit(link, source=html)
            except URLError:
                self.fail(link)
        else:
            self.fail(link)
        self.history[base(link.url)] = t
        self.visited[link.url] = True
        return True
    return False
