'Returns the next Link queued to visit and removes it from the queue.
Links on a recently visited (sub)domain are skipped until Crawler.delay has elapsed.'

def pop(self, remove=True):

    now = time.time()
    for (i, (priority, dt, link)) in enumerate(self._queue):
        if (self.delay <= (now - self.history.get(base(link.url), 0))):
            if (remove is True):
                self._queue.pop(i)
                self._queued.pop(link.url, None)
            return link
