'A crawler can be used to browse the web in an automated manner.
It visits the list of starting URLs, parses links from their content, visits those, etc.
- Links can be prioritized by overriding Crawler.priority().
- Links can be ignored by overriding Crawler.follow().
- Each visited link is passed to Crawler.visit(), which can be overridden.'

def __init__(self, links=[], domains=[], delay=20.0, parse=HTMLLinkParser().parse, sort=FIFO):

    self.parse = parse
    self.delay = delay
    self.domains = domains
    self.history = {}
    self.visited = {}
    self._queue = []
    self._queued = {}
    self.QUEUE = 10000
    self.sort = sort
    for link in ((isinstance(links, basestring) and [links]) or links):
        self.push(link, priority=1.0, sort=FIFO)
