def spider(init, max=-1, ignore_qs=False, post_func=None,
           excluded_func=None, hosts=None):
  """
  Spider a request by following some links.

  init    - The initial request(s)
  max       - The maximum of request to execute
  post_func - A hook to be executed after each new page fetched
  hosts     - A lists of authorised hosts to spider on. By default
              only the hostname of r_init is allowed.
  excluded_func - A predicate that must indicates if a Request should
                  be executed.
  """
  nb = 0
  checked = []
  if isinstance(init, Request):
    q = deque([init, ])
    hs = [ init.hostname, ]
  elif isinstance(init, RequestSet):
    q = deque(init)
    hs = list(set(init.extract("hostname")))
  else:
    raise TypeError("init must be a Request or a RequestSet")
  if hosts:
    hs += hosts
  try:
    while nb != max and q:
      to_add = []
      r = q.popleft()
      print str(len(checked)) + "/" + str(len(q)),
      clear_line()
      if not r.response:
        r()
      if r.response.content_type:
        if re.match(r'text/html', r.response.content_type):
          to_add += _follow_redirect(r)
          to_add += _get_links(r)
        else:
          print "\nIgnoring", r.response.content_type
      checked.append(r)
      if post_func:
        post_func(r)
      for nr in to_add:
        if nr.hostname not in hs:
          continue
        if excluded_func and excluded_func(nr):
          continue
        if not ignore_qs and any(nr == rc for rc in checked + list(q)):
          continue
        if ignore_qs and any(nr.similar(rc) for rc in checked + list(q)):
          continue
        q.append(nr)
      nb += 1
  except __HOLE__:
    print str(len(checked)) + "/" + str(len(q))
  return RequestSet(checked)
dataset/ETHPy150Open tweksteen/burst/burst/spider.py/spider