text: import aiohttp
  import asyncio
  import time
  from bs4 import BeautifulSoup
  from urllib.request import urljoin
  import re
  import multiprocessing as mp
  
  base_url = "https://mofanpy.com/"
  # base_url = "http://127.0.0.1:4000/"
  
  # DON'T OVER CRAWL THE WEBSITE OR YOU MAY NEVER VISIT AGAIN
  if base_url != "http://127.0.0.1:4000/":
      restricted_crawl = True
  else:
      restricted_crawl = False
  
  seen = set()
  unseen = set([base_url])
  
  
  def parse(html):
      soup = BeautifulSoup(html, 'html.parser')
      urls = soup.find_all('a', {"href": re.compile('^/.+?/$')})
      title = soup.find('h1').get_text().strip()
      page_urls = set([urljoin(base_url, url['href']) for url in urls])
      url = soup.find('meta', {'property': "og:url"})['content']
      return title, page_urls, url
  
  
  async def crawl(url, session):
      r = await session.get(url)
      html = await r.text()
      await asyncio.sleep(0.1)        # slightly delay for downloading
      return html
  
  
  async def main(loop):
      pool = mp.Pool(2)               # slightly affected
      async with aiohttp.ClientSession() as session:
          count = 1
          while len(unseen) != 0:
              if restricted_crawl and len(seen) > 20:
                  break
              tasks = [loop.create_task(crawl(url, session)) for url in unseen]
              finished, unfinished = await asyncio.wait(tasks)
              htmls = [f.result() for f in finished]
  
              parse_jobs = [pool.apply_async(parse, args=(html,)) for html in htmls]
              results = [j.get() for j in parse_jobs]
  
              seen.update(unseen)
              unseen.clear()
              for title, page_urls, url in results:
                  print(count, title, url)
                  unseen.update(page_urls - seen)
                  count += 1
  
  if __name__ == "__main__":
      t1 = time.time()
      loop = asyncio.get_event_loop()
      loop.run_until_complete(main(loop))
      loop.close()
      print("Async total time: ", time.time() - t1)