text: import re
  import tempfile
  
  from canrevan import DEFAULT_USER_AGENT_STRING
  from canrevan.crawling import Crawler
  
  _dummy_urls = [
      f"https://news.naver.com/main/list.nhn?mode=LSD&mid=shm"
      f"&sid1=100&date=20200501&page={page}"
      for page in range(1, 3)
  ]
  
  
  def _get_current_page_from_html(content: str) -> int:
      content = content[content.find('<div class="paging">') :]
      content = content[: content.find("</div>")]
  
      return int(re.search(r"<strong>(.*?)</strong>", content).group(1))
  
  
  def test_crawl_reduce_array():
      crawler = Crawler(
          concurrent_tasks=500,
          request_headers={"user-agent": DEFAULT_USER_AGENT_STRING},
          request_timeout=1,
      )
      current_pages = crawler.reduce_to_array(
          _dummy_urls, parse_fn=_get_current_page_from_html
      )
  
      for page in current_pages:
          # Note that we only crawled up to 3 pages from each category and date.
          assert isinstance(page, int)
          assert page < 3
  
  
  def test_crawl_reduce_file():
      crawler = Crawler(
          concurrent_tasks=500,
          request_headers={"user-agent": DEFAULT_USER_AGENT_STRING},
          request_timeout=1,
      )
  
      with tempfile.TemporaryDirectory() as tdir:
          written = crawler.reduce_to_file(
              _dummy_urls,
              filename=f"{tdir}/crawled.txt",
              parse_fn=_get_current_page_from_html,
          )
  
          # With a high probability, there must be successfully crawled data.
          assert written > 0
  
          with open(f"{tdir}/crawled.txt", "r") as fp:
              for page in fp:
                  page = int(page.strip())
  
                  # Note that we only crawled up to 3 pages from each category
                  # and date.
                  assert page < 3
  