crawl ys curl url
  | ys ^. maxPages <= 0 = return ()
  | otherwise = do
    let settings = ys ^. siteSettings
    verbPrint Low ys ["Crawling", url]
    body <- getInternalPage ys curl url
    case body of
      Nothing -> error $ "Failed to crawl " ++ url
      Just body' -> do
        verbPrint Debug ys ['\n' : body']
        pa@(nextPage, groups) <- parsePage ys body'
        when (ys ^. logVerbosity >= High) (prettyPage pa)
        verbPrint Low ys ["Have", show . sum $ map (length . _torrents) groups
                         , "torrents pre-filter."]
        let filtered' = torrentFilter (_filterFunc settings) groups
        let tPaths = concatMap (buildTorrentPaths settings) filtered'
        verbPrint Low ys ["Have", show $ length tPaths, "torrents post-filter."]
        mapM_ (\(fp, url') -> download fp url' ys) tPaths
        crawl (ys { _maxPages = _maxPages ys - 1 }) curl nextPage

-- | We take settings for the site and a torrent group listing and we try to
-- assign a file path to each torrent in the group to which the download
-- will be made.
crawl ys curl url
  | ys ^. maxPages <= 0 = return ()
  | otherwise = do
    let settings = ys ^. siteSettings
    verbPrint Low ys ["Crawling", url]
    body <- getInternalPage ys curl url
    case body of
      Nothing -> error $ "Failed to crawl " ++ url
      Just body' -> do
        verbPrint Debug ys ['\n' : body']
        pa@(nextPage, groups) <- parsePage ys body'
        when (ys ^. logVerbosity >= High) (prettyPage pa)
        verbPrint Low ys ["Have", show . sum $ map (length . _torrents) groups
                         , "torrents pre-filter."]
        let filtered' = torrentFilter (_filterFunc settings) groups
        let tPaths = concatMap (buildTorrentPaths settings) filtered'
        verbPrint Low ys ["Have", show $ length tPaths, "torrents post-filter."]
        mapM_ (\(fp, url') -> download fp url' ys) tPaths
        crawl (ys { _maxPages = _maxPages ys - 1 }) curl nextPage

-- | We take settings for the site and a torrent group listing and we try to
-- assign a file path to each torrent in the group to which the download
-- will be made.
crawl ys curl url
  | ys ^. maxPages <= 0 = return ()
  | otherwise = do
    let settings = ys ^. siteSettings
    verbPrint Low ys ["Crawling", url]
    body <- getInternalPage ys curl url
    case body of
      Nothing -> error $ "Failed to crawl " ++ url
      Just body' -> do
        verbPrint Debug ys ['\n' : body']
        pa@(nextPage, groups) <- parsePage ys body'
        when (ys ^. logVerbosity >= High) (prettyPage pa)
        verbPrint Low ys ["Have", show . sum $ map (length . _torrents) groups
                         , "torrents pre-filter."]
        let filtered' = torrentFilter (_filterFunc settings) groups
        let tPaths = concatMap (buildTorrentPaths settings) filtered'
        verbPrint Low ys ["Have", show $ length tPaths, "torrents post-filter."]
        mapM_ (\(fp, url') -> download fp url' ys) tPaths
        crawl (ys { _maxPages = _maxPages ys - 1 }) curl nextPage

-- | We take settings for the site and a torrent group listing and we try to
-- assign a file path to each torrent in the group to which the download
-- will be made.