 : spider-page? ( spider -- ? )
     {
         [ todo>> deque>> deque-empty? not ]
-        [ [ todo>> peek-url depth>> ] [ max-depth>> ] bi < ]
+        [ [ todo>> peek-url depth>> ] [ max-depth>> ] bi <= ]
         [ [ count>> ] [ max-count>> ] bi < ]
     } 1&& ;
 

        now >>fetched-at drop ;

:: spider-page ( spider spider-result -- )
    spider quiet>> [ spider-result print-spidering ] unless
    spider spider-result fill-spidered-result
    spider quiet>> [ spider-result describe ] unless
    spider spider-result add-spidered ;

\ spider-page ERROR add-error-logging

spider.unique-deque combinators concurrency.semaphores ;
IN: spider

TUPLE: spider
    base
    { count integer initial: 0 }
    { max-count number initial: 1/0. }
    sleep
    { max-depth integer initial: 0 }
    initial-links
    filters
    spidered
    todo
    nonmatching
    quiet?
    currently-spidering
    { #threads integer initial: 1 }
    semaphore
    follow-robots?
    robots ;

TUPLE: spider-result url depth headers
fetched-in parsed-html links processed-in fetched-at ;
