         over >>currently-spidering
         swap 0 <unique-deque> [ push-url ] keep >>todo
         <unique-deque> >>nonmatching
-        0 >>max-depth
-        0 >>count
-        1/0. >>max-count
         H{ } clone >>spidered
-        1 [ >>#threads ] [ <semaphore> >>semaphore ] bi ;
+        1 <semaphore> >>semaphore ;
 
 : <spider-result> ( url depth -- spider-result )
     spider-result new

spider.unique-deque combinators concurrency.semaphores ;
IN: spider

TUPLE: spider base count max-count sleep max-depth initial-links
filters spidered todo nonmatching quiet currently-spidering
#threads semaphore follow-robots? robots ;

TUPLE: spider-result url depth headers
fetched-in parsed-html links processed-in fetched-at ;

: spider-page? ( spider -- ? )
    {
        [ todo>> deque>> deque-empty? not ]
        [ [ todo>> peek-url depth>> ] [ max-depth>> ] bi <= ]
        [ [ count>> ] [ max-count>> ] bi < ]
    } 1&& ;

