         0 >>count
         1/0. >>max-count
         H{ } clone >>spidered
-        1 >>#threads ;
+        1 [ >>#threads ] [ <semaphore> >>semaphore ] bi ;
+
+: <spider-result> ( url depth -- spider-result )
+    spider-result new
+        swap >>depth
+        swap >>url ;
 
 <PRIVATE
 

: normalize-hrefs ( base links -- links' )
    [ derive-url ] with map ;

: print-spidering ( url depth -- )
    "depth: " write number>string write
    ", spidering: " write . yield ;

:: new-spidered-result ( spider url depth -- spider-result )
    f url spider spidered>> set-at
    [ url http-get ] benchmark :> fetched-at :> html :> headers
    [
        html parse-html
        spider currently-spidering>>
        over find-all-links normalize-hrefs
    ] benchmark :> processing-time :> links :> parsed-html
    url depth headers fetched-at parsed-html links processing-time
    now spider-result boa ;

:: spider-page ( spider url depth -- )
    spider quiet>> [ url depth print-spidering ] unless
    spider url depth new-spidered-result :> spidered-result
    spider quiet>> [ spidered-result describe ] unless
    spider spidered-result add-spidered ;

\ spider-page ERROR add-error-logging


        [ [ count>> ] [ max-count>> ] bi < ]
    } 1&& ;

: setup-next-url ( spider -- spider spider-result )
    dup todo>> peek-url url>> >>currently-spidering
    dup todo>> pop-url [ url>> ] [ depth>> ] bi <spider-result> ;

: spider-next-page ( spider -- )
    setup-next-url spider-page ;
