 
 \ spider-page ERROR add-error-logging
 
-: spider-sleep ( spider -- )
-    sleep>> [ sleep ] when* ;
+: spider-sleep ( spider -- ) sleep>> [ sleep ] when* ;
 
-:: queue-initial-links ( spider -- spider )
-    spider initial-links>> spider normalize-hrefs 0 spider add-todo spider ;
+: queue-initial-links ( spider -- )
+    [
+        [ currently-spidering>> ] [ initial-links>> ] bi normalize-hrefs 0
+    ] keep add-todo ;
 
 : spider-page? ( spider -- ? )
     {

        0 >>max-depth
        0 >>count
        1/0. >>max-count
        H{ } clone >>spidered ;

<PRIVATE


    f url spider spidered>> set-at
    [ url http-get ] benchmark :> fetch-time :> html :> headers
    [
        html parse-html
        spider currently-spidering>>
        over find-all-links normalize-hrefs
    ] benchmark :> processing-time :> links :> parsed-html
    url depth headers fetch-time parsed-html links processing-time
    now spider-result boa ;
