     [ add-nonmatching ]
     [ tuck [ apply-filters ] 2dip add-todo ] 2bi ;
 
-: url-absolute? ( url -- ? )
-    present "http://" head? ;
-
-: normalize-hrefs ( links spider -- links' )
-    currently-spidering>> present swap
-    [ [ >url ] bi@ derive-url ] with map ;
+: normalize-hrefs ( base links -- links' )
+    [ derive-url ] with map ;
 
 : print-spidering ( url depth -- )
     "depth: " write number>string write


\ spider-page ERROR add-error-logging

: spider-sleep ( spider -- )
    sleep>> [ sleep ] when* ;

:: queue-initial-links ( spider -- spider )
    spider initial-links>> spider normalize-hrefs 0 spider add-todo spider ;

: spider-page? ( spider -- ? )
    {

http.client kernel tools.time sets assocs sequences
concurrency.combinators io threads namespaces math multiline
math.parser inspector urls logging combinators.short-circuit
continuations calendar prettyprint dlists deques locals ;
IN: spider

TUPLE: spider base count max-count sleep max-depth initial-links
filters spidered todo nonmatching quiet currently-spidering
#threads follow-robots ;

TUPLE: spider-result url depth headers
fetched-in parsed-html links processed-in fetched-at ;

TUPLE: todo-url url depth ;

