 http.client kernel tools.time sets assocs sequences
 concurrency.combinators io threads namespaces math multiline
 math.parser inspector urls logging combinators.short-circuit
-continuations calendar prettyprint dlists deques locals ;
+continuations calendar prettyprint dlists deques locals
+present ;
 IN: spider
 
 TUPLE: spider base count max-count sleep max-depth initial-links
-filters spidered todo nonmatching quiet ;
+filters spidered todo nonmatching quiet currently-spidering ;
 
 TUPLE: spider-result url depth headers fetch-time parsed-html
 links processing-time timestamp ;


: run-spider-loop ( spider -- )
    dup spider-page? [
        [ spider-next-page ] [ run-spider-loop ] bi
    ] [
        drop
    ] if ;

    [ add-nonmatching ]
    [ tuck [ apply-filters ] 2dip add-todo ] 2bi ;

: url-absolute? ( url -- ? )
    present "http://" head? ;

: normalize-hrefs ( links spider -- links' )
    currently-spidering>> present swap
    [ dup url-absolute? [ derive-url ] [ url-append-path >url ] if ] with map ;

: print-spidering ( url depth -- )
    "depth: " write number>string write
