 http.client kernel tools.time sets assocs sequences
 concurrency.combinators io threads namespaces math multiline
 math.parser inspector urls logging combinators.short-circuit
-continuations calendar prettyprint dlists deques locals ;
+continuations calendar prettyprint dlists deques locals
+present ;
 IN: spider
 
 TUPLE: spider base count max-count sleep max-depth initial-links
-filters spidered todo nonmatching quiet ;
+filters spidered todo nonmatching quiet currently-spidering ;
 
 TUPLE: spider-result url depth headers fetch-time parsed-html
 links processing-time timestamp ;

: <unique-deque> ( -- unique-deque )
    H{ } clone <dlist> unique-deque boa ;

: push-url ( url depth unique-deque -- )
    [ <todo-url> ] dip
    [ [ [ t ] dip url>> ] [ assoc>> ] bi* set-at ]
    [ deque>> push-back ] 2bi ;

: pop-url ( unique-deque -- todo-url ) deque>> pop-front ;


    f url spider spidered>> set-at
    [ url http-get ] benchmark :> fetch-time :> html :> headers
    [
        html parse-html [ ] [ find-all-links spider normalize-hrefs ] bi
    ] benchmark :> processing-time :> links :> parsed-html
    url depth headers fetch-time parsed-html links processing-time
    now spider-result boa ;
