 : <unique-deque> ( -- unique-deque )
     H{ } clone <dlist> unique-deque boa ;
 
+: url-exists? ( url unique-deque -- ? )
+    [ url>> ] [ assoc>> ] bi* key? ;
+
 : push-url ( url depth unique-deque -- )
-    [ <todo-url> ] dip
-    [ [ [ t ] dip url>> ] [ assoc>> ] bi* set-at ]
-    [ deque>> push-back ] 2bi ;
+    [ <todo-url> ] dip 2dup url-exists? [
+        2drop
+    ] [
+        [ [ [ t ] dip url>> ] [ assoc>> ] bi* set-at ]
+        [ deque>> push-back ] 2bi
+    ] if ;
 
 : pop-url ( unique-deque -- todo-url ) deque>> pop-front ;
 

    [ add-nonmatching ]
    [ tuck [ apply-filters ] 2dip add-todo ] 2bi ;

: normalize-hrefs ( links spider -- links' )
    [ [ >url ] map ] dip
    base>> swap [ derive-url ] with map ;

: print-spidering ( url depth -- )
    "depth: " write number>string write

http.client kernel tools.time sets assocs sequences
concurrency.combinators io threads namespaces math multiline
math.parser inspector urls logging combinators.short-circuit
continuations calendar prettyprint dlists deques locals
present ;
IN: spider

TUPLE: spider base count max-count sleep max-depth initial-links
filters spidered todo nonmatching quiet currently-spidering ;

TUPLE: spider-result url depth headers fetch-time parsed-html
links processing-time timestamp ;
