 concurrency.combinators io threads namespaces math multiline
 math.parser inspector urls logging combinators.short-circuit
 continuations calendar prettyprint dlists deques locals
-spider.unique-deque ;
+spider.unique-deque combinators concurrency.semaphores ;
 IN: spider
 
 TUPLE: spider base count max-count sleep max-depth initial-links
 filters spidered todo nonmatching quiet currently-spidering
-#threads follow-robots? robots ;
+#threads semaphore follow-robots? robots ;
 
 TUPLE: spider-result url depth headers
 fetched-in parsed-html links processed-in fetched-at ;

        [ [ count>> ] [ max-count>> ] bi < ]
    } 1&& ;

: setup-next-url ( spider -- spider url depth )
    dup todo>> peek-url url>> >>currently-spidering
    dup todo>> pop-url [ url>> ] [ depth>> ] bi ;

: spider-next-page ( spider -- )
    setup-next-url spider-page ;

: normalize-hrefs ( base links -- links' )
    [ derive-url ] with map ;

: print-spidering ( spider-result -- )
    [ url>> ] [ depth>> ] bi
    "depth: " write number>string write
    ", spidering: " write . yield ;

:: fill-spidered-result ( spider spider-result -- )
    f spider-result url>> spider spidered>> set-at
    [ spider-result url>> http-get ] benchmark :> fetched-in :> html :> headers
    [
        html parse-html
        spider currently-spidering>>
        over find-all-links normalize-hrefs
    ] benchmark :> processed-in :> links :> parsed-html
    spider-result
        headers >>headers
        fetched-in >>fetched-in
        parsed-html >>parsed-html
        links >>links
        processed-in >>processed-in
        now >>fetched-at drop ;

:: spider-page ( spider spider-result -- )
    spider quiet>> [ spider-result print-spidering ] unless
    spider spider-result fill-spidered-result
    spider quiet>> [ spider-result describe ] unless
    spider spider-result add-spidered ;

\ spider-page ERROR add-error-logging

