procIfRobotsOK :: Lock -> GASupply -> Connection -> GAddress -> IO () -> IO ()
procIfRobotsOK lock gasupply c item action =
              do r <- if (path item /= "robots.txt")
                          then checkRobots lock gasupply c item
                          else -- Don't try to check if robots.txt itself is ok
                               return RobotsOK
                 case r of
                    RobotsOK -> action -- OK per robots.txt; run it.
                    RobotsDeny -> do msg $ "Excluded by robots.txt: " ++ 
                                             (show item)
                                     updateItem lock c item Excluded ""
                    RobotsError -> do msg $ "Problem getting robots.txt: " ++ 
                                          host item
                                      -- Next line not necessary; procItem
                                      -- on robots.txt will have done it
                                      -- already.
                                      --noteErrorOnHost lock c (host item) (show msg)

-- TODO: better crash handling on robots.txt

{- | OK, we have an item.  If it's OK according to robots.txt, download
and process it. -}
procIfRobotsOK :: Lock -> GASupply -> Connection -> GAddress -> IO () -> IO ()
procIfRobotsOK lock gasupply c item action =
              do r <- if (path item /= "robots.txt")
                          then checkRobots lock gasupply c item
                          else -- Don't try to check if robots.txt itself is ok
                               return RobotsOK
                 case r of
                    RobotsOK -> action -- OK per robots.txt; run it.
                    RobotsDeny -> do msg $ "Excluded by robots.txt: " ++ 
                                             (show item)
                                     updateItem lock c item Excluded ""
                    RobotsError -> do msg $ "Problem getting robots.txt: " ++ 
                                          host item
                                      -- Next line not necessary; procItem
                                      -- on robots.txt will have done it
                                      -- already.
                                      --noteErrorOnHost lock c (host item) (show msg)

-- TODO: better crash handling on robots.txt

{- | OK, we have an item.  If it's OK according to robots.txt, download
and process it. -}
procIfRobotsOK lock gasupply c item action =
              do r <- if (path item /= "robots.txt")
                          then checkRobots lock gasupply c item
                          else -- Don't try to check if robots.txt itself is ok
                               return RobotsOK
                 case r of
                    RobotsOK -> action -- OK per robots.txt; run it.
                    RobotsDeny -> do msg $ "Excluded by robots.txt: " ++ 
                                             (show item)
                                     updateItem lock c item Excluded ""
                    RobotsError -> do msg $ "Problem getting robots.txt: " ++ 
                                          host item
                                      -- Next line not necessary; procItem
                                      -- on robots.txt will have done it
                                      -- already.
                                      --noteErrorOnHost lock c (host item) (show msg)

-- TODO: better crash handling on robots.txt

{- | OK, we have an item.  If it's OK according to robots.txt, download
and process it. -}