-- | This module provides a function for downloading robots.txt of any domain via HTTP.
module YesWeScan.RobotsTxt (robotsTxt) where

import Network.HTTP (simpleHTTP, getRequest, getResponseBody)

-- | Get the contents of a `robots.txt` associated with a URL.
robotsTxt :: String -> IO String
robotsTxt url =
    getResponseBody =<< simpleHTTP
        (getRequest $ url ++
         if last url == '/'
            then []
            else "/" ++ "robots.txt")
