lib/input-generators/robots.js
function getRobotsUrl (options, callback) {
  request({
    url: options.source,
    timeout: options.timeout()
  }, function (err, res, body) {
    var error = err || common.checkResponse(res, "text/plain");

    if (error) {
      callback(common.prependMsgToErr(error, options.source, true));
    } else {
      body.toString().split('\n').every(function(line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          error = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });

      callback(error);
    }
  });
}
function getRobotsUrl (options, callback) {
  request({
    url: options.source,
    timeout: options.timeout()
  }, function (err, res, body) {
    var error = err || common.checkResponse(res, "text/plain");

    if (error) {
      callback(common.prependMsgToErr(error, options.source, true));
    } else {
      body.toString().split('\n').every(function(line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          error = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });

      callback(error);
    }
  });
}
function getRobotsUrl ( options , callback ) { request ( { url : options . source , timeout : options . timeout ( ) } , function ( err , res , body ) { var error = err || common . checkResponse ( res , "text/plain" ) ; if ( error ) { callback ( common . prependMsgToErr ( error , options . source , true ) ) ; } else { body . toString ( ) . split ( '\n' ) . every ( function ( line ) { // Process the line input, but break if base.input returns false. // For now, this can only happen if no outputDir is defined, //   which is a fatal bad option problem and will happen immediately. if ( ! oneline ( line , options ) ) { error = common . prependMsgToErr ( base . generatorError ( ) , line , true ) ; return false ; } return true ; } ) ; callback ( error ) ; } } ) ; }
Retrieves robots.txt from url and parses it.

@param {Object} options - Robots.txt options
@param {String} options.source - The URL to a robots.txt
@param {Function} options.timeout - Returns the robots.txt request timeout.
@param {Function} callback - A completion callback.
Retrieves robots . txt from url and parses it .