nl_tokens: Generate the snapshot arguments from a robots . txt file . Each line that has Allow : contains a url we need a snapshot for . This can return true on error for true async . An async error is supplied to listener in this case via _abort .
pl_tokens: function generateInput ( <mask> ) { return nodeCall ( common . isUrl ( options . source ) ? getRobotsUrl : getRobotsFile , options ) . catch ( function ( err ) { options . _abort ( err ) ; } ) . then ( function ( ) { base . EOI ( robots ) ; } ) ; }