nl_tokens: Generate input for one line of a simple robots . txt file Does not support wildcards .
pl_tokens: function oneline ( line , options ) { var <mask> = "Allow: " , index = line . indexOf ( key ) ; if ( index !== - 1 ) { var page = line . substr ( index + key . length ) . replace ( / ^\s+|\s+$ / g , "" ) ; return page . indexOf ( "*" ) === - 1 && base . input ( options , page ) ; } return true ; }