nl_tokens: class Tokenizer
pl_tokens: function ( rules , flag ) { flag = flag ? "g" + flag : "g" ; this . rules = rules ; this . regExps = { } ; this . matchMappings = { } ; for ( var key <mask> this . rules ) { var rule = this . rules [ key ] ; var state = rule ; var ruleRegExps = [ ] ; var matchTotal = 0 ; var mapping = this . matchMappings [ key ] = { } ; for ( var i = 0 ; i < state . length ; i ++ ) { if ( state [ i ] . regex instanceof RegExp ) state [ i ] . regex = state [ i ] . regex . toString ( ) . slice ( 1 , - 1 ) ; // Count number of matching groups. 2 extra groups from the full match // And the catch-all on the end (used to force a match); var matchcount = new RegExp ( "(?:(" + state [ i ] . regex + ")|(.))" ) . exec ( "a" ) . length - 2 ; // Replace any backreferences and offset appropriately. var adjustedregex = state [ i ] . regex . replace ( / \\([0-9]+) / g , function ( match , digit ) { return "\\" + ( parseInt ( digit , 10 ) + matchTotal + 1 ) ; } ) ; if ( matchcount > 1 && state [ i ] . token . length !== matchcount - 1 ) throw new Error ( "Matching groups and length of the token array don't match in rule #" + i + " of state " + key ) ; mapping [ matchTotal ] = { rule : i , len : matchcount } ; matchTotal += matchcount ; ruleRegExps . push ( adjustedregex ) ; } this . regExps [ key ] = new RegExp ( "(?:(" + ruleRegExps . join ( ")|(" ) + ")|(.))" , flag ) ; } }