nl_tokens: Parse a line in dictionary .
pl_tokens: function parseLine ( line , options , dict ) { var word var codes var result var hashOffset var slashOffset // Find offsets. slashOffset = line . indexOf ( slash ) while ( slashOffset !== - 1 && line . charAt ( slashOffset - 1 ) === backslash ) { line = line . slice ( 0 , slashOffset - 1 ) + line . slice ( slashOffset ) slashOffset = line . indexOf ( slash , slashOffset ) } hashOffset = line . indexOf ( numberSign ) // Handle hash and slash offsets.  Note that hash can be a valid flag, so we // should not just discard all string after it. if ( hashOffset >= 0 ) { if ( slashOffset >= 0 && slashOffset < hashOffset ) { word = line . slice ( 0 , slashOffset ) whiteSpaceExpression . lastIndex = slashOffset + 1 result = whiteSpaceExpression . exec ( line ) codes = line . slice ( slashOffset + 1 , result ? result . <mask> : undefined ) } else { word = line . slice ( 0 , hashOffset ) } } else if ( slashOffset >= 0 ) { word = line . slice ( 0 , slashOffset ) codes = line . slice ( slashOffset + 1 ) } else { word = line } word = trim ( word ) if ( word ) { codes = parseCodes ( options . flags , codes && trim ( codes ) ) add ( dict , word , codes , options ) } }