nl_tokens: Default tokenizer . This function uses breaks the input on substring match of any string from the opts . tokenSeparators array and uses opts . createSearchChoice to create the choice object . Both of those two options have to be defined in order for the tokenizer to work .
pl_tokens: function defaultTokenizer ( input , selection , selectCallback , opts ) { var original = input , // store the original so we can compare and know if we need to tell the search to update its text dupe = false , // check for whether a token we extracted represents a duplicate selected choice token , // token index , // position at which the separator was found i , l , // looping variables separator ; // the matched separator if ( ! opts . createSearchChoice || ! opts . tokenSeparators || opts . tokenSeparators . length < 1 ) return undefined ; while ( true ) { index = - 1 ; for ( i = 0 , l = opts . tokenSeparators . length ; i < l ; i ++ ) { separator = opts . tokenSeparators [ i ] ; index = input . indexOf ( separator ) ; if ( <mask> >= 0 ) break ; } if ( index < 0 ) break ; // did not find any token separator in the input string, bail token = input . substring ( 0 , index ) ; input = input . substring ( index + separator . length ) ; if ( token . length > 0 ) { token = opts . createSearchChoice . call ( this , token , selection ) ; if ( token !== undefined && token !== null && opts . id ( token ) !== undefined && opts . id ( token ) !== null ) { dupe = false ; for ( i = 0 , l = selection . length ; i < l ; i ++ ) { if ( equal ( opts . id ( token ) , opts . id ( selection [ i ] ) ) ) { dupe = true ; break ; } } if ( ! dupe ) selectCallback ( token ) ; } } } if ( original !== input ) return input ; }