nl_tokens: Codepage single - byte encodings .
pl_tokens: function ( options ) { // Prepare chars if needed if ( ! options . chars || ( options . chars . length !== 128 && options . chars . length !== 256 ) ) throw new Error ( "Encoding '" + options . type + "' has incorrect 'chars' (must be of len 128 or 256)" ) ; if ( options . chars . length === 128 ) options . chars = asciiString + options . chars ; if ( ! options . charsBuf ) { options . charsBuf = new Buffer ( options . chars , 'ucs2' ) ; } if ( ! options . revCharsBuf ) { options . revCharsBuf = <mask> Buffer ( 65536 ) ; var defChar = iconv . defaultCharSingleByte . charCodeAt ( 0 ) ; for ( var i = 0 ; i < options . revCharsBuf . length ; i ++ ) options . revCharsBuf [ i ] = defChar ; for ( var i = 0 ; i < options . chars . length ; i ++ ) options . revCharsBuf [ options . chars . charCodeAt ( i ) ] = i ; } return { toEncoding : function ( str ) { str = ensureString ( str ) ; var buf = new Buffer ( str . length ) ; var revCharsBuf = options . revCharsBuf ; for ( var i = 0 ; i < str . length ; i ++ ) buf [ i ] = revCharsBuf [ str . charCodeAt ( i ) ] ; return buf ; } , fromEncoding : function ( buf ) { buf = ensureBuffer ( buf ) ; // Strings are immutable in JS -> we use ucs2 buffer to speed up computations. var charsBuf = options . charsBuf ; var newBuf = new Buffer ( buf . length * 2 ) ; var idx1 = 0 , idx2 = 0 ; for ( var i = 0 , _len = buf . length ; i < _len ; i ++ ) { idx1 = buf [ i ] * 2 ; idx2 = i * 2 ; newBuf [ idx2 ] = charsBuf [ idx1 ] ; newBuf [ idx2 + 1 ] = charsBuf [ idx1 + 1 ] ; } return newBuf . toString ( 'ucs2' ) ; } } ; }