nl_tokens: Replace each instance of non - ascii characters by one two three or four escape sequences representing the UTF - 8 encoding of the character and return the expanded char code list .
pl_tokens: function getExpandedCharCodes ( input ) { var output = [ ] ; var length = input . length ; for ( var i = 0 ; i < length ; i ++ ) { var charCode = <mask> . charCodeAt ( i ) ; // handel utf8 if ( charCode < 0x80 ) { output . push ( charCode ) ; } else if ( charCode < 0x800 ) { output . push ( ( charCode >> 6 ) | 192 ) ; output . push ( ( charCode & 63 ) | 128 ) ; } else if ( charCode < 0x10000 ) { output . push ( ( charCode >> 12 ) | 224 ) ; output . push ( ( ( charCode >> 6 ) & 63 ) | 128 ) ; output . push ( ( charCode & 63 ) | 128 ) ; } else if ( charCode < 0x20000 ) { output . push ( ( charCode >> 18 ) | 240 ) ; output . push ( ( ( charCode >> 12 ) & 63 ) | 128 ) ; output . push ( ( ( charCode >> 6 ) & 63 ) | 128 ) ; output . push ( ( charCode & 63 ) | 128 ) ; } else { ts . Debug . assert ( false , "Unexpected code point" ) ; } } return output ; }