import { arrayLast } from "./exports" ; module export type TokenType = ( '::' | ':' | '=' | ':=' | '#' | '.' | ',' | ';' | '{' | '}' | '(' | ')' | '=|>' | 'on-update' | 'write' | '->' | '<-' | 'updatable' | '[' | ']' | '[]' | 'string' | 'number' | '###' | 'name' | 'end' | '\n' | 'call' | 'arg1' | 'arg2' | 'input' | 'record' | 'choice' | 'table' | 'array' | 'do' | 'builtin' | 'anything' | 'nil' | 'try' | 'check' | 'not' | 'else' | 'reject' | 'let' | 'export' | 'that' | 'include' | 'with' | 'find?' | 'find!' | 'tracked' | 'for-all' | 'such-that' | 'all?' | 'all!' | 'none?' | 'none!' | 'accumulate' | 'extend' | 'selection' | 'any' | 'selecting' | 'link' | 'via' | 'register' | '::replace' | '::insert' | '::append' | '::convert' | '::delete' | '::move' | '::move-insert' | '::move-append' | '::wrap-record' | '::wrap-array' | '::unwrap' | '::nochange' ) ; export class Token { constructor ( readonly type , readonly start , readonly end , readonly source ) { } /** Copy an existing token giving a different type */ static fake ( type , token ) { return new Token ( type , token . start , token . end , token . source ) ; } /** start of a character literal, including the single quote */ static readonly characterLiteralStart = "character'" ; get text ( ) { return this . toString ( ) ; } ; toString ( ) { return this . source . substring ( this . start , this . end ) ; } /** Hack to special case parsing of operator names */ get isOperator ( ) : boolean { if ( this . type !== 'name' ) return false ; if ( this . text . startsWith ( 'not=' ) ) return true ; switch ( this . text [ 0 ] ) { case '+' : case '-' : case '*' : case '/' : case '=' : case '<' : case '>' : case '&' : case '&&' : return true ; default : return false ; } } } throws export function tokenize ( source ) { /** Scanning cursor index in source */ let cursor = 0 ; let tokens = [ ] ; while ( cursor < source . length ) { let char = source . charCodeAt ( cursor ) ; if ( newline ( char ) ) { if ( tokens . length && arrayLast ( tokens ) . type !== '\n' ) { tokens . push ( new Token ( '\n' , cursor - 1 , cursor , source ) ) ; } cursor ++ ; continue ; } if ( char <= 0x20 ) { cursor ++ ; continue ; } let start = cursor ; let type = matchToken ( ) ; tokens . push ( new Token ( type , start , cursor , source ) ) ; continue ; } if ( ! tokens . length ) { throw new SyntaxError ( source . length , source , 'Expecting something' ) ; } return tokens ; throws function matchToken ( ) { let start = cursor ; if ( match ( "'" ) || match ( Token . characterLiteralStart ) ) { while ( true ) { if ( atNewline ( ) ) { throw syntaxError ( "expecting '" ) ; } if ( match ( "'" ) ) return 'string' ; if ( match ( '\\' ) ) { if ( atNewline ( ) ) { throw syntaxError ( 'expecting something after \\' ) ; } if ( match ( 'x' ) ) { if ( ! matchHexDigit ( ) || ! matchHexDigit ( ) ) { throw syntaxError ( 'expecting 2 hex digits after \\x' ) ; } continue ; } if ( match ( 'u' ) ) { if ( ! matchHexDigit ( ) || ! matchHexDigit ( ) || ! matchHexDigit ( ) || ! matchHexDigit ( ) ) { throw syntaxError ( 'expecting 4 hex digits after \\u' ) ; } continue ; } } cursor ++ ; continue ; } } if ( match ( '+' ) ) return 'name' ; if ( match ( '*' ) ) return 'name' ; if ( match ( '/' ) ) return 'name' ; if ( match ( '&&' ) ) return 'name' ; if ( match ( '&' ) ) return 'name' ; if ( match ( '=?' ) ) return 'name' ; if ( match ( '>?' ) ) return 'name' ; if ( match ( '<?' ) ) return 'name' ; if ( match ( '>=?' ) ) return 'name' ; if ( match ( '<=?' ) ) return 'name' ; if ( match ( 'not=?' ) ) return 'name' ; if ( match ( '=!' ) ) return 'name' ; if ( match ( '>!' ) ) return 'name' ; if ( match ( '<!' ) ) return 'name' ; if ( match ( '>=!' ) ) return 'name' ; if ( match ( '<=!' ) ) return 'name' ; if ( match ( 'not=!' ) ) return 'name' ; if ( match ( '###' ) ) return '###' ; if ( match ( '::nochange' ) ) return '::nochange' ; if ( match ( '::replace' ) ) return '::replace' ; if ( match ( '::insert' ) ) return '::insert' ; if ( match ( '::append' ) ) return '::append' ; if ( match ( '::convert' ) ) return '::convert' ; if ( match ( '::delete' ) ) return '::delete' ; if ( match ( '::move-insert' ) ) return '::move-insert' ; if ( match ( '::move-append' ) ) return '::move-append' ; if ( match ( '::move' ) ) return '::move' ; if ( match ( '::wrap-record' ) ) return '::wrap-record' ; if ( match ( '::wrap-array' ) ) return '::wrap-array' ; if ( match ( '::unwrap' ) ) return '::unwrap' ; if ( match ( '.' ) ) return '.' ; if ( match ( ',' ) ) return ',' ; if ( match ( ';' ) ) return ';' ; if ( match ( ':=' ) ) return ':=' ; if ( match ( '#' ) ) return '#' ; if ( match ( '::' ) ) return '::' ; if ( match ( ':' ) ) return ':' ; if ( match ( '=|>' ) ) return '=|>' ; if ( match ( '=' ) ) return '=' ; if ( match ( '->' ) ) return '->' ; if ( match ( '<-' ) ) return '<-' ; if ( match ( '{' ) ) return '{' ; if ( match ( '}' ) ) return '}' ; if ( match ( '(' ) ) return '(' ; if ( match ( ')' ) ) return ')' ; if ( match ( '[]' ) ) return '[]' ; if ( match ( '[' ) ) return '[' ; if ( match ( ']' ) ) return ']' ; let minus = match ( '-' ) ; if ( matchNumber ( ) ) return 'number' ; if ( minus ) return 'name' ; if ( matchAlpha ( ) || match ( '^' ) || match ( '~' ) ) { while ( true ) { if ( match ( '-' ) || match ( '_' ) || match ( '&' ) ) { while ( match ( '-' ) || match ( '_' ) || match ( '&' ) ) ; if ( matchAlpha ( ) || matchDigit ( ) ) continue ; throw syntaxError ( 'expecting alphanumeric character' ) ; } if ( match ( '?' ) ) { if ( matchAlpha ( ) || matchDigit ( ) || match ( '-' ) || match ( '_' ) ) { throw syntaxError ( '? can only be at end of name' ) ; } break ; } if ( match ( '!' ) ) { if ( matchAlpha ( ) || matchDigit ( ) || match ( '-' ) || match ( '_' ) ) { throw syntaxError ( '! can only be at end of name' ) ; } break ; } if ( ! ( matchAlpha ( ) || matchDigit ( ) ) ) break ; continue ; } let name = source . substring ( start , cursor ) ; switch ( name ) { case 'record' : case 'do' : case 'with' : case 'choice' : case 'try' : case 'check' : case 'let' : case 'export' : case 'not' : case 'else' : case 'reject' : case 'table' : case 'array' : case 'tracked' : case 'anything' : case 'nil' : case 'builtin' : case 'that' : case 'include' : case 'find?' : case 'find!' : case 'for-all' : case 'such-that' : case 'selecting' : case 'all?' : case 'all!' : case 'none?' : case 'none!' : case 'accumulate' : case 'on-update' : case 'updatable' : case 'write' : case 'extend' : case 'selection' : case 'any' : case 'link' : case 'via' : case 'register' : return name ; } return 'name' ; } throw syntaxError ( 'not recognized' ) ; } throws function matchNumber ( ) { if ( ! matchDigit ( ) ) return false ; while ( matchDigit ( ) ) ; if ( match ( '.' ) ) { if ( ! matchDigit ( ) ) { throw syntaxError ( 'expecting fractional number' ) ; } while ( matchDigit ( ) ) ; } if ( match ( 'e' ) || match ( 'E' ) ) { match ( '+' ) || match ( '-' ) ; if ( ! matchDigit ( ) ) { throw syntaxError ( 'expecting exponent' ) ; } while ( matchDigit ( ) ) ; } return true ; } /**
     * Matches alphabetic character, advancing cursor
     */ function matchAlpha ( ) { let char = source . charCodeAt ( cursor ) ; if ( ( char >= 0x41 && char <= 0x5a ) || ( char >= 0x61 && char <= 0x7a ) ) { cursor ++ ; return true ; } return false ; } /**
     * Matches numeric digit, advancing cursor
     */ function matchDigit ( ) { let char = source . charCodeAt ( cursor ) ; if ( char >= 0x30 && char <= 0x39 ) { cursor ++ ; return true ; } return false ; } /**
     * Matches hex digit, advancing cursor
     */ function matchHexDigit ( ) { let char = source . charCodeAt ( cursor ) ; if ( ( char >= 0x30 && char <= 0x39 ) || ( char >= 0x41 && char <= 0x46 ) || ( char >= 0x61 && char <= 0x66 ) ) { cursor ++ ; return true ; } return false ; } /**
     * Match a string in `source` at `cursor`.
     * On success advances `cursor` and returns true.
    */ function match ( s ) { for ( let i = 0 ; i < s . length ; i ++ ) { if ( s [ i ] !== source [ cursor + i ] ) { return false ; } } cursor += s . length ; return true ; } function syntaxError ( message ) { return new SyntaxError ( cursor , source , message ) ; } function atNewline ( ) { return cursor >= source . length || newline ( source . charCodeAt ( cursor ) ) ; } } function newline ( char ) { return char >= 0xa && char <= 0xd ; } /** whether string is an operator */ export function isOperator ( s ) { switch ( s [ 0 ] ) { case '+' : case '-' : case '*' : case '/' : case '=' : case '<' : case '>' : return true ; } return s . startsWith ( 'not=' ) ; } export class SyntaxError extends Error { constructor ( readonly cursor , readonly source , readonly description = "unrecognized syntax" ) { super ( description + ' [' + cursor + ']' + source . slice ( cursor , cursor + 10 ) ) ; this . name = 'SyntaxError' ; } }