import { TextRange } from '../common/textRange' ; export const enum TokenType { Invalid , EndOfStream , NewLine , Indent , Dedent , String , Number , Identifier , Keyword , Operator , Colon , Semicolon , Comma , OpenParenthesis , CloseParenthesis , OpenBracket , CloseBracket , OpenCurlyBrace , CloseCurlyBrace , Ellipsis , Dot , Arrow , Backtick } export const enum NewLineType { CarriageReturn , LineFeed , CarriageReturnLineFeed , Implied } export const enum OperatorType { Add , AddEqual , Assign , BitwiseAnd , BitwiseAndEqual , BitwiseInvert , BitwiseOr , BitwiseOrEqual , BitwiseXor , BitwiseXorEqual , Divide , DivideEqual , Equals , FloorDivide , FloorDivideEqual , GreaterThan , GreaterThanOrEqual , LeftShift , LeftShiftEqual , LessOrGreaterThan , LessThan , LessThanOrEqual , MatrixMultiply , MatrixMultiplyEqual , Mod , ModEqual , Multiply , MultiplyEqual , NotEquals , Power , PowerEqual , RightShift , RightShiftEqual , Subtract , SubtractEqual , Walrus , And , Or , Not , Is , IsNot , In , NotIn } export const enum OperatorFlags { Unary = 1 << 0 , Binary = 1 << 1 , Assignment = 1 << 2 , Comparison = 1 << 3 , Deprecated = 1 << 4 } export const enum KeywordType { And , As , Assert , Async , Await , Break , Case , Class , Continue , Debug , Def , Del , Elif , Else , Except , False , Finally , For , From , Global , If , Import , In , Is , Lambda , Match , None , Nonlocal , Not , Or , Pass , Raise , Return , True , Try , While , With , Yield } export const enum StringTokenFlags { None = 0 , SingleQuote = 1 << 0 , DoubleQuote = 1 << 1 , Triplicate = 1 << 2 , Raw = 1 << 3 , Unicode = 1 << 4 , Bytes = 1 << 5 , Format = 1 << 6 , Unterminated = 1 << 16 , ExceedsMaxSize = 1 << 17 } export interface Comment extends TextRange { readonly value : string ; readonly start : number ; readonly length : number ; } export namespace Comment { export function create ( start , length , value ) { const comment = { start , length , value , } ; return comment ; } } export interface TokenBase extends TextRange { readonly type : TokenType ; readonly comments ? : Comment [ ] | undefined ; } export interface Token extends TokenBase { } export namespace Token { export function create ( type , start , length , comments ) { const token = { start , length , type , comments , } ; return token ; } } export interface IndentToken extends Token { readonly type : TokenType . Indent ; readonly indentAmount : number ; readonly isIndentAmbiguous : boolean ; } export namespace IndentToken { export function create ( start , length , indentAmount , isIndentAmbiguous , comments ) { const token = { start , length , type : TokenType . Indent , isIndentAmbiguous , comments , indentAmount , } ; return token ; } } export interface DedentToken extends Token { readonly type : TokenType . Dedent ; readonly indentAmount : number ; readonly matchesIndent : boolean ; readonly isDedentAmbiguous : boolean ; } export namespace DedentToken { export function create ( start , length , indentAmount , matchesIndent , isDedentAmbiguous , comments ) { const token = { start , length , type : TokenType . Dedent , comments , indentAmount , matchesIndent , isDedentAmbiguous , } ; return token ; } } export interface NewLineToken extends Token { readonly type : TokenType . NewLine ; readonly newLineType : NewLineType ; } export namespace NewLineToken { export function create ( start , length , newLineType , comments ) { const token = { start , length , type : TokenType . NewLine , comments , newLineType , } ; return token ; } } export interface KeywordToken extends Token { readonly type : TokenType . Keyword ; readonly keywordType : KeywordType ; } export namespace KeywordToken { export function create ( start , length , keywordType , comments ) { const token = { start , length , type : TokenType . Keyword , comments , keywordType , } ; return token ; } } export interface StringToken extends Token { readonly type : TokenType . String ; readonly flags : StringTokenFlags ; readonly escapedValue : string ; readonly prefixLength : number ; readonly quoteMarkLength : number ; } export namespace StringToken { export function create ( start , length , flags , escapedValue , prefixLength , comments ) { const token = { start , length , type : TokenType . String , flags , escapedValue , prefixLength , quoteMarkLength : flags & StringTokenFlags . Triplicate ? 3 : 1 , comments , } ; return token ; } } export interface NumberToken extends Token { readonly type : TokenType . Number ; readonly value : number ; readonly isInteger : boolean ; readonly isImaginary : boolean ; } export namespace NumberToken { export function create ( start , length , value , isInteger , isImaginary , comments ) { const token = { start , length , type : TokenType . Number , isInteger , isImaginary , value , comments , } ; return token ; } } export interface OperatorToken extends Token { readonly type : TokenType . Operator ; readonly operatorType : OperatorType ; } export namespace OperatorToken { export function create ( start , length , operatorType , comments ) { const token = { start , length , type : TokenType . Operator , operatorType , comments , } ; return token ; } } export interface IdentifierToken extends Token { readonly type : TokenType . Identifier ; readonly value : string ; } export namespace IdentifierToken { export function create ( start , length , value , comments ) { const token = { start , length , type : TokenType . Identifier , value , comments , } ; return token ; } }