parser-core: Add ignoreWhitespaceTokens property to abstract skipping whitespace
MODIFY
packages/@romejs/codec-json/parse.ts
packages/@romejs/codec-json/parse.ts
@@ -115,6 +115,7 @@ export default createParser(
constructor(opts: JSONParserOptions) {
super(opts, '@romejs/codec-json');
this.options = opts;
+ this.ignoreWhitespaceTokens = true;
this.hasExtensions =
this.path !== undefined && this.path.getBasename().endsWith('.rjson');
@@ -271,13 +272,6 @@ export default createParser(
case ']':
return this.finishToken('BracketClose');
-
- // Skip all whitespace characters
- case ' ':
- case '\t':
- case '\r':
- case '\n':
- return this.lookaheadToken(inc(index));
}
// Numbers

MODIFY
packages/@romejs/parser-core/index.ts
packages/@romejs/parser-core/index.ts
@@ -124,6 +124,7 @@ export class ParserCore<Tokens extends TokensShape, State> {
this.currentToken = SOF_TOKEN;
this.prevToken = SOF_TOKEN;
this.state = initialState;
+ this.ignoreWhitespaceTokens = false;
}
offsetPosition: undefined | Position;
@@ -136,6 +137,7 @@ export class ParserCore<Tokens extends TokensShape, State> {
prevToken: TokenValues<Tokens>;
currentToken: TokenValues<Tokens>;
eofToken: EOFToken;
+ ignoreWhitespaceTokens: boolean;
path: undefined | UnknownFilePath;
filename: undefined | string;
@@ -204,6 +206,24 @@ export class ParserCore<Tokens extends TokensShape, State> {
}
}
+ _tokenizeWithState(
+ index: Number0,
+ input: string,
+ state: State,
+ ): undefined | {token: TokenValues<Tokens>; state: State} {
+ if (this.ignoreWhitespaceTokens) {
+ switch (input[get0(index)]) {
+ case ' ':
+ case '\t':
+ case '\r':
+ case '\n':
+ return this._tokenizeWithState(inc(index), input, state);
+ }
+ }
+
+ return this.tokenizeWithState(index, input, state);
+ }
+
// Get the current token
getToken(): TokenValues<Tokens> {
const {currentToken} = this;
@@ -304,7 +324,7 @@ export class ParserCore<Tokens extends TokensShape, State> {
this.tokenizing = true;
// Tokenize and do some validation
- const nextToken = this.tokenizeWithState(index, this.input, this.state);
+ const nextToken = this._tokenizeWithState(index, this.input, this.state);
if (nextToken === undefined) {
throw this.unexpected({
start: this.getPositionFromIndex(index),

parser-core: Add ignoreWhitespaceTokens property to abstract skipping whitespace