Bring scanner changes over from javascript
@@ -5,13 +5,12 @@ enum TokenType {
AUTOMATIC_SEMICOLON,
};
-void *tree_sitter_typescript_external_scanner_create() { return NULL; }
-void tree_sitter_typescript_external_scanner_destroy(void *payload) {}
-void tree_sitter_typescript_external_scanner_reset(void *payload) {}
-unsigned tree_sitter_typescript_external_scanner_serialize(void *payload, char *state) { return 0; }
-void tree_sitter_typescript_external_scanner_deserialize(void *payload, const char *state, unsigned length) {}
+void *tree_sitter_javascript_external_scanner_create() { return NULL; }
+void tree_sitter_javascript_external_scanner_destroy(void *p) {}
+void tree_sitter_javascript_external_scanner_reset(void *p) {}
+unsigned tree_sitter_javascript_external_scanner_serialize(void *p, char *buffer) { return 0; }
+void tree_sitter_javascript_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
-static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static bool scan_whitespace_and_comments(TSLexer *lexer) {
@@ -50,19 +49,19 @@ static bool scan_whitespace_and_comments(TSLexer *lexer) {
}
}
-bool tree_sitter_typescript_external_scanner_scan(void *payload, TSLexer *lexer,
+bool tree_sitter_javascript_external_scanner_scan(void *payload, TSLexer *lexer,
const bool *valid_symbols) {
lexer->result_symbol = AUTOMATIC_SEMICOLON;
+ lexer->mark_end(lexer);
for (;;) {
if (lexer->lookahead == 0) return true;
if (lexer->lookahead == '}') return true;
if (!iswspace(lexer->lookahead)) return false;
if (lexer->lookahead == '\n') break;
- skip(lexer);
+ advance(lexer);
}
- lexer->mark_end(lexer);
advance(lexer);
if (!scan_whitespace_and_comments(lexer)) return false;
@@ -70,8 +69,6 @@ bool tree_sitter_typescript_external_scanner_scan(void *payload, TSLexer *lexer,
switch (lexer->lookahead) {
case ',':
case '.':
- case '+':
- case '-':
case '*':
case '%':
case '>':
@@ -86,6 +83,14 @@ bool tree_sitter_typescript_external_scanner_scan(void *payload, TSLexer *lexer,
case '/':
return false;
+ // Insert a semicolon before `--` and `++`, but not before binary `+` or `-`.
+ case '+':
+ advance(lexer);
+ return lexer->lookahead == '+';
+ case '-':
+ advance(lexer);
+ return lexer->lookahead == '-';
+
// Don't insert a semicolon before `!=`, but do insert one before a unary `!`.
case '!':
advance(lexer);
