@Test void testCoordLiteral() { var tokenzier = fromString("0_5_5_5_5"); assertEquals(tokenzier.parse().getKind(), COORDGRID); }
public Token<Kind> parse() { if (!state.fallback.isEmpty()) { return state.fallback.removeFirst(); } final var builder = state.builder; final var state_kind = state.kind; char current, next; while (true) { if (state.mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (state.mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); state.mode = Mode.IDENTIFIER; } else if (current == '\"') { state.mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); state.mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); state.mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); state.mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); state.lines = new ArrayList<>(); state.mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { if (state_kind == State.StateKind.INTERPOLATION && current == '>') { popState(); state.mode = Mode.ISTRING_LITERAL; continue; } else if (table.isOperatorStart(current)) { builder.append(current); for (var index = 1; index < table.getOperatorSize(); index++) { if (!stream.hasRemaining() || stream.peek() == '\r' && stream.peek() == '\n') { break; } builder.append(stream.take()); } while (builder.length() > 0) { var sequence = builder.toString(); if (table.isOperator(sequence)) { return createToken(table.lookupOperator(sequence), sequence); } builder.setLength(builder.length() - 1); stream.rollback(1); } } addLexicalError("Unexpected character: " + current); } } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: case ISTRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '"': builder.append('\"'); break; case '\\': builder.append('\\'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); break; } } else if (current == '\"') { if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '<') { if (state.mode == Mode.ISTRING_LITERAL) { pushState(State.StateKind.INTERPOLATION); return createToken(STRING, builder.toString()); } else { pushState(State.StateKind.INTERPOLATION); feed(createToken(STRING, builder.toString())); return createToken(CONCATB); } } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (state.mode == Mode.NUMBER_LITERAL && current == '_') { state.mode = Mode.COORDGRID_LITERAL; } var coordgrid = state.mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); state.mode = Mode.IDENTIFIER; stream.mark(); } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", state.lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (state.lines.size() != 0 || line.length() != 0) { state.lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", state.lines)); } else { state.builder.append(current); } break; } } }
Tokenizer extends TokenizerBase { public Token<Kind> parse() { if (!state.fallback.isEmpty()) { return state.fallback.removeFirst(); } final var builder = state.builder; final var state_kind = state.kind; char current, next; while (true) { if (state.mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (state.mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); state.mode = Mode.IDENTIFIER; } else if (current == '\"') { state.mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); state.mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); state.mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); state.mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); state.lines = new ArrayList<>(); state.mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { if (state_kind == State.StateKind.INTERPOLATION && current == '>') { popState(); state.mode = Mode.ISTRING_LITERAL; continue; } else if (table.isOperatorStart(current)) { builder.append(current); for (var index = 1; index < table.getOperatorSize(); index++) { if (!stream.hasRemaining() || stream.peek() == '\r' && stream.peek() == '\n') { break; } builder.append(stream.take()); } while (builder.length() > 0) { var sequence = builder.toString(); if (table.isOperator(sequence)) { return createToken(table.lookupOperator(sequence), sequence); } builder.setLength(builder.length() - 1); stream.rollback(1); } } addLexicalError("Unexpected character: " + current); } } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: case ISTRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '"': builder.append('\"'); break; case '\\': builder.append('\\'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); break; } } else if (current == '\"') { if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '<') { if (state.mode == Mode.ISTRING_LITERAL) { pushState(State.StateKind.INTERPOLATION); return createToken(STRING, builder.toString()); } else { pushState(State.StateKind.INTERPOLATION); feed(createToken(STRING, builder.toString())); return createToken(CONCATB); } } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (state.mode == Mode.NUMBER_LITERAL && current == '_') { state.mode = Mode.COORDGRID_LITERAL; } var coordgrid = state.mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); state.mode = Mode.IDENTIFIER; stream.mark(); } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", state.lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (state.lines.size() != 0 || line.length() != 0) { state.lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", state.lines)); } else { state.builder.append(current); } break; } } } }
Tokenizer extends TokenizerBase { public Token<Kind> parse() { if (!state.fallback.isEmpty()) { return state.fallback.removeFirst(); } final var builder = state.builder; final var state_kind = state.kind; char current, next; while (true) { if (state.mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (state.mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); state.mode = Mode.IDENTIFIER; } else if (current == '\"') { state.mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); state.mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); state.mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); state.mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); state.lines = new ArrayList<>(); state.mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { if (state_kind == State.StateKind.INTERPOLATION && current == '>') { popState(); state.mode = Mode.ISTRING_LITERAL; continue; } else if (table.isOperatorStart(current)) { builder.append(current); for (var index = 1; index < table.getOperatorSize(); index++) { if (!stream.hasRemaining() || stream.peek() == '\r' && stream.peek() == '\n') { break; } builder.append(stream.take()); } while (builder.length() > 0) { var sequence = builder.toString(); if (table.isOperator(sequence)) { return createToken(table.lookupOperator(sequence), sequence); } builder.setLength(builder.length() - 1); stream.rollback(1); } } addLexicalError("Unexpected character: " + current); } } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: case ISTRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '"': builder.append('\"'); break; case '\\': builder.append('\\'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); break; } } else if (current == '\"') { if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '<') { if (state.mode == Mode.ISTRING_LITERAL) { pushState(State.StateKind.INTERPOLATION); return createToken(STRING, builder.toString()); } else { pushState(State.StateKind.INTERPOLATION); feed(createToken(STRING, builder.toString())); return createToken(CONCATB); } } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (state.mode == Mode.NUMBER_LITERAL && current == '_') { state.mode = Mode.COORDGRID_LITERAL; } var coordgrid = state.mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); state.mode = Mode.IDENTIFIER; stream.mark(); } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", state.lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (state.lines.size() != 0 || line.length() != 0) { state.lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", state.lines)); } else { state.builder.append(current); } break; } } } Tokenizer(ErrorReporter errorReporter, LexicalTable<Kind> table, CharStream stream); }
Tokenizer extends TokenizerBase { public Token<Kind> parse() { if (!state.fallback.isEmpty()) { return state.fallback.removeFirst(); } final var builder = state.builder; final var state_kind = state.kind; char current, next; while (true) { if (state.mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (state.mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); state.mode = Mode.IDENTIFIER; } else if (current == '\"') { state.mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); state.mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); state.mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); state.mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); state.lines = new ArrayList<>(); state.mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { if (state_kind == State.StateKind.INTERPOLATION && current == '>') { popState(); state.mode = Mode.ISTRING_LITERAL; continue; } else if (table.isOperatorStart(current)) { builder.append(current); for (var index = 1; index < table.getOperatorSize(); index++) { if (!stream.hasRemaining() || stream.peek() == '\r' && stream.peek() == '\n') { break; } builder.append(stream.take()); } while (builder.length() > 0) { var sequence = builder.toString(); if (table.isOperator(sequence)) { return createToken(table.lookupOperator(sequence), sequence); } builder.setLength(builder.length() - 1); stream.rollback(1); } } addLexicalError("Unexpected character: " + current); } } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: case ISTRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '"': builder.append('\"'); break; case '\\': builder.append('\\'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); break; } } else if (current == '\"') { if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '<') { if (state.mode == Mode.ISTRING_LITERAL) { pushState(State.StateKind.INTERPOLATION); return createToken(STRING, builder.toString()); } else { pushState(State.StateKind.INTERPOLATION); feed(createToken(STRING, builder.toString())); return createToken(CONCATB); } } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (state.mode == Mode.NUMBER_LITERAL && current == '_') { state.mode = Mode.COORDGRID_LITERAL; } var coordgrid = state.mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); state.mode = Mode.IDENTIFIER; stream.mark(); } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", state.lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (state.lines.size() != 0 || line.length() != 0) { state.lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", state.lines)); } else { state.builder.append(current); } break; } } } Tokenizer(ErrorReporter errorReporter, LexicalTable<Kind> table, CharStream stream); Token<Kind> parse(); Range range(); }
Tokenizer extends TokenizerBase { public Token<Kind> parse() { if (!state.fallback.isEmpty()) { return state.fallback.removeFirst(); } final var builder = state.builder; final var state_kind = state.kind; char current, next; while (true) { if (state.mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (state.mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); state.mode = Mode.IDENTIFIER; } else if (current == '\"') { state.mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); state.mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); state.mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); state.mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); state.lines = new ArrayList<>(); state.mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { if (state_kind == State.StateKind.INTERPOLATION && current == '>') { popState(); state.mode = Mode.ISTRING_LITERAL; continue; } else if (table.isOperatorStart(current)) { builder.append(current); for (var index = 1; index < table.getOperatorSize(); index++) { if (!stream.hasRemaining() || stream.peek() == '\r' && stream.peek() == '\n') { break; } builder.append(stream.take()); } while (builder.length() > 0) { var sequence = builder.toString(); if (table.isOperator(sequence)) { return createToken(table.lookupOperator(sequence), sequence); } builder.setLength(builder.length() - 1); stream.rollback(1); } } addLexicalError("Unexpected character: " + current); } } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: case ISTRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '"': builder.append('\"'); break; case '\\': builder.append('\\'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); break; } } else if (current == '\"') { if (state.mode == Mode.ISTRING_LITERAL) { if (builder.length() > 0) { feed(createToken(CONCATE)); } else { return createToken(CONCATE); } } return createToken(STRING, builder.toString()); } else if (current == '<') { if (state.mode == Mode.ISTRING_LITERAL) { pushState(State.StateKind.INTERPOLATION); return createToken(STRING, builder.toString()); } else { pushState(State.StateKind.INTERPOLATION); feed(createToken(STRING, builder.toString())); return createToken(CONCATB); } } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (state.mode == Mode.NUMBER_LITERAL && current == '_') { state.mode = Mode.COORDGRID_LITERAL; } var coordgrid = state.mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); state.mode = Mode.IDENTIFIER; stream.mark(); } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", state.lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (state.lines.size() != 0 || line.length() != 0) { state.lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { state.lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", state.lines)); } else { state.builder.append(current); } break; } } } Tokenizer(ErrorReporter errorReporter, LexicalTable<Kind> table, CharStream stream); Token<Kind> parse(); Range range();  }