@Test void testEof() throws IOException { var lexer = fromString("\r\n\t "); assertEquals(Kind.EOF, lexer.parse().getKind()); assertEquals(Kind.EOF, lexer.parse().getKind()); }
public Token<Kind> parse() { char current, next; while (true) { if (mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); } if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); mode = Mode.IDENTIFIER; } else if (current == '\"') { mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); lines.clear(); mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { addLexicalError("Unexpected character: " + current); } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '\\': builder.append('\\'); break; case '\"': builder.append('"'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); } } else if (current == '\"') { return createToken(STRING, builder.toString()); } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (mode == Mode.NUMBER_LITERAL && current == '_') { mode = Mode.COORDGRID_LITERAL; } var coordgrid = mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); mode = Mode.IDENTIFIER; stream.mark(); continue; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (lines.size() != 0 || line.length() != 0) { lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", lines)); } else { builder.append(current); } break; } } }
Tokenizer extends TokenizerBase { public Token<Kind> parse() { char current, next; while (true) { if (mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); } if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); mode = Mode.IDENTIFIER; } else if (current == '\"') { mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); lines.clear(); mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { addLexicalError("Unexpected character: " + current); } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '\\': builder.append('\\'); break; case '\"': builder.append('"'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); } } else if (current == '\"') { return createToken(STRING, builder.toString()); } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (mode == Mode.NUMBER_LITERAL && current == '_') { mode = Mode.COORDGRID_LITERAL; } var coordgrid = mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); mode = Mode.IDENTIFIER; stream.mark(); continue; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (lines.size() != 0 || line.length() != 0) { lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", lines)); } else { builder.append(current); } break; } } } }
Tokenizer extends TokenizerBase { public Token<Kind> parse() { char current, next; while (true) { if (mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); } if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); mode = Mode.IDENTIFIER; } else if (current == '\"') { mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); lines.clear(); mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { addLexicalError("Unexpected character: " + current); } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '\\': builder.append('\\'); break; case '\"': builder.append('"'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); } } else if (current == '\"') { return createToken(STRING, builder.toString()); } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (mode == Mode.NUMBER_LITERAL && current == '_') { mode = Mode.COORDGRID_LITERAL; } var coordgrid = mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); mode = Mode.IDENTIFIER; stream.mark(); continue; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (lines.size() != 0 || line.length() != 0) { lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", lines)); } else { builder.append(current); } break; } } } Tokenizer(ErrorReporter errorReporter, LexicalTable<Kind> table, CharStream stream); }
Tokenizer extends TokenizerBase { public Token<Kind> parse() { char current, next; while (true) { if (mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); } if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); mode = Mode.IDENTIFIER; } else if (current == '\"') { mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); lines.clear(); mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { addLexicalError("Unexpected character: " + current); } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '\\': builder.append('\\'); break; case '\"': builder.append('"'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); } } else if (current == '\"') { return createToken(STRING, builder.toString()); } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (mode == Mode.NUMBER_LITERAL && current == '_') { mode = Mode.COORDGRID_LITERAL; } var coordgrid = mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); mode = Mode.IDENTIFIER; stream.mark(); continue; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (lines.size() != 0 || line.length() != 0) { lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", lines)); } else { builder.append(current); } break; } } } Tokenizer(ErrorReporter errorReporter, LexicalTable<Kind> table, CharStream stream); Token<Kind> parse(); Range range(); }
Tokenizer extends TokenizerBase { public Token<Kind> parse() { char current, next; while (true) { if (mode == Mode.NONE) { mark(); } current = stream.take(); next = stream.peek(); switch (mode) { case NONE: if (Character.isWhitespace(current)) { continue; } else { resetBuilder(); } if (current == NULL) { return createToken(EOF); } else if (isIdentifierStart(current)) { builder.append(current); stream.mark(); mode = Mode.IDENTIFIER; } else if (current == '\"') { mode = Mode.STRING_LITERAL; } else if (current == '0' && next == 'x') { builder.append(current); builder.append(next); stream.take(); stream.mark(); mode = Mode.HEX_LITERAL; } else if (Character.isDigit(current) || (current == '-' || current == '+') && Character.isDigit(next)) { builder.append(current); stream.mark(); mode = Mode.NUMBER_LITERAL; } else if (current == '/' && next == '/') { stream.take(); mode = Mode.LINE_COMMENT; } else if (current == '/' && next == '*') { stream.take(); lines.clear(); mode = Mode.MULTI_COMMENT; } else if (table.isSeparator(current)) { return createToken(table.lookupSeparator(current), Character.toString(current)); } else { addLexicalError("Unexpected character: " + current); } break; case IDENTIFIER: if (isIdentifierPart(current)) { builder.append(current); stream.mark(); } else { stream.reset(); var word = builder.toString(); return createToken(table.isKeyword(word) ? table.lookupKeyword(word) : IDENTIFIER, builder.toString()); } break; case STRING_LITERAL: if (current == NULL || current == '\n') { addLexicalError("String literal is not properly closed by a double-quote"); return createToken(STRING, builder.toString()); } else if (current == '\\') { stream.take(); switch (next) { case 'b': builder.append('\b'); break; case 't': builder.append('\t'); break; case 'n': builder.append('\n'); break; case 'f': builder.append('\f'); break; case '\\': builder.append('\\'); break; case '\"': builder.append('"'); break; case '<': builder.append('<'); break; case '>': builder.append('>'); break; default: addLexicalError("Invalid escape sequence (valid ones are \\b \\t \\n \\f \\r \\\" \\\\ \\< \\>)"); } } else if (current == '\"') { return createToken(STRING, builder.toString()); } else { builder.append(current); } break; case HEX_LITERAL: if (Character.isDigit(current) || (current >= 'a' && current <= 'f') || (current >= 'A' && current <= 'F')) { builder.append(current); stream.mark(); } else { var kind = INTEGER; if (current == 'L' || current == 'l') { kind = LONG; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case NUMBER_LITERAL: case COORDGRID_LITERAL: if (mode == Mode.NUMBER_LITERAL && current == '_') { mode = Mode.COORDGRID_LITERAL; } var coordgrid = mode == Mode.COORDGRID_LITERAL; if (Character.isDigit(current) || current == '_') { builder.append(current); stream.mark(); } else { var kind = coordgrid ? COORDGRID : INTEGER; if (!coordgrid && (current == 'L' || current == 'l')) { kind = LONG; } else if (isIdentifierPart(current)) { builder.append(current); mode = Mode.IDENTIFIER; stream.mark(); continue; } else if (current != NULL) { stream.reset(); } return createToken(kind, builder.toString()); } break; case LINE_COMMENT: if (current == NULL || current == '\n') { return createToken(COMMENT, trimComment(builder.toString(), false)); } else { builder.append(current); } break; case MULTI_COMMENT: if (current == NULL) { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); addLexicalError("Unexpected end of comment"); return createToken(COMMENT, String.join("\n", lines)); } else if (current == '\n') { var line = trimComment(builder.toString(), true); if (lines.size() != 0 || line.length() != 0) { lines.add(line); } resetBuilder(); } else if (current == '*' && next == '/') { var line = trimComment(builder.toString(), true); if (line.length() > 0) { lines.add(line); } resetBuilder(); stream.take(); return createToken(COMMENT, String.join("\n", lines)); } else { builder.append(current); } break; } } } Tokenizer(ErrorReporter errorReporter, LexicalTable<Kind> table, CharStream stream); Token<Kind> parse(); Range range();  }