@Test void validRule() { final Lexer lexer = new Lexer( "reduction TestRule_123:\n" + " x + y * z = -(a_123 +- 3 / 2.2) -> [\n" + " x^a_123 = cos(pi) - log(e, e), " undefined, ]\n", errors::add ); final List<Token> expected = Arrays.asList( new Token(REDUCTION, "reduction", 0), new Token(IDENTIFIER, "TestRule_123", 10), new Token(COLON, ":", 22), new Token(IDENTIFIER, "x", 26), new Token(PLUS, "+", 28), new Token(IDENTIFIER, "y", 30), new Token(TIMES, "*", 32), new Token(IDENTIFIER, "z", 34), new Token(EQUALS, "=", 36), new Token(MINUS, "-", 38), new Token(LEFT_PAREN, "(", 39), new Token(IDENTIFIER, "a_123", 40), new Token(PLUS_MINUS, "+-", 46), new Token(NUMBER, "3", 49), new Token(DIVIDE, "/", 51), new Token(NUMBER, "2.2", 53), new Token(RIGHT_PAREN, ")", 56), new Token(ARROW, "->", 58), new Token(LEFT_BRACKET, "[", 61), new Token(IDENTIFIER, "x", 67), new Token(POWER, "^", 68), new Token(IDENTIFIER, "a_123", 69), new Token(EQUALS, "=", 75), new Token(COS, "cos", 77), new Token(LEFT_PAREN, "(", 80), new Token(PI, "pi", 81), new Token(RIGHT_PAREN, ")", 83), new Token(MINUS, "-", 85), new Token(LOG, "log", 87), new Token(LEFT_PAREN, "(", 90), new Token(E, "e", 91), new Token(COMMA, ",", 92), new Token(E, "e", 94), new Token(RIGHT_PAREN, ")", 95), new Token(COMMA, ",", 96), new Token(UNDEFINED, "undefined", 113), new Token(COMMA, ",", 122), new Token(RIGHT_BRACKET, "]", 138), new Token(EOF, "", 140) ); assertEquals(expected, lexer.lex()); assertTrue(errors.isEmpty()); }
public List<Token> lex() { if (used) { throw new IllegalStateException("Lexer.lex can only be called once per instance"); } used = true; while (!atEnd()) { startOfLexeme = curPosition; lexAndHandleErrors(); } reportAndClearUnexpectedCharacters(); tokens.add(new Token(EOF, "", source.length())); return tokens; }
Lexer { public List<Token> lex() { if (used) { throw new IllegalStateException("Lexer.lex can only be called once per instance"); } used = true; while (!atEnd()) { startOfLexeme = curPosition; lexAndHandleErrors(); } reportAndClearUnexpectedCharacters(); tokens.add(new Token(EOF, "", source.length())); return tokens; } }
Lexer { public List<Token> lex() { if (used) { throw new IllegalStateException("Lexer.lex can only be called once per instance"); } used = true; while (!atEnd()) { startOfLexeme = curPosition; lexAndHandleErrors(); } reportAndClearUnexpectedCharacters(); tokens.add(new Token(EOF, "", source.length())); return tokens; } Lexer(final String source, final Consumer<? super DslError> errorReporter); }
Lexer { public List<Token> lex() { if (used) { throw new IllegalStateException("Lexer.lex can only be called once per instance"); } used = true; while (!atEnd()) { startOfLexeme = curPosition; lexAndHandleErrors(); } reportAndClearUnexpectedCharacters(); tokens.add(new Token(EOF, "", source.length())); return tokens; } Lexer(final String source, final Consumer<? super DslError> errorReporter); List<Token> lex(); }
Lexer { public List<Token> lex() { if (used) { throw new IllegalStateException("Lexer.lex can only be called once per instance"); } used = true; while (!atEnd()) { startOfLexeme = curPosition; lexAndHandleErrors(); } reportAndClearUnexpectedCharacters(); tokens.add(new Token(EOF, "", source.length())); return tokens; } Lexer(final String source, final Consumer<? super DslError> errorReporter); List<Token> lex();  }