LexerImpl implements Lexer { private void tokenizeExpression() { logger.trace("Tokenizing Expression"); String token; this.source.advanceThroughWhitespace(); Matcher matcher = this.regexOperators.matcher(this.source); if (matcher.lookingAt()) { token = this.source.substring(matcher.end()); this.pushToken(Token.Type.OPERATOR, token); this.source.advance(matcher.end()); return; } matcher = REGEX_NAME.matcher(this.source); if (matcher.lookingAt()) { token = this.source.substring(matcher.end()); this.pushToken(Token.Type.NAME, token); this.source.advance(matcher.end()); return; } matcher = REGEX_LONG.matcher(this.source); if (matcher.lookingAt()) { token = this.source.substring(matcher.end() - 1); this.pushToken(Token.Type.LONG, token); this.source.advance(matcher.end()); return; } matcher = REGEX_NUMBER.matcher(this.source); if (matcher.lookingAt()) { token = this.source.substring(matcher.end()); this.pushToken(Token.Type.NUMBER, token); this.source.advance(matcher.end()); return; } if (PUNCTUATION.indexOf(this.source.charAt(0)) >= 0) { String character = String.valueOf(this.source.charAt(0)); if ("([{".contains(character)) { this.brackets.push(new Pair<>(character, this.source.getLineNumber())); } else if (")]}".contains(character)) { if (this.brackets.isEmpty()) { throw new ParserException(null, "Unexpected \"" + character + "\"", this.source.getLineNumber(), this.source.getFilename()); } else { HashMap<String, String> validPairs = new HashMap<>(); validPairs.put("(", ")"); validPairs.put("[", "]"); validPairs.put("{", "}"); String lastBracket = this.brackets.pop().getLeft(); String expected = validPairs.get(lastBracket); if (!expected.equals(character)) { throw new ParserException(null, "Unclosed \"" + expected + "\"", this.source.getLineNumber(), this.source.getFilename()); } } } this.pushToken(Token.Type.PUNCTUATION, character); this.source.advance(1); return; } matcher = REGEX_STRING_PLAIN.matcher(this.source); if (matcher.lookingAt()) { token = this.source.substring(matcher.end()); this.source.advance(matcher.end()); token = this.unquoteAndUnescape(token); this.pushToken(Token.Type.STRING, token); return; } matcher = REGEX_DOUBLEQUOTE.matcher(this.source); if (matcher.lookingAt()) { this.brackets.push(new Pair<>("\"", this.source.getLineNumber())); this.lexerStateStack.push(State.STRING); this.source.advance(matcher.end()); return; } throw new ParserException(null, String.format("Unexpected character [%s]", this.source.charAt(0)), this.source.getLineNumber(), this.source.getFilename()); } LexerImpl(Syntax syntax, Collection<UnaryOperator> unaryOperators,
      Collection<BinaryOperator> binaryOperators); @Override TokenStream tokenize(Reader reader, String name);  }
@Test void testTokenizeExpression() { Loader<String> loader = new StringLoader(); Reader templateReader = loader.getReader("{{ whatever }}"); TokenStream tokenStream = this.lexer.tokenize(templateReader, this.TEMPLATE_NAME); assertThat(tokenStream.peek(0).getType()).isEqualTo(Type.PRINT_START); assertThat(tokenStream.peek(0).getValue()).isNull(); assertThat(tokenStream.peek(1).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(1).getValue()).isEqualTo("whatever"); assertThat(tokenStream.peek(2).getType()).isEqualTo(Type.PRINT_END); assertThat(tokenStream.peek(2).getValue()).isEqualTo("}}"); assertThat(tokenStream.peek(3).getType()).isEqualTo(Type.EOF); assertThat(tokenStream.peek(3).getValue()).isNull(); }