LexerImpl implements Lexer { @Override public TokenStream tokenize(Reader reader, String name) { this.buildOperatorRegex(); try { this.source = new TemplateSource(reader, name); } catch (IOException e) { throw new ParserException(e, "Can not convert template Reader into a String", 0, name); } this.tokens = new ArrayList<>(); this.lexerStateStack = new ArrayDeque<>(); this.brackets = new LinkedList<>(); this.lexerStateStack.push(State.DATA); while (this.source.length() > 0) { switch (this.lexerStateStack.peek()) { case DATA: this.tokenizeData(); break; case EXECUTE: this.tokenizeBetweenExecuteDelimiters(); break; case PRINT: this.tokenizeBetweenPrintDelimiters(); break; case COMMENT: this.tokenizeComment(); break; case STRING: this.tokenizeString(); break; case STRING_INTERPOLATION: this.tokenizeStringInterpolation(); break; default: break; } } this.pushToken(Token.Type.EOF); this.popState(); if (!this.brackets.isEmpty()) { String expected = this.brackets.pop().getLeft(); throw new ParserException(null, String.format("Unclosed \"%s\"", expected), this.source.getLineNumber(), this.source.getFilename()); } return new TokenStream(this.tokens, this.source.getFilename()); } LexerImpl(Syntax syntax, Collection<UnaryOperator> unaryOperators,
      Collection<BinaryOperator> binaryOperators); @Override TokenStream tokenize(Reader reader, String name);  }
@Test void testTokenizeText() { Loader<String> loader = new StringLoader(); Reader templateReader = loader.getReader(" template content "); TokenStream tokenStream = this.lexer.tokenize(templateReader, this.TEMPLATE_NAME); assertThat(tokenStream.peek(0).getType()).isEqualTo(Type.TEXT); assertThat(tokenStream.peek(0).getValue()).isEqualTo(" template content "); assertThat(tokenStream.peek(1).getType()).isEqualTo(Type.EOF); }
@Test void testPunctuation() { StringBuilder stringBuilder = new StringBuilder(); stringBuilder .append("{% if item.itemType %}"); Loader<String> loader = new StringLoader(); Reader templateReader = loader.getReader(stringBuilder.toString()); TokenStream tokenStream = this.lexer.tokenize(templateReader, this.TEMPLATE_NAME); int i = 0; assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("if"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("item"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.PUNCTUATION); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("."); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("itemType"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EOF); assertThat(tokenStream.peek(i++).getValue()).isNull(); }
@Test void testIfStatementWithOperatorAndStringToken() { StringBuilder stringBuilder = new StringBuilder(); stringBuilder .append("{% if item equals \"string1\" %}"); Loader<String> loader = new StringLoader(); Reader templateReader = loader.getReader(stringBuilder.toString()); TokenStream tokenStream = this.lexer.tokenize(templateReader, this.TEMPLATE_NAME); int i = 0; assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("if"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("item"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.OPERATOR); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("equals"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.STRING); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("string1"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); }
@Test void testIfStatement() { StringBuilder stringBuilder = new StringBuilder(); stringBuilder .append("{% if item equals \"whatever\" %}\n") .append("some text\n") .append("{% endif %}"); Loader<String> loader = new StringLoader(); Reader templateReader = loader.getReader(stringBuilder.toString()); TokenStream tokenStream = this.lexer.tokenize(templateReader, this.TEMPLATE_NAME); int i = 0; assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("if"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("item"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.OPERATOR); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("equals"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.STRING); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("whatever"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.TEXT); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("\nsome text\n"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("endif"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EOF); assertThat(tokenStream.peek(i++).getValue()).isNull(); }
@Test void testTokenizeForLoop() { StringBuilder stringBuilder = new StringBuilder(); stringBuilder .append("{% for item in items %}\n") .append("stuff\n") .append("{% endfor %}"); Loader<String> loader = new StringLoader(); Reader templateReader = loader.getReader(stringBuilder.toString()); TokenStream tokenStream = this.lexer.tokenize(templateReader, this.TEMPLATE_NAME); assertThat(tokenStream.peek(0).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(0).getValue()).isNull(); assertThat(tokenStream.peek(1).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(1).getValue()).isEqualTo("for"); assertThat(tokenStream.peek(2).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(2).getValue()).isEqualTo("item"); assertThat(tokenStream.peek(3).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(3).getValue()).isEqualTo("in"); assertThat(tokenStream.peek(4).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(4).getValue()).isEqualTo("items"); assertThat(tokenStream.peek(5).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(5).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(6).getType()).isEqualTo(Type.TEXT); assertThat(tokenStream.peek(6).getValue()).isEqualTo("\nstuff\n"); assertThat(tokenStream.peek(7).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(7).getValue()).isNull(); assertThat(tokenStream.peek(8).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(8).getValue()).isEqualTo("endfor"); assertThat(tokenStream.peek(9).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(9).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(10).getType()).isEqualTo(Type.EOF); assertThat(tokenStream.peek(10).getValue()).isNull(); }
@Test void testIfStatementWithWhitespaceControl() { StringBuilder stringBuilder = new StringBuilder(); stringBuilder .append("{% if item equals \"whatever\" -%}\n") .append("some text\n") .append("{%- endif %}"); Loader<String> loader = new StringLoader(); Reader templateReader = loader.getReader(stringBuilder.toString()); TokenStream tokenStream = this.lexer.tokenize(templateReader, this.TEMPLATE_NAME); int i = 0; assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("if"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("item"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.OPERATOR); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("equals"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.STRING); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("whatever"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.TEXT); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("some text"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("endif"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EOF); assertThat(tokenStream.peek(i++).getValue()).isNull(); }
@Test void testComplexTemplate() { StringBuilder stringBuilder = new StringBuilder(); stringBuilder .append("text before for loop followed by blank line\n") .append("{% for item in items %}\n") .append("{% if item.itemType equals \"ITEM_TYPE1\" -%}\n") .append("Item 1\n") .append("{% elseif item.itemType equals \"ITEM_TYPE2\" -%}\n") .append("Item 2\n") .append("{%- endif -%}") .append("{% endfor -%}") .append("text after for loop preceded by blank line"); Loader<String> loader = new StringLoader(); Reader templateReader = loader.getReader(stringBuilder.toString()); TokenStream tokenStream = this.lexer.tokenize(templateReader, this.TEMPLATE_NAME); int i = 0; assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.TEXT); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("text before for loop followed by blank line\n"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("for"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("item"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("in"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("items"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.TEXT); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("\n"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("if"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("item"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.PUNCTUATION); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("."); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("itemType"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.OPERATOR); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("equals"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.STRING); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("ITEM_TYPE1"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.TEXT); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("Item 1\n"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("elseif"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("item"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.PUNCTUATION); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("."); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("itemType"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.OPERATOR); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("equals"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.STRING); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("ITEM_TYPE2"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.TEXT); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("Item 2"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("endif"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_START); assertThat(tokenStream.peek(i++).getValue()).isNull(); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.NAME); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("endfor"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EXECUTE_END); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("%}"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.TEXT); assertThat(tokenStream.peek(i++).getValue()).isEqualTo("text after for loop preceded by blank line"); assertThat(tokenStream.peek(i).getType()).isEqualTo(Type.EOF); assertThat(tokenStream.peek(i++).getValue()).isNull(); }