Indentation { List<Token> analyzeIndentations(List<Token> tokens, Object lf) { if (tokens.isEmpty()) { return tokens; } int size = tokens.size(); List<Token> result = Lists.arrayList(size + size / 16); Stack<Integer> indentations = new Stack<Integer>(); boolean freshLine = true; int lfIndex = 0; for (Token token : tokens) { if (freshLine) { int indentation = token.index() - lfIndex; if (Objects.equals(token.value(), lf)) { indentation = 0; } newLine(token, indentations, indentation, result); } if (Objects.equals(token.value(), lf)) { freshLine = true; lfIndex = token.index() + token.length(); } else { freshLine = false; result.add(token); } } Token lastToken = tokens.get(tokens.size() - 1); int endIndex = lastToken.index() + lastToken.length(); Token outdentToken = pseudoToken(endIndex, outdent); for (int i = 0; i < indentations.size() - 1; i++) { result.add(outdentToken); } return result; } Indentation(Object indent, Object outdent);  Indentation(); Parser<Token> indent(); Parser<Token> outdent(); Parser<List<Token>> lexer(Parser<?> tokenizer, Parser<?> delim); static final Parser<Void> WHITESPACES; }
@Test public void testAnalyzeIndentations() { assertEquals(tokenList(), analyze()); assertEquals(tokenList("foo"), analyze("foo")); assertEquals(tokenList("foo", "bar"), analyze("foo", "bar")); assertEquals(tokenList("foo", 1, "bar"), analyze("foo", "\n", "bar")); assertEquals(tokenList("foo", 4, "bar"), analyze("foo", "\n", 2, "\n", "bar")); assertEquals(tokenList("foo", 2, Indentation.Punctuation.INDENT, "bar", 1, Indentation.Punctuation.OUTDENT, 1, "baz"), analyze("foo", "\n", 1, "bar", "\n", "\n", "baz")); assertEquals(tokenList(2, "foo", "bar"), analyze(2, "foo", "bar")); assertEquals(tokenList(2, "foo", 2, "bar"), analyze(2, "foo", "\n", 1, "bar")); assertEquals(tokenList("foo", 2, Indentation.Punctuation.INDENT, "bar", Indentation.Punctuation.OUTDENT), analyze("foo", "\n", 1, "bar")); assertEquals(tokenList("foo", 2, Indentation.Punctuation.INDENT, "bar", 1, Indentation.Punctuation.OUTDENT, 2, Indentation.Punctuation.INDENT, "baz", Indentation.Punctuation.OUTDENT), analyze("foo", "\n", 1, "bar", "\n", "\n", 1, "baz")); assertEquals(tokenList("foo", 2, Indentation.Punctuation.INDENT, "bar", 2, "bar2", 4, Indentation.Punctuation.INDENT, "baz", Indentation.Punctuation.OUTDENT, Indentation.Punctuation.OUTDENT), analyze("foo", "\n", 1, "bar", "\n", 1, "bar2", "\n", 3, "baz")); assertEquals(tokenList( "foo", 2, Indentation.Punctuation.INDENT, "bar", 4, Indentation.Punctuation.INDENT, "baz", 3, Indentation.Punctuation.OUTDENT, Indentation.Punctuation.INDENT, "end", Indentation.Punctuation.OUTDENT, Indentation.Punctuation.OUTDENT), analyze("foo", "\n", 1, "bar", "\n", 3, "baz", "\n", 2, "end")); }