@Test(dataProvider = "tokens") public void tokenize(final String tree, final String[] tokens) { final List<Token> tokenize = ParenthesesTreeParser.tokenize(tree); Assert.assertEquals( tokenize.stream() .map(t -> t.seq) .toArray(String[]::new), tokens ); }
static List<Token> tokenize(final String value) { final List<Token> tokens = new ArrayList<>(); char pc = '\0'; int pos = 0; final StringBuilder token = new StringBuilder(); for (int i = 0; i < value.length(); ++i) { final char c = value.charAt(i); if (isTokenSeparator(c) && pc != ESCAPE_CHAR) { tokens.add(new Token(unescape(token.toString()), pos)); tokens.add(new Token(Character.toString(c), i)); token.setLength(0); pos = i; } else { token.append(c); } pc = c; } if (token.length() > 0) { tokens.add(new Token(unescape(token.toString()), pos)); } return tokens; }
ParenthesesTreeParser { static List<Token> tokenize(final String value) { final List<Token> tokens = new ArrayList<>(); char pc = '\0'; int pos = 0; final StringBuilder token = new StringBuilder(); for (int i = 0; i < value.length(); ++i) { final char c = value.charAt(i); if (isTokenSeparator(c) && pc != ESCAPE_CHAR) { tokens.add(new Token(unescape(token.toString()), pos)); tokens.add(new Token(Character.toString(c), i)); token.setLength(0); pos = i; } else { token.append(c); } pc = c; } if (token.length() > 0) { tokens.add(new Token(unescape(token.toString()), pos)); } return tokens; } }
ParenthesesTreeParser { static List<Token> tokenize(final String value) { final List<Token> tokens = new ArrayList<>(); char pc = '\0'; int pos = 0; final StringBuilder token = new StringBuilder(); for (int i = 0; i < value.length(); ++i) { final char c = value.charAt(i); if (isTokenSeparator(c) && pc != ESCAPE_CHAR) { tokens.add(new Token(unescape(token.toString()), pos)); tokens.add(new Token(Character.toString(c), i)); token.setLength(0); pos = i; } else { token.append(c); } pc = c; } if (token.length() > 0) { tokens.add(new Token(unescape(token.toString()), pos)); } return tokens; } private  ParenthesesTreeParser(); }
ParenthesesTreeParser { static List<Token> tokenize(final String value) { final List<Token> tokens = new ArrayList<>(); char pc = '\0'; int pos = 0; final StringBuilder token = new StringBuilder(); for (int i = 0; i < value.length(); ++i) { final char c = value.charAt(i); if (isTokenSeparator(c) && pc != ESCAPE_CHAR) { tokens.add(new Token(unescape(token.toString()), pos)); tokens.add(new Token(Character.toString(c), i)); token.setLength(0); pos = i; } else { token.append(c); } pc = c; } if (token.length() > 0) { tokens.add(new Token(unescape(token.toString()), pos)); } return tokens; } private  ParenthesesTreeParser();  }
ParenthesesTreeParser { static List<Token> tokenize(final String value) { final List<Token> tokens = new ArrayList<>(); char pc = '\0'; int pos = 0; final StringBuilder token = new StringBuilder(); for (int i = 0; i < value.length(); ++i) { final char c = value.charAt(i); if (isTokenSeparator(c) && pc != ESCAPE_CHAR) { tokens.add(new Token(unescape(token.toString()), pos)); tokens.add(new Token(Character.toString(c), i)); token.setLength(0); pos = i; } else { token.append(c); } pc = c; } if (token.length() > 0) { tokens.add(new Token(unescape(token.toString()), pos)); } return tokens; } private  ParenthesesTreeParser();   }