@Test public void testTokenize1() { List<Token> tokens = tokenizer.tokenize("Lorem ipsum ${a}${b} dolor ${c} sit amet."); assertEquals(8, tokens.size()); assertToken(LiteralToken.class, "Lorem", tokens.get(0)); assertToken(LiteralToken.class, "ipsum", tokens.get(1)); assertToken(VariableToken.class, "a", tokens.get(2)); assertToken(VariableToken.class, "b", tokens.get(3)); assertToken(LiteralToken.class, "dolor", tokens.get(4)); assertToken(VariableToken.class, "c", tokens.get(5)); assertToken(LiteralToken.class, "sit", tokens.get(6)); assertToken(LiteralToken.class, "amet", tokens.get(7)); }
public TokenList tokenize(String statementName) { assertStatementIsValid(statementName); String splitCriterion = String.format(SPLIT_TOKEN_PATTERN_TEMPLATE, Pattern.quote(prefix), Pattern.quote(suffix)); rawTokens = statementName.split(splitCriterion); findTokens(); return mergedTokens; }
Tokenizer { public TokenList tokenize(String statementName) { assertStatementIsValid(statementName); String splitCriterion = String.format(SPLIT_TOKEN_PATTERN_TEMPLATE, Pattern.quote(prefix), Pattern.quote(suffix)); rawTokens = statementName.split(splitCriterion); findTokens(); return mergedTokens; } }
Tokenizer { public TokenList tokenize(String statementName) { assertStatementIsValid(statementName); String splitCriterion = String.format(SPLIT_TOKEN_PATTERN_TEMPLATE, Pattern.quote(prefix), Pattern.quote(suffix)); rawTokens = statementName.split(splitCriterion); findTokens(); return mergedTokens; } Tokenizer();  Tokenizer(String prefix, String suffix, String escapeCharacter); }
Tokenizer { public TokenList tokenize(String statementName) { assertStatementIsValid(statementName); String splitCriterion = String.format(SPLIT_TOKEN_PATTERN_TEMPLATE, Pattern.quote(prefix), Pattern.quote(suffix)); rawTokens = statementName.split(splitCriterion); findTokens(); return mergedTokens; } Tokenizer();  Tokenizer(String prefix, String suffix, String escapeCharacter); TokenList tokenize(String statementName); }
Tokenizer { public TokenList tokenize(String statementName) { assertStatementIsValid(statementName); String splitCriterion = String.format(SPLIT_TOKEN_PATTERN_TEMPLATE, Pattern.quote(prefix), Pattern.quote(suffix)); rawTokens = statementName.split(splitCriterion); findTokens(); return mergedTokens; } Tokenizer();  Tokenizer(String prefix, String suffix, String escapeCharacter); TokenList tokenize(String statementName); @SuppressWarnings("WeakerAccess")
    static final String DEFAULT_PREFIX; @SuppressWarnings("WeakerAccess")
    static final String DEFAULT_SUFFIX; @SuppressWarnings("WeakerAccess")
    static final String DEFAULT_ESCAPE_CHARACTER; }