@Test public void tokenizeSubstringWithWholeTokens() { List<Token> tokens = tokenizer.tokenizeSubstring("1 + 2 + 3", 2, 6); assertTokensEqual(of(PLUS, integer(2)), tokens); }
public List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound) { TokensCollector tc = new TokensCollector(); for (int i = beginIndex; i < rightBound; i++) { tc.append(input.charAt(i)); } tc.collectLastToken(); return tc.tokens; }
CompletionTokenizer { public List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound) { TokensCollector tc = new TokensCollector(); for (int i = beginIndex; i < rightBound; i++) { tc.append(input.charAt(i)); } tc.collectLastToken(); return tc.tokens; } }
CompletionTokenizer { public List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound) { TokensCollector tc = new TokensCollector(); for (int i = beginIndex; i < rightBound; i++) { tc.append(input.charAt(i)); } tc.collectLastToken(); return tc.tokens; } CompletionTokenizer(TokenCompletion spec); }
CompletionTokenizer { public List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound) { TokensCollector tc = new TokensCollector(); for (int i = beginIndex; i < rightBound; i++) { tc.append(input.charAt(i)); } tc.collectLastToken(); return tc.tokens; } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound); }
CompletionTokenizer { public List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound) { TokensCollector tc = new TokensCollector(); for (int i = beginIndex; i < rightBound; i++) { tc.append(input.charAt(i)); } tc.collectLastToken(); return tc.tokens; } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound);  }