@Test public void simpleExpression() { List<Token> tokens = tokenizer.tokenize("1+2"); assertTokensEqual(of(integer(1), PLUS, integer(2)), tokens); }
public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound);  }