@Test public void valueTokens() { List<Token> tokens = tokenizer.tokenize("value * aaaa + posValue"); assertTokensEqual(of(value(), MUL, complex(), PLUS, pos()), tokens); }
public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound);  }