@Test public void longCorrectExpression() { List<Token> tokens = tokenizer.tokenize("\t( 10) * 5! + id.value ++ + '\"text 1\"' + \"text 2\""); assertTokensEqual(of(LP, integer(10), RP, MUL, integer(5), FACTORIAL, PLUS, ID, DOT, value(), INCREMENT, PLUS, singleQtd("\"text 1\""), PLUS, doubleQtd("text 2")), tokens); }
public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound);  }