@Test public void stringLiterals() { final String strBody = "These items must not be tokenized: 10 + 17! * id.value"; List<Token> tokens = tokenizer.tokenize("id + \"" + strBody + "\"'" + strBody + "'"); assertTokensEqual(of(ID, PLUS, doubleQtd(strBody), singleQtd(strBody)), tokens); }
public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound); }
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound);  }