AnyTokenizer implements Tokenizer { @Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { CharSequence text = sourceCode.getCodeBuffer(); Matcher matcher = pattern.matcher(text); int lineNo = 1; int lastLineStart = 0; try { while (matcher.find()) { String image = matcher.group(); if (isComment(image)) { continue; } else if (StringUtils.isWhitespace(image)) { lineNo++; lastLineStart = matcher.end(); continue; } int bline = lineNo; int bcol = 1 + matcher.start() - lastLineStart; int ecol = StringUtil.columnNumberAt(image, image.length()); if (ecol == image.length() + 1) { ecol = bcol + image.length(); } else { lineNo += StringUtil.lineNumberAt(image, image.length()) - 1; lastLineStart = matcher.start() + image.length() - ecol + 1; } ecol = ecol - 1; tokenEntries.add(new TokenEntry(image, sourceCode.getFileName(), bline, bcol, ecol)); } } finally { tokenEntries.add(TokenEntry.getEOF()); } } AnyTokenizer();  AnyTokenizer(String eolCommentStart); private  AnyTokenizer(Pattern pattern, String commentStart); @Override void tokenize(SourceCode sourceCode, Tokens tokenEntries);  }
@Test public void testTokenPosition() { AnyTokenizer tokenizer = new AnyTokenizer(); SourceCode code = new SourceCode(new SourceCode.StringCodeLoader("a;\nbbbb\n;")); Tokens tokens = new Tokens(); tokenizer.tokenize(code, tokens); TokenEntry bbbbToken = tokens.getTokens().get(2); assertEquals(2, bbbbToken.getBeginLine()); assertEquals(1, bbbbToken.getBeginColumn()); assertEquals(4, bbbbToken.getEndColumn()); }