@Test public void testIncrementToken() throws Exception { String inputText = "alpha/beta/gamma/delta.ext"; String[] expectedTokens = inputText.split("[/.]"); PathTokenizer tokenizer = new PathTokenizer(); tokenizer.setReader(new StringReader(inputText)); CharTermAttribute term = tokenizer.addAttribute(CharTermAttribute.class); OffsetAttribute offset = tokenizer.addAttribute(OffsetAttribute.class); int count = 0; int dots = 0; tokenizer.reset(); while (tokenizer.incrementToken()) { if ( term.toString().equals(".") ) { dots++;break; } assertTrue("too many tokens", count < expectedTokens.length); String expected = expectedTokens[count]; assertEquals("term", expected, term.toString()); assertEquals("start", inputText.indexOf(expected), offset.startOffset()); assertEquals("end", inputText.indexOf(expected) + expected.length(), offset.endOffset()); count++; } tokenizer.end(); tokenizer.close(); assertEquals("wrong number of tokens", expectedTokens.length, count+dots); }
@Override public final boolean incrementToken() throws IOException { clearAttributes(); if (dot) { dot = false; termAtt.setEmpty(); termAtt.append(cdot); termAtt.setLength(1); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + 1)); startPosition++; return true; } char[] buf = new char[64]; int c; int i = 0; do { c = input.read(); charsRead++; if (c == -1) { return false; } } while (c == delimiter); do { if (i >= buf.length) { buf = Arrays.copyOf(buf, buf.length * 2); } buf[i++] = (char) c; c = input.read(); charsRead++; } while (c != delimiter && c != cdot && !Character.isWhitespace(c) && c != -1); if (c == cdot) { dot = true; } String bufLcase = String.valueOf(buf, 0, i).toLowerCase(Locale.ROOT); i = bufLcase.length(); termAtt.append(bufLcase); termAtt.setLength(i); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + i)); startPosition = startPosition + i + 1; return true; }
PathTokenizer extends Tokenizer { @Override public final boolean incrementToken() throws IOException { clearAttributes(); if (dot) { dot = false; termAtt.setEmpty(); termAtt.append(cdot); termAtt.setLength(1); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + 1)); startPosition++; return true; } char[] buf = new char[64]; int c; int i = 0; do { c = input.read(); charsRead++; if (c == -1) { return false; } } while (c == delimiter); do { if (i >= buf.length) { buf = Arrays.copyOf(buf, buf.length * 2); } buf[i++] = (char) c; c = input.read(); charsRead++; } while (c != delimiter && c != cdot && !Character.isWhitespace(c) && c != -1); if (c == cdot) { dot = true; } String bufLcase = String.valueOf(buf, 0, i).toLowerCase(Locale.ROOT); i = bufLcase.length(); termAtt.append(bufLcase); termAtt.setLength(i); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + i)); startPosition = startPosition + i + 1; return true; } }
PathTokenizer extends Tokenizer { @Override public final boolean incrementToken() throws IOException { clearAttributes(); if (dot) { dot = false; termAtt.setEmpty(); termAtt.append(cdot); termAtt.setLength(1); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + 1)); startPosition++; return true; } char[] buf = new char[64]; int c; int i = 0; do { c = input.read(); charsRead++; if (c == -1) { return false; } } while (c == delimiter); do { if (i >= buf.length) { buf = Arrays.copyOf(buf, buf.length * 2); } buf[i++] = (char) c; c = input.read(); charsRead++; } while (c != delimiter && c != cdot && !Character.isWhitespace(c) && c != -1); if (c == cdot) { dot = true; } String bufLcase = String.valueOf(buf, 0, i).toLowerCase(Locale.ROOT); i = bufLcase.length(); termAtt.append(bufLcase); termAtt.setLength(i); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + i)); startPosition = startPosition + i + 1; return true; } PathTokenizer(); }
PathTokenizer extends Tokenizer { @Override public final boolean incrementToken() throws IOException { clearAttributes(); if (dot) { dot = false; termAtt.setEmpty(); termAtt.append(cdot); termAtt.setLength(1); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + 1)); startPosition++; return true; } char[] buf = new char[64]; int c; int i = 0; do { c = input.read(); charsRead++; if (c == -1) { return false; } } while (c == delimiter); do { if (i >= buf.length) { buf = Arrays.copyOf(buf, buf.length * 2); } buf[i++] = (char) c; c = input.read(); charsRead++; } while (c != delimiter && c != cdot && !Character.isWhitespace(c) && c != -1); if (c == cdot) { dot = true; } String bufLcase = String.valueOf(buf, 0, i).toLowerCase(Locale.ROOT); i = bufLcase.length(); termAtt.append(bufLcase); termAtt.setLength(i); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + i)); startPosition = startPosition + i + 1; return true; } PathTokenizer(); @Override final boolean incrementToken(); @Override final void end(); @Override void reset(); @Override final void close(); }
PathTokenizer extends Tokenizer { @Override public final boolean incrementToken() throws IOException { clearAttributes(); if (dot) { dot = false; termAtt.setEmpty(); termAtt.append(cdot); termAtt.setLength(1); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + 1)); startPosition++; return true; } char[] buf = new char[64]; int c; int i = 0; do { c = input.read(); charsRead++; if (c == -1) { return false; } } while (c == delimiter); do { if (i >= buf.length) { buf = Arrays.copyOf(buf, buf.length * 2); } buf[i++] = (char) c; c = input.read(); charsRead++; } while (c != delimiter && c != cdot && !Character.isWhitespace(c) && c != -1); if (c == cdot) { dot = true; } String bufLcase = String.valueOf(buf, 0, i).toLowerCase(Locale.ROOT); i = bufLcase.length(); termAtt.append(bufLcase); termAtt.setLength(i); offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + i)); startPosition = startPosition + i + 1; return true; } PathTokenizer(); @Override final boolean incrementToken(); @Override final void end(); @Override void reset(); @Override final void close(); static final char DEFAULT_DELIMITER; }