StringTokenizer { public SubString next() { if (initDelimPos() && delimLength == 0) { pos = subject.length(); return new SubString(subject); } if (pos >= subject.length()) { return null; } int nextDelimeter = updateDelimPositions(); if (nextDelimeter == -1) { SubString ret = new SubString(subject, pos); pos = subject.length(); return ret; } if (delimPos[nextDelimeter] == pos) { pos += delim[nextDelimeter].length(); return delim[nextDelimeter]; } SubString ret = new SubString(subject, pos, delimPos[nextDelimeter]); pos = delimPos[nextDelimeter]; return ret; } StringTokenizer(String stringToTokenize, SubString... delimeters); int indexOfNearestDelimeter(int... array); SubString next();  }
@Test public void testNext() { StringTokenizer f = new StringTokenizer("aaa,bbb--ccc", COMMA, SINGLE_LINE_COMMENT, MULTI_LINE_COMMENT_OPEN, MULTI_LINE_COMMENT_CLOSE); assertEquals("aaa", toString(f.next())); assertEquals(",", toString(f.next())); assertEquals("bbb", toString(f.next())); assertEquals("--", toString(f.next())); assertEquals("ccc", toString(f.next())); assertEquals("", toString(f.next())); assertNull(toString(f.next())); }
@Test public void testNextNoDelims() { StringTokenizer f = new StringTokenizer("aaaasdasdasdasdasd", COMMA, SINGLE_LINE_COMMENT, MULTI_LINE_COMMENT_OPEN, MULTI_LINE_COMMENT_CLOSE); assertEquals("aaaasdasdasdasdasd", toString(f.next())); assertNull(toString(f.next())); }
@Test public void testNextExpectDalrgeDelimDiscarded() { StringTokenizer f = new StringTokenizer("x,xx&&&&&xxxx", COMMA, LARGE_DELIM); assertEquals("x", toString(f.next())); assertEquals(",", toString(f.next())); assertEquals("xx", toString(f.next())); assertEquals("&&&&&", toString(f.next())); assertEquals("xxxx", toString(f.next())); assertNull(toString(f.next())); }
@Test public void testNextExpectEscapedDelimeterRecognizedFirst() { StringTokenizer f = new StringTokenizer(",aaa\\,bbb,", COMMA, ESCAPED_COMMA); assertEquals(",", toString(f.next())); assertEquals("aaa", toString(f.next())); assertEquals("\\,", toString(f.next())); assertEquals("bbb", toString(f.next())); assertEquals(",", toString(f.next())); assertNull(toString(f.next())); }
@Test public void testNextSomeDelimsRemovedRightAway() { StringTokenizer f = new StringTokenizer("aaaa,sdasdasdasdasd", COMMA, SINGLE_LINE_COMMENT, MULTI_LINE_COMMENT_OPEN, MULTI_LINE_COMMENT_CLOSE); assertEquals("aaaa", toString(f.next())); assertEquals(",", toString(f.next())); assertEquals("sdasdasdasdasd", toString(f.next())); assertNull(toString(f.next())); }