@Test public void testNextTokenSimple() { DefaultJavadocTokenStream stream = stream("alphabets.txt"); for (char c = 'a'; c <= 'z'; c++) { assertEquals(String.valueOf(c), stream.nextToken().getText()); } assertEquals(EOF, stream.nextToken().getKind()); assertEquals(EOF, stream.nextToken().getKind()); }
@Override public JavadocToken nextToken() { while (true) { JavadocToken token = scanner.lookahead(0); JavadocTokenKind kind = token.getKind(); if (kind == JavadocTokenKind.LINE_BREAK) { int offset = JavadocScannerUtil.countUntilNextLineStart(scanner, 0); scanner.consume(offset); } else if (kind == JavadocTokenKind.WHITE_SPACES) { scanner.consume(1); } else { scanner.consume(1); return token; } } }
DefaultJavadocTokenStream implements JavadocTokenStream { @Override public JavadocToken nextToken() { while (true) { JavadocToken token = scanner.lookahead(0); JavadocTokenKind kind = token.getKind(); if (kind == JavadocTokenKind.LINE_BREAK) { int offset = JavadocScannerUtil.countUntilNextLineStart(scanner, 0); scanner.consume(offset); } else if (kind == JavadocTokenKind.WHITE_SPACES) { scanner.consume(1); } else { scanner.consume(1); return token; } } } }
DefaultJavadocTokenStream implements JavadocTokenStream { @Override public JavadocToken nextToken() { while (true) { JavadocToken token = scanner.lookahead(0); JavadocTokenKind kind = token.getKind(); if (kind == JavadocTokenKind.LINE_BREAK) { int offset = JavadocScannerUtil.countUntilNextLineStart(scanner, 0); scanner.consume(offset); } else if (kind == JavadocTokenKind.WHITE_SPACES) { scanner.consume(1); } else { scanner.consume(1); return token; } } } DefaultJavadocTokenStream(JavadocScanner scanner); }
DefaultJavadocTokenStream implements JavadocTokenStream { @Override public JavadocToken nextToken() { while (true) { JavadocToken token = scanner.lookahead(0); JavadocTokenKind kind = token.getKind(); if (kind == JavadocTokenKind.LINE_BREAK) { int offset = JavadocScannerUtil.countUntilNextLineStart(scanner, 0); scanner.consume(offset); } else if (kind == JavadocTokenKind.WHITE_SPACES) { scanner.consume(1); } else { scanner.consume(1); return token; } } } DefaultJavadocTokenStream(JavadocScanner scanner); @Override JavadocToken peek(); @Override JavadocToken lookahead(int k); @Override JavadocToken nextToken(); @Override void mark(); @Override void rewind(); @Override void discard(); }
DefaultJavadocTokenStream implements JavadocTokenStream { @Override public JavadocToken nextToken() { while (true) { JavadocToken token = scanner.lookahead(0); JavadocTokenKind kind = token.getKind(); if (kind == JavadocTokenKind.LINE_BREAK) { int offset = JavadocScannerUtil.countUntilNextLineStart(scanner, 0); scanner.consume(offset); } else if (kind == JavadocTokenKind.WHITE_SPACES) { scanner.consume(1); } else { scanner.consume(1); return token; } } } DefaultJavadocTokenStream(JavadocScanner scanner); @Override JavadocToken peek(); @Override JavadocToken lookahead(int k); @Override JavadocToken nextToken(); @Override void mark(); @Override void rewind(); @Override void discard();  }