       } while si < len
   }
 
+  func parseTokens(fn: (token: LexerToken) throws -> Void) throws {
+    try parse() { tt in
+      try fn(token: CommonToken(bytes: self.stream.bytes,
+            startIndex: self.stream.startIndex,
+            endIndex: self.stream.currentIndex,
+            type: tt))
+    }
+  }
+
   public var description: String {
     return "CommonLexer(status: \(status))"
   }

      throws {
    try parse() { tt in
      if let s = self.stream.currentTokenString {
        return try fn(type: tt, string: s)
      } else {
        throw CommonLexerError.String
      }

}


public struct CommonToken: LexerToken {
  public var bytes: [UInt8]
  public var startIndex: Int
  public var endIndex: Int
  public var type: TokenType
}


public class CommonLexer: Lexer, CustomStringConvertible {

  public var stream: ByteStream
  var status: CommonLexerStatus

  init(bytes: [UInt8], status: CommonLexerStatus) {
