Token tkn = new Token(); StringBuffer wsBuf = new StringBuffer();   int lastChar = in.readAgain();    int c = in.read(); boolean eol = isEndOfLine(c); c = in.readAgain();   while (strategy.getIgnoreEmptyLines() && eol && (lastChar == '\n' || lastChar == ExtendedBufferedReader.UNDEFINED) && !isEndOfFile(lastChar)) {  lastChar = c; c = in.read(); eol = isEndOfLine(c); c = in.readAgain();  if (isEndOfFile(c)) { tkn.type = TT_EOF; return tkn; } }   if (isEndOfFile(lastChar) || (lastChar != strategy.getDelimiter() && isEndOfFile(c))) { tkn.type = TT_EOF; return tkn; }   while (!tkn.isReady) {  while (isWhitespace(c) && !eol) { wsBuf.append((char) c); c = in.read(); eol = isEndOfLine(c); }  if (!strategy.isCommentingDisabled() && c == strategy.getCommentStart()) {  in.readLine(); tkn = nextToken(); } else if (c == strategy.getDelimiter()) {  tkn.type = TT_TOKEN; tkn.isReady = true; } else if (eol) {  tkn.content.append(""); tkn.type = TT_EORECORD; tkn.isReady = true; } else if (c == strategy.getEncapsulator()) {  encapsulatedTokenLexer(tkn, c); } else if (isEndOfFile(c)) {  tkn.content.append(""); tkn.type = TT_EOF; tkn.isReady = true; } else {   if (!strategy.getIgnoreLeadingWhitespaces()) { tkn.content.append(wsBuf.toString()); } simpleTokenLexer(tkn, c); } } return tkn;
return nextToken(new Token());
  protected Token nextToken() throws IOException {    Token tkn = new Token();    StringBuffer wsBuf = new StringBuffer();        // get the last read char (required for empty line detection)    int lastChar = in.readAgain();        //  read the next char and set eol    /* note: unfourtunately isEndOfLine may consumes a character silently.     *       this has no effect outside of the method. so a simple workaround     *       is to call 'readAgain' on the stream...     *       uh: might using objects instead of base-types (jdk1.5 autoboxing!)     */    int c = in.read();    boolean eol = isEndOfLine(c);    c = in.readAgain();         //  empty line detection: eol AND (last char was EOL or beginning)    while (strategy.getIgnoreEmptyLines() && eol       && (lastChar == '\n'       || lastChar == ExtendedBufferedReader.UNDEFINED)       && !isEndOfFile(lastChar)) {      // go on char ahead ...      lastChar = c;      c = in.read();      eol = isEndOfLine(c);      c = in.readAgain();      // reached end of file without any content (empty line at the end)      if (isEndOfFile(c)) {        tkn.type = TT_EOF;        return tkn;      }    }    // did we reached eof during the last iteration already ? TT_EOF    if (isEndOfFile(lastChar) || (lastChar != strategy.getDelimiter() && isEndOfFile(c))) {      tkn.type = TT_EOF;      return tkn;    }         //  important: make sure a new char gets consumed in each iteration    while (!tkn.isReady) {      // ignore whitespaces at beginning of a token      while (isWhitespace(c) && !eol) {        wsBuf.append((char) c);        c = in.read();        eol = isEndOfLine(c);      }      // ok, start of token reached: comment, encapsulated, or token      if (!strategy.isCommentingDisabled() && c == strategy.getCommentStart()) {        // ignore everything till end of line and continue (incr linecount)        in.readLine();        tkn = nextToken();      } else if (c == strategy.getDelimiter()) {        // empty token return TT_TOKEN("")        tkn.type = TT_TOKEN;        tkn.isReady = true;      } else if (eol) {        // empty token return TT_EORECORD("")        tkn.content.append("");        tkn.type = TT_EORECORD;        tkn.isReady = true;      } else if (c == strategy.getEncapsulator()) {        // consume encapsulated token        encapsulatedTokenLexer(tkn, c);      } else if (isEndOfFile(c)) {        // end of file return TT_EOF()        tkn.content.append("");        tkn.type = TT_EOF;        tkn.isReady = true;      } else {        // next token must be a simple token        // add removed blanks when not ignoring whitespace chars...        if (!strategy.getIgnoreLeadingWhitespaces()) {          tkn.content.append(wsBuf.toString());        }        simpleTokenLexer(tkn, c);      }    }    return tkn;    }