public TextToken nextToken(boolean wildcards) { try { TextToken token = nextTerminalToken(wildcards); TextToken next; StringBuffer buf; int oldPos = pos; boolean found; char LA1 = LA(1); switch (token.getType()) { case TextToken.EOF : return null; case TextToken.ALPHA : found = false;  if (LA1 == '\'') { consume(); buf = new StringBuffer(token.getText()); next = nextTerminalToken(wildcards); if (next != null && next.getType() == TextToken.ALPHA) { buf.append('\''); buf.append(next.getText()); return new TextToken( TextToken.ALPHA, buf.toString()); } pos = oldPos; }   switch (LA1) { case '_' : case ':' : case '.' : case '/' : if (LA(2) == (char) - 1 || Character.isWhitespace(LA(2))) { consume(); break; } found = false; buf = new StringBuffer(token.getText()); while ((next = nextTerminalToken(wildcards)) != null) { if (next.getType() == TextToken.EOF || next.getType() == TextToken.WS) break; if(next.getType() == TextToken.P && (LA(1) == (char)-1 || Character.isWhitespace(LA(1)))) break; if(next.getType() == TextToken.ALPHANUM) found = true; buf.append(next.getText()); } if (found) token = new TextToken( TextToken.ALPHANUM, buf.toString()); else pos = oldPos; } return token; case TextToken.ALPHANUM : switch (LA1) { case '*' : case ',' : case '-' : case '_' : case ':' : case '.' : case '@' : if (LA(2) == (char) - 1 || Character.isWhitespace(LA(2))) { consume(); break; } buf = new StringBuffer(token.getText()); while ((next = nextTerminalToken(wildcards)) != null) { if (next.getType() == TextToken.EOF || next.getType() == TextToken.WS) break; buf.append(next.getText()); } token = new TextToken( TextToken.ALPHANUM, buf.toString()); } return token; default : return nextToken(wildcards); } } catch (Exception e) { System.out.println("text: " + text); e.printStackTrace(); return null; }
public TextToken nextToken() { return nextToken(false);
	public TextToken nextToken(boolean wildcards) {		try {			TextToken token = nextTerminalToken(wildcards);			TextToken next;			StringBuffer buf;			int oldPos = pos;			boolean found;			char LA1 = LA(1);			switch (token.getType()) {				case TextToken.EOF :					return null;				case TextToken.ALPHA :					found = false;					// text with apostrophe like Peter's					if (LA1 == '\'') {						consume();						buf = new StringBuffer(token.getText());						next = nextTerminalToken(wildcards);						if (next != null							&& next.getType() == TextToken.ALPHA) {							buf.append('\'');							buf.append(next.getText());							return new TextToken(								TextToken.ALPHA,								buf.toString());						}						pos = oldPos;					}					// text with some alphanumeric sequence attached 					// like Q/22/A4.5 or 12/09/1989					switch (LA1) {						case '_' :						case ':' :						case '.' :						case '/' :							if (LA(2) == (char) - 1								|| Character.isWhitespace(LA(2))) {								consume();								break;							}							found = false;							buf = new StringBuffer(token.getText());							while ((next = nextTerminalToken(wildcards))								!= null) {								if (next.getType() == TextToken.EOF									|| next.getType() == TextToken.WS)									break;								if(next.getType() == TextToken.P &&								   (LA(1) == (char)-1 || Character.isWhitespace(LA(1))))								   break;								if(next.getType() == TextToken.ALPHANUM)									found = true;								buf.append(next.getText());							}							if (found)								token =									new TextToken(										TextToken.ALPHANUM,										buf.toString());							else								pos = oldPos;					}					return token;				case TextToken.ALPHANUM :					switch (LA1) {						case '*' :						case ',' :						case '-' :						case '_' :						case ':' :						case '.' :						case '@' :							if (LA(2) == (char) - 1								|| Character.isWhitespace(LA(2))) {								consume();								break;							}							buf = new StringBuffer(token.getText());							while ((next = nextTerminalToken(wildcards))								!= null) {								if (next.getType() == TextToken.EOF									|| next.getType() == TextToken.WS)									break;								buf.append(next.getText());							}							token =								new TextToken(									TextToken.ALPHANUM,									buf.toString());					}					return token;				default :					return nextToken(wildcards);			}		} catch (Exception e) {			System.out.println("text: " + text);			e.printStackTrace();			return null;		}	}