code: /**
   * 
   */
  package edu.berkeley.nlp.io;
  
  /* The following code was generated by JFlex 1.3.5 on 12/2/02 9:02 PM */
  
  
  /**
   * doesn't handle \"can not\" to \"cannot\".
   */
  
  /**
   * This class is a scanner generated by <a href="http://www.jflex.de/">JFlex</a>
   * 1.3.5 on 12/2/02 9:02 PM from the specification file
   * <tt>file:/nlp/u1/jsmarr/cs276a/src/googleling/PTB2TextLexer.flex</tt>
   */
  class PTB2TextLexer {
  
  	/** This character denotes the end of file */
  	final public static int YYEOF = -1;
  
  	/** initial size of the lookahead buffer */
  	final private static int YY_BUFFERSIZE = 16384;
  
  	/** lexical states */
  	final public static int YYINITIAL = 0;
  
  	/**
  	 * Translates characters to character classes
  	 */
  	final private static String yycmap_packed = "\12\0\1\6\25\0\1\1\1\5\2\0\1\25\1\24\1\0\1\2"
  			+ "\1\17\1\20\2\0\1\5\1\13\1\4\13\0\1\5\1\5\3\0"
  			+ "\1\5\2\0\1\16\1\21\10\0\1\14\1\0\1\11\3\0\1\15"
  			+ "\1\0\1\12\6\0\1\22\1\0\1\23\2\0\1\3\15\0\1\7"
  			+ "\5\0\1\10\uff8b\0";
  
  	/**
  	 * Translates characters to character classes
  	 */
  	final private static char[] yycmap = yy_unpack_cmap(yycmap_packed);
  
  	/**
  	 * Translates a state to a row index in the transition table
  	 */
  	final private static int yy_rowMap[] = { 0, 22, 44, 66, 88, 110, 132, 154,
  			176, 198, 220, 88, 242, 264, 286, 88, 88, 88, 88, 308, 330, 88, 88,
  			88, 88, 352, 374, 396, 418, 440, 462, 88, 88, 484, 506, 528, 550,
  			572, 594 };
  
  	/**
  	 * The packed transition table of the DFA (part 0)
  	 */
  	final private static String yy_packed0 = "\1\2\1\3\1\2\1\4\2\2\1\5\4\2\1\6"
  			+ "\3\2\1\7\2\2\1\10\2\2\1\11\1\2\1\0"
  			+ "\4\2\1\0\17\2\2\0\1\12\1\0\1\13\1\14"
  			+ "\1\0\1\15\1\0\1\16\1\0\1\17\4\0\1\20"
  			+ "\2\0\1\21\1\22\1\0\1\2\1\23\1\2\1\24"
  			+ "\2\2\1\0\17\2\26\0\1\2\1\0\4\2\1\0"
  			+ "\5\2\1\25\12\2\1\26\4\2\1\0\20\2\1\27"
  			+ "\4\2\1\0\20\2\1\30\4\2\1\0\17\2\1\14"
  			+ "\1\0\1\31\3\14\1\0\17\14\4\0\1\32\23\0"
  			+ "\1\33\25\0\1\34\40\0\1\35\10\0\1\2\1\31"
  			+ "\4\2\1\0\20\2\1\0\4\2\1\0\6\2\1\36"
  			+ "\3\2\1\37\4\2\4\0\1\14\31\0\1\40\27\0"
  			+ "\1\41\30\0\1\42\3\0\1\43\4\0\1\2\1\0"
  			+ "\4\2\1\0\7\2\1\44\10\2\1\0\4\2\1\0"
  			+ "\7\2\1\45\7\2\16\0\1\46\25\0\1\47\7\0"
  			+ "\1\2\1\0\4\2\1\0\4\2\1\7\13\2\1\0"
  			+ "\4\2\1\0\4\2\1\10\12\2\13\0\1\20\25\0" + "\1\21\12\0";
  
  	/**
  	 * The transition table of the DFA
  	 */
  	final private static int yytrans[] = yy_unpack();
  
  	/* error codes */
  	final private static int YY_UNKNOWN_ERROR = 0;
  	final private static int YY_ILLEGAL_STATE = 1;
  	final private static int YY_NO_MATCH = 2;
  	final private static int YY_PUSHBACK_2BIG = 3;
  
  	/* error messages for the codes above */
  	final private static String YY_ERROR_MSG[] = {
  			"Unkown internal scanner error", "Internal error: unknown state",
  			"Error: could not match input",
  			"Error: pushback value was too large" };
  
  	/**
  	 * YY_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
  	 */
  	private final static byte YY_ATTRIBUTE[] = { 0, 1, 1, 1, 9, 1, 1, 1, 1, 0,
  			1, 9, 0, 0, 0, 9, 9, 9, 9, 1, 1, 9, 9, 9, 9, 0, 0, 0, 0, 1, 1, 9,
  			9, 0, 0, 1, 1, 0, 0 };
  
  	/** the input device */
  	private java.io.Reader yy_reader;
  
  	/** the current state of the DFA */
  	private int yy_state;
  
  	/** the current lexical state */
  	private int yy_lexical_state = YYINITIAL;
  
  	/**
  	 * this buffer contains the current text to be matched and is the source of
  	 * the yytext() string
  	 */
  	private char yy_buffer[] = new char[YY_BUFFERSIZE];
  
  	/** the textposition at the last accepting state */
  	private int yy_markedPos;
  
  	/** the textposition at the last state to be included in yytext */
  	private int yy_pushbackPos;
  
  	/** the current text position in the buffer */
  	private int yy_currentPos;
  
  	/** startRead marks the beginning of the yytext() string in the buffer */
  	private int yy_startRead;
  
  	/**
  	 * endRead marks the last character in the buffer, that has been read from
  	 * input
  	 */
  	private int yy_endRead;
  
  	/** number of newlines encountered up to the start of the matched text */
  	private int yyline;
  
  	/** the number of characters up to the start of the matched text */
  	private int yychar;
  
  	/**
  	 * the number of characters from the last newline up to the start of the
  	 * matched text
  	 */
  	private int yycolumn;
  
  	/**
  	 * yy_atBOL == true <=> the scanner is currently at the beginning of a line
  	 */
  	private boolean yy_atBOL = true;
  
  	/** yy_atEOF == true <=> the scanner is at the EOF */
  	private boolean yy_atEOF;
  
  	/* user code: */
  
  	/*
  	 * "'T WAS" { return("'TWAS"); } "'T was" { return("'Twas"); } "'t was" {
  	 * return("'twas"); } "'T IS" { return("'TIS"); } "'T is" { return("'Tis");
  	 * } "'t is" { return("'tis"); }
  	 */
  
  	/**
  	 * Creates a new scanner There is also a java.io.InputStream version of this
  	 * constructor.
  	 * 
  	 * @param in
  	 *            the java.io.Reader to read input from.
  	 */
  	PTB2TextLexer(java.io.Reader in) {
  		this.yy_reader = in;
  	}
  
  	/**
  	 * Creates a new scanner. There is also java.io.Reader version of this
  	 * constructor.
  	 * 
  	 * @param in
  	 *            the java.io.Inputstream to read input from.
  	 */
  	PTB2TextLexer(java.io.InputStream in) {
  		this(new java.io.InputStreamReader(in));
  	}
  
  	/**
  	 * Unpacks the split, compressed DFA transition table.
  	 * 
  	 * @return the unpacked transition table
  	 */
  	private static int[] yy_unpack() {
  		int[] trans = new int[616];
  		int offset = 0;
  		offset = yy_unpack(yy_packed0, offset, trans);
  		return trans;
  	}
  
  	/**
  	 * Unpacks the compressed DFA transition table.
  	 * 
  	 * @param packed
  	 *            the packed transition table
  	 * @return the index of the last entry
  	 */
  	private static int yy_unpack(String packed, int offset, int[] trans) {
  		int i = 0; /* index in packed string */
  		int j = offset; /* index in unpacked array */
  		int l = packed.length();
  		while (i < l) {
  			int count = packed.charAt(i++);
  			int value = packed.charAt(i++);
  			value--;
  			do
  				trans[j++] = value;
  			while (--count > 0);
  		}
  		return j;
  	}
  
  	/**
  	 * Unpacks the compressed character translation table.
  	 * 
  	 * @param packed
  	 *            the packed character translation table
  	 * @return the unpacked character translation table
  	 */
  	private static char[] yy_unpack_cmap(String packed) {
  		char[] map = new char[0x10000];
  		int i = 0; /* index in packed string */
  		int j = 0; /* index in unpacked array */
  		while (i < 86) {
  			int count = packed.charAt(i++);
  			char value = packed.charAt(i++);
  			do
  				map[j++] = value;
  			while (--count > 0);
  		}
  		return map;
  	}
  
  	/**
  	 * Refills the input buffer.
  	 * 
  	 * @return <code>false</code>, iff there was new input.
  	 * 
  	 * @exception java.io.IOException
  	 *                if any I/O-Error occurs
  	 */
  	private boolean yy_refill() throws java.io.IOException {
  
  		/* first: make room (if you can) */
  		if (yy_startRead > 0) {
  			System.arraycopy(yy_buffer, yy_startRead, yy_buffer, 0, yy_endRead
  					- yy_startRead);
  
  			/* translate stored positions */
  			yy_endRead -= yy_startRead;
  			yy_currentPos -= yy_startRead;
  			yy_markedPos -= yy_startRead;
  			yy_pushbackPos -= yy_startRead;
  			yy_startRead = 0;
  		}
  
  		/* is the buffer big enough? */
  		if (yy_currentPos >= yy_buffer.length) {
  			/* if not: blow it up */
  			char newBuffer[] = new char[yy_currentPos * 2];
  			System.arraycopy(yy_buffer, 0, newBuffer, 0, yy_buffer.length);
  			yy_buffer = newBuffer;
  		}
  
  		/* finally: fill the buffer with new input */
  		int numRead = yy_reader.read(yy_buffer, yy_endRead, yy_buffer.length
  				- yy_endRead);
  
  		if (numRead < 0) {
  			return true;
  		} else {
  			yy_endRead += numRead;
  			return false;
  		}
  	}
  
  	/**
  	 * Closes the input stream.
  	 */
  	final public void yyclose() throws java.io.IOException {
  		yy_atEOF = true; /* indicate end of file */
  		yy_endRead = yy_startRead; /* invalidate buffer */
  
  		if (yy_reader != null)
  			yy_reader.close();
  	}
  
  	/**
  	 * Closes the current stream, and resets the scanner to read from a new
  	 * input stream.
  	 * 
  	 * All internal variables are reset, the old input stream <b>cannot</b> be
  	 * reused (internal buffer is discarded and lost). Lexical state is set to
  	 * <tt>YY_INITIAL</tt>.
  	 * 
  	 * @param reader
  	 *            the new input stream
  	 */
  	final public void yyreset(java.io.Reader reader) throws java.io.IOException {
  		yyclose();
  		yy_reader = reader;
  		yy_atBOL = true;
  		yy_atEOF = false;
  		yy_endRead = yy_startRead = 0;
  		yy_currentPos = yy_markedPos = yy_pushbackPos = 0;
  		yyline = yychar = yycolumn = 0;
  		yy_lexical_state = YYINITIAL;
  	}
  
  	/**
  	 * Returns the current lexical state.
  	 */
  	final public int yystate() {
  		return yy_lexical_state;
  	}
  
  	/**
  	 * Enters a new lexical state
  	 * 
  	 * @param newState
  	 *            the new lexical state
  	 */
  	final public void yybegin(int newState) {
  		yy_lexical_state = newState;
  	}
  
  	/**
  	 * Returns the text matched by the current regular expression.
  	 */
  	final public String yytext() {
  		return new String(yy_buffer, yy_startRead, yy_markedPos - yy_startRead);
  	}
  
  	/**
  	 * Returns the character at position <tt>pos</tt> from the matched text.
  	 * 
  	 * It is equivalent to yytext().charAt(pos), but faster
  	 * 
  	 * @param pos
  	 *            the position of the character to fetch. A value from 0 to
  	 *            yylength()-1.
  	 * 
  	 * @return the character at position pos
  	 */
  	final public char yycharat(int pos) {
  		return yy_buffer[yy_startRead + pos];
  	}
  
  	/**
  	 * Returns the length of the matched text region.
  	 */
  	final public int yylength() {
  		return yy_markedPos - yy_startRead;
  	}
  
  	/**
  	 * Reports an error that occured while scanning.
  	 * 
  	 * In a wellformed scanner (no or only correct usage of yypushback(int) and
  	 * a match-all fallback rule) this method will only be called with things
  	 * that "Can't Possibly Happen". If this method is called, something is
  	 * seriously wrong (e.g. a JFlex bug producing a faulty scanner etc.).
  	 * 
  	 * Usual syntax/scanner level error handling should be done in error
  	 * fallback rules.
  	 * 
  	 * @param errorCode
  	 *            the code of the errormessage to display
  	 */
  	private void yy_ScanError(int errorCode) {
  		String message;
  		try {
  			message = YY_ERROR_MSG[errorCode];
  		} catch (ArrayIndexOutOfBoundsException e) {
  			message = YY_ERROR_MSG[YY_UNKNOWN_ERROR];
  		}
  
  		throw new Error(message);
  	}
  
  	/**
  	 * Pushes the specified amount of characters back into the input stream.
  	 * 
  	 * They will be read again by then next call of the scanning method
  	 * 
  	 * @param number
  	 *            the number of characters to be read again. This number must
  	 *            not be greater than yylength()!
  	 */
  	private void yypushback(int number) {
  		if (number > yylength())
  			yy_ScanError(YY_PUSHBACK_2BIG);
  
  		yy_markedPos -= number;
  	}
  
  	/**
  	 * Resumes scanning until the next regular expression is matched, the end of
  	 * input is encountered or an I/O-Error occurs.
  	 * 
  	 * @return the next token
  	 * @exception java.io.IOException
  	 *                if any I/O-Error occurs
  	 */
  	public String next() throws java.io.IOException {
  		int yy_input;
  		int yy_action;
  
  		// cached fields:
  		int yy_currentPos_l;
  		int yy_startRead_l;
  		int yy_markedPos_l;
  		int yy_endRead_l = yy_endRead;
  		char[] yy_buffer_l = yy_buffer;
  		char[] yycmap_l = yycmap;
  
  		int[] yytrans_l = yytrans;
  		int[] yy_rowMap_l = yy_rowMap;
  		byte[] yy_attr_l = YY_ATTRIBUTE;
  
  		while (true) {
  			yy_markedPos_l = yy_markedPos;
  
  			yy_action = -1;
  
  			yy_startRead_l = yy_currentPos_l = yy_currentPos = yy_startRead = yy_markedPos_l;
  
  			yy_state = yy_lexical_state;
  
  			yy_forAction: {
  				while (true) {
  
  					if (yy_currentPos_l < yy_endRead_l)
  						yy_input = yy_buffer_l[yy_currentPos_l++];
  					else if (yy_atEOF) {
  						yy_input = YYEOF;
  						break yy_forAction;
  					} else {
  						// store back cached positions
  						yy_currentPos = yy_currentPos_l;
  						yy_markedPos = yy_markedPos_l;
  						boolean eof = yy_refill();
  						// get translated positions and possibly new buffer
  						yy_currentPos_l = yy_currentPos;
  						yy_markedPos_l = yy_markedPos;
  						yy_buffer_l = yy_buffer;
  						yy_endRead_l = yy_endRead;
  						if (eof) {
  							yy_input = YYEOF;
  							break yy_forAction;
  						} else {
  							yy_input = yy_buffer_l[yy_currentPos_l++];
  						}
  					}
  					int yy_next = yytrans_l[yy_rowMap_l[yy_state]
  							+ yycmap_l[yy_input]];
  					if (yy_next == -1)
  						break yy_forAction;
  					yy_state = yy_next;
  
  					int yy_attributes = yy_attr_l[yy_state];
  					if ((yy_attributes & 1) == 1) {
  						yy_action = yy_state;
  						yy_markedPos_l = yy_currentPos_l;
  						if ((yy_attributes & 8) == 8)
  							break yy_forAction;
  					}
  
  				}
  			}
  
  			// store back cached position
  			yy_markedPos = yy_markedPos_l;
  
  			switch (yy_action) {
  
  			case 32: {
  				return ("N'T");
  			}
  			case 40:
  				break;
  			case 31: {
  				return ("n't");
  			}
  			case 41:
  				break;
  			case 10:
  			case 11: {
  				return (yytext().substring(1, yytext().length()));
  			}
  			case 42:
  				break;
  			case 1:
  			case 3:
  			case 5:
  			case 6:
  			case 7:
  			case 8:
  			case 19:
  			case 20:
  			case 29:
  			case 30:
  			case 35:
  			case 36: {
  				return (yytext());
  			}
  			case 43:
  				break;
  			case 2: {
  				return (yytext());
  			}
  			case 44:
  				break;
  			case 4: {
  				return (yytext());
  			}
  			case 45:
  				break;
  			case 24: {
  				return ("\"");
  			}
  			case 46:
  				break;
  			case 23: {
  				return ("$");
  			}
  			case 47:
  				break;
  			case 22: {
  				return ("[");
  			}
  			case 48:
  				break;
  			case 21: {
  				return ("(");
  			}
  			case 49:
  				break;
  			case 15: {
  				return (")");
  			}
  			case 50:
  				break;
  			case 16: {
  				return ("]");
  			}
  			case 51:
  				break;
  			case 17: {
  				return ("%");
  			}
  			case 52:
  				break;
  			case 18: {
  				return ("`");
  			}
  			case 53:
  				break;
  			default:
  				if (yy_input == YYEOF && yy_startRead == yy_currentPos) {
  					yy_atEOF = true;
  					{
  						return (null);
  					}
  				} else {
  					yy_ScanError(YY_NO_MATCH);
  				}
  			}
  		}
  	}
  
  	/**
  	 * Runs the scanner on input files.
  	 * 
  	 * This is a standalone scanner, i.e. it will print any unmatched text to
  	 * System.out unchanged.
  	 * 
  	 * @param argv
  	 *            the command line, contains the filenames to run the scanner
  	 *            on.
  	 */
  	public static void main(String argv[]) {
  		if (argv.length == 0) {
  			System.out.println("Usage : java PTB2TextLexer <inputfile>");
  		} else {
  			for (int i = 0; i < argv.length; i++) {
  				PTB2TextLexer scanner = null;
  				try {
  					scanner = new PTB2TextLexer(new java.io.FileReader(argv[i]));
  					while (!scanner.yy_atEOF)
  						scanner.next();
  				} catch (java.io.FileNotFoundException e) {
  					System.out.println("File not found : \"" + argv[i] + "\"");
  				} catch (java.io.IOException e) {
  					System.out.println("IO error scanning file \"" + argv[i]
  							+ "\"");
  					System.out.println(e);
  				} catch (Exception e) {
  					System.out.println("Unexpected exception:");
  					e.printStackTrace();
  				}
  			}
  		}
  	}
  
  }
  