code: /**
   * 
   */
  package edu.berkeley.nlp.io;
  
  import java.io.IOException;
  import java.util.LinkedList;
  import java.util.List;
  
  /**
   * Similar to PTBLexer. However, instead of reading from a Reader this class is
   * given a line and returns a list of tokenized Strings.
   * 
   * @author petrov
   * 
   */
  public class PTBLineLexer extends PTBLexer {
  
  	public PTBLineLexer() {
  		super((java.io.Reader) null);
  	}
  
  	public List<String> tokenizeLine(String line) throws IOException {
  		LinkedList<String> tokenized = new LinkedList<String>();
  		if (line == null)
  			return tokenized;
  		int nEl = line.length();
  		char[] array = line.toCharArray();
  		zzBuffer = line.toCharArray();// new char[nEl+1];
  		// for(int i=0;i<nEl;i++) yy_buffer[i] = array[i];
  		// yy_buffer[nEl] = (char)YYEOF;
  		zzStartRead = 0;
  		zzEndRead = zzBuffer.length;
  		zzAtBOL = true;
  		zzAtEOF = false;
  		zzCurrentPos = zzMarkedPos = zzPushbackPos = 0;
  		yyline = yychar = yycolumn = 0;
  		zzLexicalState = YYINITIAL;
  		while (zzMarkedPos < zzEndRead) {
  			FeatureLabel token = next();
  			if (token != null)
  				tokenized.add(token.word());
  		}
  		return tokenized;
  	}
  
  	private boolean zzRefill() throws java.io.IOException {
  		return true;
  	}
  
  }
  