context: // StreamTokenizer.cs
  // 
  // Copyright (C) 2002-2004 Ryan Seghers
  //
  // This software is provided AS IS. No warranty is granted, 
  // neither expressed nor implied. USE THIS SOFTWARE AT YOUR OWN RISK.
  // NO REPRESENTATION OF MERCHANTABILITY or FITNESS FOR ANY 
  // PURPOSE is given.
  //
  // License to use this software is limited by the following terms:
  // 1) This code may be used in any program, including programs developed
  //    for commercial purposes, provided that this notice is included verbatim.
  //    
  // Also, in return for using this code, please attempt to make your fixes and
  // updates available in some way, such as by sending your updates to the
  // author.
  //
  // To-do:
  //		make type exclusivity explict, and enforce:
  //			digit can be word
  //			word can't be whitespace
  //			etc.
  //      large-integer handling is imprecise, fix it
  //          (most 19 digit decimal numbers can be Int64's but I'm 
  //           using float anyway)
  //		add more mangled float test cases
  //
  // Later:
  //		reconfigurable vs Unicode support
  //		add NUnit wrap of built-in tests
  
  using System;
  using System.Collections;
  using System.IO;
  using System.Text;
  
  namespace RTools_NTS.Util
  {
  	// ---------------------------------------------------------------------
  	#region Exceptions
  	// ---------------------------------------------------------------------
  
  	/// <summary>
  	/// Exception class for unterminated tokens.
  	/// </summary>
  	public class StreamTokenizerUntermException : Exception
  	{
  		/// <summary>
  		/// Construct with a particular message.
  		/// </summary>
  		/// <param name="msg">The message to store in this object.</param>
  		public StreamTokenizerUntermException(string msg) : base(msg) {}
  	}
  
  	/// <summary>
  	/// Exception class for unterminated quotes.
  	/// </summary>
  	public class StreamTokenizerUntermQuoteException : StreamTokenizerUntermException
  	{
  		/// <summary>
  		/// Construct with a particular message.
  		/// </summary>
  		/// <param name="msg">The message to store in this object.</param>
  		public StreamTokenizerUntermQuoteException(string msg) : base(msg) {}
  	}
  
  	/// <summary>
  	/// Exception class for unterminated block comments.
  	/// </summary>
  	public class StreamTokenizerUntermCommentException : StreamTokenizerUntermException
  	{
  		/// <summary>
  		/// Construct with a particular message.
  		/// </summary>
  		/// <param name="msg">The message to store in this object.</param>
  		public StreamTokenizerUntermCommentException(string msg) : base(msg) {}
  	}
  
  	#endregion
  
  	// ---------------------------------------------------------------------
  	#region Enumerations
  	// ---------------------------------------------------------------------
  
  	/// <summary>
  	/// Bitwise enumeration for character types.
  	/// </summary>
  	[Flags]
  	public enum CharTypeBits : byte
  	{
  		/// <summary>word characters (usually alpha, digits, and domain specific)</summary>
  		Word = 1,
  		/// <summary># or something for line comments</summary>
  		Comment = 2,
  		/// <summary>whitespace</summary>
  		Whitespace = 4,
  		/// <summary>' or " type</summary>
  		Quote = 8,
  		/// <summary>usually 0 to 9</summary>
  		Digit = 16,
  		/// <summary>usually 0 to 9, a-f and A-F</summary>
  		HexDigit = 32,
  		/// <summary>eof char</summary>
  		Eof = 64
  	}
  
  	#endregion
  
  	/// <summary>
  	/// This contains the settings that control the behavior of the tokenizer.
  	/// This is separated from the StreamTokenizer so that common settings
  	/// are easy to package and keep together.
  	/// </summary>
  	[Serializable]
  	public class StreamTokenizerSettings
  	{
  		// ---------------------------------------------------------------------
  		#region Properties
  		// ---------------------------------------------------------------------
  
  		private byte[] charTypes;
  		/// <summary>
  		/// This is the character type table.  Each byte is bitwise encoded
  		/// with the character attributes, such as whether that character is
  		/// word or whitespace.
  		/// </summary>
  		public byte[] CharTypes { get { return(charTypes); } }
  
  		bool grabWhitespace;
  		/// <summary>
  		/// Whether or not to return whitespace tokens.  If not, they're ignored.
  		/// </summary>
  		public bool GrabWhitespace { get { return(grabWhitespace); } set { grabWhitespace = value; } }
  
  		bool grabEol;
  		/// <summary>
  		/// Whether or not to return EolTokens on end of line.  Eol tokens will not
  		/// break up other tokens which can be multi-line.  For example block comments 
  		/// and quotes will not be broken by Eol tokens.  Therefore the number of
  		/// Eol tokens does not give you the line count of a stream.
  		/// </summary>
  		public bool GrabEol { get { return(grabEol); } set { grabEol = value; } }
  
  		bool slashSlashComments;
  		/// <summary>
  		/// Whether or not to look for // comments
  		/// </summary>
  		public bool SlashSlashComments { get { return(slashSlashComments); } set { slashSlashComments = value; } }
  
  		bool slashStarComments;
  		/// <summary>
  		/// Whether or not to look for /* */ block comments.
  		/// </summary>
  		public bool SlashStarComments { get { return(slashStarComments); } set { slashStarComments = value; } }
  
  		bool grabComments;
  		/// <summary>
  		/// Whether or not to return comments.
  		/// </summary>
  		public bool GrabComments { get { return(grabComments); } set { grabComments = value; } }
  
  		bool doUntermCheck;
  		/// <summary>
  		/// Whether or not to check for unterminated quotes and block comments.
  		/// If true, and one is encoutered, an exception is thrown of the appropriate type.
  		/// </summary>
  		public bool DoUntermCheck { get { return(doUntermCheck); } set { doUntermCheck = value; } }
  
  		bool parseNumbers;
  		/// <summary>
  		/// Whether or not digits are specified as Digit type in the
  		/// character table.
  		/// This setting is based on the character types table, so this
  		/// setting interacts with character type table manipulation.
  		/// This setting may become incorrect if you modify the character
  		/// types table directly.
  		/// </summary>
  		public bool ParseNumbers
  		{
  			get { return(parseNumbers); }
  /* dropped for speed, this means this property isn't accurate if
   * character types table is modified directly.
   * 			{ 
  				for (int i = '0'; i <= '9'; i++)
  				{
  					if (!IsCharType((char)i, CharTypeBits.Digit)) 
  					{
  						return(false);
  					}
  				}
  
  				return(true); 
  			}
  */
  			set 
  			{
  				if (value)
  				{
  					for (int i = '0'; i <= '9'; i++) 
  						charTypes[i] |= (byte)CharTypeBits.Digit;
  				}
  				else
  				{
  					byte digit = (byte)CharTypeBits.Digit;
  
  					for (int i = '0'; i <= '9'; i++) 
  					{
  						charTypes[i] &= (byte)(~digit); // not digit
  					}
  				}
  				parseNumbers = value;
  			}
  		}
  
  		bool parseHexNumbers;
  
  		/// <summary>
  		/// Whether or not to parse Hex (0xABCD...) numbers.
  		/// This setting is based on the character types table, so this
  		/// setting interacts with character type table manipulation.
  		/// </summary>
  		public bool ParseHexNumbers
  		{
  			get 
  			{ 
  				return(parseHexNumbers);
  //				for (int i = 'A'; i <= 'F'; i++)
  //				{
  //					if (!IsCharType((char)i, CharTypeBits.Digit)) 
  //					{
  //						return(false);
  //					}
  //				}
  //				for (int i = 'a'; i <= 'f'; i++)
  //				{
  //					if (!IsCharType((char)i, CharTypeBits.Digit)) 
  //					{
  //						return(false);
  //					}
  //				}
  //				if (!IsCharType('x', CharTypeBits.Digit)) return(false);
  //
  //				return(true); 
  			}
  			set 
  			{
  				parseHexNumbers = value;
  				if (parseHexNumbers)
  				{
  					for (int i = '0'; i <= '9'; i++)
  						charTypes[i] |= (byte)CharTypeBits.HexDigit;
  					for (int i = 'A'; i <= 'F'; i++) 
  						charTypes[i] |= (byte)CharTypeBits.HexDigit;
  					for (int i = 'a'; i <= 'f'; i++) 
  						charTypes[i] |= (byte)CharTypeBits.HexDigit;
  					charTypes[(int)'x'] |= (byte)CharTypeBits.HexDigit;
  				}
  				else
  				{
  					byte digit = (byte)CharTypeBits.HexDigit;
  
  					for (int i = 'A'; i <= 'F'; i++) 
  					{
  						charTypes[i] &= (byte)(~digit); // not digit
  					}
  					for (int i = 'a'; i <= 'f'; i++) 
  					{
  						charTypes[i] &= (byte)(~digit); // not digit
  					}
  					charTypes[(int)'x'] &= (byte)(~digit);
  				}
  			}
  		}
  
  		#endregion
  
  		// ---------------------------------------------------------------------
  		#region Constructors/Destructor
  		// ---------------------------------------------------------------------
  
  		/// <summary>
  		/// Default constructor.
  		/// </summary>
  		public StreamTokenizerSettings()
  		{
  			charTypes = new byte[StreamTokenizer.NChars + 1];  // plus an EOF entry
  			SetDefaults();
  		}
  
  		/// <summary>
  		/// Copy constructor.
  		/// </summary>
  		public StreamTokenizerSettings(StreamTokenizerSettings other)
  		{
  			Copy(other);
  		}
  
  		/// <summary>
  		/// Sets this object to be the same as the specified object.
  		/// Note that some settings which are entirely embodied by the character
  		/// type table.
  		/// </summary>
  		public void Copy(StreamTokenizerSettings other)
  		{
  			charTypes = new byte[StreamTokenizer.NChars + 1];  // plus an EOF entry
  			Array.Copy(other.charTypes, 0, charTypes, 0, charTypes.Length);
  			
  			grabWhitespace = other.grabWhitespace;
  			grabEol = other.grabEol;
  			slashSlashComments = other.slashSlashComments;
  			slashStarComments = other.slashStarComments;
  			grabComments = other.grabComments;
  			doUntermCheck = other.doUntermCheck;
  
  			parseHexNumbers = other.parseHexNumbers;
  		}
  
  		#endregion
  
  		// ---------------------------------------------------------------------
  		#region main Setup
  		// ---------------------------------------------------------------------
  
  		/// <summary>
  		/// Setup default parse behavior.
  		/// This resets to same behavior as on construction.
  		/// </summary>
  		/// <returns>bool - true for success.</returns>
  		public bool SetDefaults()
  		{
  			slashStarComments = false;
  			grabComments = false;
  			slashSlashComments = false;
  			grabWhitespace = false;
  			doUntermCheck = true;
  			grabEol = false;
  
  			// setup table
  			ResetCharTypeTable();
  			ParseNumbers = true;
  			ParseHexNumbers = true;
  			WordChars('A', 'Z');
  			WordChars('a', 'z');
  			WhitespaceChars(0, ' ');
  			QuoteChar('\'');
  			QuoteChar('"');
  			WordChars('0', '9');
  
  			return(true);
  		}
  
  		/// <summary>
  		/// Apply settings which are commonly used for code parsing
  		/// C-endCapStyle code, including C++, C#, and Java.
  		/// </summary>
  		/// <returns></returns>
  		public bool SetupForCodeParse()
  		{
  			GrabWhitespace = true;
  			GrabComments = true;
  			SlashSlashComments = true;
  			DoUntermCheck = true;
  			SlashStarComments = true;
  			WordChar('_');
  			ParseNumbers = true;
  			ParseHexNumbers = true;
  			return(true);
  		}
  
  		#endregion
  
  		// ---------------------------------------------------------------------
  		#region Character Table Setup
  		// ---------------------------------------------------------------------
  
  		/// <summary>
  		/// Clear the character type settings.  This leaves them unset,
  		/// as opposed to the default.  Use SetDefaults() for default
  		/// settings.
  		/// </summary>
  		public void ResetCharTypeTable() 
  		{
  			Array.Clear(charTypes, 0, charTypes.Length);
  			charTypes[StreamTokenizer.NChars] = (byte)CharTypeBits.Eof; // last entry for Eof
  		}
  
  		/// <summary>
  		/// Specify that a particular character is a word character.
  		/// Character table type manipulation method.
  		/// This adds the type to the char(s), rather
  		/// than overwriting other types.
  		/// </summary>
  		/// <param name="c">The character.</param>
  		public void WordChar(int c) 
  		{
  			charTypes[c] |= (byte)CharTypeBits.Word;
  		}
  
  		/// <summary>
  		/// Specify that a range of characters are word characters.
  		/// Character table type manipulation method.
  		/// This adds the type to the char(s), rather
  		/// than overwriting other types.
  		/// </summary>
  		/// <param name="startChar">First character.</param>
  		/// <param name="endChar">Last character.</param>
  		public void WordChars(int startChar, int endChar) 
  		{
  			for (int i = startChar; i <= endChar; i++)
  			{
  				charTypes[i] |= (byte)CharTypeBits.Word;
  			}
  		}
  
  		/// <summary>
  		/// Specify that a string of characters are word characters.
  		/// Character table type manipulation method.
  		/// This adds the type to the char(s), rather
  		/// than overwriting other types.
  		/// </summary>
  		/// <param name="s"></param>
  		public void WordChars(string s)
  		{
  			for (int i = 0; i < s.Length; i++)
  				charTypes[s[i]] |= (byte)CharTypeBits.Word;
  		}
  
  		/// <summary>
  		/// Specify that a character is a whitespace character.
  		/// Character table type manipulation method.
  		/// This type is exclusive with other types.
  		/// </summary>
  		/// <param name="c">The character.</param>
  		public void WhitespaceChar(int c) 
  		{
  			charTypes[c] = (byte)CharTypeBits.Whitespace;
  		}
  
  		/// <summary>
  		/// Specify that a range of characters are whitespace characters.
  		/// Character table type manipulation method.
  		/// This adds the characteristic to the char(s), rather
  		/// than overwriting other characteristics.
  		/// </summary>
  		/// <param name="startChar">First character.</param>
  		/// <param name="endChar">Last character.</param>
  		public void WhitespaceChars(int startChar, int endChar) 
  		{
  			for (int i = startChar; i <= endChar; i++)
  				charTypes[i] = (byte)CharTypeBits.Whitespace;
  		}
  
  		/// <summary>
  		/// Remove other type settings from a range of characters.
  		/// Character table type manipulation method.
  		/// </summary>
  		/// <param name="startChar"></param>
  		/// <param name="endChar"></param>
  		public void OrdinaryChars(int startChar, int endChar) 
  		{
  			for (int i = startChar; i <= endChar; i++)
  				charTypes[i] = 0;
  		}
  
  		/// <summary>
  		/// Remove other type settings from a character.
  		/// Character table type manipulation method.
  		/// </summary>
  		/// <param name="c"></param>
  		public void OrdinaryChar(int c) 
  		{
  			charTypes[c] = 0;
  		}
  
  		/// <summary>
  		/// Specify that a particular character is a comment-starting character.
  		/// Character table type manipulation method.
  		/// </summary>
  		/// <param name="c"></param>
  		public void CommentChar(int c) 
  		{
  			charTypes[c] = (byte)CharTypeBits.Comment;
  		}
  
  		/// <summary>
  		/// Specify that a particular character is a quote character.
  		/// Character table type manipulation method.
  		/// </summary>
  		/// <param name="c"></param>
  		public void QuoteChar(int c) 
  		{
  			charTypes[c] = (byte)CharTypeBits.Quote;
  		}
  
  		#endregion
  
  		// ---------------------------------------------------------------------
  		#region Utility Methods
  		// ---------------------------------------------------------------------
  
  		/// <summary>
  		/// Return a string representation of a character type setting.
  		/// Since the type setting is bitwise encoded, a character
  		/// can have more than one type.
  		/// </summary>
  		/// <param name="ctype">The character type byte.</param>
  		/// <returns>The string representation of the type flags.</returns>
  		public string CharTypeToString(byte ctype)
  		{
  			StringBuilder str = new StringBuilder();
  
  			if (IsCharType(ctype, CharTypeBits.Quote)) str.Append('q');
  			if (IsCharType(ctype, CharTypeBits.Comment)) str.Append('m');
  			if (IsCharType(ctype, CharTypeBits.Whitespace)) str.Append('w');
  			if (IsCharType(ctype, CharTypeBits.Digit)) str.Append('d');
  			if (IsCharType(ctype, CharTypeBits.Word)) str.Append('a');
  			if (IsCharType(ctype, CharTypeBits.Eof)) str.Append('e');
  			if (str.Length == 0)
  			{
  				str.Append('c');
  			}
  			return(str.ToString());
  		}
  
  		/// <summary>
  		/// Check whether the specified char type byte has a 
  		/// particular type flag set.
  		/// </summary>
  		/// <param name="ctype">The char type byte.</param>
  		/// <param name="type">The CharTypeBits entry to compare to.</param>
  		/// <returns>bool - true or false</returns>
  		public bool IsCharType(byte ctype, CharTypeBits type)
  		{
  			return((ctype & (byte)type) != 0);
  		}
  
  		/// <summary>
  		/// Check whether the specified char has a 
  		/// particular type flag set.
  		/// </summary>
  		/// <param name="c">The character.</param>
  		/// <param name="type">The CharTypeBits entry to compare to.</param>
  		/// <returns>bool - true or false</returns>
  		public bool IsCharType(char c, CharTypeBits type)
  		{
  			return((charTypes[c] & (byte)type) != 0);
  		}
  
  		/// <summary>
  		/// Check whether the specified char has a 
  		/// particular type flag set.
  		/// </summary>
  		/// <param name="c">The character.</param>
  		/// <param name="type">The CharTypeBits entry to compare to.</param>
  		/// <returns>bool - true or false</returns>
  		public bool IsCharType(int c, CharTypeBits type)
  		{
  			return((charTypes[c] & (byte)type) != 0);
  		}
  
  		#endregion
  
  		// ---------------------------------------------------------------------
  		#region Standard Methods
  		// ---------------------------------------------------------------------
  
  		/// <summary>
  		/// Display the state of this object.
  		/// </summary>
  		public void Display()
  		{
  			Display(String.Empty);
  		}
  
  		/// <summary>
  		/// Display the state of this object, with a per-line prefix.
  		/// </summary>
  		/// <param name="prefix">The pre-line prefix.</param>
  		public void Display(string prefix)
  		{			
  		}
  		#endregion
  	}
  
  	/// <summary>
  	/// A StreamTokenizer similar to Java's.  This breaks an input stream
  	/// (coming from a TextReader) into Tokens based on various settings.  The settings
  	/// are stored in the TokenizerSettings property, which is a
  	/// StreamTokenizerSettings instance.
  	/// </summary>
  	/// <remarks>
  	/// <para>
  	/// This is configurable in that you can modify TokenizerSettings.CharTypes[] array
  	/// to specify which characters are which type, along with other settings
  	/// such as whether to look for comments or not.
  	/// </para>
  	/// <para>
  	/// WARNING: This is not internationalized.  This treats all characters beyond
  	/// the 7-bit ASCII range (decimal 127) as Word characters.
  	/// </para>
  	/// <para>
  	/// There are two main ways to use this: 1) Parse the entire stream at
  	/// once and get an ArrayList of Tokens (see the Tokenize* methods), 
  	/// and 2) call NextToken() successively.
  	/// This reads from a TextReader, which you can set directly, and this
  	/// also provides some convenient methods to parse files and strings.
  	/// This returns an Eof token if the end of the input is reached.
  	/// </para>
  	/// <para>
  	/// Here's an example of the NextToken() endCapStyle of use:
  	/// <code>
  	/// StreamTokenizer tokenizer = new StreamTokenizer();
  	/// tokenizer.GrabWhitespace = true;
  	/// tokenizer.Verbosity = VerbosityLevel.Debug; // just for debugging
  	/// tokenizer.TextReader = File.OpenText(fileName);
  	/// Token token;
  	/// while (tokenizer.NextToken(out token)) log.Info("Token = '{0}'", token);
  	/// </code>
  	/// </para>
  	/// <para>
  	/// Here's an example of the Tokenize... endCapStyle of use:
  	/// <code>
  	/// StreamTokenizer tokenizer = new StreamTokenizer("some string");
  	/// ArrayList tokens = new ArrayList();
  	/// if (!tokenizer.Tokenize(tokens)) 
  	/// { 
  	///		// error handling
  	/// }
  	/// foreach (Token t in tokens) Console.WriteLine("t = {0}", t);
  	/// </code>
  	/// </para>
  	/// <para>
  	/// Comment delimiters are hardcoded (// and /*), not affected by char type table.
  	/// </para>
  	/// <para>
  	/// This sets line numbers in the tokens it produces.  These numbers are normally
  	/// the line on which the token starts.
  	/// There is one known caveat, and that is that when GrabWhitespace setting
  	/// is true, and a whitespace token contains a newline, that token's line number
  	/// will be set to the following line rather than the line on which the token
  	/// started.
  	/// </para>
  	/// </remarks>
  	public class StreamTokenizer
  	{
  		// ----------------------------------------------------------------
  		#region Constants
  		// ----------------------------------------------------------------
  
  		/// <summary>
  		/// This is the number of characters in the character table.
  		/// </summary>
  		public static readonly int NChars = 128;
  		private static readonly int Eof = NChars;
  		#endregion
  
  		// ----------------------------------------------------------------
  		#region Private Fields
  		// ----------------------------------------------------------------
  
  		// A class for verbosity/message handling
  		private Logger log;
  
  		// The TextReader we're reading from
  		private TextReader textReader;
  
  		// buffered wrap of reader
  		//private BufferedTextReader bufferedReader; // was slower
  
  		// keep track of current line number during parse
  		private int lineNumber;  
  
  		// used to back up in the stream
  		private CharBuffer backString;  
  
  		// used to collect characters of the current (next to be
  		// emitted) token
  		private CharBuffer nextTokenSb;
  
  		// for speed, construct these once and re-use
  		private CharBuffer tmpSb;
  		private CharBuffer expSb;
  
  		#endregion
  
  		// ----------------------------------------------------------------------
  		#region Properties
  		// ----------------------------------------------------------------------
  
  		/// <summary>
  		/// This is the TextReader that this object will read from.
  		/// Set this to set the input reader for the parse.
  		/// </summary>
  		public TextReader TextReader
  		{
  			get { return(textReader); }
  			set { textReader = value; }
  		}
  
  		private StreamTokenizerSettings settings;
  		/// <summary>
  		/// The settings which govern the behavior of the tokenization.
  		/// </summary>
  		public StreamTokenizerSettings Settings { get { return(settings); } }
  
  		/// <summary>
  		/// The verbosity level for this object's Logger.
  		/// </summary>
  		public VerbosityLevel Verbosity 
  		{ 
  			get { return(log.Verbosity); } 
  			set { log.Verbosity = value; } 
  		}
  
  		#endregion
  
  		// ---------------------------------------------------------------------
  		#region Constructors/Destructor
  		// ---------------------------------------------------------------------
  
  		/// <summary>
  		/// Default constructor.
  		/// </summary>
  		public StreamTokenizer()
  		{
  			Initialize();
  		}
  
  		/// <summary>
  		/// Construct and set this object's TextReader to the one specified.
  		/// </summary>
  		/// <param name="sr">The TextReader to read from.</param>
  		public StreamTokenizer(TextReader sr)
  		{
  			Initialize();
  			textReader = sr;
  		}
  
  		/// <summary>
  		/// Construct and set a string to tokenize.
  		/// </summary>
  		/// <param name="str">The string to tokenize.</param>
  		public StreamTokenizer(string str)
  		{
  			Initialize();
  			textReader = new StringReader(str);
  		}
  
  		/// <summary>
  		/// Utility function, things common to constructors.
  		/// </summary>
  		void Initialize()
  		{
  			log = new Logger("StreamTokenizer");
  			log.Verbosity = VerbosityLevel.Warn;
  			backString = new CharBuffer(32);
  			nextTokenSb = new CharBuffer(1024);
  
  			InitializeStream();
  			settings = new StreamTokenizerSettings();
  			settings.SetDefaults();
  
  			expSb = new CharBuffer();
  			tmpSb = new CharBuffer();
  		}
  
  		/// <summary>
  		/// Clear the stream settings.
  		/// </summary>
  		void InitializeStream()
  		{
  			lineNumber = 1; // base 1 line numbers
  			textReader = null;
  		}
  
  		#endregion
  
  		// ---------------------------------------------------------------------
  		#region Standard Methods
  		// ---------------------------------------------------------------------
  
  		/// <summary>
  		/// Display the state of this object.
  		/// </summary>
  		public void Display()
  		{
  			Display(String.Empty);
  		}
  
  		/// <summary>
  		/// Display the state of this object, with a per-line prefix.
  		/// </summary>
  		/// <param name="prefix">The pre-line prefix.</param>
  		public void Display(string prefix)
  		{
  			log.WriteLine(prefix + "StreamTokenizer display:");
  			log.WriteLine(prefix + "    textReader: {0}", (textReader == null ? "null" : "non-null"));
  			log.WriteLine(prefix + "    backString: {0}", backString);
  
  			if (settings != null) settings.Display(prefix + "    ");
  		}
  
  		#endregion
  
  		// ---------------------------------------------------------------------
  		#region NextToken (the state machine)
  		// ---------------------------------------------------------------------
  
  		/// <summary>
  		/// The states of the state machine.
  		/// </summary>
  		private enum NextTokenState
  		{
  			Start,
  			Whitespace,
  			Word,
  			Quote,
  			EndQuote,
  			MaybeNumber, // could be number or word
  			MaybeComment, // after first slash, might be comment or not
  			MaybeHex, // after 0, may be hex
  			HexGot0x, // after 0x, may be hex
  			HexNumber,
  			LineComment,
  			BlockComment,
  			EndBlockComment,
  			Char,
  			Eol,
  			Eof,
  			Invalid
  		}
  
  		/// <summary>
  		/// Pick the next state given just a single character.  This is used
  		/// at the start of a new token.
  		/// </summary>
  		/// <param name="ctype">The type of the character.</param>
  		/// <param name="c">The character.</param>
  		/// <returns>The state.</returns>
  		private NextTokenState PickNextState(byte ctype, int c)
  		{
  			return(PickNextState(ctype, c, NextTokenState.Start));
  		}
  
  		/// <summary>
  		/// Pick the next state given just a single character.  This is used
  		/// at the start of a new token.
  		/// </summary>
  		/// <param name="ctype">The type of the character.</param>
  		/// <param name="c">The character.</param>
  		/// <param name="excludeState">Exclude this state from the possible next state.</param>
  		/// <returns>The state.</returns>
  		private NextTokenState PickNextState(byte ctype, int c, NextTokenState excludeState)
  		{
  			if (c == '/') 
  			{
  				return(NextTokenState.MaybeComment); // overrides all other cats
  			}
  			else if ((excludeState != NextTokenState.MaybeHex) 
  				&& settings.ParseHexNumbers && (c == '0')) 
  			{
  				return(NextTokenState.MaybeHex);
  			}
  			else if ((excludeState != NextTokenState.MaybeNumber) && settings.ParseNumbers 
  				&& (settings.IsCharType(ctype, CharTypeBits.Digit) || (c == '-') || (c == '.'))) 
  			{
  				return(NextTokenState.MaybeNumber);
  			}
  			else if (settings.IsCharType(ctype, CharTypeBits.Word)) return(NextTokenState.Word);
  			else if (settings.GrabEol && (c == 10)) return(NextTokenState.Eol);
  			else if (settings.IsCharType(ctype, CharTypeBits.Whitespace)) return(NextTokenState.Whitespace);
  			else if (settings.IsCharType(ctype, CharTypeBits.Comment)) return(NextTokenState.LineComment);
  			else if (settings.IsCharType(ctype, CharTypeBits.Quote)) return(NextTokenState.Quote);
  			else if ((c == Eof) || (settings.IsCharType(ctype, CharTypeBits.Eof))) return(NextTokenState.Eof);
  			return(NextTokenState.Char);
  		}
  
  		/// <summary>
  		/// Read the next character from the stream, or from backString
  		/// if we backed up.
  		/// </summary>
  		/// <returns>The next character.</returns>
  		private int GetNextChar()
  		{
  			int c;
  
  			// consume from backString if possible
  			if (backString.Length > 0)
  			{
  				c = backString[0];
  				backString.Remove(0, 1);
  				#if DEBUG
  				log.Debug("Backup char '{0}'", (char)c);
  				#endif
  				return(c);
  			}
  
  			if (textReader == null) return(Eof);
  
  			try
  			{
  				while((c = textReader.Read()) == 13) {} // skip LF (13)
  			}
  			catch(Exception)
  			{
  				return(Eof);
  			}
  
  			if (c == 10) 
  			{
  				lineNumber++;
  				#if DEBUG
  				log.Debug("Line number incremented to {0}", lineNumber);
  				#endif
  			}
  			else if (c < 0) 
  			{
  				c = Eof;
  			}
  
  			#if DEBUG
  			log.Debug("Read char '{0}' ({1})", (char)c, c);
  			#endif
  			return(c);
  		}
  
  		/// <summary>
  		/// Get the next token.  The last token will be an EofToken unless
  		/// there's an unterminated quote or unterminated block comment
  		/// and Settings.DoUntermCheck is true, in which case this throws
  		/// an exception of type StreamTokenizerUntermException or sub-class.
  		/// </summary>
  		/// <param name="token">The output token.</param>
  		/// <returns>bool - true for success, false for failure.</returns>
  		public bool NextToken(out Token token)
  		{
  			token = null;
  			int thisChar = 0; // current character
  			byte ctype; // type of this character
  
  			NextTokenState state = NextTokenState.Start;
  			int prevChar = 0; // previous character
  			byte prevCtype = (byte)CharTypeBits.Eof;
  
  			// get previous char from nextTokenSb if there
  			// (nextTokenSb is a StringBuilder containing the characters
  			//  of the next token to be emitted)
  			if (nextTokenSb.Length > 0) 
  			{
  				prevChar = nextTokenSb[nextTokenSb.Length - 1];
  				prevCtype = settings.CharTypes[prevChar];
  				state = PickNextState(prevCtype, prevChar);
  			}
  
  			// extra state for number parse
  			int seenDot = 0; // how many .'s in the number
  			int seenE = 0; // how many e's or E's have we seen in the number
  			bool seenDigit = false; // seen any digits (numbers can start with -)
  
  			// lineNumber can change with each GetNextChar()
  			// tokenLineNumber is the line on which the token started
  			int tokenLineNumber = lineNumber;
  
  			// State Machine: Produces a single token.
  			// Enter a state based on a single character.
  			// Generally, being in a state means we're currently collecting chars 
  			// in that type of token.
  			// We do state machine until it builds a token (Eof is a token), then
  			// return that token.
  			thisChar = prevChar;  // for first iteration, since prevChar is set to this 
  			bool done = false; // optimization
  			while (!done)
  			{
  				prevChar = thisChar;
  				thisChar = GetNextChar();
  				if (thisChar >= settings.CharTypes.Length)
  				{
  					// greater than 7-bit ascii, treat as word character
  					ctype = (byte)CharTypeBits.Word;
  				}
  				else ctype = settings.CharTypes[thisChar];
  
  				#if DEBUG
  				log.Debug("Before switch: state = {0}, thisChar = '{1}'", state, (char)thisChar);
  				#endif
  
  				// see if we need to change states, or emit a token
  				switch(state)
  				{
  					case NextTokenState.Start:
  						// RESET
  						state = PickNextState(ctype, thisChar);
  						tokenLineNumber = lineNumber;
  						break;
  
  					case NextTokenState.Char:
  						token = new CharToken((char)prevChar, tokenLineNumber);
  						done = true;
  						nextTokenSb.Length = 0;
  						break;
  
  					case NextTokenState.Word:
  						if ((!settings.IsCharType(ctype, CharTypeBits.Word))
  							&& (!settings.IsCharType(ctype, CharTypeBits.Digit)))
  						{
  							// end of word, emit
  							token = new WordToken(nextTokenSb.ToString(), tokenLineNumber);
  							done = true;
  							nextTokenSb.Length = 0;
  						}
  						break;
  
  					case NextTokenState.Whitespace:
  						if (!settings.IsCharType(ctype, CharTypeBits.Whitespace)
  
gt: 							|| (settings.GrabEol && (thisChar == 10)))