Writing a Tokenizer class in C# for a BASIC Interpreter
<p>For a bit of fun, I'm attempting to write a BASIC interpreter in C#. Following is my  Tokenizer class (with just a few keywords). I'm after any suggestions or comments... Code clarity is more important to me than efficiency. Many thanks.</p>

<pre><code>class Tokenizer
    {

        const string Operators = "+-/*%&lt;&gt;=&amp;|";
        private List&lt;string&gt; Keywords = new List&lt;string&gt;{"LET", "DIM", "PRINT", "REM"};

        private List&lt;string&gt; tokens = new List&lt;string&gt;();
        private List&lt;string&gt; tokenTypes = new List&lt;string&gt;();

        private int tokenIndex;

        // Turn command string into tokens        
        public void Tokenize(string cmdLine)
        {
            string token = "";
            char lastc = ' ';
            bool inString = false;

            tokens.Clear();
            tokenTypes.Clear();

            // Step through line and split into tokens
            foreach (char c in cmdLine)
            {
                if (c == '"') inString = !inString;

                if (!inString)
                {
                    if (IsOperator(lastc)) AddToken(ref token);
                    if (IsWhitespace(c)) AddToken(ref token);
                    if (IsOperator(c)) AddToken(ref token);
                    if (IsNumber(c) &amp;&amp; !IsNumber(lastc)) AddToken(ref token);

                    if (!IsWhitespace(c)) token += c;
                }
                else
                    token += c;

                lastc = c;
            }

            // Add last token
            AddToken(ref token);

            tokenIndex = 0;

        }

        public string Token()
        {
            return tokens[tokenIndex];
        }

        public string TokenType()
        {
            return tokenTypes[tokenIndex];
        }

        public void NextToken()
        {            
           tokenIndex++;            
        }

        public bool TokensLeft()
        {
            return tokenIndex &lt; tokens.Count;
        }

        // Add a token to the collection
        private void AddToken(ref string token)
        {
            if (token.Trim() != "")
            {
                // Determine token type
                string tokenType = "Identifier";
                if (IsOperator(token[0])) tokenType = "Operator";
                if (IsNumber(token[0])) tokenType = "Number";
                if (token[0] == '"') tokenType = "String";
                if (Keywords.Contains(token.ToUpper())) tokenType = "Keyword";

                tokens.Add(token);
                tokenTypes.Add(tokenType);
                token = "";
            }
        }

        private bool IsWhitespace(char c)
        {
            return (c.ToString() != c.ToString().Trim());
        }

        private bool IsOperator(char c)
        {
            return Operators.Contains(c);
        }

        private bool IsNumber(char c)
        {
            return Char.IsNumber(c);
        }
    }
</code></pre>
