context: // This file is part of nMars - Core War MARS for .NET 
  // Whole solution including it's license could be found at
  // http://sourceforge.net/projects/nmars/
  // 2006 Pavel Savara
  
  using System;
  using System.Collections.Generic;
  using System.IO;
  using System.Reflection;
  using System.Text.RegularExpressions;
  using com.calitha.goldparser;
  using nMars.Parser.Expressions;
  using nMars.Parser.Statements;
  using nMars.RedCode;
  using ParserException=nMars.RedCode.ParserException;
  
  namespace nMars.Parser
  {
      public abstract class ParserTokens : BaseParser
      {
          #region Initialization
  
          protected ParserTokens()
          {
              Assembly assembly = typeof(ParserTokens).Assembly;
              Stream redCode = assembly.GetManifestResourceStream("nMars.Parser.Properties.RedCode.cgt");
              if (redCode == null)
              {
                  redCode = assembly.GetManifestResourceStream("RedCode.cgt");
              }
              if (redCode == null)
              {
                  throw new ApplicationException("Couldn't load parser structures.");
              }
              Init(redCode);
          }
  
  
          private void Init(Stream stream)
          {
              CGTReader reader = new CGTReader(stream);
              parser = reader.CreateNewParser();
              parser.TrimReductions = false;
              parser.StoreTokens = LALRParser.StoreTokensMode.NoUserObject;
  
              parser.OnReduce += new LALRParser.ReduceHandler(OnReduce);
              parser.OnTokenRead += new LALRParser.TokenReadHandler(OnTokenRead);
              parser.OnShift += new LALRParser.ShiftHandler(OnShift);
              parser.OnTokenError += new LALRParser.TokenErrorHandler(TokenErrorEvent);
              parser.OnParseError += new LALRParser.ParseErrorHandler(ParseErrorEvent);
          }
  
          #endregion
  
          #region Preprocessing
  
          protected ContainerStatement ParseInternal(string source)
          {
              //;redcode
              Match sta = start.Match(source);
              if (sta != null && sta.Success && sta.Index > 0)
              {
                  string pre = source.Substring(0, sta.Index+1);
                  string prere = Regex.Replace(pre, "^.?", ";", RegexOptions.IgnoreCase | RegexOptions.Multiline);
                  source = 
                      prere + 
                      source.Substring(sta.Index+1);
              }
              sta = for0.Match(source);
              if (sta != null && sta.Success && sta.Index > 0)
              {
                  string pre = source.Substring(0, sta.Index);
                  string mid = source.Substring(sta.Index);
                  Match end = rof.Match(mid);
                  if (end != null && end.Success && end.Index > 0)
                  {
                      mid = mid.Substring(0,end.Index+3);
                      string post = source.Substring(sta.Index + end.Index + 3);
                      source = pre + Regex.Replace(mid, "^.?", ";", RegexOptions.IgnoreCase | RegexOptions.Multiline) + post;
                  }
              }
              if (!source.EndsWith("\n"))
                  source = source + "\n";
              source = Regex.Replace(source, "^;assert", "@assert", RegexOptions.IgnoreCase | RegexOptions.Multiline);
              endSymbol = 0;
  
              return (ContainerStatement)parser.Parse(source).UserObject;
          }
  
          private static Regex start;
          private static Regex for0;
          private static Regex rof;
          static ParserTokens()
          {
              start = new Regex("^;redcode", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Multiline);
              for0 = new Regex("^.*for 0", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Multiline);
              rof = new Regex("rof", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Multiline);
          }
  
          #endregion
  
          #region 
  
          protected void ProcessComments(List<string> comments)
          {
              foreach (string comment in comments)
              {
                  string lcomment = comment.ToLower();
                  if (lcomment.StartsWith("assert "))
                  {
                      CheckAssert(lcomment.Substring(7));
                  }
                  else if (lcomment.StartsWith("name "))
                  {
                      warriorName = comment.Substring(5).Trim();
                  }
                  else if (lcomment.StartsWith("author "))
                  {
                      authorName = comment.Substring(7).Trim();
                  }
              }
          }
  
          protected void CheckAssert(string assert)
          {
              //TODO
          }
  
          private void OnTokenRead(LALRParser argParser, TokenReadEventArgs args)
          {
              if (endSymbol == 2)
              {
                  TerminalToken eofToken =
                      new TerminalToken(SymbolCollection.EOF, SymbolCollection.EOF.Name, args.Token.Location);
                  argParser.lookahead = eofToken;
                  endSymbol = 3;
                  return;
              }
              args.Token.UserObject = CreateObject(args.Token, argParser);
          }
  
          private void OnReduce(LALRParser argParser, ReduceEventArgs args)
          {
              args.Token.UserObject = CreateObject(args.Token, argParser);
              if (endSymbol == 2)
              {
                  TerminalToken eofToken =
                      new TerminalToken(SymbolCollection.EOF, SymbolCollection.EOF.Name, args.Token.Location);
                  argParser.lookahead = eofToken;
                  endSymbol = 3;
              }
          }
  
          void OnShift(LALRParser argParser, ShiftEventArgs args)
          {
              if (endSymbol == 2)
              {
                  TerminalToken eofToken =
                      new TerminalToken(SymbolCollection.EOF, SymbolCollection.EOF.Name, args.Token.Location);
                  argParser.lookahead = eofToken;
                  endSymbol = 3;
              }
          }
  
          #endregion
  
          #region Terminal
  
          private Object CreateObject(TerminalToken token, LALRParser argParser)
          {
              switch (token.Symbol.Id)
              {
                  case (int)SymbolConstants.SYMBOL_INTEGER:
                      //Integer
                      return token.Text;
  
                  case (int)SymbolConstants.SYMBOL_NEWLINE:
                      if (endSymbol == 1)
                      {
                          endSymbol = 2;
                      }
                      return token.UserObject;
  
                  case (int)SymbolConstants.SYMBOL_LABEL:
                      //Variable
                      if (token.Text.EndsWith(":"))
                      {
                          return token.Text.Substring(0, token.Text.Length - 1);
                      }
                      return token.Text;
                  case (int)SymbolConstants.SYMBOL_END:
                      //end
                      endSymbol = 1;
                      return null;
                  default:
                      return null;
              }
          }
  
          #endregion
  
          #region Nonterminal
  
          public Object CreateObject(NonterminalToken token, LALRParser argParser)
          {
              Expression expression;
              Operation op;
              Modifier mod;
              Parameter paramA;
              Parameter paramB;
              EquStatement equ;
              Statement statement;
              ContainerStatement statements;
              ForRofContainerStatement forrof;
              List<LabelName> labels;
              List<string> comments;
              string labell;
  
              switch (token.Rule.Id)
              {
                      #region Comment
  
                  case (int)RuleConstants.RULE_EOL:
                      //<eol> ::= <eolSingle>
                      comments = new List<string>();
                      if (token.Tokens[0].UserObject != null)
                      {
                          string c = (string)token.Tokens[0].UserObject;
                          comments.Add(c.Replace("\r", ""));
                      }
                      return comments;
  
                  case (int)RuleConstants.RULE_EOL2:
                      //<eol> ::= <eol> <eolSingle>
                      comments = (List<string>)token.Tokens[0].UserObject;
                      if (token.Tokens[1].UserObject != null)
                      {
                          string c = (string)token.Tokens[1].UserObject;
                          comments.Add(c.Replace("\r", ""));
                      }
                      return comments;
  
                  case (int)RuleConstants.RULE_EOLSINGLE_NEWLINE:
                      //<eolSingle> ::= NewLine
                      return token.Tokens[0].UserObject;
  
                  case (int)RuleConstants.RULE_EOLOPTIONAL2:
                      //<eolOptional> ::= 
                      comments = new List<string>();
                      return comments;
  
                  case (int)RuleConstants.RULE_EOLOPTIONAL:
                      //<eolOptional> ::= <eol>
                      return token.Tokens[0].UserObject;
  
                      #endregion
  
                      #region Null
  
                  case (int)RuleConstants.RULE_EOLSINGLE_COMMENT_LINE:
                      //<eolSingle> ::= Comment Line
  
                  case (int)RuleConstants.RULE_STATEMENT:
                      //<Statement> ::= <Pin>
  
                  case (int)RuleConstants.RULE_STATEMENT2:
                      //<Statement> ::= <Org>
  
                  case (int)RuleConstants.RULE_EXPRESSIONOPTIONAL:
                      // <ExpressionOptional> ::= <Expression>
  
                  case (int)RuleConstants.RULE_EXPRESSIONOPTIONAL2:
                      // <ExpressionOptional> ::= 
  
  
                      return null;
  
                      #endregion
  
                      #region Forward
  
                  case (int)RuleConstants.RULE_MODEOPTIONAL:
                      // <ModeOptional> ::= <Mode>
  
                  case (int)RuleConstants.RULE_OPERATION:
                      //<Operation> ::= <Operation0>
  
                  case (int)RuleConstants.RULE_OPERATION2:
                      //<Operation> ::= <Operation1>
  
                  case (int)RuleConstants.RULE_OPERATION3:
                      //<Operation> ::= <Operation2>
  
                  case (int)RuleConstants.RULE_EXPRESSION:
                      //<Expression> ::= <Or Exp>
  
                  case (int)RuleConstants.RULE_OREXP:
                      //<Or Exp> ::= <And Exp>
  
                  case (int)RuleConstants.RULE_ANDEXP:
                      //<And Exp> ::= <BinOr Exp>
  
                  case (int)RuleConstants.RULE_BINOREXP:
                      //<BinOr Exp> ::= <BinXor Exp>
  
                  case (int)RuleConstants.RULE_BINXOREXP:
                      //<BinXor Exp> ::= <BinAnd Exp>
  
                  case (int)RuleConstants.RULE_BINANDEXP:
                      // <BinAnd Exp> ::= <Equate Exp>
  
                  case (int)RuleConstants.RULE_EQUATEEXP:
                      //<Equate Exp> ::= <Compare Exp>
  
                  case (int)RuleConstants.RULE_COMPAREEXP:
                      //<Compare Exp> ::= <Shift Exp>
  
                  case (int)RuleConstants.RULE_SHIFTEXP:
                      //<Shift Exp> ::= <Add Exp>
  
                  case (int)RuleConstants.RULE_ADDEXP:
                      //<Add Exp> ::= <Mult Exp>
  
                  case (int)RuleConstants.RULE_MULTEXP:
                      //<Mult Exp> ::= <Negate Exp>
  
                  case (int)RuleConstants.RULE_NEGATEEXP:
                      //<Negate Exp> ::= <Value>
  
                  case (int)RuleConstants.RULE_VALUE:
                      // <Value> ::= <Label>
  
                  case (int)RuleConstants.RULE_LABELSOPTIONAL:
                      //<LabelsOptional> ::= <Labels>
  
                  case (int)RuleConstants.RULE_STATEMENT3:
                      //<Statement> ::= <Equ>
  
                  case (int)RuleConstants.RULE_STATEMENT6:
                      // <Statement> ::= <Assert>
  
                      return token.Tokens[0].UserObject;
  
                      #endregion
  
                      #region Statements
  
                  case (int)RuleConstants.RULE_START:
                      //<Start> ::= <eolOptional> <AllStatements> <EndOptional>
                      comments = (List<string>)token.Tokens[0].UserObject;
                      ProcessComments(comments);
                      if (endSymbol<2)
                          endSymbol = 2;
                      statements = (ContainerStatement)token.Tokens[1].UserObject;
                      statement = (Statement)token.Tokens[2].UserObject;
                      if (statement!=null)
                      {
                          statements.Add(statement);
                      }
                      return statements;
  
                  case (int)RuleConstants.RULE_ALLSTATEMENTS:
                      //<AllStatements> ::= <Statement>
                      statement = (Statement)token.Tokens[0].UserObject;
                      statements = new ContainerStatement(statement, token.Location);
                      return statements;
  
                  case (int)RuleConstants.RULE_ALLSTATEMENTSOPTIONAL:
                      //<AllStatementsOptional> ::= <AllStatements> <eol>
                      statements = (ContainerStatement)token.Tokens[0].UserObject;
                      comments = (List<string>)token.Tokens[1].UserObject;
                      statements.AppendComments(comments);
                      return statements;
  
                  case (int)RuleConstants.RULE_ALLSTATEMENTS3:
                      //<AllStatements> ::= <AllStatements> <eol> <Statement>
                      statements = (ContainerStatement)token.Tokens[0].UserObject;
                      statement = (Statement)token.Tokens[2].UserObject;
                      comments = (List<string>)token.Tokens[1].UserObject;
                      statements.AppendComments(comments);
                      statements.Add(statement);
                      return statements;
  
                  case (int)RuleConstants.RULE_ALLSTATEMENTS4:
                      //<AllStatements> ::= <AllStatements> <eol> <For>
                      statements = (ContainerStatement)token.Tokens[0].UserObject;
                      comments = (List<string>)token.Tokens[1].UserObject;
                      statements.AppendComments(comments);
                      forrof = (ForRofContainerStatement)token.Tokens[2].UserObject;
                      statements.Add(forrof);
                      return statements;
  
                  case (int)RuleConstants.RULE_ALLSTATEMENTS2:
                      //<AllStatements> ::= <For>
                      statements = new ContainerStatement(token.Location);
                      forrof = (ForRofContainerStatement)token.Tokens[0].UserObject;
                      statements.Add(forrof);
                      return statements;
  
                  case (int)RuleConstants.RULE_ALLSTATEMENTSOPTIONAL2:
                      //<AllStatementsOptional> ::= 
                      statements = new ContainerStatement(token.Location);
                      return statements;
  
                  case (int)RuleConstants.RULE_FOR_FOR_ROF2:
                      // <For> ::= <Labels> <eol> for <Expression> <eol> <AllStatementsOptional> rof <ExpressionOptional>
                      labels = ((List<LabelName>)token.Tokens[0].UserObject);
                      expression = (Expression)token.Tokens[3].UserObject;
                      comments = (List<string>)token.Tokens[4].UserObject;
                      statement = (Statement)token.Tokens[5].UserObject;
                      forrof = CreateForrof(comments, expression, statement, labels, token);
                      return forrof;
  
                  case (int)RuleConstants.RULE_FOR_FOR_ROF:
                      // <For> ::= <LabelsOptional> for <Expression> <eol> <AllStatementsOptional> rof <ExpressionOptional>
                      labels = ((List<LabelName>)token.Tokens[0].UserObject);
                      expression = (Expression)token.Tokens[2].UserObject;
                      comments = (List<string>)token.Tokens[3].UserObject;
                      statement = (Statement)token.Tokens[4].UserObject;
                      forrof = CreateForrof(comments, expression, statement, labels, token);
                      return forrof;
                  case (int)RuleConstants.RULE_ASSERT_ATASSERT:
                      // <Assert> ::= @assert <Expression>
                      expression = (Expression)token.Tokens[1].UserObject;
                      return new AssertStatement(expression, token.Location);
                      #endregion
  
                      #region Operation Statements
  
                  case (int)RuleConstants.RULE_STATEMENT4:
                      //<Statement> ::= <LabelsOptional> <Operation>
                      labels = (List<LabelName>)token.Tokens[0].UserObject;
                      statement = (InstructionStatement)token.Tokens[1].UserObject;
                      statement.Labels = labels;
                      foreach (LabelName label in labels)
                      {
                          variables[label.Name] = label;
                      }
                      return statement;
  
                  case (int)RuleConstants.RULE_STATEMENT5:
                      //<Statement> ::= <Labels> <eol> <Operation>
                      labels = (List<LabelName>)token.Tokens[0].UserObject;
                      statement = (InstructionStatement)token.Tokens[2].UserObject;
                      statement.Labels = labels;
                      foreach (LabelName label in labels)
                      {
                          variables[label.Name] = label;
                      }
                      return statement;
  
                      #endregion
  
                      #region Simple Statements
  
                  case (int)RuleConstants.RULE_MODEOPTIONAL2:
                  // <ModeOptional> ::= 
                      return Mode.NULL;
  
                  case (int)RuleConstants.RULE_PIN_PIN:
                      //<Pin> ::= Pin <Expression>
  
gt:                     pin = (Expression)token.Tokens[1].UserObject;