context: #region License
  /* **********************************************************************************
   * Copyright (c) Roman Ivantsov
   * This source code is subject to terms and conditions of the MIT License
   * for Irony. A copy of the license can be found in the License.txt file
   * at the root of this distribution. 
   * By using this source code in any fashion, you are agreeing to be bound by the terms of the 
   * MIT License.
   * You must not remove this notice from this software.
   * **********************************************************************************/
  #endregion
  
  using System;
  using System.Collections.Generic;
  using System.Text;
  using System.Collections;
  using System.Reflection;
  
  
  namespace Irony.Compiler {
    // Parser class implements LALR(1) parser DFM. Its behavior is controlled by the state transition graph
    // with root in Data.InitialState. Each state contains a dictionary of parser actions indexed by input 
    // element (token or non-terminal node). 
    public class Parser {
  
      #region Constructors
      public Parser(GrammarData data) {
        Data = data;
      }
      #endregion
  
      #region Properties and fields: Data, Stack, _context, Input, CurrentState, LineCount, TokenCount
      public readonly GrammarData Data;
      public readonly ParserStack Stack = new ParserStack();
  
      private CompilerContext _context;
      private bool _caseSensitive;
  
      public IEnumerator<Token> Input {
        [System.Diagnostics.DebuggerStepThrough]
        get { return _input; }
      } IEnumerator<Token> _input;
  
      public Token CurrentToken  {
        [System.Diagnostics.DebuggerStepThrough]
        get { return _currentToken; }
      } Token  _currentToken;
  
      public ParserState CurrentState {
        [System.Diagnostics.DebuggerStepThrough]
        get { return _currentState; }
      } ParserState  _currentState;
  
  
      public int LineCount {
        [System.Diagnostics.DebuggerStepThrough]
        get { return _currentLine; }
      } int  _currentLine;
  
      public int TokenCount  {
        [System.Diagnostics.DebuggerStepThrough]
        get { return _tokenCount; }
      } int  _tokenCount;
  
      #endregion
  
      #region Events: ParserAction, TokenReceived
      public event EventHandler<ParserActionEventArgs> ActionSelected;
      public event EventHandler<TokenEventArgs> TokenReceived;
      TokenEventArgs _tokenArgs = new TokenEventArgs(null); //declare as field and reuse it to avoid generating garbage
  
      protected void OnTokenReceived(Token token) {
        if (TokenReceived == null) return;
        _tokenArgs.Token = token;
        TokenReceived(this, _tokenArgs);
      }
      #endregion
  
      #region Parsing methods
      private void Reset() {
        Stack.Reset();
        _currentState = Data.InitialState;
        _currentLine = 0;
        _tokenCount = 0;
        _context.Errors.Clear();
      }
  
  
      TokenList _previewBuffer = new TokenList();
      private Token ReadToken() {
        while (_input.MoveNext()) {
          Token result = _input.Current;
          _tokenCount++;
          _currentLine = result.Span.Start.Line + 1;
          if (TokenReceived != null)
            OnTokenReceived(result);
          if (result.Terminal.IsSet(TermOptions.IsNonGrammar))
            continue;
          return result;
        }//while
        //Return EOF
        return null; 
      }
  
      private void NextToken() {
        if (_previewBuffer.Count > 0) {
          _currentToken = _previewBuffer[0];
          _previewBuffer.RemoveAt(0);
          return; 
        }
        _currentToken = ReadToken();
        //  if null, we reached end of file; return EOF token.
        if (_currentToken == null) 
          _currentToken = Token.Create(Grammar.Eof, _context, new SourceLocation(0, _currentLine - 1, 0), string.Empty); 
      }//method
  
      public Token PreviewSymbols(StringList symbols) {
        //First check the preview buffer
        foreach (Token token in _previewBuffer) {
          if (symbols.Contains(token.Text)) return token; 
        }
        //Now read from input while saving all tokens in preview buffer
        Token tkn;
        while((tkn = ReadToken()) != null) {
          _previewBuffer.Add(tkn);
          if (symbols.Contains(tkn.Text)) return tkn; 
        } 
        return null;
      }//method
  
      public AstNode Parse(CompilerContext context, IEnumerable<Token> tokenStream) {
        _context = context;
        _caseSensitive = _context.Compiler.Grammar.CaseSensitive;
        Reset();
        _input = tokenStream.GetEnumerator();
        NextToken();
        while (true) {
          if (_currentState == Data.FinalState) {
            AstNode result = Stack[0].Node;
            Stack.Reset();
            return result;
          }
          //check for scammer error
          if (_currentToken.Terminal.Category == TokenCategory.Error) {
            ReportScannerError();
            if (!Recover()) 
              return null; 
            continue;
          }
          //Get action
          ActionRecord action = GetCurrentAction();
          if (action == null) {
            ReportParserError();
            if (!Recover())
              return null; //did not recover
            continue;
          }//action==null
  
          if (action.HasConflict())
            action = Data.Grammar.OnActionConflict(this, _currentToken, action);
          this.OnActionSelected(_currentToken, action);
          switch (action.ActionType) {
            case ParserActionType.Operator:
              if (GetActionTypeForOperation(_currentToken) == ParserActionType.Shift)
                goto case ParserActionType.Shift;
              else
                goto case ParserActionType.Reduce;
  
            case ParserActionType.Shift:
              ExecuteShiftAction(action);
              break;
  
            case ParserActionType.Reduce:
              ExecuteReduceAction(action);
              break;
          }//switch
        }//while
      }//Parse
      #endregion
  
      #region Error reporting and recovery
      private void ReportError(SourceLocation location, string message, params object[] args) {
        if (args != null && args.Length > 0)
          message = string.Format(message, args);
        _context.AddError(location, message, _currentState);
      }
  
      private void ReportScannerError() {
        _context.AddError(_currentToken.Location, _currentToken.Text, _currentState);
      }
      
      private void ReportParserError() {
        if (_currentToken.Terminal == Grammar.Eof) {
          ReportError(_currentToken.Location, "Unexpected end of file.");
          return;
        }
        StringList expectedList = GetCurrentExpectedSymbols();
        string message = this.Data.Grammar.GetSyntaxErrorMessage(_context, expectedList); 
        if (message == null) 
          message = "Syntax error" + (expectedList.Count == 0 ? "." : ", expected: " + TextUtils.Cleanup(expectedList.ToString(" ")));
        ReportError(_currentToken.Location, message);
      }
  
      #region Comment
      //TODO: This needs more work. Currently it reports all individual symbols most of the time, in a message like
      //  "Syntax error, expected: + - < > = ..."; the better method is to group operator symbols under one alias "operator". 
      // The reason is that code picks expected key list at current(!) state only, 
      // slightly tweaking it for non-terminals, without exploring Reduce roots
      // It is quite difficult to discover grouping non-terminals like "operator" in current structure. 
      // One possible solution would be to introduce "ExtendedLookaheads" in ParserState which would include 
      // all NonTerminals that might follow the current position. This list would be calculated at start up, 
      // in addition to normal lookaheads. 
      #endregion
      private StringList GetCurrentExpectedSymbols() {
        BnfTermList inputElements = new BnfTermList();
        StringSet inputKeys = new StringSet();
        inputKeys.AddRange(_currentState.Actions.Keys);
        //First check all NonTerminals
        foreach (NonTerminal nt in Data.NonTerminals) {
  
gt:         if (!inputKeys.Contains(nt.Key)) continue; 