text: // Copyright 2017-2020 The Verible Authors.
  //
  // Licensed under the Apache License, Version 2.0 (the "License");
  // you may not use this file except in compliance with the License.
  // You may obtain a copy of the License at
  //
  //      http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing, software
  // distributed under the License is distributed on an "AS IS" BASIS,
  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  // See the License for the specific language governing permissions and
  // limitations under the License.
  
  // Unit-tests for (Verilog) LexicalContext.
  //
  // Testing strategy:
  // LexicalContext is just a means of disambiguation for overloaded tokens.
  // What is most important is that the transformed tokens are correct.
  // The vast majority of tokens pass through un-modified, so focus testing
  // on those transfomations, and the state functions that directly support them.
  // Testing exhaustively is counter-productive because many aspects of the
  // class's internal details are subject to change.
  
  #include "verilog/parser/verilog_lexical_context.h"
  
  #include <array>
  #include <initializer_list>
  #include <iterator>
  #include <memory>
  #include <stack>
  #include <string>
  #include <vector>
  
  #include "absl/memory/memory.h"
  #include "absl/strings/match.h"
  #include "absl/strings/string_view.h"
  #include "common/text/text_structure.h"
  #include "common/text/token_info.h"
  #include "common/text/token_stream_view.h"
  #include "common/util/logging.h"
  #include "gmock/gmock.h"
  #include "gtest/gtest.h"
  #include "verilog/analysis/verilog_analyzer.h"  // only used for lexing
  #include "verilog/parser/verilog_parser.h"      // only used for diagnostics
  #include "verilog/parser/verilog_token_enum.h"
  
  #undef EXPECT_OK
  #define EXPECT_OK(value) EXPECT_TRUE((value).ok())
  
  namespace verilog {
  namespace {
  
  using verible::TokenInfo;
  using verible::TokenSequence;
  using verible::TokenStreamReferenceView;
  
  // TODO(fangism): move this to a test-only library
  // Compare value and reason.
  // We define it as a macro so that when it fails, you get a meaningful line
  // number.
  //
  // If this were a function, its signature would be:
  // template <typename T>
  // inline void EXPECT_EQ_REASON(const verible::WithReason<T>& r, const T&
  // expected, absl::string_view pattern);
  #define EXPECT_EQ_REASON(expr, expected, pattern)                        \
    {                                                                      \
      const auto& r = expr; /* evaluate expr once, auto-extend lifetime */ \
      EXPECT_EQ(r.value, expected) << r.reason;                            \
      /* value could be correct, but reason could be wrong. */             \
      EXPECT_TRUE(absl::StrContains(absl::string_view(r.reason), pattern)) \
          << "value: " << r.value << "\nreason: " << r.reason;             \
    }
  
  template <class SM>
  void ExpectStateMachineTokenSequence(
      SM& sm, verible::TokenStreamView::const_iterator* token_iter,
      std::initializer_list<int> expect_token_enums) {
    int i = 0;
    for (int expect_token_enum : expect_token_enums) {
      const TokenInfo& token(***token_iter);
      const int token_enum = token.token_enum();
      const int interpreted_enum = sm.InterpretToken(token_enum);
      const char* raw_symbol = verilog_symbol_name(token_enum);
      const char* interpreted_symbol = verilog_symbol_name(interpreted_enum);
      const char* expected_symbol = verilog_symbol_name(expect_token_enum);
      VLOG(1) << "token[" << i << "] enum: " << raw_symbol << " -> "
              << interpreted_symbol;
      EXPECT_EQ(interpreted_enum, expect_token_enum)
          << " (" << interpreted_symbol << " vs. " << expected_symbol << ')';
      sm.UpdateState(token_enum);
      ++*token_iter;
      ++i;
    }
  }
  
  // Tests for null state of state machine.
  TEST(KeywordLabelStateMachineTest, NoKeywords) {
    VerilogAnalyzer analyzer("1, 2; 3;", "", false);
    EXPECT_OK(analyzer.Tokenize());
    analyzer.FilterTokensForSyntaxTree();
    const auto& tokens_view = analyzer.Data().GetTokenStreamView();
    EXPECT_EQ(tokens_view.size(), 7);  // including EOF
  
    _KeywordLabelStateMachine b;
    EXPECT_TRUE(b.ItemMayStart());
    int i = 0;
    for (auto iter : tokens_view) {
      b.UpdateState(iter->token_enum());
      EXPECT_FALSE(b.ItemMayStart())
          << "Error at index " << i << ", after: " << *iter;
      ++i;
    }
  }
  
  // Test for state transitions of state machine, no labels.
  TEST(KeywordLabelStateMachineTest, KeywordsWithoutLabels) {
    VerilogAnalyzer analyzer("1 2 begin end begin end 3 begin 4 5 end 6", "",
                             false);
    const std::array<bool, 13> expect_item_may_start{
        {false, false, true, true, true, true, true, true, true, false, true,
         true, false}};
    EXPECT_OK(analyzer.Tokenize());
    analyzer.FilterTokensForSyntaxTree();
    const auto& tokens_view = analyzer.Data().GetTokenStreamView();
    EXPECT_EQ(tokens_view.size(), expect_item_may_start.size());
  
    _KeywordLabelStateMachine b;
    EXPECT_TRUE(b.ItemMayStart());
    auto expect_iter = expect_item_may_start.begin();
    for (auto iter : tokens_view) {
      b.UpdateState(iter->token_enum());
      EXPECT_EQ(b.ItemMayStart(), *expect_iter)
          << "Error at index "
          << std::distance(expect_item_may_start.begin(), expect_iter)
          << ", after: " << *iter;
      ++expect_iter;
    }
  }
  
  // Test for state transitions of state machine, with labels.
  TEST(KeywordLabelStateMachineTest, KeywordsWithLabels) {
    VerilogAnalyzer analyzer("1 begin:a end:a begin:b end:b 2", "", false);
    const std::array<bool, 15> expect_item_may_start{
        {false, true, false, true, true, false, true, true, false, true, true,
         false, true, false, false}};
    EXPECT_OK(analyzer.Tokenize());
    analyzer.FilterTokensForSyntaxTree();
    const auto& tokens_view = analyzer.Data().GetTokenStreamView();
    EXPECT_EQ(tokens_view.size(), expect_item_may_start.size());  // including EOF
  
    _KeywordLabelStateMachine b;
    EXPECT_TRUE(b.ItemMayStart());
    auto expect_iter = expect_item_may_start.cbegin();
    for (auto iter : tokens_view) {
      b.UpdateState(iter->token_enum());
      EXPECT_EQ(b.ItemMayStart(), *expect_iter)
          << "Error at index "
          << std::distance(expect_item_may_start.begin(), expect_iter)
          << ", after: " << *iter;
      ++expect_iter;
    }
  }
  
  // Test for state transitions of state machine, with some labels, some items.
  TEST(KeywordLabelStateMachineTest, ItemsInsideBlocks) {
    VerilogAnalyzer analyzer("begin:a 1 end:a 2 begin 3 end", "", false);
    const std::array<bool, 12> expect_item_may_start{{true, false, true, false,
                                                      true, false, true, false,
                                                      true, true, true, true}};
    EXPECT_OK(analyzer.Tokenize());
    analyzer.FilterTokensForSyntaxTree();
    const auto& tokens_view = analyzer.Data().GetTokenStreamView();
    EXPECT_EQ(tokens_view.size(), expect_item_may_start.size());  // including EOF
  
    _KeywordLabelStateMachine b;
    EXPECT_TRUE(b.ItemMayStart());
    auto expect_iter = expect_item_may_start.cbegin();
    for (auto iter : tokens_view) {
      b.UpdateState(iter->token_enum());
      EXPECT_EQ(b.ItemMayStart(), *expect_iter)
          << "Error at index "
          << std::distance(expect_item_may_start.begin(), expect_iter)
          << ", after: " << *iter;
      ++expect_iter;
    }
  }
  
  class LastSemicolonStateMachineTest : public ::testing::Test,
                                        public _LastSemicolonStateMachine {
   public:
    LastSemicolonStateMachineTest()
        : _LastSemicolonStateMachine(
              TK_property, TK_endproperty,
              SemicolonEndOfAssertionVariableDeclarations) {}
  };
  
  // Tests that the one and only semicolon in the range of interest is updated.
  TEST_F(LastSemicolonStateMachineTest, LifeCycleOneSemicolon) {
    EXPECT_EQ(state_, kNone);
    EXPECT_TRUE(semicolons_.empty());
  
    // Purely synthesized token sequence for testing:
    // Only enums matter, not text.
    constexpr absl::string_view text("don't care");
    TokenInfo tokens[] = {
        TokenInfo(TK_module, text),
        TokenInfo(SymbolIdentifier, text),
        TokenInfo(';', text),
        TokenInfo(TK_property, text),
        TokenInfo(SymbolIdentifier, text),
        TokenInfo(';', text),  // only this one should be modified
        TokenInfo(SymbolIdentifier, text),
        TokenInfo(TK_endproperty, text),
        TokenInfo(';', text),
    };
  
    UpdateState(&tokens[0]);
    EXPECT_EQ(state_, kNone);
  
    UpdateState(&tokens[1]);
    UpdateState(&tokens[2]);
    EXPECT_EQ(state_, kNone);
  
    UpdateState(&tokens[3]);  // TK_property
    EXPECT_EQ(state_, kActive);
  
    UpdateState(&tokens[4]);  // SymbolIdentifier
    EXPECT_EQ(state_, kActive);
  
    UpdateState(&tokens[5]);  // ';'
    EXPECT_EQ(state_, kActive);
    EXPECT_EQ(semicolons_.top(), &tokens[5]);
  
    UpdateState(&tokens[6]);  // SymbolIdentifier
    EXPECT_EQ(state_, kActive);
  
    UpdateState(&tokens[7]);  // TK_endproperty
    EXPECT_EQ(state_, kNone);
    EXPECT_TRUE(semicolons_.empty());
    EXPECT_EQ(tokens[2].token_enum(), ';');  // unmodified
    EXPECT_EQ(tokens[5].token_enum(),
              SemicolonEndOfAssertionVariableDeclarations);
  
    UpdateState(&tokens[8]);  // ';'
    EXPECT_EQ(state_, kNone);
    EXPECT_TRUE(semicolons_.empty());
    EXPECT_EQ(tokens[8].token_enum(), ';');  // unmodified
  }
  
  // Tests that only the last semicolon in the range of interest is updated.
  TEST_F(LastSemicolonStateMachineTest, LifeCycleFinalSemicolon) {
    EXPECT_EQ(state_, kNone);
    EXPECT_TRUE(semicolons_.empty());
  
    // Purely synthesized token sequence for testing:
    // Only enums matter, not text.
    constexpr absl::string_view text("don't care");
    TokenInfo tokens[] = {
        TokenInfo(TK_module, text),
        TokenInfo(SymbolIdentifier, text),
        TokenInfo(';', text),
        TokenInfo(TK_property, text),
        TokenInfo(SymbolIdentifier, text),
        TokenInfo(';', text),
        TokenInfo(SymbolIdentifier, text),
        TokenInfo(';', text),  // only this one should be modified
        TokenInfo(SymbolIdentifier, text),
        TokenInfo(TK_endproperty, text),
        TokenInfo(';', text),
    };
  
    UpdateState(&tokens[0]);
    EXPECT_EQ(state_, kNone);
  
    UpdateState(&tokens[1]);
    UpdateState(&tokens[2]);
    EXPECT_EQ(state_, kNone);
  
    UpdateState(&tokens[3]);  // TK_property
    EXPECT_EQ(state_, kActive);
  
    UpdateState(&tokens[4]);  // SymbolIdentifier
    EXPECT_EQ(state_, kActive);
  
    UpdateState(&tokens[5]);  // ';'
    EXPECT_EQ(state_, kActive);
    EXPECT_EQ(semicolons_.top(), &tokens[5]);
  
    UpdateState(&tokens[6]);  // SymbolIdentifier
    EXPECT_EQ(state_, kActive);
  
    UpdateState(&tokens[7]);  // ';'
    EXPECT_EQ(state_, kActive);
    EXPECT_EQ(semicolons_.top(), &tokens[7]);
  
    UpdateState(&tokens[8]);  // SymbolIdentifier
    EXPECT_EQ(state_, kActive);
  
    UpdateState(&tokens[9]);  // TK_endproperty
    EXPECT_EQ(state_, kNone);
    EXPECT_TRUE(semicolons_.empty());
    EXPECT_EQ(tokens[2].token_enum(), ';');  // unmodified
    EXPECT_EQ(tokens[5].token_enum(), ';');  // unmodified
    EXPECT_EQ(tokens[7].token_enum(),
              SemicolonEndOfAssertionVariableDeclarations);
  
    UpdateState(&tokens[10]);  // ';'
    EXPECT_EQ(state_, kNone);
    EXPECT_TRUE(semicolons_.empty());
    EXPECT_EQ(tokens[10].token_enum(), ';');  // unmodified
  }
  
  // TODO(fangism): move this into a test_util library
  struct StateMachineTestBase : public ::testing::Test {
    // Lexes code and initializes token_iter to point to the first token.
    void Tokenize(const std::string& code) {
      analyzer = absl::make_unique<VerilogAnalyzer>(code, "", false);
      EXPECT_OK(analyzer->Tokenize());
      analyzer->FilterTokensForSyntaxTree();
      token_iter = analyzer->Data().GetTokenStreamView().cbegin();
    }
  
    template <class SM>
    void ExpectTokenSequence(SM& sm, std::initializer_list<int> expect_enums) {
      ExpectStateMachineTokenSequence(sm, &token_iter, expect_enums);
    }
  
    // Parser, used only for lexing.
    std::unique_ptr<VerilogAnalyzer> analyzer;
  
    // Iterator over the filtered token stream.
    verible::TokenStreamView::const_iterator token_iter;
  };
  
  struct ConstraintBlockStateMachineTest : public StateMachineTestBase {
    void ExpectTokenSequence(std::initializer_list<int> expect_enums) {
      StateMachineTestBase::ExpectTokenSequence(sm, expect_enums);
    }
  
    // Instance of the state machine under test.
    _ConstraintBlockStateMachine sm;
  };
  
  // Test initial conditions of _ConstraintBlockStateMachine.
  TEST_F(ConstraintBlockStateMachineTest, Initialization) {
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that empty constraint block is balanced.
  TEST_F(ConstraintBlockStateMachineTest, EmptyBlock) {
    Tokenize("{}");
  
    ExpectTokenSequence({'{'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine parses a soft expression.
  TEST_F(ConstraintBlockStateMachineTest, SoftExpression) {
    Tokenize(R"(
    {
      soft a -> b;
    }
    )");
  
    ExpectTokenSequence({'{',  //
                         TK_soft, SymbolIdentifier /* a */, TK_LOGICAL_IMPLIES,
                         SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine parses a soft expression, with extra parens.
  TEST_F(ConstraintBlockStateMachineTest, SoftExpressionExtraParens) {
    Tokenize(R"(
    {
      soft (a -> b);
    }
    )");
  
    ExpectTokenSequence({'{',  //
                         TK_soft, '(', SymbolIdentifier /* a */,
                         TK_LOGICAL_IMPLIES, SymbolIdentifier, ')', ';'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine parses an unexpected '}'.
  TEST_F(ConstraintBlockStateMachineTest, InvalidSoftUnexpectedCloseBrace) {
    Tokenize(R"(
    {
      soft  // missing expression and ';'
    }
    )");
  
    ExpectTokenSequence({'{', TK_soft});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine parses a uniqueness constraint.
  TEST_F(ConstraintBlockStateMachineTest, UniquenessConstraint) {
    Tokenize(R"(
    {
      unique {[0:1],[3:4]};
    }
    )");
  
    ExpectTokenSequence(                                          //
        {'{',                                                     //
         TK_unique, '{',                                          //
         '[', TK_DecNumber /* 0 */, ':', TK_DecNumber, ']', ',',  //
         '[', TK_DecNumber /* 3 */, ':', TK_DecNumber, ']',       //
         '}', ';'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine parses a solve-before item.
  TEST_F(ConstraintBlockStateMachineTest, SolveItem) {
    Tokenize(R"(
    {
      solve a before b;
    }
    )");
  
    ExpectTokenSequence(
        {'{', TK_solve, SymbolIdentifier, TK_before, SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine parses a solve-before, with multiple variables.
  TEST_F(ConstraintBlockStateMachineTest, SolveItemMultiple) {
    Tokenize(R"(
    {
      solve a, b before c, d;
    }
    )");
  
    ExpectTokenSequence({'{',                                                //
                         TK_solve, SymbolIdentifier, ',', SymbolIdentifier,  //
                         TK_before, SymbolIdentifier, ',', SymbolIdentifier,
                         ';'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine parses a solve-before, with hierarchical variables.
  TEST_F(ConstraintBlockStateMachineTest, SolveItemHierarchical) {
    Tokenize(R"(
    {
      solve a.b before c.d;
    }
    )");
  
    ExpectTokenSequence({'{',                                                //
                         TK_solve, SymbolIdentifier, '.', SymbolIdentifier,  //
                         TK_before, SymbolIdentifier, '.', SymbolIdentifier,
                         ';'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine parses a disable-soft.
  TEST_F(ConstraintBlockStateMachineTest, DisableSoft) {
    Tokenize(R"(
    {
      disable soft x.y;
    }
    )");
  
    ExpectTokenSequence(
        {'{', TK_disable, TK_soft, SymbolIdentifier, '.', SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly balances, even with a missing ';'.
  TEST_F(ConstraintBlockStateMachineTest, BalanceConstraintSetMissingSemicolon) {
    Tokenize(R"(
    {
      f -> { g -> h }  // missing ';' after 'h'
    }
    )");
  
    ExpectTokenSequence(  //
        {'{', SymbolIdentifier /* f */, TK_CONSTRAINT_IMPLIES, '{',
         SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, '}'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly balances a nested constraint set.
  TEST_F(ConstraintBlockStateMachineTest, InterpretRightArrowConstraintSetRHS) {
    Tokenize(R"(
    {
      f -> { g -> h; }
    }
    )");
  
    ExpectTokenSequence(                                   //
        {'{',                                              //
         SymbolIdentifier /* f */, TK_CONSTRAINT_IMPLIES,  //
         '{', SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';',
         '}'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly balances a nested constraint set.
  TEST_F(ConstraintBlockStateMachineTest,
         InterpretRightArrowConstraintSetNested) {
    Tokenize(R"(
    {
      f -> {
        g -> {h;}
      }
    }
    )");
  
    ExpectTokenSequence(                                   //
        {'{',                                              //
         SymbolIdentifier /* f */, TK_CONSTRAINT_IMPLIES,  //
         '{', SymbolIdentifier, TK_CONSTRAINT_IMPLIES, '{', SymbolIdentifier, ';',
         '}', '}'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly balances parentheses.
  TEST_F(ConstraintBlockStateMachineTest, InterpretRightArrowDeepParens) {
    Tokenize(R"(
    {
      (((f -> g))) -> ((j -> k)) -> ((p -> q));
    }
    )");
  
    // clang-format off
    ExpectTokenSequence(
        {'{',
         '(', '(', '(',
         SymbolIdentifier /* f */, TK_LOGICAL_IMPLIES, SymbolIdentifier,
         ')', ')', ')',
         TK_CONSTRAINT_IMPLIES,
         '(', '(',
         SymbolIdentifier /* j */, TK_LOGICAL_IMPLIES, SymbolIdentifier,
         ')', ')',
         TK_CONSTRAINT_IMPLIES,
         '(', '(',
         SymbolIdentifier /* p */, TK_LOGICAL_IMPLIES, SymbolIdentifier,
         ')', ')', ';'
        });
    // clang-format on
  
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly balances braces.
  TEST_F(ConstraintBlockStateMachineTest, InterpretRightArrowDeepBraces) {
    Tokenize(R"(
    {
      {{a -> b, (g -> h)}} -> {{(j -> k), l -> m}};  // concatenation expressions
    }
    )");
  
    // clang-format off
    ExpectTokenSequence(
        {'{',
         '{', '{',
         SymbolIdentifier /* a */, TK_LOGICAL_IMPLIES, SymbolIdentifier,
         ',',
         '(', SymbolIdentifier /* g */, TK_LOGICAL_IMPLIES, SymbolIdentifier, ')',
         '}', '}',
         TK_CONSTRAINT_IMPLIES,
         '{', '{',
         '(', SymbolIdentifier /* j */, TK_LOGICAL_IMPLIES, SymbolIdentifier, ')',
         ',',
         SymbolIdentifier /* l */, TK_LOGICAL_IMPLIES, SymbolIdentifier,
         '}', '}', ';'});
    // clang-format on
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->'.
  TEST_F(ConstraintBlockStateMachineTest, InterpretRightArrow) {
    Tokenize(R"(
    {
      a -> b;
      (c -> d) -> e;
      f -> { (g -> h) -> i }  // missing ';' after 'i'
    }
    )");
  
    ExpectTokenSequence({'{', SymbolIdentifier /* a */, TK_CONSTRAINT_IMPLIES,
                         SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'(', SymbolIdentifier /* c */, TK_LOGICAL_IMPLIES,
                         SymbolIdentifier, ')', TK_CONSTRAINT_IMPLIES,
                         SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({SymbolIdentifier /* f */, TK_CONSTRAINT_IMPLIES, '{',
                         '(', SymbolIdentifier, TK_LOGICAL_IMPLIES,
                         SymbolIdentifier, ')', TK_CONSTRAINT_IMPLIES,
                         SymbolIdentifier, '}', '}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->'.
  TEST_F(ConstraintBlockStateMachineTest, InterpretRightArrow2) {
    Tokenize(R"(
    {
      a -> (b -> c);
      d -> {
        e -> (f -> g);
        (h -> i) -> j;
      }
    }
    )");
  
    ExpectTokenSequence({'{', SymbolIdentifier /* a */, TK_CONSTRAINT_IMPLIES,
                         '(', SymbolIdentifier, TK_LOGICAL_IMPLIES,
                         SymbolIdentifier, ')', ';'});
    ExpectTokenSequence({SymbolIdentifier /* d */, TK_CONSTRAINT_IMPLIES, '{',
                         SymbolIdentifier /* e */, TK_CONSTRAINT_IMPLIES, '(',
                         SymbolIdentifier, TK_LOGICAL_IMPLIES, SymbolIdentifier,
                         ')', ';'});
    ExpectTokenSequence({'(', SymbolIdentifier /* h */, TK_LOGICAL_IMPLIES,
                         SymbolIdentifier, ')', TK_CONSTRAINT_IMPLIES,
                         SymbolIdentifier, ';', '}', '}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' with balanced {}
  // expressions.
  TEST_F(ConstraintBlockStateMachineTest, InterpretRightArrowBracedExpressions) {
    Tokenize(R"(
    {
      {2{4'h0}} -> {2{4'h1}};
      ({2{4'h2}} -> {2{4'h3}}) -> {2{4'h4}};
      {2{4'h5}} -> ({2{4'h6}} -> {2{4'h7}});
    }
    )");
  
    ExpectTokenSequence({'{'});
    EXPECT_TRUE(sm.IsActive());
  
    // Re-use the same expected token sequence for {N{M'hx}}
  #define CONSTANT_CONCAT_SEQUENCE \
    '{', TK_DecNumber, '{', TK_DecNumber, TK_HexBase, TK_HexDigits, '}', '}'
  
    ExpectTokenSequence({CONSTANT_CONCAT_SEQUENCE, TK_CONSTRAINT_IMPLIES,
                         CONSTANT_CONCAT_SEQUENCE, ';'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'(', CONSTANT_CONCAT_SEQUENCE, TK_LOGICAL_IMPLIES,
                         CONSTANT_CONCAT_SEQUENCE, ')', TK_CONSTRAINT_IMPLIES,
                         CONSTANT_CONCAT_SEQUENCE, ';'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({CONSTANT_CONCAT_SEQUENCE, TK_CONSTRAINT_IMPLIES, '(',
                         CONSTANT_CONCAT_SEQUENCE, TK_LOGICAL_IMPLIES,
                         CONSTANT_CONCAT_SEQUENCE, ')', ';'});
    EXPECT_TRUE(sm.IsActive());
  
  #undef CONSTANT_CONCAT_SEQUENCE
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine recovers from a bad if construct.
  TEST_F(ConstraintBlockStateMachineTest, InvalidIf) {
    Tokenize(R"(
    {
      if
    }
    )");
  
    ExpectTokenSequence({'{', TK_if});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in if blocks.
  TEST_F(ConstraintBlockStateMachineTest, IfConstraintSingle) {
    Tokenize(R"(
    {
      if (a -> b) c -> d;
    }
    )");
  
    ExpectTokenSequence({'{', TK_if, '(', SymbolIdentifier, TK_LOGICAL_IMPLIES,
                         SymbolIdentifier, ')', SymbolIdentifier,
                         TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in if clauses.
  TEST_F(ConstraintBlockStateMachineTest, IfConstraintSingleParenExpressions) {
    Tokenize(R"(
    {
      if (a -> b)
        (c -> d) -> (e -> f);
    }
    )");
  
    ExpectTokenSequence(  //
        {'{', TK_if, '(', SymbolIdentifier, TK_LOGICAL_IMPLIES, SymbolIdentifier,
         ')'});
    ExpectTokenSequence(  //
        {'(', SymbolIdentifier /* c */, TK_LOGICAL_IMPLIES, SymbolIdentifier, ')',
         TK_CONSTRAINT_IMPLIES,  //
         '(', SymbolIdentifier /* e */, TK_LOGICAL_IMPLIES, SymbolIdentifier, ')',
         ';'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in constraint set
  // if-clause.
  TEST_F(ConstraintBlockStateMachineTest, IfConstraintBlock) {
    Tokenize(R"(
    {
      if (a -> b) { c -> d; }
    }
    )");
  
    ExpectTokenSequence(                                          //
        {'{', TK_if, '(',                                         //
         SymbolIdentifier, TK_LOGICAL_IMPLIES, SymbolIdentifier,  //
         ')',                                                     //
         '{', SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';',
         '}'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in if-else clauses.
  TEST_F(ConstraintBlockStateMachineTest, IfElseConstraintSingle) {
    Tokenize(R"(
    {
      if (a -> b) c -> d;
      else e -> f;
    }
    )");
  
    ExpectTokenSequence(  //
        {'{',             //
         TK_if, '(',      //
         SymbolIdentifier, TK_LOGICAL_IMPLIES, SymbolIdentifier,
         ')',  //
         SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence(  //
        {TK_else,         //
         SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in if-else blocks.
  TEST_F(ConstraintBlockStateMachineTest, IfElseConstraintBlocks) {
    Tokenize(R"(
    {
      if (a -> b) { c -> d; }
      else { e -> f; }
    }
    )");
  
    ExpectTokenSequence(                                             //
        {'{',                                                        //
         TK_if, '(',                                                 //
         SymbolIdentifier, TK_LOGICAL_IMPLIES, SymbolIdentifier,     //
         ')', '{',                                                   //
         SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier,  //
         ';', '}'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence(                                             //
        {TK_else, '{',                                               //
         SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier,  //
         ';', '}'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in if-else nested blocks.
  TEST_F(ConstraintBlockStateMachineTest, IfElseConstraintBlocksNested) {
    Tokenize(R"(
    {
      if (a -> b)
        if (p -> q) { c -> d; }
        else { e -> f; }
      else { r -> s; }
    }
    )");
  
    ExpectTokenSequence(  //
        {'{',             //
         TK_if, '(', SymbolIdentifier /* a */, TK_LOGICAL_IMPLIES,
         SymbolIdentifier, ')'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence(                                                     //
        {TK_if, '(',                                                         //
         SymbolIdentifier /* p */, TK_LOGICAL_IMPLIES, SymbolIdentifier,     //
         ')', '{',                                                           //
         SymbolIdentifier /* c */, TK_CONSTRAINT_IMPLIES, SymbolIdentifier,  //
         ';', '}'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({TK_else, '{', SymbolIdentifier /* e */,
                         TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';', '}'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({TK_else, '{', SymbolIdentifier /* r */,
                         TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';', '}'});
    EXPECT_TRUE(sm.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine recovers from a bad foreach construct.
  TEST_F(ConstraintBlockStateMachineTest, InvalidForeach) {
    Tokenize(R"(
    {
      foreach
    }
    )");
  
    ExpectTokenSequence({'{', TK_foreach});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in foreach blocks.
  TEST_F(ConstraintBlockStateMachineTest, ForeachSingleSimple) {
    Tokenize(R"(
    {
      foreach (a[i]) c -> d;
    }
    )");
  
    ExpectTokenSequence(                                                      //
        {'{',                                                                 //
         TK_foreach, '(', SymbolIdentifier, '[', SymbolIdentifier, ']', ')',  //
         SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in foreach blocks.
  TEST_F(ConstraintBlockStateMachineTest, ForeachSingleHierarchical) {
    Tokenize(R"(
    {
      foreach (a.b[i,j]) c -> d;
    }
    )");
  
    ExpectTokenSequence(                                             //
        {'{',                                                        //
         TK_foreach, '(',                                            //
         SymbolIdentifier /* a */, '.', SymbolIdentifier,            //
         '[', SymbolIdentifier /* i */, ',', SymbolIdentifier, ']',  //
         ')',                                                        //
         SymbolIdentifier /* c */, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in foreach blocks.
  TEST_F(ConstraintBlockStateMachineTest, ForeachBlock) {
    Tokenize(R"(
    {
      foreach (a[i]) { c -> d; }
    }
    )");
  
    ExpectTokenSequence(                                                  //
        {'{',                                                             //
         TK_foreach, '(',                                                 //
         SymbolIdentifier, '[', SymbolIdentifier, ']',                    //
         ')', '{',                                                        //
         SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';',  //
         '}'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in foreach blocks.
  TEST_F(ConstraintBlockStateMachineTest, ForeachBlockNested) {
    Tokenize(R"(
    {
      foreach (a[i])
        foreach (b[j])
          { c -> d; }
    }
    )");
  
    // clang-format off
    ExpectTokenSequence(
        {'{',
         TK_foreach, '(',
         SymbolIdentifier /* a */, '[', SymbolIdentifier, ']',
         ')',
         TK_foreach, '(',
         SymbolIdentifier /* b */, '[', SymbolIdentifier, ']',
         ')',
         '{',
         SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';',
         '}'});
    // clang-format on
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Tests that state machine correctly interprets '->' in foreach/if blocks.
  TEST_F(ConstraintBlockStateMachineTest, ForeachIfMixed) {
    Tokenize(R"(
    {
      foreach (a[i])
        if (b -> j)
          { c -> d; }
      if (e -> f)
        foreach (g[i])
          { j -> k; }
    }
    )");
  
    // clang-format off
    ExpectTokenSequence(
        {'{', TK_foreach, '(',
         SymbolIdentifier /* a */, '[', SymbolIdentifier, ']', ')',
         TK_if, '(',
         SymbolIdentifier /* b */, TK_LOGICAL_IMPLIES, SymbolIdentifier, ')',
         '{', SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';',
         '}'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence(
        {TK_if, '(',
         SymbolIdentifier /* e */, TK_LOGICAL_IMPLIES, SymbolIdentifier, ')',
         TK_foreach, '(',
         SymbolIdentifier /* g */, '[', SymbolIdentifier, ']', ')',
         '{', SymbolIdentifier, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';',
         '}'});
    // clang-format on
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({'}'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  struct RandomizeCallStateMachineTest : public StateMachineTestBase {
    void ExpectTokenSequence(std::initializer_list<int> expect_enums) {
      StateMachineTestBase::ExpectTokenSequence(sm, expect_enums);
    }
  
    // Instance of the state machine under test.
    _RandomizeCallStateMachine sm;
  };
  
  // Test that _RandomizeCallStateMachine initializes in inactive state.
  TEST_F(RandomizeCallStateMachineTest, Initialization) {
    EXPECT_FALSE(sm.IsActive());
    sm.UpdateState(TK_randomize);
    EXPECT_TRUE(sm.IsActive());
  }
  
  // Test that _RandomizeCallStateMachine updates correctly with plain call.
  TEST_F(RandomizeCallStateMachineTest, ParseStdCall) {
    Tokenize(R"(
    x = std::randomize;
    )");
  
    ExpectTokenSequence({
        SymbolIdentifier /* x */, '=', TK_randomize /* std::randomize */
    });
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({';'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Test that _RandomizeCallStateMachine updates correctly with method call.
  TEST_F(RandomizeCallStateMachineTest, ParseMethodCall) {
    Tokenize(R"(
    x = y.randomize;
    )");
    ExpectTokenSequence(  //
        {SymbolIdentifier /* x */, '=', SymbolIdentifier, '.', TK_randomize});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({';'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Test that _RandomizeCallStateMachine updates correctly with empty variables.
  TEST_F(RandomizeCallStateMachineTest, ParseMethodCallEmptyVariables) {
    Tokenize(R"(
    x = y.randomize();
    )");
    ExpectTokenSequence(                 //
        {SymbolIdentifier /* x */, '=',  //
         SymbolIdentifier, '.', TK_randomize, '(', ')'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({';'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Test that _RandomizeCallStateMachine updates correctly with one variable.
  TEST_F(RandomizeCallStateMachineTest, ParseMethodCallOneVariable) {
    Tokenize(R"(
    x = y.randomize(z);
    )");
    ExpectTokenSequence({SymbolIdentifier /* x */, '=', SymbolIdentifier, '.',
                         TK_randomize, '(', SymbolIdentifier, ')'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({';'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Test that _RandomizeCallStateMachine updates correctly with multiple
  // variables.
  TEST_F(RandomizeCallStateMachineTest, ParseMethodCallMultiVariables) {
    Tokenize(R"(
    x = y.randomize(z, w);
    )");
    ExpectTokenSequence(                            //
        {SymbolIdentifier /* x */, '=',             //
         SymbolIdentifier, '.', TK_randomize, '(',  //
         SymbolIdentifier, ',', SymbolIdentifier, ')'});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({';'});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Test that _RandomizeCallStateMachine updates correctly as a predicate.
  TEST_F(RandomizeCallStateMachineTest, ParseMethodCallPredicate) {
    Tokenize(R"(
    if (y.randomize) begin
    )");
    ExpectTokenSequence(  //
        {TK_if, '(',      //
         SymbolIdentifier /* y */, '.', TK_randomize});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({')'});
    EXPECT_FALSE(sm.IsActive());
    ExpectTokenSequence({TK_begin});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Test that _RandomizeCallStateMachine updates correctly with constraint_block.
  TEST_F(RandomizeCallStateMachineTest, ParseMethodCallWithConstraintBlock) {
    Tokenize(R"(
    if (y.randomize with {a -> b;}) begin
    )");
    ExpectTokenSequence(  //
        {TK_if, '(',      //
         SymbolIdentifier /* y */, '.', TK_randomize});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence(  //
        {TK_with, '{',    //
         SymbolIdentifier /* a */, TK_CONSTRAINT_IMPLIES, SymbolIdentifier, ';',
         '}'});
    EXPECT_FALSE(sm.IsActive());
    ExpectTokenSequence({')', TK_begin});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Test that _RandomizeCallStateMachine updates correctly with constraint_block
  // with empty variable list.
  TEST_F(RandomizeCallStateMachineTest,
         ParseMethodCallWithConstraintBlockAndEmptyVariableList) {
    Tokenize(R"(
    if (y.randomize with () {a -> b;}) begin
    )");
    ExpectTokenSequence(  //
        {TK_if, '(',      //
         SymbolIdentifier /* y */, '.', TK_randomize});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence(                                                     //
        {TK_with, '(', ')', '{',                                             //
         SymbolIdentifier /* a */, TK_CONSTRAINT_IMPLIES, SymbolIdentifier,  //
         ';', '}'});
    EXPECT_FALSE(sm.IsActive());
    ExpectTokenSequence({')', TK_begin});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Test that _RandomizeCallStateMachine updates correctly with constraint_block
  // with non-empty variable list.
  TEST_F(RandomizeCallStateMachineTest,
         ParseMethodCallWithConstraintBlockAndVariableList) {
    Tokenize(R"(
    if (y.randomize with (j, k) {a -> b;}) begin
    )");
    ExpectTokenSequence({TK_if, '(',  //
                         SymbolIdentifier /* y */, '.', TK_randomize});
    EXPECT_TRUE(sm.IsActive());
    ExpectTokenSequence({TK_with, '(', SymbolIdentifier /* j */, ',',
                         SymbolIdentifier /* k */, ')', '{',
                         SymbolIdentifier /* a */, TK_CONSTRAINT_IMPLIES,
                         SymbolIdentifier /* b */, ';', '}'});
    EXPECT_FALSE(sm.IsActive());
    ExpectTokenSequence({')', TK_begin});
    EXPECT_FALSE(sm.IsActive());
  }
  
  // Class for testing some internal methods of LexicalContext.
  class LexicalContextTest : public ::testing::Test, public LexicalContext {
   protected:
    LexicalContextTest() {}
  
    void CheckInitialState() const {
      EXPECT_EQ(previous_token_, nullptr);
      EXPECT_FALSE(in_extern_declaration_);
      EXPECT_FALSE(in_function_declaration_);
      EXPECT_FALSE(in_function_body_);
      EXPECT_FALSE(in_task_declaration_);
      EXPECT_FALSE(in_task_body_);
      EXPECT_FALSE(in_module_declaration_);
      EXPECT_FALSE(in_module_body_);
      EXPECT_FALSE(randomize_call_tracker_.IsActive());
      EXPECT_FALSE(constraint_declaration_tracker_.IsActive());
      EXPECT_FALSE(InAnyDeclaration());
      EXPECT_FALSE(InAnyDeclarationHeader());
      EXPECT_TRUE(flow_control_stack_.empty());
      EXPECT_TRUE(keyword_label_tracker_.ItemMayStart());
      EXPECT_TRUE(balance_stack_.empty());
      EXPECT_TRUE(block_stack_.empty());
      EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "first token");
    }
  
    void AdvanceToken() {
      TokenSequence::iterator iter(*token_iter_);
      TokenInfo& token(*iter);
      _AdvanceToken(&token);
      ++token_iter_;
    }
  
    static void ExpectCurrentTokenEnum(TokenStreamReferenceView::iterator iter,
                                       int expect_token_enum) {
      const int got_token_enum = (*iter)->token_enum();
      EXPECT_EQ(got_token_enum, expect_token_enum)
          << " from token " << **iter << " ("
          << verilog_symbol_name(got_token_enum) << " vs. "
          << verilog_symbol_name(expect_token_enum) << ')';
    }
  
    // Advances the token iterator once for every element in token_enums,
    // verifying token enumerations along the way.
    // Use this helper method to quickly advance through a sequence of tokens
    // without checking other interesting properties.  This also verifies that
    // the token was *not* transformed by the LexicalContext.
    void ExpectTokenSequence(std::initializer_list<int> token_enums) {
      for (const int token_enum : token_enums) {
        ExpectCurrentTokenEnum(token_iter_, token_enum);
        AdvanceToken();
        EXPECT_EQ(previous_token_->token_enum(), token_enum)
            << " from token " << **token_iter_ << " ("
            << verilog_symbol_name(previous_token_->token_enum()) << " vs. "
            << verilog_symbol_name(token_enum) << ')';
      }
    }
  
    // Advances the token iterator once, verifying the token enumeration before
    // and after advancement.
    void ExpectTransformedToken(int token_enum_before, int token_enum_after) {
      ExpectCurrentTokenEnum(token_iter_, token_enum_before);
      AdvanceToken();
      EXPECT_EQ(previous_token_->token_enum(), token_enum_after)
          << " (" << verilog_symbol_name(previous_token_->token_enum()) << " vs. "
          << verilog_symbol_name(token_enum_after) << ')';
    }
  
    // Lexes code and initializes token_iter to point to the first token.
    void Tokenize(const std::string& code) {
      analyzer_ = absl::make_unique<VerilogAnalyzer>(code, "", false);
      EXPECT_OK(analyzer_->Tokenize());
      analyzer_->FilterTokensForSyntaxTree();
      token_refs_ = analyzer_->MutableData().MakeTokenStreamReferenceView();
      token_iter_ = token_refs_.begin();
    }
  
    // Parser, used only for lexing.
    std::unique_ptr<VerilogAnalyzer> analyzer_;
  
    // Modifiable handles into token stream.
    TokenStreamReferenceView token_refs_;
  
    // Iterator over the filtered token stream.
    TokenStreamReferenceView::iterator token_iter_;
  };
  
  // Test that construction and initialization work.
  TEST_F(LexicalContextTest, Initialization) { CheckInitialState(); }
  
  // Test that token stream initialization works.
  TEST_F(LexicalContextTest, ScanEmptyTokens) {
    Tokenize("");
    EXPECT_EQ(token_refs_.size(), 1);  // only EOF token
    AdvanceToken();
    // Don't really care what the state is after EOF, just don't crash.
  }
  
  // Test that context of function declaration is correct.
  TEST_F(LexicalContextTest, ScanEmptyFunctionDeclaration) {
    const char code[] = "function void foo; endfunction";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 6);  // including EOF token
  
    ExpectTokenSequence({TK_function});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), false,
                     "in other declaration header");
    EXPECT_FALSE(ExpectingStatement());
  
    ExpectTokenSequence({TK_void});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), false,
                     "in other declaration header");
    EXPECT_FALSE(ExpectingStatement());
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), false,
                     "in other declaration header");
    EXPECT_FALSE(ExpectingStatement());
  
    ExpectTokenSequence({';'});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_TRUE(in_function_body_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTokenSequence({TK_endfunction});
    EXPECT_FALSE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "item may start");
    EXPECT_FALSE(ExpectingStatement());
  }
  
  // Test for correct context in function declaration with empty ports.
  TEST_F(LexicalContextTest, ScanFunctionDeclarationEmptyPorts) {
    const char code[] = "function void foo(); endfunction";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 8);
  
    ExpectTokenSequence({TK_function});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
  
    ExpectTokenSequence({TK_void});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_TRUE(balance_stack_.empty());
  
    ExpectTokenSequence({'('});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_EQ(balance_stack_.size(), 1);
  
    ExpectTokenSequence({')'});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_TRUE(balance_stack_.empty());
  
    ExpectTokenSequence({';'});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_TRUE(in_function_body_);
    EXPECT_TRUE(ExpectingStatement());
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
  
    ExpectTokenSequence({TK_endfunction});
    EXPECT_FALSE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingStatement());
  }
  
  // Test for correct context in function declaration with ports.
  TEST_F(LexicalContextTest, ScanFunctionDeclarationWithPorts) {
    const char code[] = "function void foo(int a, int b); endfunction";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 13);
  
    ExpectTokenSequence({TK_function});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
  
    ExpectTokenSequence({TK_void});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_TRUE(balance_stack_.empty());
  
    ExpectTokenSequence({'('});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_EQ(balance_stack_.size(), 1);
  
    ExpectTokenSequence({TK_int});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_EQ(balance_stack_.size(), 1);
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_EQ(balance_stack_.size(), 1);
  
    ExpectTokenSequence({','});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_EQ(balance_stack_.size(), 1);
  
    ExpectTokenSequence({TK_int});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_EQ(balance_stack_.size(), 1);
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_EQ(balance_stack_.size(), 1);
  
    ExpectTokenSequence({')'});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(ExpectingBodyItemStart().value);
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_TRUE(balance_stack_.empty());
  
    ExpectTokenSequence({';'});
    EXPECT_TRUE(in_function_declaration_);
    EXPECT_TRUE(in_function_body_);
    EXPECT_TRUE(ExpectingStatement());
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
  
    ExpectTokenSequence({TK_endfunction});
    EXPECT_FALSE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "item may start");
    EXPECT_FALSE(ExpectingStatement());
  }
  
  // Test that '->' is correctly disambiguated inside a function.
  TEST_F(LexicalContextTest, ScanFunctionDeclarationWithRightArrows) {
    const char code[] = R"(
    function void foo;
      -> z;  // event-trigger
      if (a -> b) -> y;  // implies, event-trigger
      -> w;  // event-trigger
      for (; c -> d; ) begin  // implies
        -> y;  // event-trigger
      end
    endfunction
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 34);  // including EOF token
  
    ExpectTokenSequence({TK_function, TK_void, SymbolIdentifier, ';'});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier, ';'});
    EXPECT_FALSE(InFlowControlHeader());
  
    ExpectTokenSequence({TK_if});
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({'(', SymbolIdentifier});
  
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({')'});
    EXPECT_FALSE(InFlowControlHeader());
    EXPECT_FALSE(InAnyDeclarationHeader());
    EXPECT_TRUE(in_function_body_);
    EXPECT_TRUE(previous_token_finished_header_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* y */, ';'});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* w */, ';', TK_for, '(', ';',
                         SymbolIdentifier /* c */});
  
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier /* d */, ';', ')'});
    EXPECT_FALSE(InFlowControlHeader());
    EXPECT_FALSE(InAnyDeclarationHeader());
    EXPECT_TRUE(in_function_body_);
    EXPECT_TRUE(previous_token_finished_header_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTokenSequence({TK_begin});
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "item may start");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* y */, ';', TK_end, TK_endfunction});
    EXPECT_FALSE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
  }
  
  // Test that '->' is correctly disambiguated, handling keyword labels.
  TEST_F(LexicalContextTest,
         ScanFunctionDeclarationWithRightArrowsControlLabels) {
    const char code[] = R"(
    function void foo;
      if (a -> b) begin : bar
        -> y;  // implies, event-trigger
      end : bar
      -> z;  // event-trigger
    endfunction
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 24);  // including EOF token
  
    ExpectTokenSequence({TK_function, TK_void, SymbolIdentifier, ';', TK_if});
  
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({'(', SymbolIdentifier});
  
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({')'});
    EXPECT_FALSE(InFlowControlHeader());
    EXPECT_FALSE(InAnyDeclarationHeader());
    EXPECT_TRUE(in_function_body_);
    EXPECT_TRUE(previous_token_finished_header_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTokenSequence({TK_begin, ':', SymbolIdentifier /* bar */});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence(
        {SymbolIdentifier /* y */, ';', TK_end, ':', SymbolIdentifier /* bar */});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* w */, ';', TK_endfunction});
    EXPECT_FALSE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
  }
  
  // Test that extern function declaration does not expect a declaration body.
  TEST_F(LexicalContextTest, ScanExternMethodDeclaration) {
    const char code[] = R"(
  class n;
    extern function foo;
  endclass
  )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 9);  // including EOF token
  
    ExpectTokenSequence({TK_class, SymbolIdentifier, ';', TK_extern});
    EXPECT_TRUE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_function, SymbolIdentifier, ';'});
    // Make sure not in function body context here because of extern declaration.
    EXPECT_FALSE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_endclass});
  }
  
  // Test that extern function declaration does not expect a declaration body.
  TEST_F(LexicalContextTest, ScanExternMethodDeclarationWithEmptyPorts) {
    const char code[] = R"(
  class n;
    extern function foo();
  endclass
  )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 11);  // including EOF token
  
    ExpectTokenSequence({TK_class, SymbolIdentifier, ';', TK_extern});
    EXPECT_TRUE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_function, SymbolIdentifier, '(', ')', ';'});
  
    // Make sure not in function body context here because of extern declaration.
    EXPECT_FALSE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_endclass});
  }
  
  // Test that extern function declaration does not expect a declaration body.
  TEST_F(LexicalContextTest, ScanExternMethodDeclarationWithSomePorts) {
    const char code[] = R"(
  class n;
    extern function foo(int bar);
  endclass
  )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 13);  // including EOF token
  
    ExpectTokenSequence({TK_class, SymbolIdentifier, ';', TK_extern});
    EXPECT_TRUE(in_extern_declaration_);
  
    ExpectTokenSequence(
        {TK_function, SymbolIdentifier, '(', TK_int, SymbolIdentifier, ')', ';'});
  
    // Make sure not in function body context here because of extern declaration.
    EXPECT_FALSE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_endclass});
  }
  
  // Test that extern task declaration does not expect a declaration body.
  TEST_F(LexicalContextTest, ScanExternTaskDeclaration) {
    const char code[] = R"(
  class n;
    extern task foo;
  endclass
  )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 9);  // including EOF token
  
    ExpectTokenSequence({TK_class, SymbolIdentifier, ';', TK_extern});
    EXPECT_TRUE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_task, SymbolIdentifier, ';'});
    // Make sure not in task body context here because of extern declaration.
    EXPECT_FALSE(in_task_declaration_);
    EXPECT_FALSE(in_task_body_);
    EXPECT_FALSE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_endclass});
  }
  
  // Test that extern constraint prototype does not expect a constraint_block.
  TEST_F(LexicalContextTest, ScanExternConstraintPrototype) {
    const char code[] = R"(
  class n;
    extern constraint foo;
  endclass
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 9);  // including EOF token
  
    ExpectTokenSequence({TK_class, SymbolIdentifier, ';', TK_extern});
    EXPECT_TRUE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_constraint});
    // constraint_prototype should not activate declaration tracker.
    EXPECT_FALSE(constraint_declaration_tracker_.IsActive());
    EXPECT_FALSE(in_extern_declaration_);  // reset
  
    ExpectTokenSequence({SymbolIdentifier, ';'});
    EXPECT_FALSE(constraint_declaration_tracker_.IsActive());
  
    ExpectTokenSequence({TK_endclass});
  }
  
  // Test that extern function declaration does not expect a declaration body,
  // and that "->" is correctly interpreted as constraint-implication.
  TEST_F(LexicalContextTest,
         ScanExternMethodDeclarationFollowedByConstraintImplies) {
    const char code[] = R"(
  class n;
    extern function foo;
  endclass
  
  constraint v {
    m -> {
      x != y;
    }
  }
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 21);  // including EOF token
  
    ExpectTokenSequence({TK_class, SymbolIdentifier, ';', TK_extern});
    EXPECT_TRUE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_function, SymbolIdentifier, ';'});
    // Make sure not in function body context here because of extern declaration.
    EXPECT_FALSE(in_function_declaration_);
    EXPECT_FALSE(in_function_body_);
    EXPECT_FALSE(in_extern_declaration_);
  
    ExpectTokenSequence({TK_endclass, TK_constraint});
    EXPECT_TRUE(constraint_declaration_tracker_.IsActive());
    ExpectTokenSequence({SymbolIdentifier, '{', SymbolIdentifier});
  
    ExpectTransformedToken(_TK_RARROW, TK_CONSTRAINT_IMPLIES);
  
    ExpectTokenSequence(
        {'{', SymbolIdentifier, TK_NE, SymbolIdentifier, ';', '}', '}'});
    EXPECT_FALSE(constraint_declaration_tracker_.IsActive());
  }
  
  // Test that '->' is correctly disambiguated, handling randomize-with.
  TEST_F(LexicalContextTest, ScanRandomizeWithConstraintBlock) {
    const char code[] = R"(
  function void rat(seq_item item);
    if (!item.randomize() with
        {
          (x -> y) -> {
            a inside {[1 : 2]};
          }
        }) begin
      `uvm_fatal("rat", "failed")
    end
  endfunction : rat
    )";
    Tokenize(code);
    CheckInitialState();
  
    ExpectTokenSequence({TK_function, TK_void, SymbolIdentifier, '(',
                         SymbolIdentifier, SymbolIdentifier, ')', ';'});
    ExpectTokenSequence({TK_if, '(', '!', SymbolIdentifier, '.', TK_randomize});
    EXPECT_TRUE(randomize_call_tracker_.IsActive());
  
    ExpectTokenSequence({'(', ')', TK_with, '{', '(', SymbolIdentifier});
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
    ExpectTokenSequence({SymbolIdentifier, ')'});
    ExpectTransformedToken(_TK_RARROW, TK_CONSTRAINT_IMPLIES);
  
    ExpectTokenSequence({'{', SymbolIdentifier, TK_inside, '{', '[', TK_DecNumber,
                         ':', TK_DecNumber, ']', '}', ';', '}', '}'});
    EXPECT_FALSE(randomize_call_tracker_.IsActive());
    ExpectTokenSequence({')', TK_begin});
  }
  
  // Test that '->' is correctly disambiguated inside a task.
  TEST_F(LexicalContextTest, ScanTaskDeclarationWithRightArrows) {
    const char code[] = R"(
    task foo;
      -> z;  // event-trigger
      if (a -> b) -> y;  // implies, event-trigger
      -> w;  // event-trigger
      for (; c -> d; ) begin  // implies
        -> y;  // event-trigger
      end
    endtask
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 33);  // including EOF token
  
    ExpectTokenSequence({TK_task});
    EXPECT_TRUE(in_task_declaration_);
    EXPECT_FALSE(in_task_body_);
  
    ExpectTokenSequence({SymbolIdentifier, ';'});
    EXPECT_TRUE(in_task_declaration_);
    EXPECT_TRUE(in_task_body_);
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier, ';'});
    EXPECT_FALSE(InFlowControlHeader());
  
    ExpectTokenSequence({TK_if});
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({'(', SymbolIdentifier});
  
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({')'});
    EXPECT_FALSE(InFlowControlHeader());
    EXPECT_FALSE(InAnyDeclarationHeader());
    EXPECT_TRUE(in_task_body_);
    EXPECT_TRUE(previous_token_finished_header_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* y */, ';'});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* w */, ';', TK_for, '(', ';',
                         SymbolIdentifier /* c */});
  
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier /* d */, ';', ')'});
    EXPECT_FALSE(InFlowControlHeader());
    EXPECT_FALSE(InAnyDeclarationHeader());
    EXPECT_TRUE(in_task_body_);
    EXPECT_TRUE(previous_token_finished_header_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTokenSequence({TK_begin});
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "item may start");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* y */, ';', TK_end, TK_endtask});
    EXPECT_FALSE(in_task_declaration_);
    EXPECT_FALSE(in_task_body_);
  }
  
  // Test that '->' is correctly disambiguated inside task,
  // handling keyword labels.
  TEST_F(LexicalContextTest, ScanTaskDeclarationWithRightArrowsControlLabels) {
    const char code[] = R"(
    task foo;
      if (a -> b) begin : bar
        -> y;  // implies, event-trigger
      end : bar
      -> z;  // event-trigger
    endtask
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 23);  // including EOF token
  
    ExpectTokenSequence({TK_task, SymbolIdentifier, ';', TK_if});
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({'(', SymbolIdentifier});
  
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({')'});
    EXPECT_FALSE(InFlowControlHeader());
    EXPECT_FALSE(InAnyDeclarationHeader());
    EXPECT_TRUE(in_task_body_);
    EXPECT_TRUE(previous_token_finished_header_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTokenSequence({TK_begin, ':', SymbolIdentifier /* bar */});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence(
        {SymbolIdentifier /* y */, ';', TK_end, ':', SymbolIdentifier /* bar */});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* w */, ';', TK_endtask});
  }
  
  // Test that '->' is correctly disambiguated inside initial blocks.
  TEST_F(LexicalContextTest, ScanInitialStatementEventTrigger) {
    const char code[] = R"(
    module foo;
    initial -> x;  // -> should be event trigger
    endmodule
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 9);  // including EOF token
  
    ExpectTokenSequence({TK_module});
    EXPECT_TRUE(in_module_declaration_);
    EXPECT_FALSE(in_module_body_);
  
    ExpectTokenSequence({SymbolIdentifier, ';'});
    EXPECT_TRUE(in_module_declaration_);
    EXPECT_TRUE(in_module_body_);
  
    ExpectTokenSequence({TK_initial});
    EXPECT_TRUE(in_initial_always_final_construct_);
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier, ';'});
    EXPECT_FALSE(in_initial_always_final_construct_);
  
    ExpectTokenSequence({TK_endmodule});
    EXPECT_FALSE(in_module_declaration_);
    EXPECT_FALSE(in_module_body_);
  }
  
  // Test that '->' is correctly interpreted as a logical implication.
  TEST_F(LexicalContextTest, AssignmentToLogcalImplicationExpression) {
    const char code[] = R"(
    module foo;
    assign a = b -> x;
    endmodule
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 12);  // including EOF token
  
    ExpectTokenSequence({TK_module});
    EXPECT_TRUE(in_module_declaration_);
    EXPECT_FALSE(in_module_body_);
  
    ExpectTokenSequence({SymbolIdentifier, ';'});
    EXPECT_TRUE(in_module_declaration_);
    EXPECT_TRUE(in_module_body_);
  
    ExpectTokenSequence({TK_assign});
    EXPECT_FALSE(in_initial_always_final_construct_);
  
    ExpectTokenSequence(
        {SymbolIdentifier /* a */, '=', SymbolIdentifier /* b */});
  
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier /* x */, ';'});
    EXPECT_FALSE(in_initial_always_final_construct_);
  
    ExpectTokenSequence({TK_endmodule});
    EXPECT_FALSE(in_module_declaration_);
    EXPECT_FALSE(in_module_body_);
  }
  
  // Test that '->' is correctly interpreted as a logical implication.
  TEST_F(LexicalContextTest, AssignmentToLogcalImplicationExpressionInSeqBlock) {
    const char code[] = R"(
    module foo;
      initial begin
        a = b -> x;
      end
    endmodule
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 14);  // including EOF token
  
    ExpectTokenSequence({TK_module});
    EXPECT_TRUE(in_module_declaration_);
    EXPECT_FALSE(in_module_body_);
  
    ExpectTokenSequence({SymbolIdentifier, ';'});
    EXPECT_TRUE(in_module_declaration_);
    EXPECT_TRUE(in_module_body_);
  
    ExpectTokenSequence({TK_initial});
    EXPECT_TRUE(ExpectingStatement());
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "initial");
    EXPECT_TRUE(in_initial_always_final_construct_);
  
    ExpectTokenSequence({TK_begin});
    EXPECT_TRUE(ExpectingStatement());
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "item may start");
  
    ExpectTokenSequence({SymbolIdentifier /* a */});
    ExpectTokenSequence({'='});
    ExpectTokenSequence({SymbolIdentifier /* b */});
  
    EXPECT_FALSE(ExpectingStatement());
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), false, "(default)");
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier /* x */, ';'});
    EXPECT_TRUE(in_initial_always_final_construct_);
  
    ExpectTokenSequence({TK_end});
    EXPECT_FALSE(in_initial_always_final_construct_);
  
    ExpectTokenSequence({TK_endmodule});
    EXPECT_FALSE(in_module_declaration_);
    EXPECT_FALSE(in_module_body_);
  }
  
  // Test that '->' is correctly disambiguated inside initial blocks.
  TEST_F(LexicalContextTest, ScanInitialBlockWithRightArrows) {
    const char code[] = R"(
    module foo;
    initial begin
      -> x;
      if (a -> b) begin : bar
        -> y;  // implies, event-trigger
      end : bar
      -> z;  // event-trigger
    end
    endmodule
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 29);  // including EOF token
  
    ExpectTokenSequence({TK_module, SymbolIdentifier, ';', TK_initial, TK_begin});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* x */, ';', TK_if});
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({'(', SymbolIdentifier});
  
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier});
    EXPECT_TRUE(InFlowControlHeader());
  
    ExpectTokenSequence({')'});
    EXPECT_FALSE(InFlowControlHeader());
    EXPECT_FALSE(InAnyDeclarationHeader());
    EXPECT_TRUE(in_module_body_);
    EXPECT_TRUE(previous_token_finished_header_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "end of header");
    EXPECT_TRUE(ExpectingStatement());
  
    ExpectTokenSequence({TK_begin, ':', SymbolIdentifier /* bar */});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence(
        {SymbolIdentifier /* y */, ';', TK_end, ':', SymbolIdentifier /* bar */});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier /* z */, ';', TK_end, TK_endmodule});
  }
  
  // Test that '->' is correctly disambiguated as a constraint implication.
  TEST_F(LexicalContextTest, ConstraintDeclarationImplication) {
    const char code[] = R"(
    constraint c {
      a -> b;
    }
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 9);  // including EOF token
  
    ExpectTokenSequence({TK_constraint});
    EXPECT_TRUE(constraint_declaration_tracker_.IsActive());
  
    ExpectTokenSequence({SymbolIdentifier, '{'});
    EXPECT_EQ(balance_stack_.size(), 1);
  
    ExpectTokenSequence({SymbolIdentifier});
  
    ExpectTransformedToken(_TK_RARROW, TK_CONSTRAINT_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier, ';', '}'});
    EXPECT_TRUE(balance_stack_.empty());
    EXPECT_FALSE(constraint_declaration_tracker_.IsActive());
  }
  
  // Test that '->' is correctly disambiguated as logical implication.
  TEST_F(LexicalContextTest, ConstraintDeclarationLogicalImplication) {
    const char code[] = R"(
    constraint c {
      if (a -> b) {
        c -> d;
      }
    }
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 17);  // including EOF token
  
    ExpectTokenSequence({TK_constraint});
    EXPECT_TRUE(constraint_declaration_tracker_.IsActive());
  
    ExpectTokenSequence({SymbolIdentifier, '{'});
    EXPECT_EQ(balance_stack_.size(), 1);
  
    ExpectTokenSequence({TK_if, '(', SymbolIdentifier});
  
    ExpectTransformedToken(_TK_RARROW, TK_LOGICAL_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier, ')', '{', SymbolIdentifier});
  
    ExpectTransformedToken(_TK_RARROW, TK_CONSTRAINT_IMPLIES);
  
    ExpectTokenSequence({SymbolIdentifier, ';', '}'});
    EXPECT_TRUE(constraint_declaration_tracker_.IsActive());
  
    ExpectTokenSequence({'}'});
    EXPECT_TRUE(balance_stack_.empty());
    EXPECT_FALSE(constraint_declaration_tracker_.IsActive());
  }
  
  TEST_F(LexicalContextTest, MacroCallBalance) {
    const char code[] = R"(
  `so_call_me_baby()
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 4);  // including EOF token
  
    ExpectTokenSequence({MacroCallId});
    EXPECT_TRUE(balance_stack_.empty());
    ExpectTokenSequence({'('});
    EXPECT_EQ(balance_stack_.size(), 1);
    ExpectTokenSequence({MacroCallCloseToEndLine});  // ')'
    EXPECT_TRUE(balance_stack_.empty());
  }
  
  TEST_F(LexicalContextTest, MacroCallBalanceWithComment) {
    const char code[] = R"(
  `so_call_me_baby()  // comment
    )";
    Tokenize(code);
    CheckInitialState();
  
    ExpectTokenSequence({MacroCallId});
    EXPECT_TRUE(balance_stack_.empty());
    ExpectTokenSequence({'('});
    EXPECT_EQ(balance_stack_.size(), 1);
    ExpectTokenSequence({MacroCallCloseToEndLine});  // ')'
    EXPECT_TRUE(balance_stack_.empty());
    // comment token is filtered out
  }
  
  TEST_F(LexicalContextTest, MacroCallBalanceSemicolon) {
    const char code[] = R"(
  `macro1(foo+bar, `innermacro(11));
    )";
    Tokenize(code);
    CheckInitialState();
  
    ExpectTokenSequence({MacroCallId});
    EXPECT_TRUE(balance_stack_.empty());
    ExpectTokenSequence({'('});
    EXPECT_EQ(balance_stack_.size(), 1);
  
    // "foo+bar" and "`innermacro(11)" are un-lexed
    ExpectTokenSequence({MacroArg, ',', MacroArg});
  
    EXPECT_EQ(balance_stack_.size(), 1);
    ExpectTokenSequence({')'});
  
    EXPECT_TRUE(balance_stack_.empty());
    ExpectTokenSequence({';'});
  }
  
  TEST_F(LexicalContextTest, TaskEventTrigger) {
    const char code[] = R"(
  module foo;
    task bar;
      begin
        -> ack;  // should be event-trigger
      end
    endtask
  endmodule
    )";
    Tokenize(code);
    CheckInitialState();
    EXPECT_EQ(token_refs_.size(), 14);  // including EOF token
  
    ExpectTokenSequence({TK_module, SymbolIdentifier, ';'});
    EXPECT_TRUE(in_module_declaration_);
    EXPECT_TRUE(in_module_body_);
    ExpectTokenSequence({TK_task, SymbolIdentifier, ';'});
    EXPECT_TRUE(in_task_declaration_);
    EXPECT_TRUE(in_task_body_);
    ExpectTokenSequence({TK_begin});
    EXPECT_TRUE(in_task_body_);
    EXPECT_EQ_REASON(ExpectingBodyItemStart(), true, "item may start");
    EXPECT_TRUE(ExpectingStatement());
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
    ExpectTokenSequence({SymbolIdentifier, ';'});
    ExpectTokenSequence({TK_end, TK_endtask, TK_endmodule});
  }
  
  TEST_F(LexicalContextTest, TaskIfEventTrigger) {
    const char code[] = R"(
  class reset_driver;
    `macro()
  endclass: reset_driver
  task drv_interface;
    if (m_kind) begin
      -> m_event;
    end
  endtask: drv_interface
    )";
    Tokenize(code);
    CheckInitialState();
  
    ExpectTokenSequence({TK_class, SymbolIdentifier, ';'});
    ExpectTokenSequence({MacroCallId, '(', MacroCallCloseToEndLine});
    EXPECT_TRUE(balance_stack_.empty());
    ExpectTokenSequence({TK_endclass, ':', SymbolIdentifier});
    ExpectTokenSequence({TK_task, SymbolIdentifier, ';'});
    ExpectTokenSequence({TK_if, '(', SymbolIdentifier, ')', TK_begin});
  
    ExpectTransformedToken(_TK_RARROW, TK_TRIGGER);
  
    ExpectTokenSequence({SymbolIdentifier, ';', TK_end});
    ExpectTokenSequence({TK_endtask, ':', SymbolIdentifier});
  }
  
  TEST_F(LexicalContextTest, RandomizeCallWithConstraintInsideTaskDeclaration) {
    const char code[] = R"(
  task wr();
    s = m.randomize() with {
      a -> b;  // should be a constraint-implication
    };
  endtask
    )";
    Tokenize(code);
    CheckInitialState();
    ExpectTokenSequence({TK_task, SymbolIdentifier, '(', ')', ';'});
    ExpectTokenSequence({SymbolIdentifier /* s */, '=', SymbolIdentifier, '.',
                         TK_randomize, '(', ')', TK_with, '{',
                         SymbolIdentifier /* a */});
    ExpectTransformedToken(_TK_RARROW, TK_CONSTRAINT_IMPLIES);
    ExpectTokenSequence({SymbolIdentifier, ';', '}', ';', TK_endtask});
  }
  
  TEST_F(LexicalContextTest, RandomizeCallWithNestedConstraintImplication) {
    const char code[] = R"(
  function void rat();
    if (!item.randomize() with {
        x -> {
            d -> {a;}
        }
        }) begin
    end
  endfunction : rat
    )";
    Tokenize(code);
    CheckInitialState();
    ExpectTokenSequence({TK_function, TK_void, SymbolIdentifier, '(', ')', ';'});
    ExpectTokenSequence({TK_if, '(', '!', SymbolIdentifier, '.', TK_randomize,
                         '(', ')', TK_with, '{'});
    ExpectTokenSequence({SymbolIdentifier /* x */});
    ExpectTransformedToken(_TK_RARROW, TK_CONSTRAINT_IMPLIES);
    ExpectTokenSequence({'{', SymbolIdentifier /* d */});
    ExpectTransformedToken(_TK_RARROW, TK_CONSTRAINT_IMPLIES);
    ExpectTokenSequence({'{', SymbolIdentifier /* a */, ';', '}'});
    ExpectTokenSequence({'}', '}', ')', TK_begin, TK_end});
    ExpectTokenSequence({TK_endfunction, ':', SymbolIdentifier});
  }
  
  }  // namespace
  }  // namespace verilog
  