text: // Copyright 2017-2020 The Verible Authors.
  //
  // Licensed under the Apache License, Version 2.0 (the "License");
  // you may not use this file except in compliance with the License.
  // You may obtain a copy of the License at
  //
  //      http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing, software
  // distributed under the License is distributed on an "AS IS" BASIS,
  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  // See the License for the specific language governing permissions and
  // limitations under the License.
  
  // Unit tests for TokenInfo
  
  #include "common/text/token_info.h"
  
  #include <string>
  #include <vector>
  
  #include "absl/base/macros.h"
  #include "absl/strings/string_view.h"
  #include "common/text/constants.h"
  #include "common/util/range.h"
  #include "gtest/gtest.h"
  
  namespace verible {
  namespace {
  
  // Test construction with a token enum and text.
  TEST(TokenInfoTest, EnumTextConstruction) {
    constexpr absl::string_view text("string of length 19");
    TokenInfo token_info(143, text);
    EXPECT_EQ(token_info.token_enum(), 143);
    EXPECT_EQ(token_info.left(text), 0);
    EXPECT_EQ(token_info.right(text), 19);
    EXPECT_EQ(token_info.text(), text);
  }
  
  // Test updating text.
  TEST(TokenInfoTest, AdvanceText) {
    constexpr absl::string_view text = "This quick brown fox...";
    TokenInfo token_info(1, text.substr(0, 0));
    EXPECT_TRUE(BoundsEqual(token_info.text(), text.substr(0, 0)));
    token_info.AdvanceText(3);
    EXPECT_TRUE(BoundsEqual(token_info.text(), text.substr(0, 3)));
    EXPECT_EQ(token_info.text(), "Thi");
    token_info.AdvanceText(4);
    EXPECT_TRUE(BoundsEqual(token_info.text(), text.substr(3, 4)));
    EXPECT_EQ(token_info.text(), "s qu");
  }
  
  // Test operator ==.
  TEST(TokenInfoTest, Equality) {
    constexpr char text[] = "string of length 21";
    TokenInfo token_info_1(143, text);
    TokenInfo token_info_2(143, text);
    EXPECT_EQ(token_info_1, token_info_2);
    EXPECT_EQ(token_info_2, token_info_1);
  }
  
  // Test operator == EOF.
  TEST(TokenInfoTest, EOFEquality) {
    TokenInfo token_info_0 = TokenInfo::EOFToken();
    TokenInfo token_info_1(TK_EOF, "foo");
    TokenInfo token_info_2(TK_EOF, "bar");
    EXPECT_EQ(token_info_0, token_info_1);
    EXPECT_EQ(token_info_1, token_info_0);
    EXPECT_EQ(token_info_0, token_info_2);
    EXPECT_EQ(token_info_2, token_info_0);
    EXPECT_EQ(token_info_1, token_info_2);
    EXPECT_EQ(token_info_2, token_info_1);
  }
  
  TEST(TokenInfoTest, EOFWithBuffer) {
    constexpr absl::string_view text("string of length 21");
    TokenInfo token_info = TokenInfo::EOFToken(text);
    EXPECT_EQ(token_info.token_enum(), TK_EOF);
    EXPECT_EQ(token_info.text().begin(), text.end());
    EXPECT_EQ(token_info.text().end(), text.end());
  }
  
  // Test operator !=.
  TEST(TokenInfoTest, Inequality) {
    constexpr absl::string_view text("string of length 21");
    const std::vector<TokenInfo> token_infos = {
        TokenInfo(143, text),
        TokenInfo(43, text),
        TokenInfo(143, text.substr(0, 1)),  // substring length 1
        TokenInfo(143, "different string"),
    };
    const auto N = token_infos.size();
    for (size_t i = 0; i < N - 1; ++i) {
      for (size_t j = i + 1; j < N; ++j) {
        EXPECT_NE(token_infos[i], token_infos[j]);
        EXPECT_NE(token_infos[j], token_infos[i]);
      }
    }
  }
  
  TEST(TokenInfoTest, EquivalentWithoutLocation) {
    const absl::string_view foo1("foo"), foo2("foo");
    TokenInfo token_0(1, foo1);   // reference token
    TokenInfo token_1(1, "bar");  // different text
    TokenInfo token_2(1, foo2);   // different location
    TokenInfo token_3(2, foo1);   // different enum
    EXPECT_FALSE(token_0.EquivalentWithoutLocation(token_1));
    EXPECT_FALSE(token_1.EquivalentWithoutLocation(token_0));
    EXPECT_TRUE(token_0.EquivalentWithoutLocation(token_2));
    EXPECT_TRUE(token_2.EquivalentWithoutLocation(token_0));
    EXPECT_FALSE(token_0.EquivalentWithoutLocation(token_3));
    EXPECT_FALSE(token_3.EquivalentWithoutLocation(token_0));
  }
  
  TEST(TokenInfoTest, EquivalentEOF) {
    TokenInfo token_0(TK_EOF, "foo");  // reference token
    TokenInfo token_1(TK_EOF, "bar");  // different text
    TokenInfo token_2(TK_EOF, "foo");  // different location
    EXPECT_TRUE(token_0.EquivalentWithoutLocation(token_1));
    EXPECT_TRUE(token_1.EquivalentWithoutLocation(token_0));
    EXPECT_TRUE(token_0.EquivalentWithoutLocation(token_2));
    EXPECT_TRUE(token_2.EquivalentWithoutLocation(token_0));
  }
  
  TEST(TokenInfoTest, EquivalentBySpaceEmpty) {
    TokenInfo t0(1, "");
    TokenInfo t1(1, "");
    EXPECT_TRUE(t0.EquivalentBySpace(t1));
    EXPECT_TRUE(t1.EquivalentBySpace(t0));
  }
  
  TEST(TokenInfoTest, EquivalentBySpaceDiffEnum) {
    TokenInfo t0(1, "");
    TokenInfo t1(2, "");
    EXPECT_FALSE(t0.EquivalentBySpace(t1));
    EXPECT_FALSE(t1.EquivalentBySpace(t0));
  }
  
  TEST(TokenInfoTest, EquivalentBySpaceDiffLengthText) {
    TokenInfo t0(1, "a");
    TokenInfo t1(1, "aa");
    EXPECT_FALSE(t0.EquivalentBySpace(t1));
    EXPECT_FALSE(t1.EquivalentBySpace(t0));
  }
  
  TEST(TokenInfoTest, EquivalentBySpaceDiffTextEqualLength) {
    TokenInfo t0(1, "bb");
    TokenInfo t1(1, "aa");
    EXPECT_TRUE(t0.EquivalentBySpace(t1));
    EXPECT_TRUE(t1.EquivalentBySpace(t0));
  }
  
  TEST(TokenInfoTest, EquivalentBySpaceEOF) {
    // text is ignored in EOF case
    TokenInfo t0(TK_EOF, "xyz");
    TokenInfo t1(TK_EOF, "12345");
    EXPECT_TRUE(t0.EquivalentBySpace(t1));
    EXPECT_TRUE(t1.EquivalentBySpace(t0));
  }
  
  // This test verifies comparison against EOF.
  TEST(TokenInfoTest, IsEOF) {
    TokenInfo token_info = TokenInfo::EOFToken();
    EXPECT_TRUE(token_info.isEOF());
  }
  
  TEST(TokenInfoTest, IsEOFAnyString) {
    TokenInfo token_info(TK_EOF, "does not matter");
    EXPECT_TRUE(token_info.isEOF());
  }
  
  // Test string representation of token_info.
  TEST(TokenInfoTest, ToStringEOF) {
    const absl::string_view base;  // empty
    const TokenInfo::Context context(base);
    TokenInfo token_info(TK_EOF, base);
    EXPECT_EQ(token_info.ToString(context), "(#0 @0-0: \"\")");
  }
  
  TEST(TokenInfoTest, ToStringWithBase) {
    const absl::string_view base("basement cat");
    const TokenInfo::Context context(base);
    TokenInfo token_info(7, base.substr(9, 3));
    EXPECT_EQ(token_info.ToString(context), "(#7 @9-12: \"cat\")");
  }
  
  void TokenTranslator(std::ostream& stream, int e) {
    switch (e) {
      case 7:
        stream << "lucky-seven";
        break;
      default:
        stream << "???";
    }
  }
  
  TEST(TokenInfoTest, ToStringWithBaseAndTranslator) {
    const absl::string_view base("basement cat");
    const TokenInfo::Context context(base, TokenTranslator);
    TokenInfo token_info(7, base.substr(9, 3));
    EXPECT_EQ(token_info.ToString(context), "(#lucky-seven @9-12: \"cat\")");
  }
  
  TEST(TokenWithContextTest, StreamOutput) {
    const absl::string_view base("basement cat");
    const TokenInfo::Context context(base, TokenTranslator);
    TokenInfo token_info(7, base.substr(9, 3));
    std::ostringstream stream;
    stream << TokenWithContext{token_info, context};
    EXPECT_EQ(stream.str(), "(#lucky-seven @9-12: \"cat\")");
  }
  
  // RebaseStringView() tests
  
  // Test that empty string token rebases correctly.
  TEST(RebaseStringViewTest, EmptyStringsZeroOffset) {
    const std::string text = "";
    // We want another empty string, but we need to trick too smart compilers
    // to give us a different memory address.
    std::string substr = "foo";
    substr.resize(0);  // Force empty string such as 'text' but memory space
    ASSERT_NE(text.c_str(), substr.c_str()) << "Mismatch in memory assumption";
  
    EXPECT_FALSE(BoundsEqual(text, substr));
    TokenInfo token(0, text);
    EXPECT_EQ(token.left(text), 0);
    token.RebaseStringView(substr);
    EXPECT_EQ(token.left(substr), 0);
    EXPECT_EQ(token.text(), substr);
  }
  
  // Test that non-empty whole-string copy rebases correctly.
  TEST(RebaseStringViewTest, IdenticalCopy) {
    const std::string text = "hello";
    const std::string substr = "hello";  // different memory space
    EXPECT_FALSE(BoundsEqual(text, substr));
    TokenInfo token(3, text);
    EXPECT_EQ(token.left(text), 0);
    token.RebaseStringView(substr);
    EXPECT_EQ(token.left(substr), 0);
    EXPECT_EQ(token.text(), substr);
  }
  
  // Test that substring mismatch between new and old is checked.
  TEST(RebaseStringViewDeathTest, SubstringMismatch) {
    const absl::string_view text = "hell0";
    const absl::string_view substr = "hello";
    TokenInfo token(1, text);
    EXPECT_EQ(token.left(text), 0);
    EXPECT_DEATH(token.RebaseStringView(substr),
                 "only valid when the new text referenced matches the old text");
  }
  
  TEST(RebaseStringViewDeathTest, SubstringMismatch2) {
    const absl::string_view text = "hello";
    const absl::string_view substr = "Hello";
    TokenInfo token(1, text);
    EXPECT_EQ(token.left(text), 0);
    EXPECT_DEATH(token.RebaseStringView(substr),
                 "only valid when the new text referenced matches the old text");
  }
  
  // Test that substring in the middle of old string is rebased correctly.
  TEST(RebaseStringViewTest, NewSubstringNotAtFront) {
    const absl::string_view text = "hello";
    const absl::string_view new_base = "xxxhelloyyy";
    TokenInfo token(1, text);
    token.RebaseStringView(new_base.substr(3, 5));
    EXPECT_EQ(token.left(new_base), 3);
    EXPECT_EQ(token.right(new_base), 8);
    EXPECT_EQ(token.text(), text);
  }
  
  // Test that substring in the middle of old string is rebased correctly.
  TEST(RebaseStringViewTest, UsingCharPointer) {
    const absl::string_view text = "hello";
    const absl::string_view new_base = "xxxhelloyyy";
    TokenInfo token(1, text);
    token.RebaseStringView(new_base.begin() + 3);  // assume original length
    EXPECT_EQ(token.left(new_base), 3);
    EXPECT_EQ(token.right(new_base), 8);
    EXPECT_EQ(token.text(), text);
  }
  
  // Test integration with substr() function rebases correctly.
  TEST(RebaseStringViewTest, RelativeToOldBase) {
    const absl::string_view full_text = "xxxxxxhelloyyyyy";
    const absl::string_view substr = full_text.substr(6, 5);
    EXPECT_EQ(substr, "hello");
    TokenInfo token(1, substr);
    EXPECT_EQ(token.left(full_text), 6);
    EXPECT_EQ(token.text(), substr);
    const absl::string_view new_base = "aahellobbb";
    token.RebaseStringView(new_base.substr(2, substr.length()));
    EXPECT_EQ(token.left(new_base), 2);
    EXPECT_EQ(token.right(new_base), 7);
    EXPECT_EQ(token.text(), substr);
  }
  
  // Test rebasing into middle of superstring.
  TEST(RebaseStringViewTest, MiddleOfSuperstring) {
    const absl::string_view dest_text = "xxxxxxhell0yyyyy";
    const absl::string_view src_text = "ccchell0ddd";
    const int dest_offset = 6;
    const absl::string_view src_substr = src_text.substr(3, 5);
    EXPECT_EQ(src_substr, "hell0");
    TokenInfo token(2, src_substr);
    // src_text[3] lines up with dest_text[6].
    token.RebaseStringView(dest_text.substr(dest_offset, src_substr.length()));
    EXPECT_EQ(token.left(dest_text), dest_offset);
    EXPECT_EQ(token.text(), src_substr);
  }
  
  // Test rebasing into prefix superstring.
  TEST(RebaseStringViewTest, PrefixSuperstring) {
    const absl::string_view dest_text = "xxxhell0yyyyyzzzzzzz";
    const absl::string_view src_text = "ccchell0ddd";
    const int dest_offset = 3;
    const absl::string_view src_substr = src_text.substr(3, 5);
    EXPECT_EQ(src_substr, "hell0");
    TokenInfo token(1, src_substr);
    // src_text[3] lines up with dest_text[3].
    token.RebaseStringView(dest_text.substr(dest_offset, src_substr.length()));
    EXPECT_EQ(token.left(dest_text), dest_offset);
    EXPECT_EQ(token.text(), src_substr);
  }
  
  // Test that concatenation works on the degenerate case of no-tokens.
  TEST(TokenInfoConcatenateTest, EmptyTokens) {
    std::string joined;
    std::vector<TokenInfo> tokens;
    TokenInfo::Concatenate(&joined, &tokens);
    EXPECT_TRUE(joined.empty());
    EXPECT_TRUE(tokens.empty());
  }
  
  // Test that concatenation works on a single token.
  TEST(TokenInfoConcatenateTest, OneToken) {
    std::string joined;
    std::vector<TokenInfo> tokens{
        {3, "foo"},
    };
    TokenInfo::Concatenate(&joined, &tokens);
    EXPECT_EQ(joined, "foo");
    ASSERT_EQ(tokens.size(), 1);
    EXPECT_EQ(tokens[0].token_enum(), 3);
    EXPECT_EQ(tokens[0].text(), "foo");
    EXPECT_EQ(tokens[0].left(joined), 0);
    EXPECT_EQ(tokens[0].right(joined), 3);
  }
  
  // Test that concatenation works on multiple tokens.
  TEST(TokenInfoConcatenateTest, MultipleTokens) {
    std::string joined;
    std::vector<TokenInfo> tokens{
        {3, "foo"},
        {4, "  "},
        {5, "bar"},
    };
    TokenInfo::Concatenate(&joined, &tokens);
    EXPECT_EQ(joined, "foo  bar");
    EXPECT_EQ(joined.size(), 8);  // 3 + 2 + 3
    ASSERT_EQ(tokens.size(), 3);
  
    EXPECT_EQ(tokens[0].token_enum(), 3);
    EXPECT_EQ(tokens[0].text(), "foo");
    EXPECT_EQ(tokens[0].left(joined), 0);
    EXPECT_EQ(tokens[0].right(joined), 3);
  
    EXPECT_EQ(tokens[1].token_enum(), 4);
    EXPECT_EQ(tokens[1].text(), "  ");
    EXPECT_EQ(tokens[1].left(joined), 3);
    EXPECT_EQ(tokens[1].right(joined), 5);
  
    EXPECT_EQ(tokens[2].token_enum(), 5);
    EXPECT_EQ(tokens[2].text(), "bar");
    EXPECT_EQ(tokens[2].left(joined), 5);
    EXPECT_EQ(tokens[2].right(joined), 8);
  }
  
  // Test that concatenation works on multiple tokens, even empty strings.
  TEST(TokenInfoConcatenateTest, MultipleTokensWithEmptyString) {
    std::string joined;
    std::vector<TokenInfo> tokens{
        {3, "foo"},
        {6, ""},
        {5, "barr"},
    };
    TokenInfo::Concatenate(&joined, &tokens);
    EXPECT_EQ(joined, "foobarr");
    EXPECT_EQ(joined.size(), 7);  // 3 + 0 + 4
    ASSERT_EQ(tokens.size(), 3);
  
    EXPECT_EQ(tokens[0].token_enum(), 3);
    EXPECT_EQ(tokens[0].text(), "foo");
    EXPECT_EQ(tokens[0].left(joined), 0);
    EXPECT_EQ(tokens[0].right(joined), 3);
  
    EXPECT_EQ(tokens[1].token_enum(), 6);
    EXPECT_EQ(tokens[1].text(), "");
    EXPECT_EQ(tokens[1].left(joined), 3);
    EXPECT_EQ(tokens[1].right(joined), 3);
  
    EXPECT_EQ(tokens[2].token_enum(), 5);
    EXPECT_EQ(tokens[2].text(), "barr");
    EXPECT_EQ(tokens[2].left(joined), 3);
    EXPECT_EQ(tokens[2].right(joined), 7);
  }
  
  }  // namespace
  }  // namespace verible
  