text: // Copyright 2010-2016, Google Inc.
  // All rights reserved.
  //
  // Redistribution and use in source and binary forms, with or without
  // modification, are permitted provided that the following conditions are
  // met:
  //
  //     * Redistributions of source code must retain the above copyright
  // notice, this list of conditions and the following disclaimer.
  //     * Redistributions in binary form must reproduce the above
  // copyright notice, this list of conditions and the following disclaimer
  // in the documentation and/or other materials provided with the
  // distribution.
  //     * Neither the name of Google Inc. nor the names of its
  // contributors may be used to endorse or promote products derived from
  // this software without specific prior written permission.
  //
  // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  
  #include "base/util.h"
  
  #include <climits>
  #include <cstdlib>
  #include <cstring>
  #include <map>
  #include <sstream>
  #include <string>
  
  #include "base/compiler_specific.h"
  #include "base/file_stream.h"
  #include "base/file_util.h"
  #include "base/logging.h"
  #include "base/number_util.h"
  #include "testing/base/public/gunit.h"
  #include "testing/base/public/mozctest.h"
  
  namespace mozc {
  namespace {
  
  #ifndef OS_NACL
  // Disabled on NaCl since it uses a mock file system.
  void FillTestCharacterSetMap(map<char32, Util::CharacterSet> *test_map) {
    CHECK(test_map);
    const string &path = testing::GetSourceFileOrDie({
        "data", "test", "character_set", "character_set.tsv"});
    map<string, Util::CharacterSet> character_set_type_map;
    character_set_type_map["ASCII"] = Util::ASCII;
    character_set_type_map["JISX0201"] = Util::JISX0201;
    character_set_type_map["JISX0208"] = Util::JISX0208;
    character_set_type_map["JISX0212"] = Util::JISX0212;
    character_set_type_map["JISX0213"] = Util::JISX0213;
    character_set_type_map["CP932"] = Util::CP932;
    // UNICODE_ONLY should not appear in the tsv file though.
    character_set_type_map["UNICODE_ONLY"] = Util::UNICODE_ONLY;
  
    InputFileStream finput(path.c_str());
  
    // Read tsv file.
    string line;
    while (!getline(finput, line).fail()) {
      if (Util::StartsWith(line, "#")) {
        // Skip comment line.
        continue;
      }
  
      vector<string> col;
      mozc::Util::SplitStringUsing(line, "\t", &col);
      CHECK_GE(col.size(), 2) << "format error: " << line;
      const char32 ucs4 = NumberUtil::SimpleAtoi(col[0]);
      map<string, Util::CharacterSet>::const_iterator itr =
          character_set_type_map.find(col[1]);
      // We cannot use CHECK_NE here because of overload resolution.
      CHECK(character_set_type_map.end() != itr)
          << "Unknown character set type: " << col[1];
      test_map->insert(std::make_pair(ucs4, itr->second));
    }
  }
  #endif  // !OS_NACL
  
  Util::CharacterSet GetExpectedCharacterSet(
      const map<char32, Util::CharacterSet> &test_map,
      char32 ucs4) {
    map<char32, Util::CharacterSet>::const_iterator itr =
        test_map.find(ucs4);
    if (test_map.find(ucs4) == test_map.end()) {
      // If the test data does not have an entry, it should be
      // interpreted as |Util::UNICODE_ONLY|.
      return Util::UNICODE_ONLY;
    }
    return itr->second;
  }
  
  }  // namespace
  
  TEST(UtilTest, JoinStrings) {
    vector<string> input;
    input.push_back("ab");
    input.push_back("cdef");
    input.push_back("ghr");
    string output;
    Util::JoinStrings(input, ":", &output);
    EXPECT_EQ("ab:cdef:ghr", output);
  }
  
  TEST(UtilTest, JoinStringPieces) {
    {
      vector<StringPiece> input;
      input.push_back("ab");
      string output;
      Util::JoinStringPieces(input, ":", &output);
      EXPECT_EQ("ab", output);
    }
    {
      vector<StringPiece> input;
      input.push_back("ab");
      input.push_back("cdef");
      input.push_back("ghr");
      string output;
      Util::JoinStringPieces(input, ":", &output);
      EXPECT_EQ("ab:cdef:ghr", output);
    }
    {
      vector<StringPiece> input;
      input.push_back("ab");
      input.push_back("cdef");
      input.push_back("ghr");
      string output;
      Util::JoinStringPieces(input, "::", &output);
      EXPECT_EQ("ab::cdef::ghr", output);
    }
  }
  
  TEST(UtilTest, ConcatStrings) {
    string s;
  
    Util::ConcatStrings("", "", &s);
    EXPECT_TRUE(s.empty());
  
    Util::ConcatStrings("ABC", "", &s);
    EXPECT_EQ("ABC", s);
  
    Util::ConcatStrings("", "DEF", &s);
    EXPECT_EQ("DEF", s);
  
    Util::ConcatStrings("ABC", "DEF", &s);
    EXPECT_EQ("ABCDEF", s);
  }
  
  TEST(UtilTest, AppendStringWithDelimiter) {
    string result;
    string input;
    const char kDelemiter[] = ":";
  
    {
      result.clear();
      Util::AppendStringWithDelimiter(kDelemiter, "test", &result);
      EXPECT_EQ("test", result);
    }
  
    {
      result = "foo";
      Util::AppendStringWithDelimiter(kDelemiter, "test", &result);
      EXPECT_EQ("foo:test", result);
    }
  
    {
      result = "foo";
      Util::AppendStringWithDelimiter(kDelemiter, "", &result);
      EXPECT_EQ("foo:", result);
    }
  }
  
  TEST(UtilTest, SplitIterator_SingleDelimiter_SkipEmpty) {
    typedef SplitIterator<SingleDelimiter, SkipEmpty> SplitIterator;
    {
      SplitIterator iter("", " ");
      EXPECT_TRUE(iter.Done());
    }
    {
      SplitIterator iter(StringPiece(), " ");
      EXPECT_TRUE(iter.Done());
    }
    {
      const char *s = "a b cde";
      SplitIterator iter(s, " ");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("cde", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
    {
      const char *s = " a b  cde ";
      SplitIterator iter(s, " ");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("cde", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
    {
      StringPiece s("a b  cde ", 5);  // s = "a b  ";
      SplitIterator iter(s, " ");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
  }
  
  TEST(UtilTest, SplitIterator_MultiDelimiter_SkipEmpty) {
    typedef SplitIterator<MultiDelimiter, SkipEmpty> SplitIterator;
    {
      SplitIterator iter("", " \t,");
      EXPECT_TRUE(iter.Done());
    }
    {
      SplitIterator iter(StringPiece(), ",.");
      EXPECT_TRUE(iter.Done());
    }
    {
      const char *s = "a b\tcde:fg";
      SplitIterator iter(s, " \t:");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("cde", iter.Get());
      EXPECT_FALSE(iter.Done());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("fg", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
    {
      const char *s = "  \t:a b\t\tcde:fg:";
      SplitIterator iter(s, " \t:");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("cde", iter.Get());
      EXPECT_FALSE(iter.Done());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("fg", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
  }
  
  TEST(UtilTest, SplitIterator_SingleDelimiter_AllowEmpty) {
    typedef SplitIterator<SingleDelimiter, AllowEmpty> SplitIterator;
    {
      SplitIterator iter("", " ");
      EXPECT_TRUE(iter.Done());
    }
    {
      SplitIterator iter(StringPiece(), " ");
      EXPECT_TRUE(iter.Done());
    }
    {
      const char *s = "a b cde";
      SplitIterator iter(s, " ");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("cde", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
    {
      const char *s = " a b  cde ";
      SplitIterator iter(s, " ");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("cde", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
    {
      StringPiece s("a b  cde ", 5);  // s = "a b  ";
      SplitIterator iter(s, " ");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
  }
  
  TEST(UtilTest, SplitIterator_MultiDelimiter_AllowEmpty) {
    typedef SplitIterator<MultiDelimiter, AllowEmpty> SplitIterator;
    {
      SplitIterator iter("", " \t,");
      EXPECT_TRUE(iter.Done());
    }
    {
      SplitIterator iter(StringPiece(), ",.");
      EXPECT_TRUE(iter.Done());
    }
    {
      const char *s = "a b\tcde:fg";
      SplitIterator iter(s, " \t:");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("cde", iter.Get());
      EXPECT_FALSE(iter.Done());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("fg", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
    {
      const char *s = "a b\t\tcde:fg:";
      SplitIterator iter(s, " \t:");
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("a", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("b", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("cde", iter.Get());
      EXPECT_FALSE(iter.Done());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("fg", iter.Get());
      iter.Next();
      EXPECT_FALSE(iter.Done());
      EXPECT_EQ("", iter.Get());
      iter.Next();
      EXPECT_TRUE(iter.Done());
    }
  }
  
  TEST(UtilTest, SplitStringUsing) {
    {
      const string input = "a b  c def";
      vector<string> output;
      Util::SplitStringUsing(input, " ", &output);
      EXPECT_EQ(output.size(), 4);
      EXPECT_EQ("a", output[0]);
      EXPECT_EQ("b", output[1]);
      EXPECT_EQ("c", output[2]);
      EXPECT_EQ("def", output[3]);
    }
    {
      const string input = " a b  c";
      vector<string> output;
      Util::SplitStringUsing(input, " ", &output);
      EXPECT_EQ(output.size(), 3);
      EXPECT_EQ("a", output[0]);
      EXPECT_EQ("b", output[1]);
      EXPECT_EQ("c", output[2]);
    }
    {
      const string input = "a b  c ";
      vector<string> output;
      Util::SplitStringUsing(input, " ", &output);
      EXPECT_EQ(output.size(), 3);
      EXPECT_EQ("a", output[0]);
      EXPECT_EQ("b", output[1]);
      EXPECT_EQ("c", output[2]);
    }
    {
      const string input = "a:b  cd ";
      vector<string> output;
      Util::SplitStringUsing(input, ": ", &output);
      EXPECT_EQ(output.size(), 3);
      EXPECT_EQ("a", output[0]);
      EXPECT_EQ("b", output[1]);
      EXPECT_EQ("cd", output[2]);
    }
    {
      const string input = "Empty delimiter";
      vector<string> output;
      Util::SplitStringUsing(input, "", &output);
      EXPECT_EQ(output.size(), 1);
      EXPECT_EQ(input, output[0]);
    }
  }
  
  TEST(UtilTest, SplitStringAllowEmpty) {
    {
      const string input = "a b  c def";
      vector<string> output;
      Util::SplitStringAllowEmpty(input, " ", &output);
      EXPECT_EQ(output.size(), 5);
      EXPECT_EQ("a", output[0]);
      EXPECT_EQ("b", output[1]);
      EXPECT_EQ("", output[2]);
      EXPECT_EQ("c", output[3]);
      EXPECT_EQ("def", output[4]);
    }
    {
      const string input = " a b  c";
      vector<string> output;
      Util::SplitStringAllowEmpty(input, " ", &output);
      EXPECT_EQ(output.size(), 5);
      EXPECT_EQ("", output[0]);
      EXPECT_EQ("a", output[1]);
      EXPECT_EQ("b", output[2]);
      EXPECT_EQ("", output[3]);
      EXPECT_EQ("c", output[4]);
    }
    {
      const string input = "a b  c ";
      vector<string> output;
      Util::SplitStringAllowEmpty(input, " ", &output);
      EXPECT_EQ(output.size(), 5);
      EXPECT_EQ("a", output[0]);
      EXPECT_EQ("b", output[1]);
      EXPECT_EQ("", output[2]);
      EXPECT_EQ("c", output[3]);
      EXPECT_EQ("", output[4]);
    }
    {
      const string input = "a:b  c ";
      vector<string> output;
      Util::SplitStringAllowEmpty(input, ": ", &output);
      EXPECT_EQ(output.size(), 5);
      EXPECT_EQ("a", output[0]);
      EXPECT_EQ("b", output[1]);
      EXPECT_EQ("", output[2]);
      EXPECT_EQ("c", output[3]);
      EXPECT_EQ("", output[4]);
    }
    {
      const string input = "Empty delimiter";
      vector<string> output;
      Util::SplitStringAllowEmpty(input, "", &output);
      EXPECT_EQ(output.size(), 1);
      EXPECT_EQ(input, output[0]);
    }
  }
  
  TEST(UtilTest, StripWhiteSpaces) {
    // basic scenario.
    {
      const string input = "  foo   ";
      string output;
      Util::StripWhiteSpaces(input, &output);
      EXPECT_EQ("foo", output);
    }
  
    // no space means just copy.
    {
      const string input = "foo";
      string output;
      Util::StripWhiteSpaces(input, &output);
      EXPECT_EQ("foo", output);
    }
  
    // tabs and linebreaks are also spaces.
    {
      const string input = " \tfoo\n";
      string output;
      Util::StripWhiteSpaces(input, &output);
      EXPECT_EQ("foo", output);
    }
  
    // spaces in the middle remains.
    {
      const string input = " foo bar baz ";
      string output;
      Util::StripWhiteSpaces(input, &output);
      EXPECT_EQ("foo bar baz", output);
    }
  
    // all spaces means clear out output.
    {
      const string input = " \v \r ";
      string output;
      Util::StripWhiteSpaces(input, &output);
      EXPECT_TRUE(output.empty());
    }
  
    // empty input.
    {
      const string input = "";
      string output;
      Util::StripWhiteSpaces(input, &output);
      EXPECT_TRUE(output.empty());
    }
  
    // one character.
    {
      const string input = "a";
      string output;
      Util::StripWhiteSpaces(input, &output);
      EXPECT_EQ("a", output);
    }
  }
  
  TEST(UtilTest, SplitStringToUtf8Chars) {
    {
      vector<string> output;
      Util::SplitStringToUtf8Chars("", &output);
      EXPECT_EQ(0, output.size());
    }
  
    {
      // "a" "あ" "Ａ" "亜" "\n" "a"
      const string kInputs[] = {
        "a",
        "\xE3\x81\x82",
        "\xEF\xBC\xA1",
        "\xE4\xBA\x9C",
        "\n",
        "a",
      };
      string joined_string;
      for (int i = 0; i < arraysize(kInputs); ++i) {
        joined_string += kInputs[i];
      }
  
      vector<string> output;
      Util::SplitStringToUtf8Chars(joined_string, &output);
      EXPECT_EQ(arraysize(kInputs), output.size());
  
      for (size_t i = 0; i < output.size(); ++i) {
        EXPECT_EQ(kInputs[i], output[i]);
      }
    }
  }
  
  TEST(UtilTest, SplitCSV) {
    vector<string> answer_vector;
  
    Util::SplitCSV(
        "Google,x,\"Buchheit, Paul\",\"string with \"\" quote in it\"",
        &answer_vector);
    CHECK_EQ(answer_vector.size(), 4);
    CHECK_EQ(answer_vector[0], "Google");
    CHECK_EQ(answer_vector[1], "x");
    CHECK_EQ(answer_vector[2], "Buchheit, Paul");
    CHECK_EQ(answer_vector[3], "string with \" quote in it");
  
    Util::SplitCSV("Google,hello,",  &answer_vector);
    CHECK_EQ(answer_vector.size(), 3);
    CHECK_EQ(answer_vector[0], "Google");
    CHECK_EQ(answer_vector[1], "hello");
    CHECK_EQ(answer_vector[2], "");
  
    Util::SplitCSV("Google rocks,hello", &answer_vector);
    CHECK_EQ(answer_vector.size(), 2);
    CHECK_EQ(answer_vector[0], "Google rocks");
    CHECK_EQ(answer_vector[1], "hello");
  
    Util::SplitCSV(",,\"\",,", &answer_vector);
    CHECK_EQ(answer_vector.size(), 5);
    CHECK_EQ(answer_vector[0], "");
    CHECK_EQ(answer_vector[1], "");
    CHECK_EQ(answer_vector[2], "");
    CHECK_EQ(answer_vector[3], "");
    CHECK_EQ(answer_vector[4], "");
  
    // Test a string containing a comma.
    Util::SplitCSV("\",\",hello", &answer_vector);
    CHECK_EQ(answer_vector.size(), 2);
    CHECK_EQ(answer_vector[0], ",");
    CHECK_EQ(answer_vector[1], "hello");
  
    // Invalid CSV
    Util::SplitCSV("\"no,last,quote", &answer_vector);
    CHECK_EQ(answer_vector.size(), 1);
    CHECK_EQ(answer_vector[0], "no,last,quote");
  
    Util::SplitCSV("backslash\\,is,no,an,\"escape\"", &answer_vector);
    CHECK_EQ(answer_vector.size(), 5);
    CHECK_EQ(answer_vector[0], "backslash\\");
    CHECK_EQ(answer_vector[1], "is");
    CHECK_EQ(answer_vector[2], "no");
    CHECK_EQ(answer_vector[3], "an");
    CHECK_EQ(answer_vector[4], "escape");
  
    Util::SplitCSV("", &answer_vector);
    CHECK_EQ(answer_vector.size(), 0);
  }
  
  TEST(UtilTest, ReplaceString) {
    const string input = "foobarfoobar";
    string output;
    Util::StringReplace(input, "bar", "buz", true, &output);
    EXPECT_EQ("foobuzfoobuz", output);
  
    output.clear();
    Util::StringReplace(input, "bar", "buz", false, &output);
    EXPECT_EQ("foobuzfoobar", output);
  }
  
  TEST(UtilTest, LowerString) {
    string s = "TeSTtest";
    Util::LowerString(&s);
    EXPECT_EQ("testtest", s);
  
    // "ＴｅＳＴ＠ＡＢＣＸＹＺ［｀ａｂｃｘｙｚ｛"
    string s2 = "\xef\xbc\xb4\xef\xbd\x85\xef\xbc\xb3\xef\xbc\xb4\xef\xbc\xa0\xef"
                "\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3\xef\xbc\xb8\xef\xbc\xb9\xef\xbc"
                "\xba\xef\xbc\xbb\xef\xbd\x80\xef\xbd\x81\xef\xbd\x82\xef\xbd\x83"
                "\xef\xbd\x98\xef\xbd\x99\xef\xbd\x9a\xef\xbd\x9b";
    Util::LowerString(&s2);
    // "ｔｅｓｔ＠ａｂｃｘｙｚ［｀ａｂｃｘｙｚ｛"
    EXPECT_EQ("\xef\xbd\x94\xef\xbd\x85\xef\xbd\x93\xef\xbd\x94\xef\xbc\xa0\xef"
              "\xbd\x81\xef\xbd\x82\xef\xbd\x83\xef\xbd\x98\xef\xbd\x99\xef\xbd"
              "\x9a\xef\xbc\xbb\xef\xbd\x80\xef\xbd\x81\xef\xbd\x82\xef\xbd\x83"
              "\xef\xbd\x98\xef\xbd\x99\xef\xbd\x9a\xef\xbd\x9b", s2);
  }
  
  TEST(UtilTest, UpperString) {
    string s = "TeSTtest";
    Util::UpperString(&s);
    EXPECT_EQ("TESTTEST", s);
  
    // "ＴｅＳＴ＠ＡＢＣＸＹＺ［｀ａｂｃｘｙｚ｛"
    string s2 = "\xef\xbc\xb4\xef\xbd\x85\xef\xbc\xb3\xef\xbc\xb4\xef\xbc\xa0\xef"
                "\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3\xef\xbc\xb8\xef\xbc\xb9\xef\xbc"
                "\xba\xef\xbc\xbb\xef\xbd\x80\xef\xbd\x81\xef\xbd\x82\xef\xbd\x83"
                "\xef\xbd\x98\xef\xbd\x99\xef\xbd\x9a\xef\xbd\x9b";
    Util::UpperString(&s2);
    // "ＴＥＳＴ＠ＡＢＣＸＹＺ［｀ＡＢＣＸＹＺ｛"
    EXPECT_EQ("\xef\xbc\xb4\xef\xbc\xa5\xef\xbc\xb3\xef\xbc\xb4\xef\xbc\xa0\xef"
              "\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3\xef\xbc\xb8\xef\xbc\xb9\xef\xbc"
              "\xba\xef\xbc\xbb\xef\xbd\x80\xef\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3"
              "\xef\xbc\xb8\xef\xbc\xb9\xef\xbc\xba\xef\xbd\x9b", s2);
  }
  
  TEST(UtilTest, CapitalizeString) {
    string s = "TeSTtest";
    Util::CapitalizeString(&s);
    EXPECT_EQ("Testtest", s);
  
    // "ＴｅＳＴ＠ＡＢＣＸＹＺ［｀ａｂｃｘｙｚ｛"
    string s2 = "\xef\xbc\xb4\xef\xbd\x85\xef\xbc\xb3\xef\xbc\xb4\xef\xbc\xa0\xef"
                "\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3\xef\xbc\xb8\xef\xbc\xb9\xef\xbc"
                "\xba\xef\xbc\xbb\xef\xbd\x80\xef\xbd\x81\xef\xbd\x82\xef\xbd\x83"
                "\xef\xbd\x98\xef\xbd\x99\xef\xbd\x9a\xef\xbd\x9b";
    Util::CapitalizeString(&s2);
    // "Ｔｅｓｔ＠ａｂｃｘｙｚ［｀ａｂｃｘｙｚ｛"
    EXPECT_EQ("\xef\xbc\xb4\xef\xbd\x85\xef\xbd\x93\xef\xbd\x94\xef\xbc\xa0\xef"
              "\xbd\x81\xef\xbd\x82\xef\xbd\x83\xef\xbd\x98\xef\xbd\x99\xef\xbd"
              "\x9a\xef\xbc\xbb\xef\xbd\x80\xef\xbd\x81\xef\xbd\x82\xef\xbd\x83"
              "\xef\xbd\x98\xef\xbd\x99\xef\xbd\x9a\xef\xbd\x9b", s2);
  }
  
  TEST(UtilTest, IsLowerAscii) {
    EXPECT_TRUE(Util::IsLowerAscii(""));
    EXPECT_TRUE(Util::IsLowerAscii("hello"));
    EXPECT_FALSE(Util::IsLowerAscii("HELLO"));
    EXPECT_FALSE(Util::IsLowerAscii("Hello"));
    EXPECT_FALSE(Util::IsLowerAscii("HeLlO"));
    EXPECT_FALSE(Util::IsLowerAscii("symbol!"));
    EXPECT_FALSE(Util::IsLowerAscii(  // "Ｈｅｌｌｏ"
        "\xEF\xBC\xA8\xEF\xBD\x85\xEF\xBD\x8C\xEF\xBD\x8C\xEF\xBD\x8F"));
  }
  
  TEST(UtilTest, IsUpperAscii) {
    EXPECT_TRUE(Util::IsUpperAscii(""));
    EXPECT_FALSE(Util::IsUpperAscii("hello"));
    EXPECT_TRUE(Util::IsUpperAscii("HELLO"));
    EXPECT_FALSE(Util::IsUpperAscii("Hello"));
    EXPECT_FALSE(Util::IsUpperAscii("HeLlO"));
    EXPECT_FALSE(Util::IsUpperAscii("symbol!"));
    EXPECT_FALSE(Util::IsUpperAscii(  // "Ｈｅｌｌｏ"
        "\xEF\xBC\xA8\xEF\xBD\x85\xEF\xBD\x8C\xEF\xBD\x8C\xEF\xBD\x8F"));
  }
  
  TEST(UtilTest, IsCapitalizedAscii) {
    EXPECT_TRUE(Util::IsCapitalizedAscii(""));
    EXPECT_FALSE(Util::IsCapitalizedAscii("hello"));
    EXPECT_FALSE(Util::IsCapitalizedAscii("HELLO"));
    EXPECT_TRUE(Util::IsCapitalizedAscii("Hello"));
    EXPECT_FALSE(Util::IsCapitalizedAscii("HeLlO"));
    EXPECT_FALSE(Util::IsCapitalizedAscii("symbol!"));
    EXPECT_FALSE(Util::IsCapitalizedAscii(  // "Ｈｅｌｌｏ"
        "\xEF\xBC\xA8\xEF\xBD\x85\xEF\xBD\x8C\xEF\xBD\x8C\xEF\xBD\x8F"));
  }
  
  TEST(UtilTest, IsLowerOrUpperAscii) {
    EXPECT_TRUE(Util::IsLowerOrUpperAscii(""));
    EXPECT_TRUE(Util::IsLowerOrUpperAscii("hello"));
    EXPECT_TRUE(Util::IsLowerOrUpperAscii("HELLO"));
    EXPECT_FALSE(Util::IsLowerOrUpperAscii("Hello"));
    EXPECT_FALSE(Util::IsLowerOrUpperAscii("HeLlO"));
    EXPECT_FALSE(Util::IsLowerOrUpperAscii("symbol!"));
    EXPECT_FALSE(Util::IsLowerOrUpperAscii(  // "Ｈｅｌｌｏ"
        "\xEF\xBC\xA8\xEF\xBD\x85\xEF\xBD\x8C\xEF\xBD\x8C\xEF\xBD\x8F"));
  }
  
  TEST(UtilTest, IsUpperOrCapitalizedAscii) {
    EXPECT_TRUE(Util::IsUpperOrCapitalizedAscii(""));
    EXPECT_FALSE(Util::IsUpperOrCapitalizedAscii("hello"));
    EXPECT_TRUE(Util::IsUpperOrCapitalizedAscii("HELLO"));
    EXPECT_TRUE(Util::IsUpperOrCapitalizedAscii("Hello"));
    EXPECT_FALSE(Util::IsUpperOrCapitalizedAscii("HeLlO"));
    EXPECT_FALSE(Util::IsUpperOrCapitalizedAscii("symbol!"));
    EXPECT_FALSE(Util::IsUpperOrCapitalizedAscii(  // "Ｈｅｌｌｏ"
        "\xEF\xBC\xA8\xEF\xBD\x85\xEF\xBD\x8C\xEF\xBD\x8C\xEF\xBD\x8F"));
  }
  
  void VerifyUTF8ToUCS4(const string &text, char32 expected_ucs4,
                        size_t expected_len) {
    const char *begin = text.data();
    const char *end = begin + text.size();
    size_t mblen = 0;
    char32 result = Util::UTF8ToUCS4(begin, end, &mblen);
    EXPECT_EQ(expected_ucs4, result) << text << " " << expected_ucs4;
    EXPECT_EQ(expected_len, mblen) << text << " " << expected_len;
  }
  
  TEST(UtilTest, UTF8ToUCS4) {
    VerifyUTF8ToUCS4("", 0, 0);
    VerifyUTF8ToUCS4("\x01", 1, 1);
    VerifyUTF8ToUCS4("\x7F", 0x7F, 1);
    VerifyUTF8ToUCS4("\xC2\x80", 0x80, 2);
    VerifyUTF8ToUCS4("\xDF\xBF", 0x7FF, 2);
    VerifyUTF8ToUCS4("\xE0\xA0\x80", 0x800, 3);
    VerifyUTF8ToUCS4("\xEF\xBF\xBF", 0xFFFF, 3);
    VerifyUTF8ToUCS4("\xF0\x90\x80\x80", 0x10000, 4);
    VerifyUTF8ToUCS4("\xF7\xBF\xBF\xBF", 0x1FFFFF, 4);
    // do not test 5-6 bytes because it's out of spec of UTF8.
  }
  
  TEST(UtilTest, UCS4ToUTF8) {
    string output;
  
    // Do nothing if |c| is NUL. Previous implementation of UCS4ToUTF8 worked like
    // this even though the reason is unclear.
    Util::UCS4ToUTF8(0, &output);
    EXPECT_TRUE(output.empty());
  
    Util::UCS4ToUTF8(0x7F, &output);
    EXPECT_EQ("\x7F", output);
    Util::UCS4ToUTF8(0x80, &output);
    EXPECT_EQ("\xC2\x80", output);
    Util::UCS4ToUTF8(0x7FF, &output);
    EXPECT_EQ("\xDF\xBF", output);
    Util::UCS4ToUTF8(0x800, &output);
    EXPECT_EQ("\xE0\xA0\x80", output);
    Util::UCS4ToUTF8(0xFFFF, &output);
    EXPECT_EQ("\xEF\xBF\xBF", output);
    Util::UCS4ToUTF8(0x10000, &output);
    EXPECT_EQ("\xF0\x90\x80\x80", output);
    Util::UCS4ToUTF8(0x1FFFFF, &output);
    EXPECT_EQ("\xF7\xBF\xBF\xBF", output);
  
    // Buffer version.
    char buf[7];
  
    EXPECT_EQ(0, Util::UCS4ToUTF8(0, buf));
    EXPECT_EQ(0, strcmp(buf, ""));
  
    EXPECT_EQ(1, Util::UCS4ToUTF8(0x7F, buf));
    EXPECT_EQ(0, strcmp("\x7F", buf));
  
    EXPECT_EQ(2, Util::UCS4ToUTF8(0x80, buf));
    EXPECT_EQ(0, strcmp("\xC2\x80", buf));
  
    EXPECT_EQ(2, Util::UCS4ToUTF8(0x7FF, buf));
    EXPECT_EQ(0, strcmp("\xDF\xBF", buf));
  
    EXPECT_EQ(3, Util::UCS4ToUTF8(0x800, buf));
    EXPECT_EQ(0, strcmp("\xE0\xA0\x80", buf));
  
    EXPECT_EQ(3, Util::UCS4ToUTF8(0xFFFF, buf));
    EXPECT_EQ(0, strcmp("\xEF\xBF\xBF", buf));
  
    EXPECT_EQ(4, Util::UCS4ToUTF8(0x10000, buf));
    EXPECT_EQ(0, strcmp("\xF0\x90\x80\x80", buf));
  
    EXPECT_EQ(4, Util::UCS4ToUTF8(0x1FFFFF, buf));
    EXPECT_EQ(0, strcmp("\xF7\xBF\xBF\xBF", buf));
  }
  
  TEST(UtilTest, CharsLen) {
    // "私の名前は中野です"
    const string src = "\xe7\xa7\x81\xe3\x81\xae\xe5\x90\x8d\xe5\x89\x8d\xe3\x81"
                       "\xaf\xe4\xb8\xad\xe9\x87\x8e\xe3\x81\xa7\xe3\x81\x99";
    EXPECT_EQ(Util::CharsLen(src.c_str(), src.size()), 9);
  }
  
  TEST(UtilTest, SubStringPiece) {
    // "私の名前は中野です"
    const string src = "\xe7\xa7\x81\xe3\x81\xae\xe5\x90\x8d\xe5\x89\x8d\xe3\x81"
                       "\xaf\xe4\xb8\xad\xe9\x87\x8e\xe3\x81\xa7\xe3\x81\x99";
    StringPiece result;
  
    result = Util::SubStringPiece(src, 0, 2);
    // "私の"
    EXPECT_EQ("\xe7\xa7\x81\xe3\x81\xae", result);
    // |result|'s data should point to the same memory block as src.
    EXPECT_LE(src.data(), result.data());
  
    result = Util::SubStringPiece(src, 4, 1);
    // "は"
    EXPECT_EQ("\xe3\x81\xaf", result);
    EXPECT_LE(src.data(), result.data());
  
    result = Util::SubStringPiece(src, 5, 3);
    // "中野で"
    EXPECT_EQ("\xe4\xb8\xad\xe9\x87\x8e\xe3\x81\xa7", result);
    EXPECT_LE(src.data(), result.data());
  
    result = Util::SubStringPiece(src, 6, 10);
    // "野です"
    EXPECT_EQ("\xe9\x87\x8e\xe3\x81\xa7\xe3\x81\x99", result);
    EXPECT_LE(src.data(), result.data());
  
    result = Util::SubStringPiece(src, 4, 2);
    // "は中"
    EXPECT_EQ("\xe3\x81\xaf\xe4\xb8\xad", result);
    EXPECT_LE(src.data(), result.data());
  
    result = Util::SubStringPiece(src, 2, string::npos);
    // "名前は中野です"
    EXPECT_EQ("\xe5\x90\x8d\xe5\x89\x8d\xe3\x81\xaf\xe4\xb8\xad\xe9\x87"
              "\x8e\xe3\x81\xa7\xe3\x81\x99",
              result);
    EXPECT_LE(src.data(), result.data());
  
    result = Util::SubStringPiece(src, 5, string::npos);
    // "中野です"
    EXPECT_EQ("\xe4\xb8\xad\xe9\x87\x8e\xe3\x81\xa7\xe3\x81\x99", result);
    EXPECT_LE(src.data(), result.data());
  }
  
  TEST(UtilTest, SubStringPiece2) {
    // "私はGoogleです"
    const string src =
        "\xE7\xA7\x81\xE3\x81\xAF\x47\x6F\x6F\x67\x6C\x65"
        "\xE3\x81\xA7\xE3\x81\x99";
  
    StringPiece result;
  
    result = Util::SubStringPiece(src, 0);
    EXPECT_EQ(src, result);
  
    result = Util::SubStringPiece(src, 5);
    // "gleです"
    EXPECT_EQ("\x67\x6C\x65\xE3\x81\xA7\xE3\x81\x99", result);
  
    result = Util::SubStringPiece(src, 10);
    EXPECT_TRUE(result.empty());
  
    result = Util::SubStringPiece(src, 13);
    EXPECT_TRUE(result.empty());
  }
  
  TEST(UtilTest, SubString) {
    // "私の名前は中野です"
    const string src = "\xe7\xa7\x81\xe3\x81\xae\xe5\x90\x8d\xe5\x89\x8d\xe3\x81"
                       "\xaf\xe4\xb8\xad\xe9\x87\x8e\xe3\x81\xa7\xe3\x81\x99";
    string result;
  
    result.clear();
    Util::SubString(src, 0, 2, &result);
    // "私の"
    EXPECT_EQ(result, "\xe7\xa7\x81\xe3\x81\xae");
  
    result.clear();
    Util::SubString(src, 4, 1, &result);
    // "は"
    EXPECT_EQ(result, "\xe3\x81\xaf");
  
    result.clear();
    Util::SubString(src, 5, 3, &result);
    // "中野で"
    EXPECT_EQ(result, "\xe4\xb8\xad\xe9\x87\x8e\xe3\x81\xa7");
  
    result.clear();
    Util::SubString(src, 6, 10, &result);
    // "野です"
    EXPECT_EQ(result, "\xe9\x87\x8e\xe3\x81\xa7\xe3\x81\x99");
  
    result.clear();
    Util::SubString(src, 4, 2, &result);
    // "は中"
    EXPECT_EQ(result, "\xe3\x81\xaf\xe4\xb8\xad");
  
    result.clear();
    Util::SubString(src, 2, string::npos, &result);
    // "名前は中野です"
    EXPECT_EQ(result, "\xe5\x90\x8d\xe5\x89\x8d\xe3\x81\xaf\xe4\xb8\xad\xe9\x87"
                      "\x8e\xe3\x81\xa7\xe3\x81\x99");
  
    result.clear();
    Util::SubString(src, 5, string::npos, &result);
    // "中野です"
    EXPECT_EQ(result, "\xe4\xb8\xad\xe9\x87\x8e\xe3\x81\xa7\xe3\x81\x99");
  
    // Doesn't clear result and call Util::SubString
    Util::SubString(src, 5, string::npos, &result);
    // "中野です"
    EXPECT_EQ(result, "\xe4\xb8\xad\xe9\x87\x8e\xe3\x81\xa7\xe3\x81\x99");
  }
  
  TEST(UtilTest, StartsWith) {
    const string str = "abcdefg";
    EXPECT_TRUE(Util::StartsWith(str, ""));
    EXPECT_TRUE(Util::StartsWith(str, "a"));
    EXPECT_TRUE(Util::StartsWith(str, "abc"));
    EXPECT_TRUE(Util::StartsWith(str, "abcdefg"));
    EXPECT_FALSE(Util::StartsWith(str, "abcdefghi"));
    EXPECT_FALSE(Util::StartsWith(str, "foobar"));
  }
  
  TEST(UtilTest, EndsWith) {
    const string str = "abcdefg";
    EXPECT_TRUE(Util::EndsWith(str, ""));
    EXPECT_TRUE(Util::EndsWith(str, "g"));
    EXPECT_TRUE(Util::EndsWith(str, "fg"));
    EXPECT_TRUE(Util::EndsWith(str, "abcdefg"));
    EXPECT_FALSE(Util::EndsWith(str, "aaabcdefg"));
    EXPECT_FALSE(Util::EndsWith(str, "foobar"));
    EXPECT_FALSE(Util::EndsWith(str, "foobarbuzbuz"));
  }
  
  TEST(UtilTest, StripUTF8BOM) {
    string line;
  
    // Should be stripped.
    line = "\xef\xbb\xbf" "abc";
    Util::StripUTF8BOM(&line);
    EXPECT_EQ("abc", line);
  
    // Should be stripped.
    line = "\xef\xbb\xbf";
    Util::StripUTF8BOM(&line);
    EXPECT_EQ("", line);
  
    // BOM in the middle of text. Shouldn't be stripped.
    line = "a" "\xef\xbb\xbf" "bc";
    Util::StripUTF8BOM(&line);
    EXPECT_EQ("a" "\xef\xbb\xbf" "bc", line);
  
    // Incomplete BOM. Shouldn't be stripped.
    line = "\xef\xbb" "abc";
    Util::StripUTF8BOM(&line);
    EXPECT_EQ("\xef\xbb" "abc", line);
  
    // String shorter than the BOM. Do nothing.
    line = "a";
    Util::StripUTF8BOM(&line);
    EXPECT_EQ("a", line);
  
    // Empty string. Do nothing.
    line = "";
    Util::StripUTF8BOM(&line);
    EXPECT_EQ("", line);
  }
  
  TEST(UtilTest, IsUTF16BOM) {
    EXPECT_FALSE(Util::IsUTF16BOM(""));
    EXPECT_FALSE(Util::IsUTF16BOM("abc"));
    EXPECT_TRUE(Util::IsUTF16BOM("\xfe\xff"));
    EXPECT_TRUE(Util::IsUTF16BOM("\xff\xfe"));
    EXPECT_TRUE(Util::IsUTF16BOM("\xfe\xff "));
    EXPECT_TRUE(Util::IsUTF16BOM("\xff\xfe "));
    EXPECT_FALSE(Util::IsUTF16BOM(" \xfe\xff"));
    EXPECT_FALSE(Util::IsUTF16BOM(" \xff\xfe"));
    EXPECT_FALSE(Util::IsUTF16BOM("\xff\xff"));
  }
  
  TEST(UtilTest, IsAndroidPuaEmoji) {
    EXPECT_FALSE(Util::IsAndroidPuaEmoji(""));
    EXPECT_FALSE(Util::IsAndroidPuaEmoji("A"));
    EXPECT_FALSE(Util::IsAndroidPuaEmoji("a"));
  
    string str;
    Util::UCS4ToUTF8(0xFDFFF, &str);
    EXPECT_FALSE(Util::IsAndroidPuaEmoji(str));
    Util::UCS4ToUTF8(0xFE000, &str);
    EXPECT_TRUE(Util::IsAndroidPuaEmoji(str));
    Util::UCS4ToUTF8(0xFE800, &str);
    EXPECT_TRUE(Util::IsAndroidPuaEmoji(str));
    Util::UCS4ToUTF8(0xFEEA0, &str);
    EXPECT_TRUE(Util::IsAndroidPuaEmoji(str));
    Util::UCS4ToUTF8(0xFEEA1, &str);
    EXPECT_FALSE(Util::IsAndroidPuaEmoji(str));
  
    // If it has two ucs4 chars (or more), just expect false.
    Util::UCS4ToUTF8(0xFE000, &str);
    Util::UCS4ToUTF8Append(0xFE000, &str);
    EXPECT_FALSE(Util::IsAndroidPuaEmoji(str));
  }
  
  TEST(UtilTest, StringPrintf) {
    // On GCC, |EXPECT_EQ("", Util::StringPrintf(""))| may cause
    // "warning: zero-length printf format string" so we disable this check.
    MOZC_GCC_DISABLE_WARNING_INLINE(format-zero-length);
  
    // strings
    EXPECT_EQ("", Util::StringPrintf(""));
    EXPECT_EQ("", Util::StringPrintf("%s", ""));
    EXPECT_EQ("hello, world", Util::StringPrintf("hello, world"));
    EXPECT_EQ("hello, world", Util::StringPrintf("%s", "hello, world"));
    EXPECT_EQ("hello, world", Util::StringPrintf("%s, %s", "hello", "world"));
    const char kHello[] = "\xE3\x81\xAF\xE3\x82\x8D\xE3\x83\xBC"  // はろー
                          "\xE4\xB8\x96\xE7\x95\x8C";  // 世界
    EXPECT_EQ(kHello, Util::StringPrintf("%s", kHello));
  
    // 32-bit integers
    EXPECT_EQ("-2147483648", Util::StringPrintf("%d", kint32min));
    EXPECT_EQ("2147483647", Util::StringPrintf("%d", kint32max));
    EXPECT_EQ("4294967295", Util::StringPrintf("%u", kuint32max));
    EXPECT_EQ("80000000", Util::StringPrintf("%x", kint32min));
    EXPECT_EQ("7fffffff", Util::StringPrintf("%x", kint32max));
    EXPECT_EQ("FFFFFFFF", Util::StringPrintf("%X", kuint32max));
  
    // 64-bit integers
    EXPECT_EQ("-9223372036854775808",
              Util::StringPrintf("%" GG_LL_FORMAT "d", kint64min));
    EXPECT_EQ("9223372036854775807",
              Util::StringPrintf("%" GG_LL_FORMAT "d", kint64max));
    EXPECT_EQ("18446744073709551615",
              Util::StringPrintf("%" GG_LL_FORMAT "u", kuint64max));
    EXPECT_EQ("8000000000000000",
              Util::StringPrintf("%" GG_LL_FORMAT "x", kint64min));
    EXPECT_EQ("7fffffffffffffff",
              Util::StringPrintf("%" GG_LL_FORMAT "x", kint64max));
    EXPECT_EQ("FFFFFFFFFFFFFFFF",
              Util::StringPrintf("%" GG_LL_FORMAT "X", kuint64max));
  
    // Simple test for floating point numbers
    EXPECT_EQ("-1.75", Util::StringPrintf("%.2f", -1.75));
  
    // 4096 is greater than a temporary buffer size (1024 bytes)
    // which is used in StringPrintf().
    const string kLongStrA(4096, '.');
    const string kLongStrB(4096, '_');
    const string& result = Util::StringPrintf("%s\t%s\n",
                                              kLongStrA.c_str(),
                                              kLongStrB.c_str());
    EXPECT_EQ(kLongStrA + "\t" + kLongStrB + "\n", result);
  }
  
  TEST(UtilTest, HiraganaToKatakana) {
    {
      // "あいうえおぁぃぅぇぉかきくけこがぎぐげごさしすせそざじずぜぞたちつてと"
      // "だぢづでどっなにぬねのはひふへほばびぶべぼぱぴぷぺぽまみむめもやゆよゃ"
      // "ゅょらりるれろわゎをんゔ"
      const string input =
          "\xE3\x81\x82\xE3\x81\x84\xE3\x81\x86\xE3\x81\x88\xE3\x81\x8A"
          "\xE3\x81\x81\xE3\x81\x83\xE3\x81\x85\xE3\x81\x87\xE3\x81\x89"
          "\xE3\x81\x8B\xE3\x81\x8D\xE3\x81\x8F\xE3\x81\x91\xE3\x81\x93"
          "\xE3\x81\x8C\xE3\x81\x8E\xE3\x81\x90\xE3\x81\x92\xE3\x81\x94"
          "\xE3\x81\x95\xE3\x81\x97\xE3\x81\x99\xE3\x81\x9B\xE3\x81\x9D"
          "\xE3\x81\x96\xE3\x81\x98\xE3\x81\x9A\xE3\x81\x9C\xE3\x81\x9E"
          "\xE3\x81\x9F\xE3\x81\xA1\xE3\x81\xA4\xE3\x81\xA6\xE3\x81\xA8"
          "\xE3\x81\xA0\xE3\x81\xA2\xE3\x81\xA5\xE3\x81\xA7\xE3\x81\xA9"
          "\xE3\x81\xA3\xE3\x81\xAA\xE3\x81\xAB\xE3\x81\xAC\xE3\x81\xAD"
          "\xE3\x81\xAE\xE3\x81\xAF\xE3\x81\xB2\xE3\x81\xB5\xE3\x81\xB8"
          "\xE3\x81\xBB\xE3\x81\xB0\xE3\x81\xB3\xE3\x81\xB6\xE3\x81\xB9"
          "\xE3\x81\xBC\xE3\x81\xB1\xE3\x81\xB4\xE3\x81\xB7\xE3\x81\xBA"
          "\xE3\x81\xBD\xE3\x81\xBE\xE3\x81\xBF\xE3\x82\x80\xE3\x82\x81"
          "\xE3\x82\x82\xE3\x82\x84\xE3\x82\x86\xE3\x82\x88\xE3\x82\x83"
          "\xE3\x82\x85\xE3\x82\x87\xE3\x82\x89\xE3\x82\x8A\xE3\x82\x8B"
          "\xE3\x82\x8C\xE3\x82\x8D\xE3\x82\x8F\xE3\x82\x8E\xE3\x82\x92"
          "\xE3\x82\x93\xE3\x82\x94";
      string output;
      Util::HiraganaToKatakana(input, &output);
      // "アイウエオァィゥェォカキクケコガギグゲゴサシスセソザジズゼゾタチツテト"
      // "ダヂヅデドッナニヌネノハヒフヘホバビブベボパピプペポマミムメモヤユヨャ"
      // "ュョラリルレロワヮヲンヴ"
      EXPECT_EQ("\xE3\x82\xA2\xE3\x82\xA4\xE3\x82\xA6\xE3\x82\xA8\xE3\x82\xAA"
                "\xE3\x82\xA1\xE3\x82\xA3\xE3\x82\xA5\xE3\x82\xA7\xE3\x82\xA9"
                "\xE3\x82\xAB\xE3\x82\xAD\xE3\x82\xAF\xE3\x82\xB1\xE3\x82\xB3"
                "\xE3\x82\xAC\xE3\x82\xAE\xE3\x82\xB0\xE3\x82\xB2\xE3\x82\xB4"
                "\xE3\x82\xB5\xE3\x82\xB7\xE3\x82\xB9\xE3\x82\xBB\xE3\x82\xBD"
                "\xE3\x82\xB6\xE3\x82\xB8\xE3\x82\xBA\xE3\x82\xBC\xE3\x82\xBE"
                "\xE3\x82\xBF\xE3\x83\x81\xE3\x83\x84\xE3\x83\x86\xE3\x83\x88"
                "\xE3\x83\x80\xE3\x83\x82\xE3\x83\x85\xE3\x83\x87\xE3\x83\x89"
                "\xE3\x83\x83\xE3\x83\x8A\xE3\x83\x8B\xE3\x83\x8C\xE3\x83\x8D"
                "\xE3\x83\x8E\xE3\x83\x8F\xE3\x83\x92\xE3\x83\x95\xE3\x83\x98"
                "\xE3\x83\x9B\xE3\x83\x90\xE3\x83\x93\xE3\x83\x96\xE3\x83\x99"
                "\xE3\x83\x9C\xE3\x83\x91\xE3\x83\x94\xE3\x83\x97\xE3\x83\x9A"
                "\xE3\x83\x9D\xE3\x83\x9E\xE3\x83\x9F\xE3\x83\xA0\xE3\x83\xA1"
                "\xE3\x83\xA2\xE3\x83\xA4\xE3\x83\xA6\xE3\x83\xA8\xE3\x83\xA3"
                "\xE3\x83\xA5\xE3\x83\xA7\xE3\x83\xA9\xE3\x83\xAA\xE3\x83\xAB"
                "\xE3\x83\xAC\xE3\x83\xAD\xE3\x83\xAF\xE3\x83\xAE\xE3\x83\xB2"
                "\xE3\x83\xB3\xE3\x83\xB4",
                output);
    }
    {
      // "わたしのなまえはなかのですうまーよろしゅう"
      const string input = "\xe3\x82\x8f\xe3\x81\x9f\xe3\x81\x97\xe3\x81\xae\xe3"
                           "\x81\xaa\xe3\x81\xbe\xe3\x81\x88\xe3\x81\xaf\xe3\x81"
                           "\xaa\xe3\x81\x8b\xe3\x81\xae\xe3\x81\xa7\xe3\x81\x99"
                           "\xe3\x81\x86\xe3\x81\xbe\xe3\x83\xbc\xe3\x82\x88\xe3"
                           "\x82\x8d\xe3\x81\x97\xe3\x82\x85\xe3\x81\x86";
      string output;
      Util::HiraganaToKatakana(input, &output);
      // "ワタシノナマエハナカノデスウマーヨロシュウ"
      EXPECT_EQ("\xe3\x83\xaf\xe3\x82\xbf\xe3\x82\xb7\xe3\x83\x8e\xe3\x83\x8a\xe3"
                "\x83\x9e\xe3\x82\xa8\xe3\x83\x8f\xe3\x83\x8a\xe3\x82\xab\xe3\x83"
                "\x8e\xe3\x83\x87\xe3\x82\xb9\xe3\x82\xa6\xe3\x83\x9e\xe3\x83\xbc"
                "\xe3\x83\xa8\xe3\x83\xad\xe3\x82\xb7\xe3\x83\xa5\xe3\x82\xa6",
                output);
    }
    {
      // "グーグル工藤よろしくabc"
      const string input = "\xe3\x82\xb0\xe3\x83\xbc\xe3\x82\xb0\xe3\x83\xab\xe5"
                           "\xb7\xa5\xe8\x97\xa4\xe3\x82\x88\xe3\x82\x8d\xe3\x81"
                           "\x97\xe3\x81\x8f\x61\x62\x63";
      string output;
      Util::HiraganaToKatakana(input, &output);
      // "グーグル工藤ヨロシクabc"
      EXPECT_EQ("\xe3\x82\xb0\xe3\x83\xbc\xe3\x82\xb0\xe3\x83\xab\xe5\xb7\xa5\xe8"
                "\x97\xa4\xe3\x83\xa8\xe3\x83\xad\xe3\x82\xb7\xe3\x82\xaf\x61\x62"
                "\x63", output);
    }
  }
  
  TEST(UtilTest, KatakanaToHiragana) {
    {
      // "アイウエオァィゥェォカキクケコガギグゲゴサシスセソザジズゼゾタチツテト"
      // "ダヂヅデドッナニヌネノハヒフヘホバビブベボパピプペポマミムメモヤユヨャ"
      // "ュョラリルレロワヮヲンヰヱヴ"
      const string input =
          "\xE3\x82\xA2\xE3\x82\xA4\xE3\x82\xA6\xE3\x82\xA8\xE3\x82\xAA"
          "\xE3\x82\xA1\xE3\x82\xA3\xE3\x82\xA5\xE3\x82\xA7\xE3\x82\xA9"
          "\xE3\x82\xAB\xE3\x82\xAD\xE3\x82\xAF\xE3\x82\xB1\xE3\x82\xB3"
          "\xE3\x82\xAC\xE3\x82\xAE\xE3\x82\xB0\xE3\x82\xB2\xE3\x82\xB4"
          "\xE3\x82\xB5\xE3\x82\xB7\xE3\x82\xB9\xE3\x82\xBB\xE3\x82\xBD"
          "\xE3\x82\xB6\xE3\x82\xB8\xE3\x82\xBA\xE3\x82\xBC\xE3\x82\xBE"
          "\xE3\x82\xBF\xE3\x83\x81\xE3\x83\x84\xE3\x83\x86\xE3\x83\x88"
          "\xE3\x83\x80\xE3\x83\x82\xE3\x83\x85\xE3\x83\x87\xE3\x83\x89"
          "\xE3\x83\x83\xE3\x83\x8A\xE3\x83\x8B\xE3\x83\x8C\xE3\x83\x8D"
          "\xE3\x83\x8E\xE3\x83\x8F\xE3\x83\x92\xE3\x83\x95\xE3\x83\x98"
          "\xE3\x83\x9B\xE3\x83\x90\xE3\x83\x93\xE3\x83\x96\xE3\x83\x99"
          "\xE3\x83\x9C\xE3\x83\x91\xE3\x83\x94\xE3\x83\x97\xE3\x83\x9A"
          "\xE3\x83\x9D\xE3\x83\x9E\xE3\x83\x9F\xE3\x83\xA0\xE3\x83\xA1"
          "\xE3\x83\xA2\xE3\x83\xA4\xE3\x83\xA6\xE3\x83\xA8\xE3\x83\xA3"
          "\xE3\x83\xA5\xE3\x83\xA7\xE3\x83\xA9\xE3\x83\xAA\xE3\x83\xAB"
          "\xE3\x83\xAC\xE3\x83\xAD\xE3\x83\xAF\xE3\x83\xAE\xE3\x83\xB2"
          "\xE3\x83\xB3\xE3\x83\xB0\xE3\x83\xB1\xE3\x83\xB4";
      string output;
      Util::KatakanaToHiragana(input, &output);
      // "あいうえおぁぃぅぇぉかきくけこがぎぐげごさしすせそざじずぜぞたちつてと"
      // "だぢづでどっなにぬねのはひふへほばびぶべぼぱぴぷぺぽまみむめもやゆよゃ"
      // "ゅょらりるれろわゎをんゐゑゔ"
      EXPECT_EQ("\xE3\x81\x82\xE3\x81\x84\xE3\x81\x86\xE3\x81\x88\xE3\x81\x8A"
                "\xE3\x81\x81\xE3\x81\x83\xE3\x81\x85\xE3\x81\x87\xE3\x81\x89"
                "\xE3\x81\x8B\xE3\x81\x8D\xE3\x81\x8F\xE3\x81\x91\xE3\x81\x93"
                "\xE3\x81\x8C\xE3\x81\x8E\xE3\x81\x90\xE3\x81\x92\xE3\x81\x94"
                "\xE3\x81\x95\xE3\x81\x97\xE3\x81\x99\xE3\x81\x9B\xE3\x81\x9D"
                "\xE3\x81\x96\xE3\x81\x98\xE3\x81\x9A\xE3\x81\x9C\xE3\x81\x9E"
                "\xE3\x81\x9F\xE3\x81\xA1\xE3\x81\xA4\xE3\x81\xA6\xE3\x81\xA8"
                "\xE3\x81\xA0\xE3\x81\xA2\xE3\x81\xA5\xE3\x81\xA7\xE3\x81\xA9"
                "\xE3\x81\xA3\xE3\x81\xAA\xE3\x81\xAB\xE3\x81\xAC\xE3\x81\xAD"
                "\xE3\x81\xAE\xE3\x81\xAF\xE3\x81\xB2\xE3\x81\xB5\xE3\x81\xB8"
                "\xE3\x81\xBB\xE3\x81\xB0\xE3\x81\xB3\xE3\x81\xB6\xE3\x81\xB9"
                "\xE3\x81\xBC\xE3\x81\xB1\xE3\x81\xB4\xE3\x81\xB7\xE3\x81\xBA"
                "\xE3\x81\xBD\xE3\x81\xBE\xE3\x81\xBF\xE3\x82\x80\xE3\x82\x81"
                "\xE3\x82\x82\xE3\x82\x84\xE3\x82\x86\xE3\x82\x88\xE3\x82\x83"
                "\xE3\x82\x85\xE3\x82\x87\xE3\x82\x89\xE3\x82\x8A\xE3\x82\x8B"
                "\xE3\x82\x8C\xE3\x82\x8D\xE3\x82\x8F\xE3\x82\x8E\xE3\x82\x92"
                "\xE3\x82\x93\xE3\x82\x90\xE3\x82\x91\xE3\x82\x94",
                output);
    }
    {
      // "ワタシノナマエハナカノデスウマーヨロシュウ"
      const string input =
          "\xE3\x82\x8F\xE3\x81\x9F\xE3\x81\x97\xE3\x81\xAE\xE3\x81\xAA"
          "\xE3\x81\xBE\xE3\x81\x88\xE3\x81\xAF\xE3\x81\xAA\xE3\x81\x8B"
          "\xE3\x81\xAE\xE3\x81\xA7\xE3\x81\x99\xE3\x81\x86\xE3\x81\xBE"
          "\xE3\x83\xBC\xE3\x82\x88\xE3\x82\x8D\xE3\x81\x97\xE3\x82\x85"
          "\xE3\x81\x86";
      string output;
      Util::KatakanaToHiragana(input, &output);
      // "わたしのなまえはなかのですうまーよろしゅう"
      EXPECT_EQ("\xE3\x82\x8F\xE3\x81\x9F\xE3\x81\x97\xE3\x81\xAE"
                "\xE3\x81\xAA\xE3\x81\xBE\xE3\x81\x88\xE3\x81\xAF"
                "\xE3\x81\xAA\xE3\x81\x8B\xE3\x81\xAE\xE3\x81\xA7"
                "\xE3\x81\x99\xE3\x81\x86\xE3\x81\xBE\xE3\x83\xBC"
                "\xE3\x82\x88\xE3\x82\x8D\xE3\x81\x97\xE3\x82\x85"
                "\xE3\x81\x86",
                output);
    }
    {
      // "グーグル工藤ヨロシクabc"
      const string input =
          "\xE3\x82\xB0\xE3\x83\xBC\xE3\x82\xB0\xE3\x83\xAB\xE5\xB7\xA5"
          "\xE8\x97\xA4\xE3\x83\xA8\xE3\x83\xAD\xE3\x82\xB7\xE3\x82\xAF"
          "\x61\x62\x63";
      string output;
      Util::KatakanaToHiragana(input, &output);
      // "ぐーぐる工藤よろしくabc"
      EXPECT_EQ("\xE3\x81\x90\xE3\x83\xBC\xE3\x81\x90\xE3\x82\x8B"
                "\xE5\xB7\xA5\xE8\x97\xA4\xE3\x82\x88\xE3\x82\x8D"
                "\xE3\x81\x97\xE3\x81\x8F\x61\x62\x63",
                output);
    }
  }
  
  TEST(UtilTest, RomanjiToHiragana) {
    struct {
      const char *input;
      const char *expected;
    } kTestCases[] = {
      {"watasinonamaehatakahashinoriyukidesu",
       // "わたしのなまえはたかはしのりゆきです"
       "\xE3\x82\x8F\xE3\x81\x9F\xE3\x81\x97\xE3\x81\xAE\xE3\x81\xAA"
       "\xE3\x81\xBE\xE3\x81\x88\xE3\x81\xAF\xE3\x81\x9F\xE3\x81\x8B"
       "\xE3\x81\xAF\xE3\x81\x97\xE3\x81\xAE\xE3\x82\x8A\xE3\x82\x86"
       "\xE3\x81\x8D\xE3\x81\xA7\xE3\x81\x99"},
      {"majissukamajiyabexe",
       // "まじっすかまじやべぇ"
       "\xE3\x81\xBE\xE3\x81\x98\xE3\x81\xA3\xE3\x81\x99\xE3\x81\x8B"
       "\xE3\x81\xBE\xE3\x81\x98\xE3\x82\x84\xE3\x81\xB9\xE3\x81\x87"},
      {"kk",
       // "っk"
       "\xE3\x81\xA3\x6B"},
      {"xyz", "xyz"},
    };
    for (size_t i = 0; i < arraysize(kTestCases); ++i) {
      string actual;
      Util::RomanjiToHiragana(kTestCases[i].input, &actual);
      EXPECT_EQ(kTestCases[i].expected, actual);
    }
  }
  
  TEST(UtilTest, NormalizeVoicedSoundMark) {
    // "僕のう゛ぁいおりん"
    const string input = "\xe5\x83\x95\xe3\x81\xae\xe3\x81\x86\xe3\x82\x9b\xe3"
                         "\x81\x81\xe3\x81\x84\xe3\x81\x8a\xe3\x82\x8a\xe3\x82"
                         "\x93";
    string output;
    Util::NormalizeVoicedSoundMark(input, &output);
    // "僕のゔぁいおりん"
    EXPECT_EQ("\xe5\x83\x95\xe3\x81\xae\xe3\x82\x94\xe3\x81\x81\xe3\x81\x84\xe3"
              "\x81\x8a\xe3\x82\x8a\xe3\x82\x93", output);
  }
  
  TEST(UtilTest, IsFullWidthSymbolInHalfWidthKatakana) {
    // "グーグル"
    EXPECT_FALSE(Util::IsFullWidthSymbolInHalfWidthKatakana("\xe3\x82\xb0\xe3\x83"
                                                            "\xbc\xe3\x82\xb0\xe3"
                                                            "\x83\xab"));
    // "ー"
    EXPECT_TRUE(Util::IsFullWidthSymbolInHalfWidthKatakana("\xe3\x83\xbc"));
    // "。"
    EXPECT_TRUE(Util::IsFullWidthSymbolInHalfWidthKatakana("\xe3\x80\x82"));
    // "グーグル。"
    EXPECT_FALSE(Util::IsFullWidthSymbolInHalfWidthKatakana("\xe3\x82\xb0\xe3\x83"
                                                            "\xbc\xe3\x82\xb0\xe3"
                                                            "\x83\xab\xe3\x80"
                                                            "\x82"));
    // "ー。"
    EXPECT_TRUE(Util::IsFullWidthSymbolInHalfWidthKatakana(
        "\xe3\x83\xbc\xe3\x80\x82"));
    // "ーグ。"
    EXPECT_FALSE(Util::IsFullWidthSymbolInHalfWidthKatakana(
        "\xe3\x83\xbc\xe3\x82\xb0\xe3\x80\x82"));
  }
  
  TEST(UtilTest, IsHalfWidthKatakanaSymbol) {
    // "ｸﾞｰｸﾞﾙ"
    EXPECT_FALSE(Util::IsHalfWidthKatakanaSymbol("\xef\xbd\xb8\xef\xbe\x9e\xef"
                                                 "\xbd\xb0\xef\xbd\xb8\xef\xbe"
                                                 "\x9e\xef\xbe\x99"));
    // "ｰ"
    EXPECT_TRUE(Util::IsHalfWidthKatakanaSymbol("\xef\xbd\xb0"));
    // "｡"
    EXPECT_TRUE(Util::IsHalfWidthKatakanaSymbol("\xef\xbd\xa1"));
    // "､"
    EXPECT_TRUE(Util::IsHalfWidthKatakanaSymbol("\xef\xbd\xa4"));
    // "グーグル｡"
    EXPECT_FALSE(Util::IsHalfWidthKatakanaSymbol("\xe3\x82\xb0\xe3\x83\xbc\xe3"
                                                 "\x82\xb0\xe3\x83\xab\xef\xbd"
                                                 "\xa1"));
    // "､｡"
    // "not 。、"
    EXPECT_TRUE(Util::IsHalfWidthKatakanaSymbol("\xef\xbd\xa4\xef\xbd\xa1"));
  }
  
  TEST(UtilTest, FullWidthAndHalfWidth) {
    string output;
  
    Util::FullWidthToHalfWidth("", &output);
    CHECK_EQ("", output);
  
    Util::HalfWidthToFullWidth("", &output);
    CHECK_EQ("", output);
  
    Util::HalfWidthToFullWidth("abc[]?.", &output);
    // "ａｂｃ［］？．"
    CHECK_EQ("\xef\xbd\x81\xef\xbd\x82\xef\xbd\x83\xef\xbc\xbb\xef\xbc\xbd\xef"
             "\xbc\x9f\xef\xbc\x8e", output);
  
    // "ｲﾝﾀｰﾈｯﾄ｢」"
    Util::HalfWidthToFullWidth("\xef\xbd\xb2\xef\xbe\x9d\xef\xbe\x80\xef\xbd\xb0"
                               "\xef\xbe\x88\xef\xbd\xaf\xef\xbe\x84\xef\xbd\xa2"
                               "\xe3\x80\x8d", &output);
    // "インターネット「」"
    CHECK_EQ("\xe3\x82\xa4\xe3\x83\xb3\xe3\x82\xbf\xe3\x83\xbc\xe3\x83\x8d\xe3"
             "\x83\x83\xe3\x83\x88\xe3\x80\x8c\xe3\x80\x8d", output);
  
    // "ｲﾝﾀｰﾈｯﾄグーグル"
    Util::HalfWidthToFullWidth("\xef\xbd\xb2\xef\xbe\x9d\xef\xbe\x80\xef\xbd\xb0"
                               "\xef\xbe\x88\xef\xbd\xaf\xef\xbe\x84\xe3\x82\xb0"
                               "\xe3\x83\xbc\xe3\x82\xb0\xe3\x83\xab", &output);
    // "インターネットグーグル"
    CHECK_EQ("\xe3\x82\xa4\xe3\x83\xb3\xe3\x82\xbf\xe3\x83\xbc\xe3\x83\x8d\xe3"
             "\x83\x83\xe3\x83\x88\xe3\x82\xb0\xe3\x83\xbc\xe3\x82\xb0\xe3\x83"
             "\xab", output);
  
    // "ａｂｃ［］？．"
    Util::FullWidthToHalfWidth("\xef\xbd\x81\xef\xbd\x82\xef\xbd\x83\xef\xbc\xbb"
                               "\xef\xbc\xbd\xef\xbc\x9f\xef\xbc\x8e", &output);
    CHECK_EQ("abc[]?.", output);
  
    // "インターネット"
    Util::FullWidthToHalfWidth("\xe3\x82\xa4\xe3\x83\xb3\xe3\x82\xbf\xe3\x83\xbc"
                               "\xe3\x83\x8d\xe3\x83\x83\xe3\x83\x88", &output);
    // "ｲﾝﾀｰﾈｯﾄ"
    CHECK_EQ("\xef\xbd\xb2\xef\xbe\x9d\xef\xbe\x80\xef\xbd\xb0\xef\xbe\x88\xef"
             "\xbd\xaf\xef\xbe\x84", output);
  
    // "ｲﾝﾀｰﾈｯﾄグーグル"
    Util::FullWidthToHalfWidth("\xef\xbd\xb2\xef\xbe\x9d\xef\xbe\x80\xef\xbd\xb0"
                               "\xef\xbe\x88\xef\xbd\xaf\xef\xbe\x84\xe3\x82\xb0"
                               "\xe3\x83\xbc\xe3\x82\xb0\xe3\x83\xab", &output);
    // "ｲﾝﾀｰﾈｯﾄｸﾞｰｸﾞﾙ"
    CHECK_EQ("\xef\xbd\xb2\xef\xbe\x9d\xef\xbe\x80\xef\xbd\xb0\xef\xbe\x88\xef"
             "\xbd\xaf\xef\xbe\x84\xef\xbd\xb8\xef\xbe\x9e\xef\xbd\xb0\xef\xbd"
             "\xb8\xef\xbe\x9e\xef\xbe\x99", output);
  
    // spaces
    // " 　"
    Util::FullWidthToHalfWidth("\x20\xe3\x80\x80", &output);
    CHECK_EQ("  ", output);
  
    // " 　"
    Util::HalfWidthToFullWidth("\x20\xe3\x80\x80", &output);
    // "　　"
    CHECK_EQ("\xe3\x80\x80\xe3\x80\x80", output);
  
    // spaces are treated as Ascii here
    // " 　"
    Util::FullWidthAsciiToHalfWidthAscii("\x20\xe3\x80\x80", &output);
    CHECK_EQ("  ", output);
  
    // " 　"
    Util::HalfWidthAsciiToFullWidthAscii("\x20\xe3\x80\x80", &output);
    // "　　"
    CHECK_EQ("\xe3\x80\x80\xe3\x80\x80", output);
  
    // " 　"
    Util::FullWidthKatakanaToHalfWidthKatakana("\x20\xe3\x80\x80", &output);
    // " 　"
    CHECK_EQ("\x20\xe3\x80\x80", output);
  
    // " 　"
    Util::HalfWidthKatakanaToFullWidthKatakana("\x20\xe3\x80\x80", &output);
    // " 　"
    CHECK_EQ("\x20\xe3\x80\x80", output);
  }
  
  TEST(UtilTest, BracketTest) {
    static const struct BracketType {
      const char *open_bracket;
      const char *close_bracket;
    } kBracketType[] = {
      //  { "（", "）" },
      //  { "〔", "〕" },
      //  { "［", "］" },
      //  { "｛", "｝" },
      //  { "〈", "〉" },
      //  { "《", "》" },
      //  { "「", "」" },
      //  { "『", "』" },
      //  { "【", "】" },
      //  { "〘", "〙" },
      //  { "〚", "〛" },
      { "\xEF\xBC\x88", "\xEF\xBC\x89" },
      { "\xE3\x80\x94", "\xE3\x80\x95" },
      { "\xEF\xBC\xBB", "\xEF\xBC\xBD" },
      { "\xEF\xBD\x9B", "\xEF\xBD\x9D" },
      { "\xE3\x80\x88", "\xE3\x80\x89" },
      { "\xE3\x80\x8A", "\xE3\x80\x8B" },
      { "\xE3\x80\x8C", "\xE3\x80\x8D" },
      { "\xE3\x80\x8E", "\xE3\x80\x8F" },
      { "\xE3\x80\x90", "\xE3\x80\x91" },
      { "\xe3\x80\x98", "\xe3\x80\x99" },
      { "\xe3\x80\x9a", "\xe3\x80\x9b" },
      { NULL, NULL },  // sentinel
    };
  
    string pair;
    for (size_t i = 0;
         (kBracketType[i].open_bracket != NULL ||
          kBracketType[i].close_bracket != NULL);
         ++i) {
      EXPECT_TRUE(Util::IsOpenBracket(kBracketType[i].open_bracket, &pair));
      EXPECT_EQ(kBracketType[i].close_bracket, pair);
      EXPECT_TRUE(Util::IsCloseBracket(kBracketType[i].close_bracket, &pair));
      EXPECT_EQ(kBracketType[i].open_bracket, pair);
      EXPECT_FALSE(Util::IsOpenBracket(kBracketType[i].close_bracket, &pair));
      EXPECT_FALSE(Util::IsCloseBracket(kBracketType[i].open_bracket, &pair));
    }
  }
  
  TEST(UtilTest, IsEnglishTransliteration) {
    EXPECT_TRUE(Util::IsEnglishTransliteration("ABC"));
    EXPECT_TRUE(Util::IsEnglishTransliteration("Google"));
    EXPECT_TRUE(Util::IsEnglishTransliteration("Google Map"));
    EXPECT_TRUE(Util::IsEnglishTransliteration("ABC-DEF"));
    EXPECT_TRUE(Util::IsEnglishTransliteration("Foo-bar"));
    EXPECT_TRUE(Util::IsEnglishTransliteration("Foo!"));
    EXPECT_TRUE(Util::IsEnglishTransliteration("Who's"));
    EXPECT_TRUE(Util::IsEnglishTransliteration("!"));
    EXPECT_TRUE(Util::IsEnglishTransliteration("  "));
    //  EXPECT_FALSE(Util::IsEnglishTransliteration("てすと"));
    //  EXPECT_FALSE(Util::IsEnglishTransliteration("テスト"));
    //  EXPECT_FALSE(Util::IsEnglishTransliteration("東京"));
    EXPECT_FALSE(Util::IsEnglishTransliteration(
        "\xE3\x81\xA6\xE3\x81\x99\xE3\x81\xA8"));
    EXPECT_FALSE(Util::IsEnglishTransliteration(
        "\xE3\x83\x86\xE3\x82\xB9\xE3\x83\x88"));
    EXPECT_FALSE(Util::IsEnglishTransliteration(
        "\xE6\x9D\xB1\xE4\xBA\xAC"));
  }
  
  TEST(UtilTest, ChopReturns) {
    string line = "line\n";
    EXPECT_TRUE(Util::ChopReturns(&line));
    EXPECT_EQ("line", line);
  
    line = "line\r";
    EXPECT_TRUE(Util::ChopReturns(&line));
    EXPECT_EQ("line", line);
  
    line = "line\r\n";
    EXPECT_TRUE(Util::ChopReturns(&line));
    EXPECT_EQ("line", line);
  
    line = "line";
    EXPECT_FALSE(Util::ChopReturns(&line));
    EXPECT_EQ("line", line);
  
    line = "line1\nline2\n";
    EXPECT_TRUE(Util::ChopReturns(&line));
    EXPECT_EQ("line1\nline2", line);
  
    line = "line\n\n\n";
    EXPECT_TRUE(Util::ChopReturns(&line));
    EXPECT_EQ("line", line);
  }
  
  TEST(UtilTest, EncodeURI) {
    string encoded;
    // "もずく"
    Util::EncodeURI("\xe3\x82\x82\xe3\x81\x9a\xe3\x81\x8f", &encoded);
    EXPECT_EQ("%E3%82%82%E3%81%9A%E3%81%8F", encoded);
  
    encoded.clear();
    Util::EncodeURI("mozc", &encoded);
    EXPECT_EQ("mozc", encoded);
  
    encoded.clear();
    Util::EncodeURI("http://mozc/?q=Hello World", &encoded);
    EXPECT_EQ("http%3A%2F%2Fmozc%2F%3Fq%3DHello%20World", encoded);
  }
  
  TEST(UtilTest, DecodeURI) {
    string decoded;
    Util::DecodeURI("%E3%82%82%E3%81%9A%E3%81%8F", &decoded);
    // "もずく"
    EXPECT_EQ("\xe3\x82\x82\xe3\x81\x9a\xe3\x81\x8f", decoded);
  
    decoded.clear();
    Util::DecodeURI("mozc", &decoded);
    EXPECT_EQ("mozc", decoded);
  
    decoded.clear();
    Util::DecodeURI("http%3A%2F%2Fmozc%2F%3Fq%3DHello+World", &decoded);
    EXPECT_EQ("http://mozc/?q=Hello World", decoded);
  }
  
  TEST(UtilTest, AppendCGIParams) {
    vector<pair<string, string> > params;
    string url;
    Util::AppendCGIParams(params, &url);
    EXPECT_TRUE(url.empty());
  
    params.push_back(std::make_pair("foo", "b a+r"));
    url = "http://mozc.com?";
    Util::AppendCGIParams(params, &url);
    EXPECT_EQ("http://mozc.com?foo=b%20a%2Br", url);
  
    params.push_back(std::make_pair("buzz", "mozc"));
    url.clear();
    Util::AppendCGIParams(params, &url);
    EXPECT_EQ("foo=b%20a%2Br&buzz=mozc", url);
  }
  
  TEST(UtilTest, Escape) {
    string escaped;
    // "らむだ"
    Util::Escape("\xe3\x82\x89\xe3\x82\x80\xe3\x81\xa0", &escaped);
    EXPECT_EQ("\\xE3\\x82\\x89\\xE3\\x82\\x80\\xE3\\x81\\xA0", escaped);
  }
  
  TEST(UtilTest, EscapeUrl) {
    string escaped;
    // "らむだ"
    Util::EscapeUrl("\xe3\x82\x89\xe3\x82\x80\xe3\x81\xa0", &escaped);
    EXPECT_EQ("%E3%82%89%E3%82%80%E3%81%A0", escaped);
    EXPECT_EQ("%E3%82%89%E3%82%80%E3%81%A0",
              Util::EscapeUrl("\xe3\x82\x89\xe3\x82\x80\xe3\x81\xa0"));
  }
  
  TEST(UtilTest, EscapeHtml) {
    string escaped;
    Util::EscapeHtml("<>&'\"abc", &escaped);
    EXPECT_EQ("&lt;&gt;&amp;&#39;&quot;abc", escaped);
  }
  
  TEST(UtilTest, EscapeCss) {
    string escaped;
    Util::EscapeCss("<>&'\"abc", &escaped);
    EXPECT_EQ("&lt;>&'\"abc", escaped);
  }
  
  TEST(UtilTest, ScriptType) {
    // "くどう"
    EXPECT_TRUE(Util::IsScriptType("\xe3\x81\x8f\xe3\x81\xa9\xe3\x81\x86",
                                   Util::HIRAGANA));
    // "京都"
    EXPECT_TRUE(Util::IsScriptType("\xe4\xba\xac\xe9\x83\xbd", Util::KANJI));
    // "人々" (b/4201140)
    EXPECT_TRUE(Util::IsScriptType("\xE4\xBA\xBA\xE3\x80\x85", Util::KANJI));
    // "モズク"
    EXPECT_TRUE(Util::IsScriptType("\xe3\x83\xa2\xe3\x82\xba\xe3\x82\xaf",
                                   Util::KATAKANA));
    // "モズクﾓｽﾞｸ"
    EXPECT_TRUE(Util::IsScriptType("\xe3\x83\xa2\xe3\x82\xba\xe3\x82\xaf\xef\xbe"
                                   "\x93\xef\xbd\xbd\xef\xbe\x9e\xef\xbd\xb8",
                                   Util::KATAKANA));
    // "ぐーぐる"
    EXPECT_TRUE(Util::IsScriptType("\xe3\x81\x90\xe3\x83\xbc\xe3\x81\x90\xe3\x82"
                                   "\x8b", Util::HIRAGANA));
    // "グーグル"
    EXPECT_TRUE(Util::IsScriptType("\xe3\x82\xb0\xe3\x83\xbc\xe3\x82\xb0\xe3\x83"
                                   "\xab", Util::KATAKANA));
    // "ゟ" U+309F: HIRAGANA DIGRAPH YORI
    EXPECT_TRUE(Util::IsScriptType("\xE3\x82\x9F", Util::HIRAGANA));
    // "ヿ" U+30FF: KATAKANA DIGRAPH KOTO
    EXPECT_TRUE(Util::IsScriptType("\xE3\x83\xBF", Util::KATAKANA));
    // "ヷヸヹヺㇰㇱㇲㇳㇴㇵㇶㇷㇸㇹㇺㇻㇼㇽㇾㇿ"
    EXPECT_TRUE(Util::IsScriptType(
        "\xE3\x83\xB7\xE3\x83\xB8\xE3\x83\xB9\xE3\x83\xBA\xE3\x87\xB0"
        "\xE3\x87\xB1\xE3\x87\xB2\xE3\x87\xB3\xE3\x87\xB4\xE3\x87\xB5"
        "\xE3\x87\xB6\xE3\x87\xB7\xE3\x87\xB8\xE3\x87\xB9\xE3\x87\xBA"
        "\xE3\x87\xBB\xE3\x87\xBC\xE3\x87\xBD\xE3\x87\xBE\xE3\x87\xBF",
        Util::KATAKANA));
    // "𛀀" U+1B000: KATAKANA LETTER ARCHAIC E
    EXPECT_TRUE(Util::IsScriptType("\xF0\x9B\x80\x80", Util::KATAKANA));
    // "𛀁" U+1B001: HIRAGANA LETTER ARCHAIC YE
    EXPECT_TRUE(Util::IsScriptType("\xF0\x9B\x80\x81", Util::HIRAGANA));
  
    EXPECT_TRUE(Util::IsScriptType("012", Util::NUMBER));
    // "０１２012"
    EXPECT_TRUE(Util::IsScriptType("\xef\xbc\x90\xef\xbc\x91\xef\xbc\x92\x30\x31"
                                   "\x32", Util::NUMBER));
    EXPECT_TRUE(Util::IsScriptType("abcABC", Util::ALPHABET));
    // "ＡＢＣＤ"
    EXPECT_TRUE(Util::IsScriptType("\xef\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3\xef\xbc"
                                   "\xa4", Util::ALPHABET));
    EXPECT_TRUE(Util::IsScriptType("@!#", Util::UNKNOWN_SCRIPT));
  
    // "くどカう"
    EXPECT_FALSE(Util::IsScriptType("\xe3\x81\x8f\xe3\x81\xa9\xe3\x82\xab\xe3\x81"
                                    "\x86", Util::HIRAGANA));
    // "京あ都"
    EXPECT_FALSE(Util::IsScriptType("\xe4\xba\xac\xe3\x81\x82\xe9\x83\xbd",
                                    Util::KANJI));
    // "モズあク"
    EXPECT_FALSE(Util::IsScriptType("\xe3\x83\xa2\xe3\x82\xba\xe3\x81\x82\xe3\x82"
                                    "\xaf", Util::KATAKANA));
    // "モあズクﾓｽﾞｸ"
    EXPECT_FALSE(Util::IsScriptType("\xe3\x83\xa2\xe3\x81\x82\xe3\x82\xba\xe3\x82"
                                    "\xaf\xef\xbe\x93\xef\xbd\xbd\xef\xbe\x9e\xef"
                                    "\xbd\xb8", Util::KATAKANA));
    // "012あ"
    EXPECT_FALSE(Util::IsScriptType("\x30\x31\x32\xe3\x81\x82", Util::NUMBER));
    // "０１２あ012"
    EXPECT_FALSE(Util::IsScriptType("\xef\xbc\x90\xef\xbc\x91\xef\xbc\x92\xe3\x81"
                                    "\x82\x30\x31\x32", Util::NUMBER));
    // "abcABあC"
    EXPECT_FALSE(Util::IsScriptType("\x61\x62\x63\x41\x42\xe3\x81\x82\x43",
                                    Util::ALPHABET));
    // "ＡＢあＣＤ"
    EXPECT_FALSE(Util::IsScriptType("\xef\xbc\xa1\xef\xbc\xa2\xe3\x81\x82\xef\xbc"
                                    "\xa3\xef\xbc\xa4", Util::ALPHABET));
    // "ぐーぐるグ"
    EXPECT_FALSE(Util::IsScriptType("\xe3\x81\x90\xe3\x83\xbc\xe3\x81\x90\xe3\x82"
                                    "\x8b\xe3\x82\xb0", Util::HIRAGANA));
    // "グーグルぐ"
    EXPECT_FALSE(Util::IsScriptType("\xe3\x82\xb0\xe3\x83\xbc\xe3\x82\xb0\xe3\x83"
                                    "\xab\xe3\x81\x90", Util::KATAKANA));
  
    // "グーグルsuggest"
    EXPECT_TRUE(Util::ContainsScriptType("\xe3\x82\xb0\xe3\x83\xbc\xe3\x82\xb0"
                                         "\xe3\x83\xab\x73\x75\x67\x67\x65\x73"
                                         "\x74", Util::ALPHABET));
    // "グーグルサジェスト"
    EXPECT_FALSE(Util::ContainsScriptType("\xe3\x82\xb0\xe3\x83\xbc\xe3\x82\xb0"
                                          "\xe3\x83\xab\xe3\x82\xb5\xe3\x82\xb8"
                                          "\xe3\x82\xa7\xe3\x82\xb9\xe3\x83\x88",
                                          Util::ALPHABET));
  
    // "くどう"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptType("\xe3\x81\x8f\xe3\x81\xa9\xe3"
                                                  "\x81\x86"));
    // "京都"
    EXPECT_EQ(Util::KANJI, Util::GetScriptType("\xe4\xba\xac\xe9\x83\xbd"));
    // "人々" (b/4201140)
    EXPECT_EQ(Util::KANJI, Util::GetScriptType("\xE4\xBA\xBA\xE3\x80\x85"));
    // "モズク"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xe3\x83\xa2\xe3\x82\xba\xe3"
                                                  "\x82\xaf"));
    // "モズクﾓｽﾞｸ"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xe3\x83\xa2\xe3\x82\xba\xe3"
                                                  "\x82\xaf\xef\xbe\x93\xef\xbd"
                                                  "\xbd\xef\xbe\x9e\xef\xbd"
                                                  "\xb8"));
    // "ぐーぐる"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptType("\xe3\x81\x90\xe3\x83\xbc\xe3"
                                                  "\x81\x90\xe3\x82\x8b"));
    EXPECT_EQ(Util::HIRAGANA,
              Util::GetFirstScriptType("\xe3\x81\x90\xe3\x83\xbc\xe3\x81\x90"
                                       "\xe3\x82\x8b"));
  
    // "グーグル"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xe3\x82\xb0\xe3\x83\xbc\xe3"
                                                  "\x82\xb0\xe3\x83\xab"));
    EXPECT_EQ(Util::KATAKANA,
              Util::GetFirstScriptType("\xe3\x82\xb0\xe3\x83\xbc\xe3\x82\xb0"
                                       "\xe3\x83\xab"));
    // "ゟ" U+309F HIRAGANA DIGRAPH YORI
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptType("\xE3\x82\x9F"));
    EXPECT_EQ(Util::HIRAGANA, Util::GetFirstScriptType("\xE3\x82\x9F"));
  
    // "ヿ" U+30FF KATAKANA DIGRAPH KOTO
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xE3\x83\xBF"));
    // "ヷヸヹヺㇰㇱㇲㇳㇴㇵㇶㇷㇸㇹㇺㇻㇼㇽㇾㇿ"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType(
        "\xE3\x83\xB7\xE3\x83\xB8\xE3\x83\xB9\xE3\x83\xBA\xE3\x87\xB0"
        "\xE3\x87\xB1\xE3\x87\xB2\xE3\x87\xB3\xE3\x87\xB4\xE3\x87\xB5"
        "\xE3\x87\xB6\xE3\x87\xB7\xE3\x87\xB8\xE3\x87\xB9\xE3\x87\xBA"
        "\xE3\x87\xBB\xE3\x87\xBC\xE3\x87\xBD\xE3\x87\xBE\xE3\x87\xBF"));
    // "𛀀" U+1B000 KATAKANA LETTER ARCHAIC E
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xF0\x9B\x80\x80"));
    // "𛀁" U+1B001 HIRAGANA LETTER ARCHAIC YE
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptType("\xF0\x9B\x80\x81"));
  
    // "!グーグル"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\x21\xe3\x82\xb0\xe3\x83"
                                                        "\xbc\xe3\x82\xb0\xe3\x83"
                                                        "\xab"));
    // "ー"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe3\x83\xbc"));
    EXPECT_EQ(Util::KATAKANA, Util::GetFirstScriptType("\xe3\x83\xbc"));
    // "ーー"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe3\x83\xbc\xe3\x83"
                                                        "\xbc"));
    EXPECT_EQ(Util::KATAKANA, Util::GetFirstScriptType("\xe3\x83\xbc\xe3"
                                                             "\x83\xbc"));
    // "゛"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe3\x82\x9b"));
    // "゜"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe3\x82\x9c"));
  
    EXPECT_EQ(Util::NUMBER, Util::GetScriptType("012"));
    // "０１２012"
    EXPECT_EQ(Util::NUMBER, Util::GetScriptType("\xef\xbc\x90\xef\xbc\x91\xef\xbc"
                                                "\x92\x30\x31\x32"));
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptType("abcABC"));
    // "ＡＢＣＤ"
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptType("\xef\xbc\xa1\xef\xbc\xa2\xef"
                                                  "\xbc\xa3\xef\xbc\xa4"));
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("@!#"));
    // "＠！＃"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xef\xbc\xa0\xef\xbc\x81"
                                                        "\xef\xbc\x83"));
  
    // "ーひらがな"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptType("\xe3\x83\xbc\xe3\x81\xb2\xe3"
                                                  "\x82\x89\xe3\x81\x8c\xe3\x81"
                                                  "\xaa"));
    EXPECT_EQ(Util::KATAKANA, Util::GetFirstScriptType("\xe3\x83\xbc\xe3\x81\xb2"
                                                       "\xe3\x82\x89\xe3\x81\x8c"
                                                       "\xe3\x81\xaa"));
    // "ーカタカナ"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xe3\x83\xbc\xe3\x82\xab\xe3"
                                                  "\x82\xbf\xe3\x82\xab\xe3\x83"
                                                  "\x8a"));
    // "ｰｶﾀｶﾅ"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xef\xbd\xb0\xef\xbd\xb6\xef"
                                                  "\xbe\x80\xef\xbd\xb6\xef\xbe"
                                                  "\x85"));
    // "ひらがなー"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptType("\xe3\x81\xb2\xe3\x82\x89\xe3"
                                                  "\x81\x8c\xe3\x81\xaa\xe3\x83"
                                                  "\xbc"));
    // "カタカナー"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xe3\x82\xab\xe3\x82\xbf\xe3"
                                                  "\x82\xab\xe3\x83\x8a\xe3\x83"
                                                  "\xbc"));
    // "ｶﾀｶﾅｰ"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xef\xbd\xb6\xef\xbe\x80\xef"
                                                  "\xbd\xb6\xef\xbe\x85\xef\xbd"
                                                  "\xb0"));
  
    // "あ゛っ"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptType("\xe3\x81\x82\xe3\x82\x9b\xe3"
                                                  "\x81\xa3"));
    // "あ゜っ"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptType("\xe3\x81\x82\xe3\x82\x9c\xe3"
                                                  "\x81\xa3"));
    // "ア゛ッ"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xe3\x82\xa2\xe3\x82\x9b\xe3"
                                                  "\x83\x83"));
    // "ア゜ッ"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptType("\xe3\x82\xa2\xe3\x82\x9c\xe3"
                                                  "\x83\x83"));
  
    // "くどカう"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe3\x81\x8f\xe3\x81\xa9"
                                                        "\xe3\x82\xab\xe3\x81"
                                                        "\x86"));
    // "京あ都"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe4\xba\xac\xe3\x81\x82"
                                                        "\xe9\x83\xbd"));
    EXPECT_EQ(Util::KANJI, Util::GetFirstScriptType("\xe4\xba\xac\xe3\x81\x82"
                                                    "\xe9\x83\xbd"));
  
    // "モズあク"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe3\x83\xa2\xe3\x82\xba"
                                                        "\xe3\x81\x82\xe3\x82"
                                                        "\xaf"));
    EXPECT_EQ(Util::KATAKANA, Util::GetFirstScriptType("\xe3\x83\xa2\xe3\x82\xba"
                                                       "\xe3\x81\x82\xe3\x82"
                                                       "\xaf"));
  
    // "モあズクﾓｽﾞｸ"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe3\x83\xa2\xe3\x81\x82"
                                                        "\xe3\x82\xba\xe3\x82\xaf"
                                                        "\xef\xbe\x93\xef\xbd\xbd"
                                                        "\xef\xbe\x9e\xef\xbd"
                                                        "\xb8"));
    // "012あ"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\x30\x31\x32\xe3\x81"
                                                        "\x82"));
    EXPECT_EQ(Util::NUMBER, Util::GetFirstScriptType("\x30\x31\x32\xe3\x81"
                                                     "\x82"));
    // "０１２あ012"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xef\xbc\x90\xef\xbc\x91"
                                                        "\xef\xbc\x92\xe3\x81\x82"
                                                        "\x30\x31\x32"));
    EXPECT_EQ(Util::NUMBER, Util::GetFirstScriptType("\xef\xbc\x90\xef\xbc\x91"
                                                     "\xef\xbc\x92\xe3\x81\x82"
                                                     "\x30\x31\x32"));
    // "abcABあC"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\x61\x62\x63\x41\x42\xe3"
                                                        "\x81\x82\x43"));
    // "ＡＢあＣＤ"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xef\xbc\xa1\xef\xbc\xa2"
                                                        "\xe3\x81\x82\xef\xbc\xa3"
                                                        "\xef\xbc\xa4"));
    // "ぐーぐるグ"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe3\x81\x90\xe3\x83\xbc"
                                                        "\xe3\x81\x90\xe3\x82\x8b"
                                                        "\xe3\x82\xb0"));
    // "グーグルぐ"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptType("\xe3\x82\xb0\xe3\x83\xbc"
                                                        "\xe3\x82\xb0\xe3\x83\xab"
                                                        "\xe3\x81\x90"));
  
    // "龦" U+9FA6
    EXPECT_EQ(Util::KANJI, Util::GetScriptType("\xE9\xBE\xA6"));
    // "龻" U+9FBB
    EXPECT_EQ(Util::KANJI, Util::GetScriptType("\xE9\xBE\xBB"));
    // U+9FFF is not assigned yet but reserved for CJK Unified Ideographs.
    EXPECT_EQ(Util::KANJI, Util::GetScriptType("\xE9\xBF\xBF"));
    // "𠮟咤" U+20B9F U+54A4
    EXPECT_EQ(Util::KANJI, Util::GetScriptType("\xF0\xA0\xAE\x9F\xE5\x92\xA4"));
    // "𠮷野" U+20BB7 U+91CE
    EXPECT_EQ(Util::KANJI, Util::GetScriptType("\xF0\xA0\xAE\xB7\xE9\x87\x8E"));
    // "巽" U+2F884
    EXPECT_EQ(Util::KANJI, Util::GetScriptType("\xF0\xAF\xA2\x84"));
  
    // U+1F466, BOY/smile emoji
    EXPECT_EQ(Util::EMOJI, Util::GetScriptType("\xF0\x9F\x91\xA6"));
    // U+FE003, Snow-man Android PUA emoji
    EXPECT_TRUE(Util::IsAndroidPuaEmoji("\xf3\xbe\x80\x83"));
    EXPECT_EQ(Util::EMOJI, Util::GetScriptType("\xf3\xbe\x80\x83"));
  }
  
  
  TEST(UtilTest, ScriptTypeWithoutSymbols) {
    // "くど う"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptTypeWithoutSymbols(
        "\xe3\x81\x8f\xe3\x81\xa9 \xe3\x81\x86"));
    // "京 都"
    EXPECT_EQ(Util::KANJI, Util::GetScriptTypeWithoutSymbols(
        "\xe4\xba\xac \xe9\x83\xbd"));
    // "モズ ク"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptTypeWithoutSymbols(
        "\xe3\x83\xa2\xe3\x82\xba\xe3\x82\xaf"));
    // "モズ クﾓｽﾞｸ"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptTypeWithoutSymbols(
        "\xe3\x83\xa2\xe3\x82\xba \xe3\x82\xaf\xef\xbe\x93\xef\xbd"
        "\xbd\xef\xbe\x9e\xef\xbd\xb8"));
    // "Google Earth"
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols(
        "Google Earth"));
    // "Google "
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols(
        "Google "));
    // " Google"
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols(
        " Google"));
    // " Google "
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols(
        " Google "));
    // "     g"
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols(
        "     g"));
    // ""
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols(
        ""));
    // " "
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols(
        " "));
    // "  "
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols(
        "   "));
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols("Hello!"));
    // "Hello!あ"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols(
        "\x48\x65\x6c\x6c\x6f\x21\xe3\x81\x82"));
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols("CD-ROM"));
    // "CD-ROMア"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols(
        "\x43\x44\x2d\x52\x4f\x4d\xe3\x82\xa2"));
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols("-"));
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols("-A"));
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols("--A"));
    EXPECT_EQ(Util::ALPHABET, Util::GetScriptTypeWithoutSymbols("--A---"));
    // "--A-ｱ-"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols(
        "\x2d\x2d\x41\x2d\xef\xbd\xb1\x2d"));
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols("!"));
    // "・あ"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptTypeWithoutSymbols(
        "\xe3\x83\xbb\xe3\x81\x82"));
    // "・・あ"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptTypeWithoutSymbols(
        "\xe3\x83\xbb\xe3\x83\xbb\xe3\x81\x82"));
    // "コギト・エルゴ・スム"
    EXPECT_EQ(Util::KATAKANA, Util::GetScriptTypeWithoutSymbols(
        "\xe3\x82\xb3\xe3\x82\xae\xe3\x83\x88\xe3\x83\xbb\xe3\x82\xa8"
        "\xe3\x83\xab\xe3\x82\xb4\xe3\x83\xbb\xe3\x82\xb9\xe3\x83\xa0"));
    // "コギト・エルゴ・住む"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols(
        "\xe3\x82\xb3\xe3\x82\xae\xe3\x83\x88\xe3\x83\xbb\xe3\x82\xa8"
        "\xe3\x83\xab\xe3\x82\xb4\xe3\x83\xbb\xe4\xbd\x8f\xe3\x82\x80"));
    // "人☆名"
    EXPECT_EQ(Util::KANJI, Util::GetScriptTypeWithoutSymbols(
        "\xe4\xba\xba\xe2\x98\x86\xe5\x90\x8d"));
    // "ひとの☆なまえ"
    EXPECT_EQ(Util::HIRAGANA, Util::GetScriptTypeWithoutSymbols(
        "\xe3\x81\xb2\xe3\x81\xa8\xe3\x81\xae\xe2\x98\x86\xe3\x81\xaa"
        "\xe3\x81\xbe\xe3\x81\x88"));
    // "超☆最高です"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols(
        "\xe8\xb6\x85\xe2\x98\x86\xe6\x9c\x80\xe9\xab\x98\xe3\x81\xa7"
        "\xe3\x81\x99"));
    // "・--☆"
    EXPECT_EQ(Util::UNKNOWN_SCRIPT, Util::GetScriptTypeWithoutSymbols(
        "\xe3\x83\xbb\x2d\x2d\xe2\x98\x86"));
  }
  
  TEST(UtilTest, FormType) {
    // "くどう"
    EXPECT_EQ(Util::FULL_WIDTH, Util::GetFormType("\xe3\x81\x8f\xe3\x81\xa9\xe3"
                                                  "\x81\x86"));
    // "京都"
    EXPECT_EQ(Util::FULL_WIDTH, Util::GetFormType("\xe4\xba\xac\xe9\x83\xbd"));
    // "モズク"
    EXPECT_EQ(Util::FULL_WIDTH, Util::GetFormType("\xe3\x83\xa2\xe3\x82\xba\xe3"
                                                  "\x82\xaf"));
    // "ﾓｽﾞｸ"
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType("\xef\xbe\x93\xef\xbd\xbd\xef"
                                                  "\xbe\x9e\xef\xbd\xb8"));
    // "ぐーぐる"
    EXPECT_EQ(Util::FULL_WIDTH, Util::GetFormType("\xe3\x81\x90\xe3\x83\xbc\xe3"
                                                  "\x81\x90\xe3\x82\x8b"));
    // "グーグル"
    EXPECT_EQ(Util::FULL_WIDTH, Util::GetFormType("\xe3\x82\xb0\xe3\x83\xbc\xe3"
                                                  "\x82\xb0\xe3\x83\xab"));
    // "ｸﾞｰｸﾞﾙ"
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType("\xef\xbd\xb8\xef\xbe\x9e\xef"
                                                  "\xbd\xb0\xef\xbd\xb8\xef\xbe"
                                                  "\x9e\xef\xbe\x99"));
    // "ｰ"
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType("\xef\xbd\xb0"));
    // "ー"
    EXPECT_EQ(Util::FULL_WIDTH, Util::GetFormType("\xe3\x83\xbc"));
  
    // "¢£¥¦¬¯"
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType("\xc2\xa2\xc2\xa3\xc2\xa5"
                                                  "\xc2\xa6\xc2\xac\xc2\xaf"));
  
    // "￨￩￪￫￬￭￮"
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType(
        "\xef\xbf\xa8\xef\xbf\xa9\xef\xbf\xaa\xef\xbf\xab\xef\xbf\xac"
        "\xef\xbf\xad\xef\xbf\xae"));
  
    // Half-width mathematical symbols
    // [U+27E6, U+27ED], U+2985, and U+2986
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType(
        "\xe2\x9f\xa6\xe2\x9f\xa7\xe2\x9f\xa8\xe2\x9f\xa9\xe2\x9f\xaa\xe2"
        "\x9f\xab\xe2\x9f\xac\xe2\x9f\xad\xe2\xa6\x85\xe2\xa6\x86"));
  
    // Half-width hangul "ﾠﾡﾢ"
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType("\xef\xbe\xa0\xef\xbe\xa1"
                                                  "\xef\xbe\xa2"));
  
    // Half-width won "₩"
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType("\xe2\x82\xa9"));
  
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType("012"));
    // "０１２012"
    EXPECT_EQ(Util::UNKNOWN_FORM, Util::GetFormType("\xef\xbc\x90\xef\xbc\x91\xef"
                                                    "\xbc\x92\x30\x31\x32"));
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType("abcABC"));
    // "ＡＢＣＤ"
    EXPECT_EQ(Util::FULL_WIDTH, Util::GetFormType("\xef\xbc\xa1\xef\xbc\xa2\xef"
                                                  "\xbc\xa3\xef\xbc\xa4"));
    EXPECT_EQ(Util::HALF_WIDTH, Util::GetFormType("@!#"));
  }
  
  
  #ifndef OS_NACL
  // We have a snapshot of the result of |Util::GetCharacterSet(ucs4)| in
  // data/test/character_set/character_set.tsv.
  // Compare the result for each character just in case.
  //
  // Disabled on NaCl since it uses a mock file system.
  TEST(UtilTest, CharacterSetFullTest) {
    map<char32, Util::CharacterSet> test_set;
    FillTestCharacterSetMap(&test_set);
    EXPECT_FALSE(test_set.empty());
  
    // Unicode characters consist of [U+0000, U+10FFFF].
    for (char32 ucs4 = 0; ucs4 <= 0x10ffff; ++ucs4) {
      EXPECT_EQ(GetExpectedCharacterSet(test_set, ucs4),
                Util::GetCharacterSet(ucs4))
          << "Character set changed at " << ucs4;
    }
  }
  #endif  // OS_NACL
  
  TEST(UtilTest, CharacterSet_gen_character_set) {
    // [0x00, 0x7f] are ASCII
    for (size_t i = 0; i <= 0x7f; ++i) {
      EXPECT_EQ(Util::ASCII, Util::GetCharacterSet(i));
    }
    // [0x80, 0xff] are not ASCII
    for (size_t i = 0x80; i <= 0xff; ++i) {
      EXPECT_NE(Util::ASCII, Util::GetCharacterSet(i));
    }
  
    // 0213
    // "Ⅰ"
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet(0x2160));
    // "①"
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet(0x2460));
    // "㊤"
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet(0x32A4));
    // "ð ®" from UCS4 range (b/4176888)
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet(0x20B9F));
    // "ðª²" from UCS4 range (b/4176888)
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet(0x2A6B2));
  
    // only in CP932
    // "凬"
    EXPECT_EQ(Util::CP932, Util::GetCharacterSet(0x51EC));
  
    // only in Unicode
    // "￦"
    EXPECT_EQ(Util::UNICODE_ONLY, Util::GetCharacterSet(0xFFE6));
    // "ð ®·" from UCS4 range (b/4176888)
    EXPECT_EQ(Util::UNICODE_ONLY, Util::GetCharacterSet(0x20BB7));
  }
  
  TEST(UtilTest, CharacterSet) {
    // "あいうえお"
    EXPECT_EQ(Util::JISX0208, Util::GetCharacterSet("\xe3\x81\x82\xe3\x81\x84\xe3"
                                                    "\x81\x86\xe3\x81\x88\xe3\x81"
                                                    "\x8a"));
    EXPECT_EQ(Util::ASCII, Util::GetCharacterSet("abc"));
    // "abcあいう"
    EXPECT_EQ(Util::JISX0208, Util::GetCharacterSet("\x61\x62\x63\xe3\x81\x82\xe3"
                                                    "\x81\x84\xe3\x81\x86"));
  
    // half width katakana
    // "ｶﾀｶﾅ"
    EXPECT_EQ(Util::JISX0201, Util::GetCharacterSet("\xef\xbd\xb6\xef\xbe\x80\xef"
                                                    "\xbd\xb6\xef\xbe\x85"));
    // "ｶﾀｶﾅカタカナ"
    EXPECT_EQ(Util::JISX0208, Util::GetCharacterSet("\xef\xbd\xb6\xef\xbe\x80\xef"
                                                    "\xbd\xb6\xef\xbe\x85\xe3\x82"
                                                    "\xab\xe3\x82\xbf\xe3\x82\xab"
                                                    "\xe3\x83\x8a"));
  
    // 0213
    // "Ⅰ"
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet("\xe2\x85\xa0"));
    // "①"
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet("\xe2\x91\xa0"));
    // "㊤"
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet("\xe3\x8a\xa4"));
    // "ð ®" from UCS4 range (b/4176888)
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet("\xF0\xA0\xAE\x9F"));
    // "ðª²" from UCS4 range (b/4176888)
    EXPECT_EQ(Util::JISX0213, Util::GetCharacterSet("\xF0\xAA\x9A\xB2"));
  
    // only in CP932
    // "凬"
    EXPECT_EQ(Util::CP932, Util::GetCharacterSet("\xe5\x87\xac"));
  
    // only in Unicode
    // "￦"
    EXPECT_EQ(Util::UNICODE_ONLY, Util::GetCharacterSet("\xef\xbf\xa6"));
    // "ð ®·" from UCS4 range (b/4176888)
    EXPECT_EQ(Util::UNICODE_ONLY, Util::GetCharacterSet("\xF0\xA0\xAE\xB7"));
  }
  
  #ifdef OS_WIN
  TEST(UtilTest, WideCharsLen) {
    // "að ®b"
    const string input_utf8 = "a\360\240\256\237b";
    EXPECT_EQ(4, Util::WideCharsLen(input_utf8));
    EXPECT_EQ(0, Util::WideCharsLen(Util::SubString(input_utf8, 0, 0)));
    EXPECT_EQ(1, Util::WideCharsLen(Util::SubString(input_utf8, 0, 1)));
    EXPECT_EQ(3, Util::WideCharsLen(Util::SubString(input_utf8, 0, 2)));
    EXPECT_EQ(4, Util::WideCharsLen(Util::SubString(input_utf8, 0, 3)));
  }
  
  TEST(UtilTest, UTF8ToWide) {
    const string input_utf8 = "abc";
    wstring output_wide;
    Util::UTF8ToWide(input_utf8, &output_wide);
  
    string output_utf8;
    Util::WideToUTF8(output_wide, &output_utf8);
    EXPECT_EQ("abc", output_utf8);
  }
  
  TEST(UtilTest, WideToUTF8_SurrogatePairSupport) {
    // Visual C++ 2008 does not support embedding surrogate pair in string
    // literals like L"\uD842\uDF9F". This is why we use wchar_t array instead.
    // "ð ®"
    const wchar_t input_wide[] = {0xD842, 0xDF9F, 0};
    string output_utf8;
    Util::WideToUTF8(input_wide, &output_utf8);
  
    wstring output_wide;
    Util::UTF8ToWide(output_utf8, &output_wide);
  
    EXPECT_EQ("\360\240\256\237", output_utf8);
    EXPECT_EQ(input_wide, output_wide);
  }
  #endif  // OS_WIN
  
  TEST(UtilTest, IsKanaSymbolContained) {
    const string kFullstop("\xe3\x80\x82");  // "。"
    const string kSpace(" ");
    EXPECT_TRUE(Util::IsKanaSymbolContained(kFullstop));
    EXPECT_TRUE(Util::IsKanaSymbolContained(kSpace + kFullstop));
    EXPECT_TRUE(Util::IsKanaSymbolContained(kFullstop + kSpace));
    EXPECT_FALSE(Util::IsKanaSymbolContained(kSpace));
    EXPECT_FALSE(Util::IsKanaSymbolContained(""));
  }
  
  TEST(UtilTest, RandomSeedTest) {
    Util::SetRandomSeed(0);
    const int first_try = Util::Random(INT_MAX);
    const int second_try = Util::Random(INT_MAX);
    EXPECT_NE(first_try, second_try);
  
    // Reset the seed.
    Util::SetRandomSeed(0);
    EXPECT_EQ(first_try, Util::Random(INT_MAX));
  }
  
  TEST(UtilTest, SplitFirstChar32) {
    StringPiece rest;
    char32 c = 0;
  
    rest.clear();
    c = 0;
    EXPECT_FALSE(Util::SplitFirstChar32("", &c, &rest));
    EXPECT_EQ(0, c);
    EXPECT_TRUE(rest.empty());
  
    // Allow NULL to ignore the matched value.
    rest.clear();
    EXPECT_TRUE(Util::SplitFirstChar32("01", NULL, &rest));
    EXPECT_EQ("1", rest);
  
    // Allow NULL to ignore the matched value.
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("01", &c, NULL));
    EXPECT_EQ('0', c);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\x01 ", &c, &rest));
    EXPECT_EQ(1, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\x7F ", &c, &rest));
    EXPECT_EQ(0x7F, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xC2\x80 ", &c, &rest));
    EXPECT_EQ(0x80, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xDF\xBF ", &c, &rest));
    EXPECT_EQ(0x7FF, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xE0\xA0\x80 ", &c, &rest));
    EXPECT_EQ(0x800, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xEF\xBF\xBF ", &c, &rest));
    EXPECT_EQ(0xFFFF, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xF0\x90\x80\x80 ", &c, &rest));
    EXPECT_EQ(0x10000, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xF7\xBF\xBF\xBF ", &c, &rest));
    EXPECT_EQ(0x1FFFFF, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xF8\x88\x80\x80\x80 ", &c, &rest));
    EXPECT_EQ(0x200000, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xFB\xBF\xBF\xBF\xBF ", &c, &rest));
    EXPECT_EQ(0x3FFFFFF, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xFC\x84\x80\x80\x80\x80 ", &c, &rest));
    EXPECT_EQ(0x4000000, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitFirstChar32("\xFD\xBF\xBF\xBF\xBF\xBF ", &c, &rest));
    EXPECT_EQ(0x7FFFFFFF, c);
    EXPECT_EQ(" ", rest);
  
    // If there is any invalid sequence, the entire text should be treated as
    // am empty string.
    {
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xC2 ", &c, &rest));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xC2\xC2 ", &c, &rest));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xE0 ", &c, &rest));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xE0\xE0\xE0 ", &c, &rest));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xF0 ", &c, &rest));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xF0\xF0\xF0\xF0 ", &c, &rest));
      EXPECT_EQ(0, c);
    }
  
    // BOM should be treated as invalid byte.
    {
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xFF ", &c, &rest));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xFE ", &c, &rest));
      EXPECT_EQ(0, c);
    }
  
    // Invalid sequence for U+002F (redundant encoding)
    {
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xC0\xAF", &c, &rest));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xE0\x80\xAF", &c, &rest));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitFirstChar32("\xF0\x80\x80\xAF", &c, &rest));
      EXPECT_EQ(0, c);
    }
  }
  
  TEST(UtilTest, SplitLastChar32) {
    StringPiece rest;
    char32 c = 0;
  
    rest.clear();
    c = 0;
    EXPECT_FALSE(Util::SplitLastChar32("", &rest, &c));
    EXPECT_EQ(0, c);
    EXPECT_TRUE(rest.empty());
  
    // Allow NULL to ignore the matched value.
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32("01", NULL, &c));
    EXPECT_EQ('1', c);
  
    // Allow NULL to ignore the matched value.
    rest.clear();
    EXPECT_TRUE(Util::SplitLastChar32("01", &rest, NULL));
    EXPECT_EQ("0", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \x01", &rest, &c));
    EXPECT_EQ(1, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \x7F", &rest, &c));
    EXPECT_EQ(0x7F, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xC2\x80", &rest, &c));
    EXPECT_EQ(0x80, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xDF\xBF", &rest, &c));
    EXPECT_EQ(0x7FF, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xE0\xA0\x80", &rest, &c));
    EXPECT_EQ(0x800, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xEF\xBF\xBF", &rest, &c));
    EXPECT_EQ(0xFFFF, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xF0\x90\x80\x80", &rest, &c));
    EXPECT_EQ(0x10000, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xF7\xBF\xBF\xBF", &rest, &c));
    EXPECT_EQ(0x1FFFFF, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xF8\x88\x80\x80\x80", &rest, &c));
    EXPECT_EQ(0x200000, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xFB\xBF\xBF\xBF\xBF", &rest, &c));
    EXPECT_EQ(0x3FFFFFF, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xFC\x84\x80\x80\x80\x80", &rest, &c));
    EXPECT_EQ(0x4000000, c);
    EXPECT_EQ(" ", rest);
  
    rest.clear();
    c = 0;
    EXPECT_TRUE(Util::SplitLastChar32(" \xFD\xBF\xBF\xBF\xBF\xBF", &rest, &c));
    EXPECT_EQ(0x7FFFFFFF, c);
    EXPECT_EQ(" ", rest);
  
    // If there is any invalid sequence, the entire text should be treated as
    // am empty string.
    {
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32(" \xC2", &rest, &c));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32(" \xC2\xC2", &rest, &c));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32(" \xE0", &rest, &c));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32(" \xE0\xE0\xE0", &rest, &c));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32(" \xF0", &rest, &c));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32(" \xF0\xF0\xF0\xF0", &rest, &c));
      EXPECT_EQ(0, c);
    }
  
    // BOM should be treated as invalid byte.
    {
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32(" \xFF", &rest, &c));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32(" \xFE", &rest, &c));
      EXPECT_EQ(0, c);
    }
  
    // Invalid sequence for U+002F (redundant encoding)
    {
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32("\xC0\xAF", &rest, &c));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32("\xE0\x80\xAF", &rest, &c));
      EXPECT_EQ(0, c);
  
      c = 0;
      EXPECT_FALSE(Util::SplitLastChar32("\xF0\x80\x80\xAF", &rest, &c));
      EXPECT_EQ(0, c);
    }
  }
  
  }  // namespace mozc
  