text: // Tencent is pleased to support the open source community by making RapidJSON available.
  // 
  // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
  //
  // Licensed under the MIT License (the "License"); you may not use this file except
  // in compliance with the License. You may obtain a copy of the License at
  //
  // http://opensource.org/licenses/MIT
  //
  // Unless required by applicable law or agreed to in writing, software distributed 
  // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
  // CONDITIONS OF ANY KIND, either express or implied. See the License for the 
  // specific language governing permissions and limitations under the License.
  
  #include "unittest.h"
  #include "rapidjson/filereadstream.h"
  #include "rapidjson/filewritestream.h"
  #include "rapidjson/encodedstream.h"
  #include "rapidjson/stringbuffer.h"
  
  using namespace rapidjson;
  
  // Verification of encoders/decoders with Hoehrmann's UTF8 decoder
  
  // http://www.unicode.org/Public/UNIDATA/Blocks.txt
  static const unsigned kCodepointRanges[] = {
      0x0000,     0x007F,     // Basic Latin
      0x0080,     0x00FF,     // Latin-1 Supplement
      0x0100,     0x017F,     // Latin Extended-A
      0x0180,     0x024F,     // Latin Extended-B
      0x0250,     0x02AF,     // IPA Extensions
      0x02B0,     0x02FF,     // Spacing Modifier Letters
      0x0300,     0x036F,     // Combining Diacritical Marks
      0x0370,     0x03FF,     // Greek and Coptic
      0x0400,     0x04FF,     // Cyrillic
      0x0500,     0x052F,     // Cyrillic Supplement
      0x0530,     0x058F,     // Armenian
      0x0590,     0x05FF,     // Hebrew
      0x0600,     0x06FF,     // Arabic
      0x0700,     0x074F,     // Syriac
      0x0750,     0x077F,     // Arabic Supplement
      0x0780,     0x07BF,     // Thaana
      0x07C0,     0x07FF,     // NKo
      0x0800,     0x083F,     // Samaritan
      0x0840,     0x085F,     // Mandaic
      0x0900,     0x097F,     // Devanagari
      0x0980,     0x09FF,     // Bengali
      0x0A00,     0x0A7F,     // Gurmukhi
      0x0A80,     0x0AFF,     // Gujarati
      0x0B00,     0x0B7F,     // Oriya
      0x0B80,     0x0BFF,     // Tamil
      0x0C00,     0x0C7F,     // Telugu
      0x0C80,     0x0CFF,     // Kannada
      0x0D00,     0x0D7F,     // Malayalam
      0x0D80,     0x0DFF,     // Sinhala
      0x0E00,     0x0E7F,     // Thai
      0x0E80,     0x0EFF,     // Lao
      0x0F00,     0x0FFF,     // Tibetan
      0x1000,     0x109F,     // Myanmar
      0x10A0,     0x10FF,     // Georgian
      0x1100,     0x11FF,     // Hangul Jamo
      0x1200,     0x137F,     // Ethiopic
      0x1380,     0x139F,     // Ethiopic Supplement
      0x13A0,     0x13FF,     // Cherokee
      0x1400,     0x167F,     // Unified Canadian Aboriginal Syllabics
      0x1680,     0x169F,     // Ogham
      0x16A0,     0x16FF,     // Runic
      0x1700,     0x171F,     // Tagalog
      0x1720,     0x173F,     // Hanunoo
      0x1740,     0x175F,     // Buhid
      0x1760,     0x177F,     // Tagbanwa
      0x1780,     0x17FF,     // Khmer
      0x1800,     0x18AF,     // Mongolian
      0x18B0,     0x18FF,     // Unified Canadian Aboriginal Syllabics Extended
      0x1900,     0x194F,     // Limbu
      0x1950,     0x197F,     // Tai Le
      0x1980,     0x19DF,     // New Tai Lue
      0x19E0,     0x19FF,     // Khmer Symbols
      0x1A00,     0x1A1F,     // Buginese
      0x1A20,     0x1AAF,     // Tai Tham
      0x1B00,     0x1B7F,     // Balinese
      0x1B80,     0x1BBF,     // Sundanese
      0x1BC0,     0x1BFF,     // Batak
      0x1C00,     0x1C4F,     // Lepcha
      0x1C50,     0x1C7F,     // Ol Chiki
      0x1CD0,     0x1CFF,     // Vedic Extensions
      0x1D00,     0x1D7F,     // Phonetic Extensions
      0x1D80,     0x1DBF,     // Phonetic Extensions Supplement
      0x1DC0,     0x1DFF,     // Combining Diacritical Marks Supplement
      0x1E00,     0x1EFF,     // Latin Extended Additional
      0x1F00,     0x1FFF,     // Greek Extended
      0x2000,     0x206F,     // General Punctuation
      0x2070,     0x209F,     // Superscripts and Subscripts
      0x20A0,     0x20CF,     // Currency Symbols
      0x20D0,     0x20FF,     // Combining Diacritical Marks for Symbols
      0x2100,     0x214F,     // Letterlike Symbols
      0x2150,     0x218F,     // Number Forms
      0x2190,     0x21FF,     // Arrows
      0x2200,     0x22FF,     // Mathematical Operators
      0x2300,     0x23FF,     // Miscellaneous Technical
      0x2400,     0x243F,     // Control Pictures
      0x2440,     0x245F,     // Optical Character Recognition
      0x2460,     0x24FF,     // Enclosed Alphanumerics
      0x2500,     0x257F,     // Box Drawing
      0x2580,     0x259F,     // Block Elements
      0x25A0,     0x25FF,     // Geometric Shapes
      0x2600,     0x26FF,     // Miscellaneous Symbols
      0x2700,     0x27BF,     // Dingbats
      0x27C0,     0x27EF,     // Miscellaneous Mathematical Symbols-A
      0x27F0,     0x27FF,     // Supplemental Arrows-A
      0x2800,     0x28FF,     // Braille Patterns
      0x2900,     0x297F,     // Supplemental Arrows-B
      0x2980,     0x29FF,     // Miscellaneous Mathematical Symbols-B
      0x2A00,     0x2AFF,     // Supplemental Mathematical Operators
      0x2B00,     0x2BFF,     // Miscellaneous Symbols and Arrows
      0x2C00,     0x2C5F,     // Glagolitic
      0x2C60,     0x2C7F,     // Latin Extended-C
      0x2C80,     0x2CFF,     // Coptic
      0x2D00,     0x2D2F,     // Georgian Supplement
      0x2D30,     0x2D7F,     // Tifinagh
      0x2D80,     0x2DDF,     // Ethiopic Extended
      0x2DE0,     0x2DFF,     // Cyrillic Extended-A
      0x2E00,     0x2E7F,     // Supplemental Punctuation
      0x2E80,     0x2EFF,     // CJK Radicals Supplement
      0x2F00,     0x2FDF,     // Kangxi Radicals
      0x2FF0,     0x2FFF,     // Ideographic Description Characters
      0x3000,     0x303F,     // CJK Symbols and Punctuation
      0x3040,     0x309F,     // Hiragana
      0x30A0,     0x30FF,     // Katakana
      0x3100,     0x312F,     // Bopomofo
      0x3130,     0x318F,     // Hangul Compatibility Jamo
      0x3190,     0x319F,     // Kanbun
      0x31A0,     0x31BF,     // Bopomofo Extended
      0x31C0,     0x31EF,     // CJK Strokes
      0x31F0,     0x31FF,     // Katakana Phonetic Extensions
      0x3200,     0x32FF,     // Enclosed CJK Letters and Months
      0x3300,     0x33FF,     // CJK Compatibility
      0x3400,     0x4DBF,     // CJK Unified Ideographs Extension A
      0x4DC0,     0x4DFF,     // Yijing Hexagram Symbols
      0x4E00,     0x9FFF,     // CJK Unified Ideographs
      0xA000,     0xA48F,     // Yi Syllables
      0xA490,     0xA4CF,     // Yi Radicals
      0xA4D0,     0xA4FF,     // Lisu
      0xA500,     0xA63F,     // Vai
      0xA640,     0xA69F,     // Cyrillic Extended-B
      0xA6A0,     0xA6FF,     // Bamum
      0xA700,     0xA71F,     // Modifier Tone Letters
      0xA720,     0xA7FF,     // Latin Extended-D
      0xA800,     0xA82F,     // Syloti Nagri
      0xA830,     0xA83F,     // Common Indic Number Forms
      0xA840,     0xA87F,     // Phags-pa
      0xA880,     0xA8DF,     // Saurashtra
      0xA8E0,     0xA8FF,     // Devanagari Extended
      0xA900,     0xA92F,     // Kayah Li
      0xA930,     0xA95F,     // Rejang
      0xA960,     0xA97F,     // Hangul Jamo Extended-A
      0xA980,     0xA9DF,     // Javanese
      0xAA00,     0xAA5F,     // Cham
      0xAA60,     0xAA7F,     // Myanmar Extended-A
      0xAA80,     0xAADF,     // Tai Viet
      0xAB00,     0xAB2F,     // Ethiopic Extended-A
      0xABC0,     0xABFF,     // Meetei Mayek
      0xAC00,     0xD7AF,     // Hangul Syllables
      0xD7B0,     0xD7FF,     // Hangul Jamo Extended-B
      //0xD800,       0xDB7F,     // High Surrogates
      //0xDB80,       0xDBFF,     // High Private Use Surrogates
      //0xDC00,       0xDFFF,     // Low Surrogates
      0xE000,     0xF8FF,     // Private Use Area
      0xF900,     0xFAFF,     // CJK Compatibility Ideographs
      0xFB00,     0xFB4F,     // Alphabetic Presentation Forms
      0xFB50,     0xFDFF,     // Arabic Presentation Forms-A
      0xFE00,     0xFE0F,     // Variation Selectors
      0xFE10,     0xFE1F,     // Vertical Forms
      0xFE20,     0xFE2F,     // Combining Half Marks
      0xFE30,     0xFE4F,     // CJK Compatibility Forms
      0xFE50,     0xFE6F,     // Small Form Variants
      0xFE70,     0xFEFF,     // Arabic Presentation Forms-B
      0xFF00,     0xFFEF,     // Halfwidth and Fullwidth Forms
      0xFFF0,     0xFFFF,     // Specials
      0x10000,    0x1007F,    // Linear B Syllabary
      0x10080,    0x100FF,    // Linear B Ideograms
      0x10100,    0x1013F,    // Aegean Numbers
      0x10140,    0x1018F,    // Ancient Greek Numbers
      0x10190,    0x101CF,    // Ancient Symbols
      0x101D0,    0x101FF,    // Phaistos Disc
      0x10280,    0x1029F,    // Lycian
      0x102A0,    0x102DF,    // Carian
      0x10300,    0x1032F,    // Old Italic
      0x10330,    0x1034F,    // Gothic
      0x10380,    0x1039F,    // Ugaritic
      0x103A0,    0x103DF,    // Old Persian
      0x10400,    0x1044F,    // Deseret
      0x10450,    0x1047F,    // Shavian
      0x10480,    0x104AF,    // Osmanya
      0x10800,    0x1083F,    // Cypriot Syllabary
      0x10840,    0x1085F,    // Imperial Aramaic
      0x10900,    0x1091F,    // Phoenician
      0x10920,    0x1093F,    // Lydian
      0x10A00,    0x10A5F,    // Kharoshthi
      0x10A60,    0x10A7F,    // Old South Arabian
      0x10B00,    0x10B3F,    // Avestan
      0x10B40,    0x10B5F,    // Inscriptional Parthian
      0x10B60,    0x10B7F,    // Inscriptional Pahlavi
      0x10C00,    0x10C4F,    // Old Turkic
      0x10E60,    0x10E7F,    // Rumi Numeral Symbols
      0x11000,    0x1107F,    // Brahmi
      0x11080,    0x110CF,    // Kaithi
      0x12000,    0x123FF,    // Cuneiform
      0x12400,    0x1247F,    // Cuneiform Numbers and Punctuation
      0x13000,    0x1342F,    // Egyptian Hieroglyphs
      0x16800,    0x16A3F,    // Bamum Supplement
      0x1B000,    0x1B0FF,    // Kana Supplement
      0x1D000,    0x1D0FF,    // Byzantine Musical Symbols
      0x1D100,    0x1D1FF,    // Musical Symbols
      0x1D200,    0x1D24F,    // Ancient Greek Musical Notation
      0x1D300,    0x1D35F,    // Tai Xuan Jing Symbols
      0x1D360,    0x1D37F,    // Counting Rod Numerals
      0x1D400,    0x1D7FF,    // Mathematical Alphanumeric Symbols
      0x1F000,    0x1F02F,    // Mahjong Tiles
      0x1F030,    0x1F09F,    // Domino Tiles
      0x1F0A0,    0x1F0FF,    // Playing Cards
      0x1F100,    0x1F1FF,    // Enclosed Alphanumeric Supplement
      0x1F200,    0x1F2FF,    // Enclosed Ideographic Supplement
      0x1F300,    0x1F5FF,    // Miscellaneous Symbols And Pictographs
      0x1F600,    0x1F64F,    // Emoticons
      0x1F680,    0x1F6FF,    // Transport And Map Symbols
      0x1F700,    0x1F77F,    // Alchemical Symbols
      0x20000,    0x2A6DF,    // CJK Unified Ideographs Extension B
      0x2A700,    0x2B73F,    // CJK Unified Ideographs Extension C
      0x2B740,    0x2B81F,    // CJK Unified Ideographs Extension D
      0x2F800,    0x2FA1F,    // CJK Compatibility Ideographs Supplement
      0xE0000,    0xE007F,    // Tags
      0xE0100,    0xE01EF,    // Variation Selectors Supplement
      0xF0000,    0xFFFFF,    // Supplementary Private Use Area-A
      0x100000,   0x10FFFF,   // Supplementary Private Use Area-B
      0xFFFFFFFF
  };
  
  // Copyright (c) 2008-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
  // See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
  
  #define UTF8_ACCEPT 0u
  
  static const unsigned char utf8d[] = {
      // The first part of the table maps bytes to character classes that
      // to reduce the size of the transition table and create bitmasks.
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
      7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
      8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
      10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
  
      // The second part is a transition table that maps a combination
      // of a state of the automaton and a character class to a state.
      0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
      12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
      12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
      12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
      12,36,12,12,12,12,12,12,12,12,12,12, 
  };
  
  static unsigned inline decode(unsigned* state, unsigned* codep, unsigned byte) {
      unsigned type = utf8d[byte];
  
      *codep = (*state != UTF8_ACCEPT) ?
          (byte & 0x3fu) | (*codep << 6) :
      (0xffu >> type) & (byte);
  
      *state = utf8d[256 + *state + type];
      return *state;
  }
  
  //static bool IsUTF8(unsigned char* s) {
  //  unsigned codepoint, state = 0;
  //
  //  while (*s)
  //      decode(&state, &codepoint, *s++);
  //
  //  return state == UTF8_ACCEPT;
  //}
  
  TEST(EncodingsTest, UTF8) {
      StringBuffer os, os2;
      for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) {
          for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) {
              os.Clear();
              UTF8<>::Encode(os, codepoint);
              const char* encodedStr = os.GetString();
  
              // Decode with Hoehrmann
              {
                  unsigned decodedCodepoint = 0;
                  unsigned state = 0;
  
                  unsigned decodedCount = 0;
                  for (const char* s = encodedStr; *s; ++s)
                      if (!decode(&state, &decodedCodepoint, static_cast<unsigned char>(*s))) {
                          EXPECT_EQ(codepoint, decodedCodepoint);
                          decodedCount++;
                      }
  
                  if (*encodedStr) {                  // This decoder cannot handle U+0000
                      EXPECT_EQ(1u, decodedCount);    // Should only contain one code point
                  }
  
                  EXPECT_EQ(UTF8_ACCEPT, state);
                  if (UTF8_ACCEPT != state)
                      std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
              }
  
              // Decode
              {
                  StringStream is(encodedStr);
                  unsigned decodedCodepoint;
                  bool result = UTF8<>::Decode(is, &decodedCodepoint);
                  EXPECT_TRUE(result);
                  EXPECT_EQ(codepoint, decodedCodepoint);
                  if (!result || codepoint != decodedCodepoint)
                      std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
              }
  
              // Validate
              {
                  StringStream is(encodedStr);
                  os2.Clear();
                  bool result = UTF8<>::Validate(is, os2);
                  EXPECT_TRUE(result);
                  EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
              }
          }
      }
  }
  
  TEST(EncodingsTest, UTF16) {
      GenericStringBuffer<UTF16<> > os, os2;
      GenericStringBuffer<UTF8<> > utf8os;
      for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) {
          for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) {
              os.Clear();
              UTF16<>::Encode(os, codepoint);
              const UTF16<>::Ch* encodedStr = os.GetString();
  
              // Encode with Hoehrmann's code
              if (codepoint != 0) // cannot handle U+0000
              {
                  // encode with UTF8<> first
                  utf8os.Clear();
                  UTF8<>::Encode(utf8os, codepoint);
  
                  // transcode from UTF8 to UTF16 with Hoehrmann's code
                  unsigned decodedCodepoint = 0;
                  unsigned state = 0;
                  UTF16<>::Ch buffer[3], *p = &buffer[0];
                  for (const char* s = utf8os.GetString(); *s; ++s) {
                      if (!decode(&state, &decodedCodepoint, static_cast<unsigned char>(*s)))
                          break;
                  }
  
                  if (codepoint <= 0xFFFF)
                      *p++ = static_cast<UTF16<>::Ch>(decodedCodepoint);
                  else {
                      // Encode code points above U+FFFF as surrogate pair.
                      *p++ = static_cast<UTF16<>::Ch>(0xD7C0 + (decodedCodepoint >> 10));
                      *p++ = static_cast<UTF16<>::Ch>(0xDC00 + (decodedCodepoint & 0x3FF));
                  }
                  *p++ = '\0';
  
                  EXPECT_EQ(0, StrCmp(buffer, encodedStr));
              }
  
              // Decode
              {
                  GenericStringStream<UTF16<> > is(encodedStr);
                  unsigned decodedCodepoint;
                  bool result = UTF16<>::Decode(is, &decodedCodepoint);
                  EXPECT_TRUE(result);
                  EXPECT_EQ(codepoint, decodedCodepoint);         
                  if (!result || codepoint != decodedCodepoint)
                      std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
              }
  
              // Validate
              {
                  GenericStringStream<UTF16<> > is(encodedStr);
                  os2.Clear();
                  bool result = UTF16<>::Validate(is, os2);
                  EXPECT_TRUE(result);
                  EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
              }
          }
      }
  }
  
  TEST(EncodingsTest, UTF32) {
      GenericStringBuffer<UTF32<> > os, os2;
      for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) {
          for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) {
              os.Clear();
              UTF32<>::Encode(os, codepoint);
              const UTF32<>::Ch* encodedStr = os.GetString();
  
              // Decode
              {
                  GenericStringStream<UTF32<> > is(encodedStr);
                  unsigned decodedCodepoint;
                  bool result = UTF32<>::Decode(is, &decodedCodepoint);
                  EXPECT_TRUE(result);
                  EXPECT_EQ(codepoint, decodedCodepoint);         
                  if (!result || codepoint != decodedCodepoint)
                      std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
              }
  
              // Validate
              {
                  GenericStringStream<UTF32<> > is(encodedStr);
                  os2.Clear();
                  bool result = UTF32<>::Validate(is, os2);
                  EXPECT_TRUE(result);
                  EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
              }
          }
      }
  }
  
  TEST(EncodingsTest, ASCII) {
      StringBuffer os, os2;
      for (unsigned codepoint = 0; codepoint < 128; codepoint++) {
          os.Clear();
          ASCII<>::Encode(os, codepoint);
          const ASCII<>::Ch* encodedStr = os.GetString();
          {
              StringStream is(encodedStr);
              unsigned decodedCodepoint;
              bool result = ASCII<>::Decode(is, &decodedCodepoint);
              if (!result || codepoint != decodedCodepoint)
                  std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
          }
  
          // Validate
          {
              StringStream is(encodedStr);
              os2.Clear();
              bool result = ASCII<>::Validate(is, os2);
              EXPECT_TRUE(result);
              EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
          }
      }
  }
  