text: //
  //  Copyright (c) 2009-2011 Artyom Beilis (Tonkikh)
  //
  //  Distributed under the Boost Software License, Version 1.0. (See
  //  accompanying file LICENSE_1_0.txt or copy at
  //  http://www.boost.org/LICENSE_1_0.txt)
  //
  
  #ifndef BOOST_LOCALE_IMPL_WCONV_CODEPAGE_HPP
  #define BOOST_LOCALE_IMPL_WCONV_CODEPAGE_HPP
  
  
  #include <boost/locale/encoding.hpp>
  #include <algorithm>
  #include <cstring>
  #include <string>
  #include "conv.hpp"
  
  #ifndef NOMINMAX
  # define NOMINMAX
  #endif
  #include <windows.h>
  #include <vector>
  
  
  namespace boost {
  namespace locale {
  namespace conv {
  namespace impl {
  
      struct windows_encoding {
          char const *name;
          unsigned codepage;
          unsigned was_tested;
      };
  
      bool operator<(windows_encoding const &l,windows_encoding const &r)
      {
          return strcmp(l.name,r.name) < 0;
      }
  
      windows_encoding all_windows_encodings[] = {
          { "big5",       950, 0 },
          { "cp1250",     1250, 0 },
          { "cp1251",     1251, 0 },
          { "cp1252",     1252, 0 },
          { "cp1253",     1253, 0 },
          { "cp1254",     1254, 0 },
          { "cp1255",     1255, 0 },
          { "cp1256",     1256, 0 },
          { "cp1257",     1257, 0 },
          { "cp874",      874, 0 },
          { "cp932",      932, 0 },
          { "cp936",      936, 0 },
          { "eucjp",      20932, 0 },
          { "euckr",      51949, 0 },
          { "gb18030",    54936, 0 },
          { "gb2312",     20936, 0 },
          { "gbk",        936, 0 },
          { "iso2022jp",  50220, 0 },
          { "iso2022kr",  50225, 0 },
          { "iso88591",   28591, 0 },
          { "iso885913",  28603, 0 },
          { "iso885915",  28605, 0 },
          { "iso88592",   28592, 0 },
          { "iso88593",   28593, 0 },
          { "iso88594",   28594, 0 },
          { "iso88595",   28595, 0 },
          { "iso88596",   28596, 0 },
          { "iso88597",   28597, 0 },
          { "iso88598",   28598, 0 },
          { "iso88599",   28599, 0 },
          { "koi8r",      20866, 0 },
          { "koi8u",      21866, 0 },
          { "ms936",      936, 0 },
          { "shiftjis",   932, 0 },
          { "sjis",       932, 0 },
          { "usascii",    20127, 0 },
          { "utf8",       65001, 0 },
          { "windows1250",        1250, 0 },
          { "windows1251",        1251, 0 },
          { "windows1252",        1252, 0 },
          { "windows1253",        1253, 0 },
          { "windows1254",        1254, 0 },
          { "windows1255",        1255, 0 },
          { "windows1256",        1256, 0 },
          { "windows1257",        1257, 0 },
          { "windows874",         874, 0 },
          { "windows932",         932, 0 },
          { "windows936",         936, 0 },
      };
  
      size_t remove_substitutions(std::vector<char> &v)
      {
          if(std::find(v.begin(),v.end(),0) == v.end()) {
              return v.size();
          }
          std::vector<char> v2;
          v2.reserve(v.size());
          for(unsigned i=0;i<v.size();i++) {
              if(v[i]!=0)
                  v2.push_back(v[i]);
          }
          v.swap(v2);
          return v.size();
      }
  
      void multibyte_to_wide_one_by_one(int codepage,char const *begin,char const *end,std::vector<wchar_t> &buf)
      {
          buf.reserve(end-begin);
          while(begin!=end) {
              wchar_t wide_buf[4];
              int n = 0;
              int len = IsDBCSLeadByteEx(codepage,*begin) ? 2 : 1;
              if(len == 2 && begin+1==end)
                  return;
              n = MultiByteToWideChar(codepage,MB_ERR_INVALID_CHARS,begin,len,wide_buf,4);
              for(int i=0;i<n;i++)
                  buf.push_back(wide_buf[i]);
              begin+=len;
          }
      }
  
  
      void multibyte_to_wide(int codepage,char const *begin,char const *end,bool do_skip,std::vector<wchar_t> &buf)
      {
          if(begin==end)
              return;
          int n = MultiByteToWideChar(codepage,MB_ERR_INVALID_CHARS,begin,end-begin,0,0);
          if(n == 0) {
              if(do_skip) {
                  multibyte_to_wide_one_by_one(codepage,begin,end,buf);
                  return;
              }
              throw conversion_error();
          }
  
          buf.resize(n,0);
          if(MultiByteToWideChar(codepage,MB_ERR_INVALID_CHARS,begin,end-begin,&buf.front(),buf.size())==0)
              throw conversion_error();
      }
  
      void wide_to_multibyte_non_zero(int codepage,wchar_t const *begin,wchar_t const *end,bool do_skip,std::vector<char> &buf)
      {
          if(begin==end)
              return;
          BOOL substitute = FALSE;
          BOOL *substitute_ptr = codepage == 65001 || codepage == 65000 ? 0 : &substitute;
          char subst_char = 0;
          char *subst_char_ptr = codepage == 65001 || codepage == 65000 ? 0 : &subst_char;
  
          int n = WideCharToMultiByte(codepage,0,begin,end-begin,0,0,subst_char_ptr,substitute_ptr);
          buf.resize(n);
  
          if(WideCharToMultiByte(codepage,0,begin,end-begin,&buf[0],n,subst_char_ptr,substitute_ptr)==0)
              throw conversion_error();
          if(substitute) {
              if(do_skip)
                  remove_substitutions(buf);
              else
                  throw conversion_error();
          }
      }
  
      void wide_to_multibyte(int codepage,wchar_t const *begin,wchar_t const *end,bool do_skip,std::vector<char> &buf)
      {
          if(begin==end)
              return;
          buf.reserve(end-begin);
          wchar_t const *e = std::find(begin,end,L'\0');
          wchar_t const *b = begin;
          for(;;) {
              std::vector<char> tmp;
              wide_to_multibyte_non_zero(codepage,b,e,do_skip,tmp);
              size_t osize = buf.size();
              buf.resize(osize+tmp.size());
              std::copy(tmp.begin(),tmp.end(),buf.begin()+osize);
              if(e!=end) {
                  buf.push_back('\0');
                  b=e+1;
                  e=std::find(b,end,L'0');
              }
              else
                  break;
          }
      }
  
  
      int encoding_to_windows_codepage(char const *ccharset)
      {
          std::string charset = normalize_encoding(ccharset);
          windows_encoding ref;
          ref.name = charset.c_str();
          size_t n = sizeof(all_windows_encodings)/sizeof(all_windows_encodings[0]);
          windows_encoding *begin = all_windows_encodings;
          windows_encoding *end = all_windows_encodings + n;
          windows_encoding *ptr = std::lower_bound(begin,end,ref);
          if(ptr!=end && strcmp(ptr->name,charset.c_str())==0) {
              if(ptr->was_tested) {
                  return ptr->codepage;
              }
              else if(IsValidCodePage(ptr->codepage)) {
                  // the thread safety is not an issue, maximum
                  // it would be checked more then once
                  ptr->was_tested=1;
                  return ptr->codepage;
              }
              else {
                  return -1;
              }
          }
          return -1;
  
      }
  
      template<typename CharType>
      bool validate_utf16(CharType const *str,unsigned len)
      {
          CharType const *begin = str;
          CharType const *end = str+len;
          while(begin!=end) {
              utf::code_point c = utf::utf_traits<CharType,2>::template decode<CharType const *>(begin,end);
              if(c==utf::illegal || c==utf::incomplete)
                  return false;
          }
          return true;
      }
  
      template<typename CharType,typename OutChar>
      void clean_invalid_utf16(CharType const *str,unsigned len,std::vector<OutChar> &out)
      {
          out.reserve(len);
          for(unsigned i=0;i<len;i++) {
              uint16_t c = static_cast<uint16_t>(str[i]);
  
              if(0xD800 <= c && c<= 0xDBFF) {
                  i++;
                  if(i>=len)
                      return;
                  uint16_t c2=static_cast<uint16_t>(str[i]);
                  if(0xDC00 <= c2 && c2 <= 0xDFFF) {
                      out.push_back(static_cast<OutChar>(c));
                      out.push_back(static_cast<OutChar>(c2));
                  }
              }
              else if(0xDC00 <= c && c <=0xDFFF)
                  continue;
              else
                  out.push_back(static_cast<OutChar>(c));
          }
      }
  
  
      class wconv_between : public converter_between {
      public:
          wconv_between() :
              how_(skip),
              to_code_page_ (-1),
              from_code_page_ ( -1)
          {
          }
          bool open(char const *to_charset,char const *from_charset,method_type how)
          {
              how_ = how;
              to_code_page_ = encoding_to_windows_codepage(to_charset);
              from_code_page_ = encoding_to_windows_codepage(from_charset);
              if(to_code_page_ == -1 || from_code_page_ == -1)
                  return false;
              return true;
          }
          virtual std::string convert(char const *begin,char const *end)
          {
              if(to_code_page_ == 65001 && from_code_page_ == 65001)
                  return utf_to_utf<char>(begin,end,how_);
  
              std::string res;
  
              std::vector<wchar_t> tmp;   // buffer for mb2w
              std::wstring tmps;          // buffer for utf_to_utf
              wchar_t const *wbegin=0;
              wchar_t const *wend=0;
  
              if(from_code_page_ == 65001) {
                  tmps = utf_to_utf<wchar_t>(begin,end,how_);
                  if(tmps.empty())
                      return res;
                  wbegin = tmps.c_str();
                  wend = wbegin + tmps.size();
              }
              else {
                  multibyte_to_wide(from_code_page_,begin,end,how_ == skip,tmp);
                  if(tmp.empty())
                      return res;
                  wbegin = &tmp[0];
                  wend = wbegin + tmp.size();
              }
  
              if(to_code_page_ == 65001) {
                  return utf_to_utf<char>(wbegin,wend,how_);
              }
  
              std::vector<char> ctmp;
              wide_to_multibyte(to_code_page_,wbegin,wend,how_ == skip,ctmp);
              if(ctmp.empty())
                  return res;
              res.assign(&ctmp.front(),ctmp.size());
              return res;
          }
      private:
          method_type how_;
          int to_code_page_;
          int from_code_page_;
      };
  
      template<typename CharType,int size = sizeof(CharType) >
      class wconv_to_utf;
  
      template<typename CharType,int size = sizeof(CharType) >
      class wconv_from_utf;
  
      template<>
      class wconv_to_utf<char,1> : public  converter_to_utf<char> , public wconv_between {
      public:
          virtual bool open(char const *cs,method_type how)
          {
              return wconv_between::open("UTF-8",cs,how);
          }
          virtual std::string convert(char const *begin,char const *end)
          {
              return wconv_between::convert(begin,end);
          }
      };
  
      template<>
      class wconv_from_utf<char,1> : public  converter_from_utf<char> , public wconv_between {
      public:
          virtual bool open(char const *cs,method_type how)
          {
              return wconv_between::open(cs,"UTF-8",how);
          }
          virtual std::string convert(char const *begin,char const *end)
          {
              return wconv_between::convert(begin,end);
          }
      };
  
      template<typename CharType>
      class wconv_to_utf<CharType,2> : public converter_to_utf<CharType> {
      public:
          typedef CharType char_type;
  
          typedef std::basic_string<char_type> string_type;
  
          wconv_to_utf() :
              how_(skip),
              code_page_(-1)
          {
          }
  
          virtual bool open(char const *charset,method_type how)
          {
              how_ = how;
              code_page_ = encoding_to_windows_codepage(charset);
              return code_page_ != -1;
          }
  
          virtual string_type convert(char const *begin,char const *end)
          {
              if(code_page_ == 65001) {
                  return utf_to_utf<char_type>(begin,end,how_);
              }
              std::vector<wchar_t> tmp;
              multibyte_to_wide(code_page_,begin,end,how_ == skip,tmp);
              string_type res;
              if(!tmp.empty())
                  res.assign(reinterpret_cast<char_type *>(&tmp.front()),tmp.size());
              return res;
          }
  
      private:
          method_type how_;
          int code_page_;
      };
  
      template<typename CharType>
      class wconv_from_utf<CharType,2> : public converter_from_utf<CharType> {
      public:
          typedef CharType char_type;
  
          typedef std::basic_string<char_type> string_type;
  
          wconv_from_utf() :
              how_(skip),
              code_page_(-1)
          {
          }
  
          virtual bool open(char const *charset,method_type how)
          {
              how_ = how;
              code_page_ = encoding_to_windows_codepage(charset);
              return code_page_ != -1;
          }
  
          virtual std::string convert(CharType const *begin,CharType const *end)
          {
              if(code_page_ == 65001) {
                  return utf_to_utf<char>(begin,end,how_);
              }
              wchar_t const *wbegin = 0;
              wchar_t const *wend = 0;
              std::vector<wchar_t> buffer; // if needed
              if(begin==end)
                  return std::string();
              if(validate_utf16(begin,end-begin)) {
                  wbegin =  reinterpret_cast<wchar_t const *>(begin);
                  wend = reinterpret_cast<wchar_t const *>(end);
              }
              else {
                  if(how_ == stop) {
                          throw conversion_error();
                  }
                  else {
                      clean_invalid_utf16(begin,end-begin,buffer);
                      if(!buffer.empty()) {
                          wbegin = &buffer[0];
                          wend = wbegin + buffer.size();
                      }
                  }
              }
              std::string res;
              if(wbegin==wend)
                  return res;
              std::vector<char> ctmp;
              wide_to_multibyte(code_page_,wbegin,wend,how_ == skip,ctmp);
              if(ctmp.empty())
                  return res;
              res.assign(&ctmp.front(),ctmp.size());
              return res;
          }
  
      private:
          method_type how_;
          int code_page_;
      };
  
  
  
      template<typename CharType>
      class wconv_to_utf<CharType,4> : public converter_to_utf<CharType> {
      public:
          typedef CharType char_type;
  
          typedef std::basic_string<char_type> string_type;
  
          wconv_to_utf() :
              how_(skip),
              code_page_(-1)
          {
          }
  
          virtual bool open(char const *charset,method_type how)
          {
              how_ = how;
              code_page_ = encoding_to_windows_codepage(charset);
              return code_page_ != -1;
          }
  
          virtual string_type convert(char const *begin,char const *end)
          {
              if(code_page_ == 65001) {
                  return utf_to_utf<char_type>(begin,end,how_);
              }
              std::vector<wchar_t> buf;
              multibyte_to_wide(code_page_,begin,end,how_ == skip,buf);
  
              if(buf.empty())
                  return string_type();
  
              return utf_to_utf<CharType>(&buf[0],&buf[0]+buf.size(),how_);
          }
      private:
          method_type how_;
          int code_page_;
      };
  
      template<typename CharType>
      class wconv_from_utf<CharType,4> : public converter_from_utf<CharType> {
      public:
          typedef CharType char_type;
  
          typedef std::basic_string<char_type> string_type;
  
          wconv_from_utf() :
              how_(skip),
              code_page_(-1)
          {
          }
  
          virtual bool open(char const *charset,method_type how)
          {
              how_ = how;
              code_page_ = encoding_to_windows_codepage(charset);
              return code_page_ != -1;
          }
  
          virtual std::string convert(CharType const *begin,CharType const *end)
          {
              if(code_page_ == 65001) {
                  return utf_to_utf<char>(begin,end,how_);
              }
              std::wstring tmp = utf_to_utf<wchar_t>(begin,end,how_);
  
              std::vector<char> ctmp;
              wide_to_multibyte(code_page_,tmp.c_str(),tmp.c_str()+tmp.size(),how_ == skip,ctmp);
              std::string res;
              if(ctmp.empty())
                  return res;
              res.assign(&ctmp.front(),ctmp.size());
              return res;
  
          }
  
      private:
          method_type how_;
          int code_page_;
      };
  
  
  
  
  
  } // impl
  } // conv
  } // locale
  } // boost
  
  #endif
  // vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
  