Added Normalization
Introduced Normalization as per Ghabry's proposed patch to
MODIFY
src/reader_util.cpp
src/reader_util.cpp
#ifdef LCF_SUPPORT_ICU
# include <unicode/ucsdet.h>
# include <unicode/ucnv.h>
+# include <unicode/normalizer2.h>
+# include <unicode/unistr.h>
#else
# ifdef _MSC_VER
# error MSVC builds require ICU
@@ -374,3 +376,20 @@ std::string ReaderUtil::Recode(const std::string& str_to_encode,
return result;
#endif
}
+
+std::string ReaderUtil::Normalize(const std::string &str) {
+#ifdef LCF_SUPPORT_ICU
+ icu::UnicodeString& uni = icu::UnicodeString(str.c_str()).toLower();
+ UErrorCode err = U_ZERO_ERROR;
+ const icu::Normalizer2* norm = icu::Normalizer2::getNFKCInstance(err);
+ icu::UnicodeString f = norm->normalize(uni, err);
+ std::string res;
+ f.toUTF8String(res);
+ return res;
+#else
+ // LAME
+ std::string result = str;
+ std::transform(result.begin(), result.end(), result.begin(), tolower);
+ return result;
+#endif
+}

MODIFY
src/reader_util.h
src/reader_util.h
@@ -110,6 +110,15 @@ namespace ReaderUtil {
const std::string& src_enc,
const std::string& dst_enc);
+ /**
+ * Converts a UTF-8 string to lowercase and then decomposes it.
+ *
+ * @param str the string to normalize.
+ * @return the normalized string.
+ */
+ std::string Normalize(const std::string &str);
+
+
/**
* Helper function that returns an element from a vector using a 1-based
* index as usually used by LCF data structures.

Added Normalization

Introduced Normalization as per Ghabry's proposed patch to #839