src/training/unicharset_extractor.cpp
static void AddStringsToUnicharset(const std::vector<std::string> &strings, int norm_mode,
                                   UNICHARSET *unicharset) {
  for (const auto &string : strings) {
    std::vector<std::string> normalized;
    if (NormalizeCleanAndSegmentUTF8(UnicodeNormMode::kNFC, OCRNorm::kNone,
                                     static_cast<GraphemeNormMode>(norm_mode),
                                     /*report_errors*/ true, string.c_str(), &normalized)) {
      for (const std::string &normed : normalized) {
        // normed is a UTF-8 encoded string
        if (normed.empty() || IsUTF8Whitespace(normed.c_str())) {
          continue;
        }
        unicharset->unichar_insert(normed.c_str());
      }
    } else {
      tprintf("Normalization failed for string '%s'\n", string.c_str());
    }
  }
}
// Helper normalizes and segments the given strings according to norm_mode, and
// adds the segmented parts to unicharset.
Helper normalizes and segments the given strings according to norm_mode, and
adds the segmented parts to unicharset.
Helper normalizes and segments the given strings according to norm_mode and adds the segmented parts to unicharset .
static void AddStringsToUnicharset(const std::vector<std::string> &strings, int norm_mode,
                                   UNICHARSET *unicharset) {
  for (const auto &string : strings) {
    std::vector<std::string> normalized;
    if (NormalizeCleanAndSegmentUTF8(UnicodeNormMode::kNFC, OCRNorm::kNone,
                                     static_cast<GraphemeNormMode>(norm_mode),
                                      true, string.c_str(), &normalized)) {
      for (const std::string &normed : normalized) {
        if (normed.empty() || IsUTF8Whitespace(normed.c_str())) {
          continue;
        }
        unicharset->unichar_insert(normed.c_str());
      }
    } else {
      tprintf("Normalization failed for string '%s'\n", string.c_str());
    }
  }
}
static void AddStringsToUnicharset ( const std :: vector < std :: string > & strings , int norm_mode , UNICHARSET * unicharset ) { for ( const auto & string : strings ) { std :: vector < std :: string > normalized ; if ( NormalizeCleanAndSegmentUTF8 ( UnicodeNormMode :: kNFC , OCRNorm :: kNone , static_cast < GraphemeNormMode > ( norm_mode ) , true , string . c_str ( ) , & normalized ) ) { for ( const std :: string & normed : normalized ) { if ( normed . empty ( ) || IsUTF8Whitespace ( normed . c_str ( ) ) ) { continue ; } unicharset -> unichar_insert ( normed . c_str ( ) ) ; } } else { tprintf ( " \n " , string . c_str ( ) ) ; } } }
Helper normalizes and segments the given strings according to norm_mode, and
adds the segmented parts to unicharset.
Helper normalizes and segments the given strings according to norm_mode and adds the segmented parts to unicharset .
/*report_errors*/ // normed is a UTF-8 encoded string