oss_src/sframe/sarray_v2_type_encoding.cpp
static void encode_string(block_info& info, 
                          oarchive& oarc, 
                          const std::vector<flexible_type>& data) {
  bool use_dictionary_encoding = true;
  std::unordered_map<std::string, size_t> unique_values;
  std::vector<flexible_type> idx_values;
  std::vector<std::string> str_values;
  idx_values.resize(data.size(), flexible_type(flex_type_enum::INTEGER));
  size_t idxctr = 0;
  for (size_t i = 0;i < data.size(); ++i) {
    if (data[i].get_type() != flex_type_enum::UNDEFINED) {
      auto iter = unique_values.find(data[i].get<std::string>());
      if (iter != unique_values.end()) {
        idx_values[idxctr++].mutable_get<flex_int>() = iter->second;
      } else {
        // if we have too many unique values, fail.
        if (unique_values.size() >= 64) {
          use_dictionary_encoding = false;
          break;
        }
        size_t newidx = unique_values.size();
        unique_values[data[i].get<std::string>()] = newidx;
        str_values.push_back(data[i].get<std::string>());
        idx_values[idxctr++].mutable_get<flex_int>() = newidx;
      }
    }
  }
  oarc << use_dictionary_encoding;
  if (use_dictionary_encoding) {
    idx_values.resize(idxctr);

    variable_encode(oarc, str_values.size());
    for (auto& str: str_values) {
      variable_encode(oarc, str.length());
      oarc.write(str.c_str(), str.length());
    }
    encode_number(info, oarc, idx_values);
  } else {
    // encode all the lengths 
    idxctr = 0;
    for (auto& f: data) {
      if (f.get_type() != flex_type_enum::UNDEFINED) {
        idx_values[idxctr++].mutable_get<flex_int>() = f.get<std::string>().length();
      }
    }
    idx_values.resize(idxctr);
    encode_number(info, oarc, idx_values);
    for (auto& f: data) {
      if (f.get_type() != flex_type_enum::UNDEFINED) {
        oarc.write(f.get<std::string>().c_str(), f.get<flex_string>().length());
      }
    }
  }
}
/**
 * Encodes a collection of strings in data, skipping all UNDEFINED values.
 *
 * Two encoding strategies are used.
 * Strategy 1: 
 * Dictionary encode:
 *  - A dictionary of unique strings are built, and an array of numbers 
 *    mapping to the string values are constructed.
 *     - variable_encode(dictionary length)
 *     - for each entry in dictionary:
 *         - variable_encode entry length
 *         - write bytes contents for each entry
 *     - encode_number(dictionary mapping)
 * Strategy 2:
 * Direct encode:
 *  - encode_number(lengths of all the strings)
 *  - for each entry:
 *     - write byte contents for each entry
 *
 * \note The coding does not store the number of values stored. The decoder
 * \ref decode_string() requires the number of values to decode correctly.
 */
Encodes a collection of strings in data, skipping all UNDEFINED values.
Two encoding strategies are used.
Strategy 1:
Dictionary encode:
- A dictionary of unique strings are built, and an array of numbers
mapping to the string values are constructed.

\note The coding does not store the number of values stored. The decoder
\ref decode_string() requires the number of values to decode correctly.
Encodes a collection of strings in data skipping all UNDEFINED values . Two encoding strategies are used . Strategy 1 : Dictionary encode : - A dictionary of unique strings are built and an array of numbers mapping to the string values are constructed . \ note The coding does not store the number of values stored . The decoder \ ref decode_string () requires the number of values to decode correctly .
static void encode_string(block_info& info, 
                          oarchive& oarc, 
                          const std::vector<flexible_type>& data) {
  bool use_dictionary_encoding = true;
  std::unordered_map<std::string, size_t> unique_values;
  std::vector<flexible_type> idx_values;
  std::vector<std::string> str_values;
  idx_values.resize(data.size(), flexible_type(flex_type_enum::INTEGER));
  size_t idxctr = 0;
  for (size_t i = 0;i < data.size(); ++i) {
    if (data[i].get_type() != flex_type_enum::UNDEFINED) {
      auto iter = unique_values.find(data[i].get<std::string>());
      if (iter != unique_values.end()) {
        idx_values[idxctr++].mutable_get<flex_int>() = iter->second;
      } else {
        if (unique_values.size() >= 64) {
          use_dictionary_encoding = false;
          break;
        }
        size_t newidx = unique_values.size();
        unique_values[data[i].get<std::string>()] = newidx;
        str_values.push_back(data[i].get<std::string>());
        idx_values[idxctr++].mutable_get<flex_int>() = newidx;
      }
    }
  }
  oarc << use_dictionary_encoding;
  if (use_dictionary_encoding) {
    idx_values.resize(idxctr);
    variable_encode(oarc, str_values.size());
    for (auto& str: str_values) {
      variable_encode(oarc, str.length());
      oarc.write(str.c_str(), str.length());
    }
    encode_number(info, oarc, idx_values);
  } else {
    idxctr = 0;
    for (auto& f: data) {
      if (f.get_type() != flex_type_enum::UNDEFINED) {
        idx_values[idxctr++].mutable_get<flex_int>() = f.get<std::string>().length();
      }
    }
    idx_values.resize(idxctr);
    encode_number(info, oarc, idx_values);
    for (auto& f: data) {
      if (f.get_type() != flex_type_enum::UNDEFINED) {
        oarc.write(f.get<std::string>().c_str(), f.get<flex_string>().length());
      }
    }
  }
}
static void encode_string ( block_info & info , oarchive & oarc , const std :: vector < flexible_type > & data ) { bool use_dictionary_encoding = true ; std :: unordered_map < std :: string , size_t > unique_values ; std :: vector < flexible_type > idx_values ; std :: vector < std :: string > str_values ; idx_values . resize ( data . size ( ) , flexible_type ( flex_type_enum :: INTEGER ) ) ; size_t idxctr = 0 ; for ( size_t i = 0 ; i < data . size ( ) ; ++ i ) { if ( data [ i ] . get_type ( ) != flex_type_enum :: UNDEFINED ) { auto iter = unique_values . find ( data [ i ] . get < std :: string > ( ) ) ; if ( iter != unique_values . end ( ) ) { idx_values [ idxctr ++ ] . mutable_get < flex_int > ( ) = iter -> second ; } else { if ( unique_values . size ( ) >= 64 ) { use_dictionary_encoding = false ; break ; } size_t newidx = unique_values . size ( ) ; unique_values [ data [ i ] . get < std :: string > ( ) ] = newidx ; str_values . push_back ( data [ i ] . get < std :: string > ( ) ) ; idx_values [ idxctr ++ ] . mutable_get < flex_int > ( ) = newidx ; } } } oarc << use_dictionary_encoding ; if ( use_dictionary_encoding ) { idx_values . resize ( idxctr ) ; variable_encode ( oarc , str_values . size ( ) ) ; for ( auto & str : str_values ) { variable_encode ( oarc , str . length ( ) ) ; oarc . write ( str . c_str ( ) , str . length ( ) ) ; } encode_number ( info , oarc , idx_values ) ; } else { idxctr = 0 ; for ( auto & f : data ) { if ( f . get_type ( ) != flex_type_enum :: UNDEFINED ) { idx_values [ idxctr ++ ] . mutable_get < flex_int > ( ) = f . get < std :: string > ( ) . length ( ) ; } } idx_values . resize ( idxctr ) ; encode_number ( info , oarc , idx_values ) ; for ( auto & f : data ) { if ( f . get_type ( ) != flex_type_enum :: UNDEFINED ) { oarc . write ( f . get < std :: string > ( ) . c_str ( ) , f . get < flex_string > ( ) . length ( ) ) ; } } } }
Encodes a collection of strings in data, skipping all UNDEFINED values.
Encodes a collection of strings in data skipping all UNDEFINED values .
// if we have too many unique values, fail. // encode all the lengths 