fix: ensure values are different
@@ -273,11 +273,31 @@ impl SeriesSetConverter {
None
};
+ let col_values = col.values();
+ let values = col_values
+ .as_any()
+ .downcast_ref::<StringArray>()
+ .expect("Casting values column failed");
+
let mut current_val = get_key(0);
for row in 1..num_rows {
let next_val = get_key(row);
if next_val != current_val {
- debug!(?current_val, ?next_val, ?col_idx, ?row, "value transition");
+ //
+ // N.B, concatenating two Arrow dictionary arrays can
+ // result in duplicate values with differing keys.
+ // Therefore, when keys differ we should verify they are
+ // encoding different values. See:
+ // https://github.com/apache/arrow-rs/pull/15
+ //
+ if let (Some(curr), Some(next)) = (current_val, next_val) {
+ if values.value(curr as usize) == values.value(next as usize) {
+ // these values are different even though they
+ // have the same encoded key.
+ continue;
+ }
+ }
+
bitmap.add(row as u32);
current_val = next_val;
}
