Add datablock column to xor bitmap unit test
// See the License for the specific language governing permissions and
// limitations under the License.
+use common_datablocks::DataBlock;
+use common_datavalues::prelude::*;
+use common_datavalues::DataField;
+use common_datavalues::DataSchemaRefExt;
+use common_datavalues::Series;
+use common_datavalues::SeriesFrom;
use common_exception::Result;
use common_storages_index::bloom::Bloom;
use common_storages_index::bloom::XorBloom;
@@ -166,3 +172,39 @@ fn test_xor_bitmap_duplicate_string() -> Result<()> {
Ok(())
}
+
+#[test]
+fn test_xor_bitmap_data_block() -> Result<()> {
+ let seed: u64 = random();
+ let numbers = 1_000_000;
+
+ let size = 8 * numbers;
+ let mut rng = StdRng::seed_from_u64(seed);
+ let keys: Vec<i64> = (0..numbers).map(|_| rng.gen::<i64>()).collect();
+
+ let schema = DataSchemaRefExt::create(vec![DataField::new("a", i64::to_data_type())]);
+ let block = DataBlock::create(schema, vec![Series::from_data(keys)]);
+ let column = block.try_column_by_name("a")?;
+
+ let mut filter = XorBloom::create();
+ filter.add_keys(&column.to_values());
+ filter.build()?;
+
+ for key in column.to_values() {
+ assert!(filter.contains(&key), "key {} not present", key);
+ }
+
+ let val = filter.to_bytes()?;
+ let (_, n) = XorBloom::from_bytes(&val)?;
+ assert_eq!(n, val.len(), "{} {}", n, val.len());
+
+ // data block enc:1230069, raw:8000000, ratio:0.15375863
+ println!(
+ "data block enc:{}, raw:{}, ratio:{}",
+ val.len(),
+ size,
+ val.len() as f32 / size as f32
+ );
+
+ Ok(())
+}
