unit test for reduce_blocK_meta
@@ -18,12 +18,15 @@ use common_base::base::tokio;
use common_datablocks::DataBlock;
use common_datavalues::prelude::*;
use common_functions::aggregates::eval_aggr;
+use common_fuse_meta::meta::BlockMeta;
use common_fuse_meta::meta::ClusterStatistics;
use common_fuse_meta::meta::ColumnStatistics;
+use common_fuse_meta::meta::Statistics;
use common_legacy_expression::add;
use common_legacy_expression::col;
use common_legacy_expression::lit;
use common_pipeline_transforms::processors::ExpressionExecutor;
+use common_storages_fuse::statistics::reducers::reduce_block_metas;
use common_storages_fuse::statistics::Trim;
use common_storages_fuse::statistics::STATS_REPLACEMENT_CHAR;
use common_storages_fuse::statistics::STATS_STRING_PREFIX_LEN;
@@ -286,7 +289,7 @@ fn test_ft_stats_block_stats_string_columns_trimming() -> common_exception::Resu
Ok(())
};
- // let runs = 0..1000; // use this at home
+ // let runs = 0..1000; // use this setting at home
let runs = 0..100;
for _ in runs {
suite()?
@@ -406,3 +409,53 @@ fn char_len(value: &[u8]) -> usize {
.chars()
.count()
}
+
+#[test]
+fn test_reduce_block_meta() -> common_exception::Result<()> {
+ // case 1: empty input should return the default statistics
+ let block_metas: Vec<BlockMeta> = vec![];
+ let reduced = reduce_block_metas(&block_metas)?;
+ assert_eq!(Statistics::default(), reduced);
+
+ // case 2: accumulated variants of size index should be as expected
+ // reduction of ColumnStatistics, ClusterStatistics already covered by other cases
+ let mut rng = rand::thread_rng();
+ let size = 100_u64;
+ let location = ("".to_owned(), 0);
+ let mut blocks = vec![];
+ let mut acc_row_count = 0;
+ let mut acc_block_size = 0;
+ let mut acc_file_size = 0;
+ let mut acc_bloom_filter_index_size = 0;
+ for _ in 0..size {
+ let row_count = rng.gen::<u64>() / size;
+ let block_size = rng.gen::<u64>() / size;
+ let file_size = rng.gen::<u64>() / size;
+ let bloom_filter_index_size = rng.gen::<u64>() / size;
+ acc_row_count += row_count;
+ acc_block_size += block_size;
+ acc_file_size += file_size;
+ acc_bloom_filter_index_size += bloom_filter_index_size;
+ let block_meta = BlockMeta::new(
+ row_count,
+ block_size,
+ file_size,
+ HashMap::new(),
+ HashMap::new(),
+ None,
+ location.clone(),
+ None,
+ bloom_filter_index_size,
+ );
+ blocks.push(block_meta);
+ }
+
+ let stats = reduce_block_metas(&blocks)?;
+
+ assert_eq!(acc_row_count, stats.row_count);
+ assert_eq!(acc_block_size, stats.uncompressed_byte_size);
+ assert_eq!(acc_file_size, stats.compressed_byte_size);
+ assert_eq!(acc_bloom_filter_index_size, stats.index_size);
+
+ Ok(())
+}
