remove null in bloom index builder

Author: zhyass

src/query/storages/common/index/src/bloom_index.rs

@@ -398,11 +398,8 @@ impl BloomIndex {
             (column, None)
         };
 
-        let capacity = validity.map_or(column.len(), |v| v.true_count() + 1);
+        let capacity = validity.map_or(column.len(), |v| v.true_count());
         let mut result = Vec::with_capacity(capacity);
-        if validity.is_some() {
-            result.push(0);
-        }
         let column = T::try_downcast_column(column).unwrap();
         if let Some(validity) = validity {
             let column_iter = T::iter_column(&column);

src/query/storages/fuse/src/io/write/stream/block_builder.rs

@@ -308,7 +308,7 @@ impl StreamBlockBuilder {
         } else {
             None
         };
-        let bloom_distinct_count = bloom_index_state
+        let mut column_distinct_count = bloom_index_state
             .as_ref()
             .map(|i| i.column_distinct_count.clone())
             .unwrap_or_default();
@@ -317,12 +317,10 @@ impl StreamBlockBuilder {
             .meta_locations
             .block_stats_location(&block_id);
         let block_stats_state = self.block_stats_builder.finalize(block_stats_location)?;
-        let hll_distinct_count = block_stats_state
-            .as_ref()
-            .map_or(HashMap::new(), |i| i.column_distinct_count.clone());
-        let col_stats = self
-            .column_stats_state
-            .finalize(bloom_distinct_count, hll_distinct_count)?;
+        if let Some(state) = &block_stats_state {
+            column_distinct_count.extend(state.column_distinct_count.clone());
+        }
+        let col_stats = self.column_stats_state.finalize(column_distinct_count)?;
 
         let mut inverted_index_states = Vec::with_capacity(self.inverted_index_writers.len());
         for (i, inverted_index_writer) in std::mem::take(&mut self.inverted_index_writers)

src/query/storages/fuse/src/io/write/stream/column_statistics_state.rs

@@ -88,29 +88,17 @@ impl ColumnStatisticsState {
 
     pub fn finalize(
         self,
-        bloom_distinct_count: HashMap<ColumnId, usize>,
-        hll_distinct_count: HashMap<ColumnId, usize>,
+        mut column_distinct_count: HashMap<ColumnId, usize>,
     ) -> Result<StatisticsOfColumns> {
-        let mut distinct_count =
-            HashMap::with_capacity(self.distinct_columns.len() + hll_distinct_count.len());
         for (column_id, estimator) in &self.distinct_columns {
-            distinct_count.insert(*column_id, estimator.finalize());
+            column_distinct_count.insert(*column_id, estimator.finalize());
         }
-        distinct_count.extend(hll_distinct_count);
 
         let mut statistics = StatisticsOfColumns::with_capacity(self.col_stats.len());
         for (id, stats) in self.col_stats {
             let mut col_stats = stats.finalize()?;
-            if let Some(count) = distinct_count.get(&id) {
+            if let Some(count) = column_distinct_count.get(&id) {
                 col_stats.distinct_of_values = Some(*count as u64);
-            } else if let Some(&count) = bloom_distinct_count.get(&id) {
-                // value calculated by xor hash function include NULL, need to subtract one.
-                let distinct_of_values = if col_stats.null_count > 0 && count > 0 {
-                    count as u64 - 1
-                } else {
-                    count as u64
-                };
-                col_stats.distinct_of_values = Some(distinct_of_values);
             } else if col_stats.min == col_stats.max {
                 // Bloom index will skip the large string column, it also no need to calc distinct values.
                 if col_stats.min.is_null() {
@@ -188,7 +176,7 @@ mod tests {
         }
         let mut column_stats_state = ColumnStatisticsState::new(&stats_columns, &stats_columns);
         column_stats_state.add_block(&schema, &block)?;
-        let stats_1 = column_stats_state.finalize(HashMap::new(), HashMap::new())?;
+        let stats_1 = column_stats_state.finalize(HashMap::new())?;
 
         assert_eq!(stats_0, stats_1);
         Ok(())

src/query/storages/fuse/src/statistics/column_statistic.rs

@@ -124,12 +124,7 @@ pub fn gen_columns_statistics(
                     .as_ref()
                     .and_then(|v| v.get(&column_id))
                 {
-                    // value calculated by xor hash function include NULL, need to subtract one.
-                    if unset_bits > 0 && value > 0 {
-                        value as u64 - 1
-                    } else {
-                        value as u64
-                    }
+                    value as u64
                 } else {
                     calc_column_distinct_of_values(&col, rows)?
                 };