feat(query): Support fuse_virtual_column function to show virtual column size (#17910)

Author: b41sh

src/query/service/src/table_functions/table_function_factory.rs

@@ -30,6 +30,7 @@ use databend_common_storages_fuse::table_functions::FuseTimeTravelSizeFunc;
 use databend_common_storages_fuse::table_functions::FuseVacuumDropAggregatingIndex;
 use databend_common_storages_fuse::table_functions::FuseVacuumDropInvertedIndex;
 use databend_common_storages_fuse::table_functions::FuseVacuumTemporaryTable;
+use databend_common_storages_fuse::table_functions::FuseVirtualColumnFunc;
 use databend_common_storages_fuse::table_functions::SetCacheCapacity;
 use databend_common_storages_fuse::table_functions::TableFunctionTemplate;
 use databend_common_storages_iceberg::IcebergInspectTable;
@@ -191,6 +192,14 @@ impl TableFunctionFactory {
             ),
         );
 
+        creators.insert(
+            "fuse_virtual_column".to_string(),
+            (
+                next_id(),
+                Arc::new(TableFunctionTemplate::<FuseVirtualColumnFunc>::create),
+            ),
+        );
+
         creators.insert(
             "fuse_statistic".to_string(),
             (

src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs

@@ -21,12 +21,15 @@ use super::ColumnOrientedSegment;
 use crate::meta::BlockMeta;
 use crate::meta::ColumnMeta;
 use crate::meta::ColumnMetaV0;
+use crate::meta::VirtualBlockMeta;
+
 pub trait AbstractBlockMeta: Send + Sync + 'static + Sized {
     fn block_size(&self) -> u64;
     fn file_size(&self) -> u64;
     fn row_count(&self) -> u64;
     fn location_path(&self) -> String;
     fn col_metas(&self, col_ids: &HashSet<ColumnId>) -> HashMap<ColumnId, ColumnMeta>;
+    fn virtual_block_meta(&self) -> Option<VirtualBlockMeta>;
 }
 
 impl AbstractBlockMeta for BlockMeta {
@@ -54,6 +57,10 @@ impl AbstractBlockMeta for BlockMeta {
     fn location_path(&self) -> String {
         self.location.0.to_string()
     }
+
+    fn virtual_block_meta(&self) -> Option<VirtualBlockMeta> {
+        self.virtual_block_meta.clone()
+    }
 }
 
 impl AbstractBlockMeta for ColumnOrientedBlockMeta {
@@ -98,6 +105,11 @@ impl AbstractBlockMeta for ColumnOrientedBlockMeta {
             .unwrap()
             .to_string()
     }
+
+    fn virtual_block_meta(&self) -> Option<VirtualBlockMeta> {
+        // TODO
+        None
+    }
 }
 
 pub struct ColumnOrientedBlockMeta {

src/query/storages/fuse/src/io/write/block_writer.rs

@@ -222,12 +222,14 @@ impl BlockBuilder {
                 .as_ref()
                 .map(|v| v.size)
                 .unwrap_or_default(),
+            ngram_filter_index_size: bloom_index_state
+                .as_ref()
+                .map(|v| v.ngram_size)
+                .unwrap_or_default(),
             compression: self.write_settings.table_compression.into(),
             inverted_index_size,
             virtual_block_meta: None,
             create_on: Some(Utc::now()),
-            // TODO(kould): ngram index
-            ngram_filter_index_size: None,
         };
 
         let serialized = BlockSerialization {

src/query/storages/fuse/src/io/write/bloom_index_writer.rs

@@ -43,13 +43,33 @@ use crate::FuseStorageFormat;
 pub struct BloomIndexState {
     pub(crate) data: Vec<u8>,
     pub(crate) size: u64,
+    pub(crate) ngram_size: Option<u64>,
     pub(crate) location: Location,
     pub(crate) column_distinct_count: HashMap<ColumnId, usize>,
 }
 
 impl BloomIndexState {
     pub fn from_bloom_index(bloom_index: &BloomIndex, location: Location) -> Result<Self> {
         let index_block = bloom_index.serialize_to_data_block()?;
+        // Calculate ngram index size
+        let ngram_indexes = &bloom_index
+            .filter_schema
+            .fields()
+            .iter()
+            .enumerate()
+            .filter(|(_, f)| f.name.starts_with("Ngram"))
+            .map(|(i, _)| i)
+            .collect::<Vec<_>>();
+        let ngram_size = if !ngram_indexes.is_empty() {
+            let mut ngram_size = 0;
+            for i in ngram_indexes {
+                let column = index_block.get_by_offset(*i);
+                ngram_size += column.value.memory_size() as u64;
+            }
+            Some(ngram_size)
+        } else {
+            None
+        };
         let mut data = Vec::with_capacity(DEFAULT_BLOCK_INDEX_BUFFER_SIZE);
         let _ = blocks_to_parquet(
             &bloom_index.filter_schema,
@@ -61,6 +81,7 @@ impl BloomIndexState {
         Ok(Self {
             data,
             size: data_size,
+            ngram_size,
             location,
             column_distinct_count: bloom_index.column_distinct_count.clone(),
         })

src/query/storages/fuse/src/table_functions/fuse_block.rs

@@ -65,6 +65,14 @@ impl TableMetaFunc for FuseBlock {
                 "inverted_index_size",
                 TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))),
             ),
+            TableField::new(
+                "ngram_index_size",
+                TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))),
+            ),
+            TableField::new(
+                "virtual_column_size",
+                TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))),
+            ),
         ])
     }
 
@@ -86,6 +94,8 @@ impl TableMetaFunc for FuseBlock {
         let mut bloom_filter_location = vec![];
         let mut bloom_filter_size = Vec::with_capacity(len);
         let mut inverted_index_size = Vec::with_capacity(len);
+        let mut ngram_index_size = Vec::with_capacity(len);
+        let mut virtual_column_size = Vec::with_capacity(len);
 
         let segments_io = SegmentsIO::create(ctx.clone(), tbl.operator.clone(), tbl.schema());
 
@@ -113,6 +123,13 @@ impl TableMetaFunc for FuseBlock {
                     );
                     bloom_filter_size.push(block.bloom_filter_index_size);
                     inverted_index_size.push(block.inverted_index_size);
+                    ngram_index_size.push(block.ngram_filter_index_size);
+                    virtual_column_size.push(
+                        block
+                            .virtual_block_meta
+                            .as_ref()
+                            .map(|m| m.virtual_column_size),
+                    );
 
                     row_num += 1;
                     if row_num >= limit {
@@ -157,6 +174,14 @@ impl TableMetaFunc for FuseBlock {
                     DataType::Nullable(Box::new(DataType::Number(NumberDataType::UInt64))),
                     Value::Column(UInt64Type::from_opt_data(inverted_index_size)),
                 ),
+                BlockEntry::new(
+                    DataType::Nullable(Box::new(DataType::Number(NumberDataType::UInt64))),
+                    Value::Column(UInt64Type::from_opt_data(ngram_index_size)),
+                ),
+                BlockEntry::new(
+                    DataType::Nullable(Box::new(DataType::Number(NumberDataType::UInt64))),
+                    Value::Column(UInt64Type::from_opt_data(virtual_column_size)),
+                ),
             ],
             row_num,
         ))

src/query/storages/fuse/src/table_functions/fuse_segment.rs

@@ -57,6 +57,11 @@ impl TableMetaFunc for FuseSegment {
                 "bytes_compressed",
                 TableDataType::Number(NumberDataType::UInt64),
             ),
+            TableField::new("index_size", TableDataType::Number(NumberDataType::UInt64)),
+            TableField::new(
+                "virtual_block_count",
+                TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))),
+            ),
         ])
     }
 
@@ -75,6 +80,8 @@ impl TableMetaFunc for FuseSegment {
         let mut row_count: Vec<u64> = Vec::with_capacity(len);
         let mut compressed: Vec<u64> = Vec::with_capacity(len);
         let mut uncompressed: Vec<u64> = Vec::with_capacity(len);
+        let mut index_size: Vec<u64> = Vec::with_capacity(len);
+        let mut virtual_block_count: Vec<Option<u64>> = Vec::with_capacity(len);
         let mut file_location: Vec<String> = Vec::with_capacity(len);
 
         let segments_io = SegmentsIO::create(ctx.clone(), tbl.operator.clone(), tbl.schema());
@@ -95,6 +102,8 @@ impl TableMetaFunc for FuseSegment {
                 row_count.push(segment.summary.row_count);
                 compressed.push(segment.summary.compressed_byte_size);
                 uncompressed.push(segment.summary.uncompressed_byte_size);
+                index_size.push(segment.summary.index_size);
+                virtual_block_count.push(segment.summary.virtual_block_count);
                 file_location.push(segment_locations[idx].0.clone());
 
                 row_num += 1;
@@ -116,6 +125,8 @@ impl TableMetaFunc for FuseSegment {
             UInt64Type::from_data(row_count),
             UInt64Type::from_data(uncompressed),
             UInt64Type::from_data(compressed),
+            UInt64Type::from_data(index_size),
+            UInt64Type::from_opt_data(virtual_block_count),
         ]))
     }
 }