Skip to content

Commit bb768dc

Browse files
zhyasszhyass
committed
fix test
1 parent 3341cd0 commit bb768dc

File tree

10 files changed

+73
-15
lines changed

10 files changed

+73
-15
lines changed

‎src/query/ee/tests/it/storages/fuse/operations/vacuum.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ async fn test_fuse_do_vacuum_drop_tables() -> Result<()> {
6666
1,
6767
1,
6868
1,
69+
1,
6970
None,
7071
None,
7172
)
@@ -90,6 +91,7 @@ async fn test_fuse_do_vacuum_drop_tables() -> Result<()> {
9091
1,
9192
1,
9293
1,
94+
1,
9395
None,
9496
None,
9597
)
@@ -108,6 +110,7 @@ async fn test_fuse_do_vacuum_drop_tables() -> Result<()> {
108110
0,
109111
0,
110112
0,
113+
0,
111114
None,
112115
None,
113116
)

‎src/query/service/src/test_kits/check.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ use databend_common_meta_app::storage::StorageParams;
2424
use databend_common_storages_fuse::operations::load_last_snapshot_hint;
2525
use databend_common_storages_fuse::FuseTable;
2626
use databend_common_storages_fuse::FUSE_TBL_BLOCK_PREFIX;
27+
use databend_common_storages_fuse::FUSE_TBL_BLOCK_STATISTICS_PREFIX;
2728
use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX;
2829
use databend_common_storages_fuse::FUSE_TBL_SNAPSHOT_PREFIX;
2930
use databend_common_storages_fuse::FUSE_TBL_SNAPSHOT_STATISTICS_PREFIX;
@@ -79,6 +80,7 @@ pub async fn check_data_dir(
7980
segment_count: u32,
8081
block_count: u32,
8182
index_count: u32,
83+
block_stat_count: u32,
8284
check_last_snapshot: Option<()>,
8385
check_table_statistic_file: Option<()>,
8486
) -> Result<()> {
@@ -92,12 +94,14 @@ pub async fn check_data_dir(
9294
let mut sg_count = 0;
9395
let mut b_count = 0;
9496
let mut i_count = 0;
97+
let mut b_stat_count = 0;
9598
let mut table_statistic_files = vec![];
9699
let prefix_snapshot = FUSE_TBL_SNAPSHOT_PREFIX;
97100
let prefix_snapshot_statistics = FUSE_TBL_SNAPSHOT_STATISTICS_PREFIX;
98101
let prefix_segment = FUSE_TBL_SEGMENT_PREFIX;
99102
let prefix_block = FUSE_TBL_BLOCK_PREFIX;
100103
let prefix_index = FUSE_TBL_XOR_BLOOM_INDEX_PREFIX;
104+
let prefix_block_stats = FUSE_TBL_BLOCK_STATISTICS_PREFIX;
101105
for entry in WalkDir::new(root) {
102106
let entry = entry.unwrap();
103107
if entry.file_type().is_file() {
@@ -109,6 +113,8 @@ pub async fn check_data_dir(
109113
ss_count += 1;
110114
} else if path.starts_with(prefix_segment) {
111115
sg_count += 1;
116+
} else if path.starts_with(prefix_block_stats) {
117+
b_stat_count += 1;
112118
} else if path.starts_with(prefix_block) {
113119
b_count += 1;
114120
} else if path.starts_with(prefix_index) {
@@ -142,6 +148,12 @@ pub async fn check_data_dir(
142148
case_name
143149
);
144150

151+
assert_eq!(
152+
b_stat_count, block_stat_count,
153+
"case [{}], check block statistics count",
154+
case_name
155+
);
156+
145157
assert_eq!(
146158
i_count, index_count,
147159
"case [{}], check index count",

‎src/query/service/src/test_kits/fixture.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ use databend_common_sql::plans::CreateTablePlan;
6262
use databend_common_tracing::set_panic_hook;
6363
use databend_common_version::DATABEND_COMMIT_VERSION;
6464
use databend_storages_common_table_meta::meta::TableMetaTimestamps;
65+
use databend_storages_common_table_meta::table::OPT_KEY_APPROX_DISTINCT_COLUMNS;
6566
use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID;
6667
use futures::TryStreamExt;
6768
use jsonb::Number as JsonbNumber;
@@ -374,6 +375,7 @@ impl TestFixture {
374375
options: [
375376
// database id is required for FUSE
376377
(OPT_KEY_DATABASE_ID.to_owned(), "1".to_owned()),
378+
(OPT_KEY_APPROX_DISTINCT_COLUMNS.to_owned(), "all".to_owned()),
377379
]
378380
.into(),
379381
field_comments: vec!["number".to_string(), "tuple".to_string()],
@@ -401,6 +403,7 @@ impl TestFixture {
401403
options: [
402404
// database id is required for FUSE
403405
(OPT_KEY_DATABASE_ID.to_owned(), "1".to_owned()),
406+
(OPT_KEY_APPROX_DISTINCT_COLUMNS.to_owned(), "all".to_owned()),
404407
]
405408
.into(),
406409
field_comments: vec!["number".to_string(), "tuple".to_string()],

‎src/query/service/tests/it/storages/fuse/meta/column_oriented.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ async fn test_segment_cache() -> Result<()> {
405405
)
406406
.await?;
407407
let cached = cache.get(&location).unwrap();
408-
assert_eq!(cached.segment_schema.fields.len(), 10);
408+
assert_eq!(cached.segment_schema.fields.len(), 12);
409409
assert_eq!(cached.segment_schema, segment_schema(&TableSchema::empty()));
410410
check_summary(&block_metas, &cached);
411411
check_block_level_meta(&block_metas, &cached);
@@ -418,7 +418,7 @@ async fn test_segment_cache() -> Result<()> {
418418
let _column_oriented_segment =
419419
read_column_oriented_segment(operator.clone(), &location, &projection, true).await?;
420420
let cached = cache.get(&location).unwrap();
421-
assert_eq!(cached.segment_schema.fields.len(), 12);
421+
assert_eq!(cached.segment_schema.fields.len(), 14);
422422

423423
let column_1 = table_schema.field_of_column_id(col_id).unwrap();
424424
let stat_1 = column_oriented_segment
@@ -442,7 +442,7 @@ async fn test_segment_cache() -> Result<()> {
442442
read_column_oriented_segment(operator.clone(), &location, &projection, true).await?;
443443
let cached = cache.get(&location).unwrap();
444444
// column 2 does not have stats
445-
assert_eq!(cached.segment_schema.fields.len(), 13);
445+
assert_eq!(cached.segment_schema.fields.len(), 15);
446446
check_summary(&block_metas, &cached);
447447
check_block_level_meta(&block_metas, &cached);
448448
check_column_stats_and_meta(&block_metas, &cached, &[1, 2]);
@@ -456,7 +456,7 @@ async fn test_segment_cache() -> Result<()> {
456456
read_column_oriented_segment(operator.clone(), &location, &projection, true).await?;
457457
let cached = cache.get(&location).unwrap();
458458
// column 2 does not have stats
459-
assert_eq!(cached.segment_schema.fields.len(), 13);
459+
assert_eq!(cached.segment_schema.fields.len(), 15);
460460
check_summary(&block_metas, &cached);
461461
check_block_level_meta(&block_metas, &cached);
462462
check_column_stats_and_meta(&block_metas, &cached, &[1, 2]);

‎src/query/service/tests/it/storages/fuse/operations/analyze.rs‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ async fn test_fuse_snapshot_analyze() -> Result<()> {
3232
do_insertions(&fixture).await?;
3333

3434
analyze_table(&fixture).await?;
35-
check_data_dir(&fixture, case_name, 3, 1, 2, 2, 2, Some(()), None).await?;
35+
check_data_dir(&fixture, case_name, 3, 1, 2, 2, 2, 2, Some(()), None).await?;
3636

3737
// Purge will keep at least two snapshots.
3838
let table = fixture.latest_default_table().await?;
@@ -42,7 +42,7 @@ async fn test_fuse_snapshot_analyze() -> Result<()> {
4242
fuse_table
4343
.do_purge(&table_ctx, snapshot_files, None, true, false)
4444
.await?;
45-
check_data_dir(&fixture, case_name, 1, 1, 1, 1, 1, Some(()), Some(())).await?;
45+
check_data_dir(&fixture, case_name, 1, 1, 1, 1, 1, 1, Some(()), Some(())).await?;
4646

4747
Ok(())
4848
}
@@ -65,7 +65,7 @@ async fn test_fuse_snapshot_analyze_and_truncate() -> Result<()> {
6565

6666
fixture.execute_command(&qry).await?;
6767

68-
check_data_dir(&fixture, case_name, 3, 1, 2, 2, 2, None, Some(())).await?;
68+
check_data_dir(&fixture, case_name, 3, 1, 2, 2, 2, 2, None, Some(())).await?;
6969
}
7070

7171
// truncate table
@@ -103,7 +103,19 @@ async fn test_fuse_snapshot_analyze_purge() -> Result<()> {
103103
// optimize statistics three times
104104
for i in 0..3 {
105105
analyze_table(&fixture).await?;
106-
check_data_dir(&fixture, case_name, 3 + i, 1 + i, 2, 2, 2, Some(()), None).await?;
106+
check_data_dir(
107+
&fixture,
108+
case_name,
109+
3 + i,
110+
1 + i,
111+
2,
112+
2,
113+
2,
114+
2,
115+
Some(()),
116+
None,
117+
)
118+
.await?;
107119
}
108120

109121
// Purge will keep at least two snapshots.
@@ -114,7 +126,7 @@ async fn test_fuse_snapshot_analyze_purge() -> Result<()> {
114126
fuse_table
115127
.do_purge(&table_ctx, snapshot_files, None, true, false)
116128
.await?;
117-
check_data_dir(&fixture, case_name, 1, 1, 1, 1, 1, Some(()), Some(())).await?;
129+
check_data_dir(&fixture, case_name, 1, 1, 1, 1, 1, 1, Some(()), Some(())).await?;
118130

119131
Ok(())
120132
}

‎src/query/service/tests/it/storages/fuse/operations/gc.rs‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ async fn test_fuse_purge_normal_case() -> Result<()> {
5959
1, // 1 segments
6060
1, // 1 blocks
6161
1, // 1 index
62+
1, // 1 block statistic
6263
Some(()),
6364
None,
6465
)
@@ -118,9 +119,10 @@ async fn test_fuse_purge_normal_orphan_snapshot() -> Result<()> {
118119
"do_gc: there should be 1 snapshot, 0 segment/block",
119120
expected_num_of_snapshot,
120121
0, // 0 snapshot statistic
121-
1, // 0 segments
122-
1, // 0 blocks
123-
1, // 0 index
122+
1, // 1 segments
123+
1, // 1 blocks
124+
1, // 1 index
125+
1, // 1 block statistic
124126
Some(()),
125127
None,
126128
)
@@ -250,6 +252,7 @@ async fn test_fuse_purge_orphan_retention() -> Result<()> {
250252
let expected_num_of_segment = 3;
251253
let expected_num_of_blocks = 3;
252254
let expected_num_of_index = expected_num_of_blocks;
255+
let expected_num_of_block_stats = expected_num_of_blocks;
253256
check_data_dir(
254257
&fixture,
255258
"do_gc: verify retention period",
@@ -258,6 +261,7 @@ async fn test_fuse_purge_orphan_retention() -> Result<()> {
258261
expected_num_of_segment,
259262
expected_num_of_blocks,
260263
expected_num_of_index,
264+
expected_num_of_block_stats,
261265
Some(()),
262266
None,
263267
)
@@ -296,6 +300,7 @@ async fn test_fuse_purge_older_version() -> Result<()> {
296300
let expected_num_of_segment = 3;
297301
let expected_num_of_blocks = 6;
298302
let expected_num_of_index = expected_num_of_blocks;
303+
let expected_num_of_block_stats = expected_num_of_blocks;
299304
check_data_dir(
300305
&fixture,
301306
"do_gc: navigate to time point",
@@ -304,6 +309,7 @@ async fn test_fuse_purge_older_version() -> Result<()> {
304309
expected_num_of_segment,
305310
expected_num_of_blocks,
306311
expected_num_of_index,
312+
expected_num_of_block_stats,
307313
Some(()),
308314
None,
309315
)
@@ -317,7 +323,7 @@ async fn test_fuse_purge_older_version() -> Result<()> {
317323
{
318324
let table = fixture.latest_default_table().await?;
319325
compact_segment(ctx.clone(), &table).await?;
320-
check_data_dir(&fixture, "", 4, 0, 5, 7, 7, Some(()), None).await?;
326+
check_data_dir(&fixture, "", 4, 0, 5, 7, 7, 7, Some(()), None).await?;
321327
}
322328

323329
let table = fixture.latest_default_table().await?;
@@ -333,6 +339,7 @@ async fn test_fuse_purge_older_version() -> Result<()> {
333339
let expected_num_of_segment = 1;
334340
let expected_num_of_blocks = 7;
335341
let expected_num_of_index = expected_num_of_blocks;
342+
let expected_num_of_block_stats = expected_num_of_blocks;
336343
check_data_dir(
337344
&fixture,
338345
"do_gc: with older version",
@@ -341,6 +348,7 @@ async fn test_fuse_purge_older_version() -> Result<()> {
341348
expected_num_of_segment,
342349
expected_num_of_blocks,
343350
expected_num_of_index,
351+
expected_num_of_block_stats,
344352
Some(()),
345353
None,
346354
)
@@ -357,6 +365,7 @@ async fn test_fuse_purge_older_version() -> Result<()> {
357365
let expected_num_of_segment = 0;
358366
let expected_num_of_blocks = 0;
359367
let expected_num_of_index = expected_num_of_blocks;
368+
let expected_num_of_block_stats = expected_num_of_blocks;
360369
check_data_dir(
361370
&fixture,
362371
"do_gc: purge last snapshot",
@@ -365,6 +374,7 @@ async fn test_fuse_purge_older_version() -> Result<()> {
365374
expected_num_of_segment,
366375
expected_num_of_blocks,
367376
expected_num_of_index,
377+
expected_num_of_block_stats,
368378
Some(()),
369379
None,
370380
)

‎src/query/service/tests/it/storages/fuse/operations/optimize.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,12 @@ use crate::storages::fuse::utils::do_purge_test;
2424

2525
#[tokio::test(flavor = "multi_thread")]
2626
async fn test_fuse_snapshot_optimize_purge() -> Result<()> {
27-
do_purge_test("test_fuse_snapshot_optimize_purge", 1, 0, 1, 1, 1).await
27+
do_purge_test("test_fuse_snapshot_optimize_purge", 1, 0, 1, 1, 1, 1).await
2828
}
2929

3030
#[tokio::test(flavor = "multi_thread")]
3131
async fn test_fuse_snapshot_optimize_all() -> Result<()> {
32-
do_purge_test("test_fuse_snapshot_optimize_all", 1, 0, 1, 1, 1).await
32+
do_purge_test("test_fuse_snapshot_optimize_all", 1, 0, 1, 1, 1, 1).await
3333
}
3434

3535
#[tokio::test(flavor = "multi_thread")]

‎src/query/service/tests/it/storages/fuse/operations/purge_drop.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,20 @@ async fn test_fuse_snapshot_truncate_in_drop_all_stmt() -> Result<()> {
4646

4747
// ingests some test data
4848
append_sample_data(1, &fixture).await?;
49+
check_data_dir(
50+
&fixture,
51+
"drop table: there should be 1 snapshot, 0 segment/block",
52+
1, // 1 snapshot
53+
0, // 0 snapshot statistic
54+
1, // 0 segments
55+
1, // 0 blocks
56+
1, // 0 index
57+
1, // 0 block statistic
58+
None,
59+
None,
60+
)
61+
.await?;
62+
4963
// let's Drop
5064
let qry = format!("drop table {}.{} all", db, tbl);
5165
fixture.execute_command(qry.as_str()).await?;
@@ -58,6 +72,7 @@ async fn test_fuse_snapshot_truncate_in_drop_all_stmt() -> Result<()> {
5872
0, // 0 segments
5973
0, // 0 blocks
6074
0, // 0 index
75+
0, // 0 block statistic
6176
None,
6277
None,
6378
)

‎src/query/service/tests/it/storages/fuse/utils.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ pub async fn do_purge_test(
5454
segment_count: u32,
5555
block_count: u32,
5656
index_count: u32,
57+
block_stat_count: u32,
5758
) -> Result<()> {
5859
let fixture = TestFixture::setup().await?;
5960
fixture.create_default_database().await?;
@@ -81,6 +82,7 @@ pub async fn do_purge_test(
8182
segment_count,
8283
block_count,
8384
index_count,
85+
block_stat_count,
8486
Some(()),
8587
None,
8688
)

‎src/query/storages/common/table_meta/src/meta/v2/segment.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ pub struct BlockMeta {
177177
pub compression: Compression,
178178

179179
pub block_stats_location: Option<Location>,
180+
#[serde(default)]
180181
pub block_stats_size: u64,
181182

182183
// block create_on

0 commit comments

Comments
 (0)