diff --git a/tsl/test/expected/direct_compress_insert.out b/tsl/test/expected/direct_compress_insert.out index 4b3e13c9214..01aa8281dff 100644 --- a/tsl/test/expected/direct_compress_insert.out +++ b/tsl/test/expected/direct_compress_insert.out @@ -64,19 +64,16 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks ROLLBACK; -- simple test with compressed insert enabled and reversed order BEGIN; -INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz - (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- - Append (actual rows=3001.00 loops=1) - -> Custom Scan (ColumnarScan) on _hyper_1_7_chunk (actual rows=960.00 loops=1) - -> Seq Scan on compress_hyper_2_8_chunk (actual rows=1.00 loops=1) - -> Custom Scan (ColumnarScan) on _hyper_1_9_chunk (actual rows=2041.00 loops=1) - -> Seq Scan on compress_hyper_2_10_chunk (actual rows=3.00 loops=1) + Custom Scan (ColumnarScan) on _hyper_1_7_chunk (actual rows=3001.00 loops=1) + -> Seq Scan on compress_hyper_2_8_chunk (actual rows=4.00 loops=1) SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub; first | last ------------------------------+------------------------------ - Wed Jan 01 00:00:00 2025 PST | Fri Jan 03 02:00:00 2025 PST + Sun Dec 29 22:00:00 2024 PST | Wed Jan 01 00:00:00 2025 PST -- since the chunks are new status should be COMPRESSED, UNORDERED SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; @@ -92,10 +89,10 @@ INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interva EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- Append (actual rows=3001.00 loops=1) - -> Custom Scan (ColumnarScan) on _hyper_1_11_chunk (actual rows=960.00 loops=1) - -> Seq Scan on compress_hyper_2_12_chunk (actual rows=1.00 loops=1) - -> Custom Scan (ColumnarScan) on _hyper_1_13_chunk (actual rows=2041.00 loops=1) - -> Seq Scan on compress_hyper_2_14_chunk (actual rows=3.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_9_chunk (actual rows=960.00 loops=1) + -> Seq Scan on compress_hyper_2_10_chunk (actual rows=1.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_11_chunk (actual rows=2041.00 loops=1) + -> Seq Scan on compress_hyper_2_12_chunk (actual rows=3.00 loops=1) SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub; first | last @@ -114,8 +111,8 @@ BEGIN; INSERT INTO metrics SELECT '2025-01-01'::timestamptz - (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- - Custom Scan (ColumnarScan) on _hyper_1_15_chunk (actual rows=3001.00 loops=1) - -> Seq Scan on compress_hyper_2_16_chunk (actual rows=4.00 loops=1) + Custom Scan (ColumnarScan) on _hyper_1_13_chunk (actual rows=3001.00 loops=1) + -> Seq Scan on compress_hyper_2_14_chunk (actual rows=4.00 loops=1) SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub; first | last @@ -141,12 +138,12 @@ INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interva EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- Append (actual rows=6002.00 loops=1) - -> Custom Scan (ColumnarScan) on _hyper_1_17_chunk (actual rows=960.00 loops=1) - -> Seq Scan on compress_hyper_2_19_chunk (actual rows=1.00 loops=1) - -> Seq Scan on _hyper_1_17_chunk (actual rows=960.00 loops=1) - -> Custom Scan (ColumnarScan) on _hyper_1_18_chunk (actual rows=2041.00 loops=1) - -> Seq Scan on compress_hyper_2_20_chunk (actual rows=3.00 loops=1) - -> Seq Scan on _hyper_1_18_chunk (actual rows=2041.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_15_chunk (actual rows=960.00 loops=1) + -> Seq Scan on compress_hyper_2_17_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_1_15_chunk (actual rows=960.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_16_chunk (actual rows=2041.00 loops=1) + -> Seq Scan on compress_hyper_2_18_chunk (actual rows=3.00 loops=1) + -> Seq Scan on _hyper_1_16_chunk (actual rows=2041.00 loops=1) -- since the chunks are new status should be COMPRESSED, PARTIAL SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; @@ -162,8 +159,8 @@ SET timescaledb.enable_direct_compress_insert_client_sorted = true; INSERT INTO metrics SELECT '2025-01-01'::timestamptz - (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- - Custom Scan (ColumnarScan) on _hyper_1_21_chunk (actual rows=3001.00 loops=1) - -> Seq Scan on compress_hyper_2_22_chunk (actual rows=4.00 loops=1) + Custom Scan (ColumnarScan) on _hyper_1_19_chunk (actual rows=3001.00 loops=1) + -> Seq Scan on compress_hyper_2_20_chunk (actual rows=4.00 loops=1) -- since the chunks are new status should be COMPRESSED SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; @@ -179,8 +176,8 @@ SET timescaledb.enable_direct_compress_insert_client_sorted = false; INSERT INTO metrics SELECT '2025-01-01'::timestamptz - (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- - Custom Scan (ColumnarScan) on _hyper_1_23_chunk (actual rows=3001.00 loops=1) - -> Seq Scan on compress_hyper_2_24_chunk (actual rows=4.00 loops=1) + Custom Scan (ColumnarScan) on _hyper_1_21_chunk (actual rows=3001.00 loops=1) + -> Seq Scan on compress_hyper_2_22_chunk (actual rows=4.00 loops=1) -- since the chunks are new status should be COMPRESSED, UNORDERED SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; @@ -199,9 +196,9 @@ INSERT INTO metrics SELECT '2025-01-01'::timestamptz - (i || ' minute')::interva EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- Append (actual rows=3002.00 loops=1) - -> Custom Scan (ColumnarScan) on _hyper_1_25_chunk (actual rows=3001.00 loops=1) - -> Seq Scan on compress_hyper_2_26_chunk (actual rows=4.00 loops=1) - -> Seq Scan on _hyper_1_25_chunk (actual rows=1.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_23_chunk (actual rows=3001.00 loops=1) + -> Seq Scan on compress_hyper_2_24_chunk (actual rows=4.00 loops=1) + -> Seq Scan on _hyper_1_23_chunk (actual rows=1.00 loops=1) -- since the chunks are new status should be COMPRESSED, UNORDERED, PARTIAL SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; @@ -209,6 +206,92 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks -------------------------------- {COMPRESSED,UNORDERED,PARTIAL} +ROLLBACK; +-- simple test with compressed insert enabled and no presorted with partial and compressed chunks +BEGIN; +SET timescaledb.enable_direct_compress_insert = true; +SET timescaledb.enable_direct_compress_insert_client_sorted = false; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz, 'd1', 0; +WARNING: disabling direct compress because of too small batch size +INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-02'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; +--- QUERY PLAN --- + Append (actual rows=6003.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_25_chunk (actual rows=960.00 loops=1) + -> Seq Scan on compress_hyper_2_26_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_1_25_chunk (actual rows=1.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_27_chunk (actual rows=5042.00 loops=1) + -> Seq Scan on compress_hyper_2_28_chunk (actual rows=7.00 loops=1) + +SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub; + first | last +------------------------------+------------------------------ + Wed Jan 01 00:00:00 2025 PST | Sat Jan 04 02:00:00 2025 PST + +SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL order by 1 desc limit 1 \gset +-- should see overlapping batches +select _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 from :COMPRESSED_CHUNK order by 2; + _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +----------------+------------------------------+------------------------------ + 1000 | Wed Jan 01 16:00:00 2025 PST | Thu Jan 02 08:39:00 2025 PST + 1000 | Thu Jan 02 00:00:00 2025 PST | Thu Jan 02 16:39:00 2025 PST + 1000 | Thu Jan 02 08:40:00 2025 PST | Fri Jan 03 01:19:00 2025 PST + 1000 | Thu Jan 02 16:40:00 2025 PST | Fri Jan 03 09:19:00 2025 PST + 41 | Fri Jan 03 01:20:00 2025 PST | Fri Jan 03 02:00:00 2025 PST + 1000 | Fri Jan 03 09:20:00 2025 PST | Sat Jan 04 01:59:00 2025 PST + 1 | Sat Jan 04 02:00:00 2025 PST | Sat Jan 04 02:00:00 2025 PST + +-- since the chunks are new status should be COMPRESSED, UNORDERED, PARTIAL and COMPRESSED, UNORDERED +SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk order by 1; + chunk_status_text +-------------------------------- + {COMPRESSED,UNORDERED} + {COMPRESSED,UNORDERED,PARTIAL} + +ROLLBACK; +-- simple test with compressed insert enabled and presorted with partial and compressed chunks +BEGIN; +SET timescaledb.enable_direct_compress_insert = true; +SET timescaledb.enable_direct_compress_insert_client_sorted = true; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz, 'd1', 0; +WARNING: disabling direct compress because of too small batch size +INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-05'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; +--- QUERY PLAN --- + Append (actual rows=6003.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_29_chunk (actual rows=960.00 loops=1) + -> Seq Scan on compress_hyper_2_30_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_1_29_chunk (actual rows=1.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_31_chunk (actual rows=5042.00 loops=1) + -> Seq Scan on compress_hyper_2_32_chunk (actual rows=7.00 loops=1) + +SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub; + first | last +------------------------------+------------------------------ + Wed Jan 01 00:00:00 2025 PST | Tue Jan 07 02:00:00 2025 PST + +SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL order by 1 desc limit 1 \gset +-- should not see overlapping batches +select _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 from :COMPRESSED_CHUNK order by 2; + _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +----------------+------------------------------+------------------------------ + 1000 | Wed Jan 01 16:00:00 2025 PST | Thu Jan 02 08:39:00 2025 PST + 1000 | Thu Jan 02 08:40:00 2025 PST | Fri Jan 03 01:19:00 2025 PST + 41 | Fri Jan 03 01:20:00 2025 PST | Fri Jan 03 02:00:00 2025 PST + 1000 | Sun Jan 05 00:00:00 2025 PST | Sun Jan 05 16:39:00 2025 PST + 1000 | Sun Jan 05 16:40:00 2025 PST | Mon Jan 06 09:19:00 2025 PST + 1000 | Mon Jan 06 09:20:00 2025 PST | Tue Jan 07 01:59:00 2025 PST + 1 | Tue Jan 07 02:00:00 2025 PST | Tue Jan 07 02:00:00 2025 PST + +-- since the chunks are new status should be COMPRESSED, UNORDERED, PARTIAL and COMPRESSED, UNORDERED +SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk order by 1; + chunk_status_text +---------------------- + {COMPRESSED} + {COMPRESSED,PARTIAL} + ROLLBACK; -- test with segmentby BEGIN; @@ -218,8 +301,8 @@ SET timescaledb.enable_direct_compress_insert_client_sorted = true; INSERT INTO metrics SELECT '2025-01-01'::timestamptz - (i || ' minute')::interval, floor(i), i::float FROM generate_series(0.0,9.8,0.2) i; EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- - Custom Scan (ColumnarScan) on _hyper_1_27_chunk (actual rows=50.00 loops=1) - -> Seq Scan on compress_hyper_2_28_chunk (actual rows=10.00 loops=1) + Custom Scan (ColumnarScan) on _hyper_1_33_chunk (actual rows=50.00 loops=1) + -> Seq Scan on compress_hyper_2_34_chunk (actual rows=10.00 loops=1) SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL \gset -- should have 10 batches @@ -234,6 +317,74 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks ------------------- {COMPRESSED} +ROLLBACK; +-- segmentby with overlapping batches +BEGIN; +ALTER TABLE metrics SET (tsdb.segmentby = 'device'); +SET timescaledb.enable_direct_compress_insert = true; +SET timescaledb.enable_direct_compress_insert_client_sorted = false; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd'||i%2, i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-02'::timestamptz + (i || ' minute')::interval, 'd'||i%2, i::float FROM generate_series(0,3000) i; +EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; +--- QUERY PLAN --- + Append (actual rows=6002.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_35_chunk (actual rows=960.00 loops=1) + -> Seq Scan on compress_hyper_2_36_chunk (actual rows=2.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_37_chunk (actual rows=5042.00 loops=1) + -> Seq Scan on compress_hyper_2_38_chunk (actual rows=8.00 loops=1) + +SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL order by 1 desc limit 1 \gset +-- should see overlapping batches per device +select _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1, device from :COMPRESSED_CHUNK order by 4, 2; + _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 | device +----------------+------------------------------+------------------------------+-------- + 1000 | Wed Jan 01 16:00:00 2025 PST | Fri Jan 03 01:18:00 2025 PST | d0 + 1000 | Thu Jan 02 00:00:00 2025 PST | Fri Jan 03 09:18:00 2025 PST | d0 + 21 | Fri Jan 03 01:20:00 2025 PST | Fri Jan 03 02:00:00 2025 PST | d0 + 501 | Fri Jan 03 09:20:00 2025 PST | Sat Jan 04 02:00:00 2025 PST | d0 + 1000 | Wed Jan 01 16:01:00 2025 PST | Fri Jan 03 01:19:00 2025 PST | d1 + 1000 | Thu Jan 02 00:01:00 2025 PST | Fri Jan 03 09:19:00 2025 PST | d1 + 20 | Fri Jan 03 01:21:00 2025 PST | Fri Jan 03 01:59:00 2025 PST | d1 + 500 | Fri Jan 03 09:21:00 2025 PST | Sat Jan 04 01:59:00 2025 PST | d1 + +-- since the chunks are new status should be COMPRESSED, UNORDERED +SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; + chunk_status_text +------------------------ + {COMPRESSED,UNORDERED} + +ROLLBACK; +-- multikey orderby +BEGIN; +ALTER TABLE metrics SET (tsdb.orderby = 'device desc,time'); +SET timescaledb.enable_direct_compress_insert = true; +SET timescaledb.enable_direct_compress_insert_client_sorted = false; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd'||i%3, i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-02'::timestamptz - (i || ' minute')::interval, 'd'||i%3, i::float FROM generate_series(0,3000) i; +EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; +--- QUERY PLAN --- + Append (actual rows=6002.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_39_chunk (actual rows=3480.00 loops=1) + -> Seq Scan on compress_hyper_2_40_chunk (actual rows=4.00 loops=1) + -> Custom Scan (ColumnarScan) on _hyper_1_41_chunk (actual rows=2522.00 loops=1) + -> Seq Scan on compress_hyper_2_42_chunk (actual rows=4.00 loops=1) + +SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL order by 1 limit 1 \gset +-- should see overlapping batches +select _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2 from :COMPRESSED_CHUNK order by 2, 4; + _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 | _ts_meta_min_2 | _ts_meta_max_2 +----------------+----------------+----------------+------------------------------+------------------------------ + 1000 | d0 | d1 | Mon Dec 30 22:00:00 2024 PST | Wed Jan 01 15:59:00 2025 PST + 520 | d0 | d0 | Tue Dec 31 14:00:00 2024 PST | Wed Jan 01 15:57:00 2025 PST + 960 | d0 | d2 | Wed Jan 01 00:00:00 2025 PST | Wed Jan 01 15:59:00 2025 PST + 1000 | d1 | d2 | Mon Dec 30 22:01:00 2024 PST | Wed Jan 01 15:58:00 2025 PST + +-- since the chunks are new status should be COMPRESSED, UNORDERED +SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; + chunk_status_text +------------------------ + {COMPRESSED,UNORDERED} + ROLLBACK; -- test unique constraints prevent direct compress BEGIN; @@ -243,7 +394,7 @@ INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interva WARNING: disabling direct compress because the destination table has unique constraints EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- - Seq Scan on _hyper_1_29_chunk (actual rows=101.00 loops=1) + Seq Scan on _hyper_1_43_chunk (actual rows=101.00 loops=1) SELECT DISTINCT status FROM _timescaledb_catalog.chunk WHERE compressed_chunk_id IS NOT NULL; status @@ -259,7 +410,7 @@ INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interva WARNING: disabling direct compress because the destination table has triggers EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- - Seq Scan on _hyper_1_30_chunk (actual rows=101.00 loops=1) + Seq Scan on _hyper_1_44_chunk (actual rows=101.00 loops=1) SELECT DISTINCT status FROM _timescaledb_catalog.chunk WHERE compressed_chunk_id IS NOT NULL; status @@ -274,7 +425,7 @@ INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interva WARNING: disabling direct compress because the destination table has continuous aggregates EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; --- QUERY PLAN --- - Seq Scan on _hyper_1_31_chunk (actual rows=101.00 loops=1) + Seq Scan on _hyper_1_45_chunk (actual rows=101.00 loops=1) SELECT DISTINCT status FROM _timescaledb_catalog.chunk WHERE compressed_chunk_id IS NOT NULL; status @@ -359,7 +510,7 @@ SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics SELECT compress_chunk(show_chunks('metrics_status')); compress_chunk ----------------------------------------- - _timescaledb_internal._hyper_4_39_chunk + _timescaledb_internal._hyper_4_53_chunk -- status should be COMPRESSED SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk; @@ -408,7 +559,7 @@ EXPLAIN (costs off,summary off,timing off) INSERT INTO :CHUNK SELECT '2025-01-01 --- QUERY PLAN --- Custom Scan (ModifyHypertable) Direct Compress: true - -> Insert on _hyper_6_41_chunk + -> Insert on _hyper_6_55_chunk -> Function Scan on generate_series i BEGIN; @@ -430,8 +581,8 @@ EXPLAIN (analyze,buffers off,costs off,summary off,timing off) DELETE FROM :CHUN Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) Batches decompressed: 1 Tuples decompressed: 101 - -> Delete on _hyper_6_41_chunk (actual rows=0.00 loops=1) - -> Index Scan using _hyper_6_41_chunk_metrics_chunk_time_idx on _hyper_6_41_chunk (actual rows=100.00 loops=1) + -> Delete on _hyper_6_55_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_6_55_chunk_metrics_chunk_time_idx on _hyper_6_55_chunk (actual rows=100.00 loops=1) Index Cond: ("time" > 'Wed Jan 01 00:00:00 2025 PST'::timestamp with time zone) SELECT count(*) FROM :CHUNK; diff --git a/tsl/test/sql/direct_compress_insert.sql b/tsl/test/sql/direct_compress_insert.sql index 7863f05a01c..7cdefc8f425 100644 --- a/tsl/test/sql/direct_compress_insert.sql +++ b/tsl/test/sql/direct_compress_insert.sql @@ -32,7 +32,7 @@ ROLLBACK; -- simple test with compressed insert enabled and reversed order BEGIN; -INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz - (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub; -- since the chunks are new status should be COMPRESSED, UNORDERED @@ -104,6 +104,38 @@ EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; ROLLBACK; +-- simple test with compressed insert enabled and no presorted with partial and compressed chunks +BEGIN; +SET timescaledb.enable_direct_compress_insert = true; +SET timescaledb.enable_direct_compress_insert_client_sorted = false; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz, 'd1', 0; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-02'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; +SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub; +SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL order by 1 desc limit 1 \gset +-- should see overlapping batches +select _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 from :COMPRESSED_CHUNK order by 2; +-- since the chunks are new status should be COMPRESSED, UNORDERED, PARTIAL and COMPRESSED, UNORDERED +SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk order by 1; +ROLLBACK; + +-- simple test with compressed insert enabled and presorted with partial and compressed chunks +BEGIN; +SET timescaledb.enable_direct_compress_insert = true; +SET timescaledb.enable_direct_compress_insert_client_sorted = true; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz, 'd1', 0; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-05'::timestamptz + (i || ' minute')::interval, 'd1', i::float FROM generate_series(0,3000) i; +EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; +SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub; +SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL order by 1 desc limit 1 \gset +-- should not see overlapping batches +select _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 from :COMPRESSED_CHUNK order by 2; +-- since the chunks are new status should be COMPRESSED, UNORDERED, PARTIAL and COMPRESSED, UNORDERED +SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk order by 1; +ROLLBACK; + -- test with segmentby BEGIN; ALTER TABLE metrics SET (tsdb.segmentby = 'device'); @@ -118,6 +150,36 @@ SELECT count(*) FROM :COMPRESSED_CHUNK; SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; ROLLBACK; +-- segmentby with overlapping batches +BEGIN; +ALTER TABLE metrics SET (tsdb.segmentby = 'device'); +SET timescaledb.enable_direct_compress_insert = true; +SET timescaledb.enable_direct_compress_insert_client_sorted = false; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd'||i%2, i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-02'::timestamptz + (i || ' minute')::interval, 'd'||i%2, i::float FROM generate_series(0,3000) i; +EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; +SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL order by 1 desc limit 1 \gset +-- should see overlapping batches per device +select _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1, device from :COMPRESSED_CHUNK order by 4, 2; +-- since the chunks are new status should be COMPRESSED, UNORDERED +SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; +ROLLBACK; + +-- multikey orderby +BEGIN; +ALTER TABLE metrics SET (tsdb.orderby = 'device desc,time'); +SET timescaledb.enable_direct_compress_insert = true; +SET timescaledb.enable_direct_compress_insert_client_sorted = false; +INSERT INTO metrics SELECT '2025-01-01'::timestamptz + (i || ' minute')::interval, 'd'||i%3, i::float FROM generate_series(0,3000) i; +INSERT INTO metrics SELECT '2025-01-02'::timestamptz - (i || ' minute')::interval, 'd'||i%3, i::float FROM generate_series(0,3000) i; +EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics; +SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL order by 1 limit 1 \gset +-- should see overlapping batches +select _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2 from :COMPRESSED_CHUNK order by 2, 4; +-- since the chunks are new status should be COMPRESSED, UNORDERED +SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics') chunk; +ROLLBACK; + -- test unique constraints prevent direct compress BEGIN; SET timescaledb.enable_direct_compress_insert = true;