diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..861cc80f --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.wasm32-unknown-unknown] +rustflags = ["--cfg=getrandom_backend=\"wasm_js\""] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46f82937..6175e800 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,14 +90,78 @@ jobs: with: command: run args: --bin sqllogictest-test --manifest-path ./tests/sqllogictest/Cargo.toml -# codecov: -# name: Upload coverage reports to Codecov -# runs-on: ubuntu-latest -# steps: -# - name: Upload coverage reports to Codecov -# uses: codecov/codecov-action@v3 -# with: -# files: ./lcov.info -# flags: rust -# env: -# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + # 4 + wasm-tests: + name: Wasm cargo tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Install stable with wasm target + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: wasm32-unknown-unknown + override: true + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install wasm-pack + uses: jetli/wasm-pack-action@v0.4.0 + with: + version: latest + + - name: Run wasm-bindgen tests (wasm32 target) + run: wasm-pack test --node -- --package kite_sql --lib + # 5 + wasm-examples: + name: Wasm examples (nodejs) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Install stable with wasm target + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: wasm32-unknown-unknown + override: true + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install wasm-pack + uses: jetli/wasm-pack-action@v0.4.0 + with: + version: latest + + - name: Build wasm package + run: wasm-pack build --release --target nodejs + + - name: Run wasm example scripts + run: | + node examples/wasm_hello_world.test.mjs + node examples/wasm_index_usage.test.mjs + # 6 + native-examples: + name: Native examples + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Run hello_world example + run: cargo run --example hello_world + + - name: Run transaction example + run: cargo run --example transaction diff --git a/.gitignore b/.gitignore index 3e67c383..c0fc418a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,8 +16,7 @@ target/ /.obsidian .DS_Store -/hello_world -/transaction +/example_data kitesql_data kitesql_bench diff --git a/Cargo.lock b/Cargo.lock index de6cb2a3..45d97f1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,9 +141,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-trait" -version = "0.1.88" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", @@ -195,6 +195,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -517,7 +523,6 @@ version = "7.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ - "crossterm", "unicode-segmentation", "unicode-width", ] @@ -620,28 +625,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags 2.9.1", - "crossterm_winapi", - "parking_lot", - "rustix 0.38.44", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - [[package]] name = "crunchy" version = "0.2.4" @@ -1051,8 +1034,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1062,9 +1047,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1323,9 +1310,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ "once_cell", "wasm-bindgen", @@ -1337,6 +1324,7 @@ version = "0.1.3" dependencies = [ "ahash 0.8.12", "async-trait", + "base64 0.21.7", "bincode", "bumpalo", "byteorder", @@ -1350,22 +1338,26 @@ dependencies = [ "fixedbitset", "futures", "genawaiter", + "getrandom 0.2.16", + "getrandom 0.3.3", "indicatif", "itertools 0.12.1", + "js-sys", "kite_sql_serde_macros", "log", + "once_cell", "ordered-float", "parking_lot", "paste", "petgraph", "pgwire", "pprof", - "rand 0.8.5", "recursive", "regex", "rocksdb", "rust_decimal", "serde", + "serde-wasm-bindgen", "siphasher", "sqlite", "sqlparser", @@ -1374,6 +1366,9 @@ dependencies = [ "tokio", "typetag", "ulid", + "wasm-bindgen", + "wasm-bindgen-test", + "web-sys", ] [[package]] @@ -1437,6 +1432,12 @@ dependencies = [ "windows-targets 0.53.2", ] +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "libredox" version = "0.1.4" @@ -1564,6 +1565,16 @@ dependencies = [ "libc", ] +[[package]] +name = "minicov" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" +dependencies = [ + "cc", + "walkdir", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1611,6 +1622,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1647,6 +1667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1845,7 +1866,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" dependencies = [ - "base64", + "base64 0.22.1", "byteorder", "bytes", "fallible-iterator", @@ -2386,6 +2407,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_derive" version = "1.0.219" @@ -3027,35 +3059,35 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", + "wasm-bindgen-shared", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" +name = "wasm-bindgen-futures" +version = "0.4.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.104", - "wasm-bindgen-shared", + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3063,31 +3095,63 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ + "bumpalo", "proc-macro2", "quote", "syn 2.0.104", - "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-bindgen-test" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e90e66d265d3a1efc0e72a54809ab90b9c0c515915c67cdf658689d2c22c6c" +dependencies = [ + "async-trait", + "cast", + "js-sys", + "libm", + "minicov", + "nu-ansi-term", + "num-traits", + "oorandom", + "serde", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7150335716dce6028bead2b848e72f47b45e7b9422f64cccdc23bedca89affc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index ce175e43..6635a96d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ required-features = ["net"] [lib] doctest = false +crate-type = ["cdylib", "rlib"] [features] default = ["macros"] @@ -40,9 +41,8 @@ bincode = { version = "1" } bumpalo = { version = "3", features = ["allocator-api2", "collections", "std"] } byteorder = { version = "1" } chrono = { version = "0.4" } -comfy-table = { version = "7" } +comfy-table = { version = "7", default-features = false } csv = { version = "1" } -dirs = { version = "5" } fixedbitset = { version = "0.4" } itertools = { version = "0.12" } ordered-float = { version = "4", features = ["serde"] } @@ -51,7 +51,6 @@ parking_lot = { version = "0.12", features = ["arc_lock"] } petgraph = { version = "0.6" } recursive = { version = "0.1" } regex = { version = "1" } -rocksdb = { version = "0.23" } rust_decimal = { version = "1" } serde = { version = "1", features = ["derive", "rc"] } kite_sql_serde_macros = { version = "0.1.0", path = "kite_sql_serde_macros" } @@ -61,7 +60,6 @@ thiserror = { version = "1" } typetag = { version = "0.2" } ulid = { version = "1", features = ["serde"] } genawaiter = { version = "0.99" } -rand = { version = "0.8" } # Feature: net async-trait = { version = "0.1", optional = true } @@ -73,15 +71,35 @@ pgwire = { version = "0.28.0", optional = true } tokio = { version = "1.36", features = ["full"], optional = true } -[dev-dependencies] +[target.'cfg(unix)'.dev-dependencies] +pprof = { version = "0.13", features = ["flamegraph", "criterion"] } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] criterion = { version = "0.5", features = ["html_reports"] } indicatif = { version = "0.17" } tempfile = { version = "3.10" } # Benchmark sqlite = { version = "0.34" } -[target.'cfg(unix)'.dev-dependencies] -pprof = { version = "0.13", features = ["flamegraph", "criterion"] } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +dirs = { version = "5" } +rocksdb = { version = "0.23" } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +wasm-bindgen = { version = "0.2.106" } +web-sys = { version = "0.3.83", features = [ + "Storage", + "Window", +] } +base64 = { version = "0.21" } +getrandom = { version = "0.2", features = ["js"] } +getrandom_03 = { package = "getrandom", version = "0.3", features = ["wasm_js"] } +js-sys = { version = "0.3.83" } +serde-wasm-bindgen = { version = "0.6.5" } +once_cell = { version = "1" } + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +wasm-bindgen-test = "0.3.56" [workspace] members = [ @@ -92,3 +110,6 @@ members = [ [profile.release] lto = true + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/README.md b/README.md index e4e58a2d..2960b3ab 100755 --- a/README.md +++ b/README.md @@ -34,9 +34,24 @@ - All metadata and actual data in KV Storage, and there is no state component (e.g. system table) in the middle - Supports extending storage for customized workloads - Supports most of the SQL 2016 syntax +- Ships a WebAssembly build for JavaScript runtimes #### 👉[check more](docs/features.md) +## WebAssembly +- Build: `wasm-pack build --release --target nodejs` (outputs to `./pkg`; use `--target web` or `--target bundler` for browser/bundler setups). +- Usage: +```js +import { WasmDatabase } from "./pkg/kite_sql.js"; + +const db = new WasmDatabase(); +await db.execute("create table demo(id int primary key, v int)"); +await db.execute("insert into demo values (1, 2), (2, 4)"); +const rows = db.run("select * from demo").rows(); +console.log(rows.map((r) => r.values.map((v) => v.Int32 ?? v))); +``` +- In Node.js, provide a small `localStorage` shim if you enable statistics-related features (see `examples/wasm_index_usage.test.mjs`). + ## Examples ```rust diff --git a/examples/hello_world.rs b/examples/hello_world.rs index 60a0eff1..46b5a223 100644 --- a/examples/hello_world.rs +++ b/examples/hello_world.rs @@ -1,3 +1,5 @@ +#![cfg(not(target_arch = "wasm32"))] + use kite_sql::db::{DataBaseBuilder, ResultIter}; use kite_sql::errors::DatabaseError; use kite_sql::implement_from_tuple; @@ -26,21 +28,50 @@ implement_from_tuple!( #[cfg(feature = "macros")] fn main() -> Result<(), DatabaseError> { - let database = DataBaseBuilder::path("./hello_world").build()?; + let database = DataBaseBuilder::path("./example_data/hello_world").build()?; + // 1) Create table and insert multiple rows with mixed types. + database + .run( + "create table if not exists my_struct ( + c1 int primary key, + c2 varchar, + c3 int + )", + )? + .done()?; database - .run("create table if not exists my_struct (c1 int primary key, c2 int)")? + .run( + r#" + insert into my_struct values + (0, 'zero', 0), + (1, 'one', 1), + (2, 'two', 2) + "#, + )? .done()?; + + // 2) Update and delete demo. database - .run("insert into my_struct values(0, 0), (1, 1)")? + .run("update my_struct set c3 = c3 + 10 where c1 = 1")? .done()?; + database.run("delete from my_struct where c1 = 2")?.done()?; + // 3) Query and deserialize into Rust struct. let iter = database.run("select * from my_struct")?; let schema = iter.schema().clone(); for tuple in iter { println!("{:?}", MyStruct::from((&schema, tuple?))); } + + // 4) Aggregate example. + let mut agg = database.run("select count(*) from my_struct")?; + if let Some(count_row) = agg.next() { + println!("row count = {:?}", count_row?); + } + agg.done()?; + database.run("drop table my_struct")?.done()?; Ok(()) diff --git a/examples/transaction.rs b/examples/transaction.rs index ef2834b9..ff69b835 100644 --- a/examples/transaction.rs +++ b/examples/transaction.rs @@ -1,22 +1,50 @@ +#![cfg(not(target_arch = "wasm32"))] + use kite_sql::db::{DataBaseBuilder, ResultIter}; use kite_sql::errors::DatabaseError; +use kite_sql::types::tuple::Tuple; +use kite_sql::types::value::DataValue; fn main() -> Result<(), DatabaseError> { - let database = DataBaseBuilder::path("./transaction").build_optimistic()?; - let mut transaction = database.new_transaction()?; - - transaction + let database = DataBaseBuilder::path("./example_data/transaction").build_optimistic()?; + database .run("create table if not exists t1 (c1 int primary key, c2 int)")? .done()?; + let mut transaction = database.new_transaction()?; + transaction .run("insert into t1 values(0, 0), (1, 1)")? .done()?; - assert!(database.run("select * from t1").is_err()); + assert!(database.run("select * from t1")?.next().is_none()); transaction.commit()?; - assert!(database.run("select * from t1").is_ok()); + let mut iter = database.run("select * from t1")?; + assert_eq!( + iter.next().unwrap()?, + Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]) + ); + assert_eq!( + iter.next().unwrap()?, + Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(1)]) + ); + assert!(iter.next().is_none()); + + // Scenario: another transaction updates but does not commit; changes stay invisible. + let mut tx2 = database.new_transaction()?; + tx2.run("update t1 set c2 = 99 where c1 = 0")?.done()?; + assert_eq!( + database + .run("select c2 from t1 where c1 = 0")? + .next() + .unwrap()? + .values[0] + .i32(), + Some(0) + ); + // rollback + drop(tx2); database.run("drop table t1")?.done()?; diff --git a/examples/wasm_hello_world.test.mjs b/examples/wasm_hello_world.test.mjs new file mode 100644 index 00000000..34ecd2d0 --- /dev/null +++ b/examples/wasm_hello_world.test.mjs @@ -0,0 +1,49 @@ +// Simple wasm smoke test; run `wasm-pack build --target nodejs` first to generate ./pkg +import assert from "node:assert/strict"; +import { createRequire } from "module"; + +const require = createRequire(import.meta.url); +const { WasmDatabase } = require("../pkg/kite_sql.js"); + +async function main() { + const db = new WasmDatabase(); + + await db.execute("drop table if exists my_struct"); + await db.execute("create table my_struct (c1 int primary key, c2 int)"); + await db.execute("insert into my_struct values(0, 0), (1, 1)"); + + const rows = db.run("select * from my_struct").rows(); + assert.equal(rows.length, 2, "should return two rows"); + + const [first, second] = rows; + const firstVals = first.values.map((v) => v.Int32 ?? v); + const secondVals = second.values.map((v) => v.Int32 ?? v); + assert.deepEqual(firstVals, [0, 0]); + assert.deepEqual(secondVals, [1, 1]); + + await db.execute("update my_struct set c2 = c2 + 10 where c1 = 1"); + const after = db.run("select c2 from my_struct where c1 = 1").rows(); + assert.deepEqual(after[0].values.map((v) => v.Int32 ?? v), [11]); + + // Stream the rows using the iterator interface + const stream = db.run("select * from my_struct"); + const streamed = []; + let row = stream.next(); + while (row !== undefined) { + streamed.push(row.values.map((v) => v.Int32 ?? v)); + row = stream.next(); + } + stream.finish(); + assert.deepEqual(streamed, [ + [0, 0], + [1, 11], + ]); + + await db.execute("drop table my_struct"); + console.log("wasm hello_world test passed"); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/wasm_index_usage.test.mjs b/examples/wasm_index_usage.test.mjs new file mode 100644 index 00000000..8ba437f8 --- /dev/null +++ b/examples/wasm_index_usage.test.mjs @@ -0,0 +1,111 @@ +// Verify index usage in wasm build: create table, build indexes, analyze, and query with filters. +import assert from "node:assert/strict"; +import { createRequire } from "module"; + +// Minimal localStorage polyfill for wasm code paths (statistics persistence). +class LocalStorage { + constructor() { + this.map = new Map(); + } + get length() { + return this.map.size; + } + key(i) { + return Array.from(this.map.keys())[i] ?? null; + } + getItem(k) { + return this.map.has(k) ? this.map.get(k) : null; + } + setItem(k, v) { + this.map.set(String(k), String(v)); + } + removeItem(k) { + this.map.delete(k); + } + clear() { + this.map.clear(); + } +} +const windowShim = { localStorage: new LocalStorage() }; +globalThis.window = windowShim; +globalThis.localStorage = windowShim.localStorage; +globalThis.self = windowShim; +global.window = windowShim; + +const require = createRequire(import.meta.url); +const { WasmDatabase } = require("../pkg/kite_sql.js"); + +async function main() { + const db = new WasmDatabase(); + + await db.execute("drop table if exists t1"); + await db.execute("create table t1(id int primary key, c1 int, c2 int)"); + + // Insert data in bulk (20k rows) without reading from disk. + // Each row matches the old CSV pattern: id = i*3, c1 = i*3+1, c2 = i*3+2. + for (let i = 0; i < 20_000; i += 1) { + const id = i * 3; + const c1 = id + 1; + const c2 = id + 2; + await db.execute(`insert into t1 values(${id}, ${c1}, ${c2})`); + } + + // Add indexes and analyze + await db.execute("create unique index u_c1_index on t1 (c1)"); + await db.execute("create index c2_index on t1 (c2)"); + await db.execute("create index p_index on t1 (c1, c2)"); + await db.execute("analyze table t1"); + + const rowVals = (row) => { + const ints = row.values.map((v) => v.Int32 ?? v); + const id = row.pk?.Int32 ?? ints[0]; + const rest = ints.slice(1); + return [id, ...rest]; + }; + + // Basic queries to confirm rows + const first10 = db.run("select * from t1 limit 10").rows(); + assert.equal(first10.length, 10); + + // Point lookups on primary key + const pkRow = db.run("select * from t1 where id = 0").rows().map(rowVals); + assert.deepEqual(pkRow, [[0, 1, 2]]); + + // Range on primary key + const rangePk = db.run("select * from t1 where id >= 9 and id <= 15").rows().map(rowVals); + assert.deepEqual(rangePk, [ + [9, 10, 11], + [12, 13, 14], + [15, 16, 17], + ]); + + // Query hitting c1 index + const c1Eq = db.run("select * from t1 where c1 = 7 and c2 = 8").rows().map(rowVals); + assert.deepEqual(c1Eq, [[6, 7, 8]]); + + // Range query on c2 index + const c2Range = db.run("select * from t1 where c2 > 100 and c2 < 110").rows().map(rowVals); + assert.deepEqual(c2Range, [ + [99, 100, 101], + [102, 103, 104], + [105, 106, 107], + ]); + + // Update and re-query to ensure index maintenance + await db.execute("update t1 set c2 = 123456 where c1 = 7"); + const afterUpdate = db.run("select * from t1 where c2 = 123456").rows().map(rowVals); + assert.deepEqual(afterUpdate, [[6, 7, 123456]]); + + // Delete and ensure index reflects removal + await db.execute("delete from t1 where c1 = 7"); + const afterDelete = db.run("select * from t1 where c2 = 123456").rows().map(rowVals); + assert.equal(afterDelete.length, 0); + + await db.execute("drop table t1"); + console.log("wasm index usage test passed"); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/binder/create_table.rs b/src/binder/create_table.rs index a35528eb..5f71cfac 100644 --- a/src/binder/create_table.rs +++ b/src/binder/create_table.rs @@ -154,7 +154,7 @@ impl> Binder<'_, '_, T, A> } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; use crate::binder::BinderContext; diff --git a/src/binder/mod.rs b/src/binder/mod.rs index 43d5af57..52216846 100644 --- a/src/binder/mod.rs +++ b/src/binder/mod.rs @@ -524,7 +524,7 @@ pub(crate) fn is_valid_identifier(s: &str) -> bool { && !s.chars().all(|c| c == '_') } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub mod test { use crate::binder::{is_valid_identifier, Binder, BinderContext}; use crate::catalog::{ColumnCatalog, ColumnDesc, TableCatalog}; diff --git a/src/binder/select.rs b/src/binder/select.rs index 3db10633..40b7a63f 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -1189,7 +1189,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(&'static str, DataValue)]>> Binder<' } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; diff --git a/src/catalog/table.rs b/src/catalog/table.rs index 2ace0d0f..e4181445 100644 --- a/src/catalog/table.rs +++ b/src/catalog/table.rs @@ -49,7 +49,7 @@ impl TableCatalog { self.columns.get(id).map(|i| &self.schema_ref[*i]) } - #[cfg(test)] + #[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) fn get_column_id_by_name(&self, name: &str) -> Option<&ColumnId> { self.column_idxs.get(name).map(|(id, _)| id) } @@ -270,7 +270,7 @@ impl TableMeta { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; use crate::catalog::ColumnDesc; diff --git a/src/db.rs b/src/db.rs index 142cdf46..99833531 100644 --- a/src/db.rs +++ b/src/db.rs @@ -17,6 +17,8 @@ use crate::optimizer::rule::implementation::ImplementationRuleImpl; use crate::optimizer::rule::normalization::NormalizationRuleImpl; use crate::parser::parse_sql; use crate::planner::LogicalPlan; +use crate::storage::memory::MemoryStorage; +#[cfg(not(target_arch = "wasm32"))] use crate::storage::rocksdb::{OptimisticRocksStorage, RocksStorage}; use crate::storage::{StatisticsMetaCache, Storage, TableCache, Transaction, ViewCache}; use crate::types::tuple::{SchemaRef, Tuple}; @@ -44,6 +46,7 @@ pub(crate) enum MetaDataLock { } pub struct DataBaseBuilder { + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] path: PathBuf, scala_functions: ScalaFunctions, table_functions: TableFunctions, @@ -82,12 +85,31 @@ impl DataBaseBuilder { self } + pub fn build_with_storage(self, storage: T) -> Result, DatabaseError> { + Self::_build::(storage, self.scala_functions, self.table_functions) + } + + #[cfg(target_arch = "wasm32")] + pub fn build(self) -> Result, DatabaseError> { + let storage = MemoryStorage::new(); + + Self::_build::(storage, self.scala_functions, self.table_functions) + } + + #[cfg(not(target_arch = "wasm32"))] pub fn build(self) -> Result, DatabaseError> { let storage = RocksStorage::new(self.path)?; Self::_build::(storage, self.scala_functions, self.table_functions) } + pub fn build_in_memory(self) -> Result, DatabaseError> { + let storage = MemoryStorage::new(); + + Self::_build::(storage, self.scala_functions, self.table_functions) + } + + #[cfg(not(target_arch = "wasm32"))] pub fn build_optimistic(self) -> Result, DatabaseError> { let storage = OptimisticRocksStorage::new(self.path)?; @@ -467,7 +489,7 @@ impl ResultIter for TransactionIter<'_> { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) mod test { use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; use crate::db::{DataBaseBuilder, DatabaseError, ResultIter}; diff --git a/src/errors.rs b/src/errors.rs index 65ae8dbf..42fbed53 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -127,6 +127,7 @@ pub enum DatabaseError { PrimaryKeyNotFound, #[error("primaryKey only allows single or multiple values")] PrimaryKeyTooManyLayers, + #[cfg(not(target_arch = "wasm32"))] #[error("rocksdb: {0}")] RocksDB( #[source] diff --git a/src/execution/dml/analyze.rs b/src/execution/dml/analyze.rs index d9d9d7a0..ce3a87fa 100644 --- a/src/execution/dml/analyze.rs +++ b/src/execution/dml/analyze.rs @@ -5,6 +5,9 @@ use crate::execution::{build_read, spawn_executor, Executor, WriteExecutor}; use crate::expression::{BindPosition, ScalarExpression}; use crate::optimizer::core::histogram::HistogramBuilder; use crate::optimizer::core::statistics_meta::StatisticsMeta; +use crate::paths::require_statistics_base_dir; +#[cfg(target_arch = "wasm32")] +use crate::paths::{wasm_remove_storage_key, wasm_set_storage_item, wasm_storage_keys_with_prefix}; use crate::planner::operator::analyze::AnalyzeOperator; use crate::planner::LogicalPlan; use crate::storage::{StatisticsMetaCache, TableCache, Transaction, ViewCache}; @@ -16,15 +19,16 @@ use itertools::Itertools; use sqlparser::ast::CharLengthUnits; use std::borrow::Cow; use std::collections::HashSet; +#[cfg(not(target_arch = "wasm32"))] use std::ffi::OsStr; -use std::fmt::Formatter; -use std::fs::DirEntry; +use std::fmt::{self, Formatter}; +#[cfg(not(target_arch = "wasm32"))] +use std::fs::{self, DirEntry}; +#[cfg(not(target_arch = "wasm32"))] use std::path::PathBuf; use std::sync::Arc; -use std::{fmt, fs}; const DEFAULT_NUM_OF_BUCKETS: usize = 100; -const DEFAULT_STATISTICS_META_PATH: &str = "kite_sql_statistics_metas"; pub struct Analyze { table_name: TableName, @@ -120,52 +124,17 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for Analyze { } } drop(coroutine); - let mut values = Vec::with_capacity(builders.len()); - let dir_path = Self::build_statistics_meta_path(&table_name); - throw!(co, fs::create_dir_all(&dir_path).map_err(DatabaseError::IO)); - - let mut active_index_paths = HashSet::new(); - - for State { - index_id, builder, .. - } in builders - { - let index_file = OsStr::new(&index_id.to_string()).to_os_string(); - let path = dir_path.join(&index_file); - let temp_path = path.with_extension("tmp"); - let path_str: String = path.to_string_lossy().into(); - - let (histogram, sketch) = throw!(co, builder.build(DEFAULT_NUM_OF_BUCKETS)); - let meta = StatisticsMeta::new(histogram, sketch); - - throw!(co, meta.to_file(&temp_path)); - values.push(DataValue::Utf8 { - value: path_str.clone(), - ty: Utf8Type::Variable(None), - unit: CharLengthUnits::Characters, - }); - throw!( - co, - unsafe { &mut (*transaction) }.save_table_meta( - cache.2, - &table_name, - path_str, - meta - ) - ); - throw!(co, fs::rename(&temp_path, &path).map_err(DatabaseError::IO)); - - active_index_paths.insert(index_file); - } - - // clean expired index - for entry in throw!(co, fs::read_dir(dir_path).map_err(DatabaseError::IO)) { - let entry: DirEntry = throw!(co, entry.map_err(DatabaseError::IO)); + #[cfg(target_arch = "wasm32")] + let values = throw!( + co, + Self::persist_statistics_meta_wasm(&table_name, builders, cache.2, transaction) + ); - if !active_index_paths.remove(&entry.file_name()) { - throw!(co, fs::remove_file(entry.path()).map_err(DatabaseError::IO)); - } - } + #[cfg(not(target_arch = "wasm32"))] + let values = throw!( + co, + Self::persist_statistics_meta_native(&table_name, builders, cache.2, transaction) + ); co.yield_(Ok(Tuple::new(None, values))).await; }) @@ -180,11 +149,107 @@ struct State { } impl Analyze { - pub fn build_statistics_meta_path(table_name: &TableName) -> PathBuf { - dirs::home_dir() - .expect("Your system does not have a Config directory!") - .join(DEFAULT_STATISTICS_META_PATH) + #[cfg(not(target_arch = "wasm32"))] + fn persist_statistics_meta_native( + table_name: &TableName, + builders: Vec, + cache: &StatisticsMetaCache, + transaction: *mut T, + ) -> Result, DatabaseError> { + let dir_path = Self::build_statistics_meta_path(table_name)?; + fs::create_dir_all(&dir_path).map_err(DatabaseError::IO)?; + + let mut values = Vec::with_capacity(builders.len()); + let mut active_index_paths = HashSet::new(); + + for State { + index_id, builder, .. + } in builders + { + let index_file = OsStr::new(&index_id.to_string()).to_os_string(); + let path = dir_path.join(&index_file); + let temp_path = path.with_extension("tmp"); + let path_str: String = path.to_string_lossy().into(); + + let (histogram, sketch) = builder.build(DEFAULT_NUM_OF_BUCKETS)?; + let meta = StatisticsMeta::new(histogram, sketch); + + meta.to_file(&temp_path)?; + values.push(DataValue::Utf8 { + value: path_str.clone(), + ty: Utf8Type::Variable(None), + unit: CharLengthUnits::Characters, + }); + unsafe { &mut (*transaction) }.save_table_meta(cache, table_name, path_str, meta)?; + fs::rename(&temp_path, &path).map_err(DatabaseError::IO)?; + + active_index_paths.insert(index_file); + } + + // clean expired index + for entry in fs::read_dir(dir_path).map_err(DatabaseError::IO)? { + let entry: DirEntry = entry.map_err(DatabaseError::IO)?; + + if !active_index_paths.remove(&entry.file_name()) { + fs::remove_file(entry.path()).map_err(DatabaseError::IO)?; + } + } + + Ok(values) + } + + #[cfg(target_arch = "wasm32")] + fn persist_statistics_meta_wasm( + table_name: &TableName, + builders: Vec, + cache: &StatisticsMetaCache, + transaction: *mut T, + ) -> Result, DatabaseError> { + let prefix = Self::build_statistics_meta_prefix(table_name)?; + let mut values = Vec::with_capacity(builders.len()); + let mut active_keys = HashSet::new(); + + for State { + index_id, builder, .. + } in builders + { + let key = format!("{prefix}/{index_id}"); + let (histogram, sketch) = builder.build(DEFAULT_NUM_OF_BUCKETS)?; + let meta = StatisticsMeta::new(histogram, sketch); + let encoded = meta.to_storage_string()?; + + wasm_set_storage_item(&key, &encoded)?; + values.push(DataValue::Utf8 { + value: key.clone(), + ty: Utf8Type::Variable(None), + unit: CharLengthUnits::Characters, + }); + unsafe { &mut (*transaction) }.save_table_meta(cache, table_name, key.clone(), meta)?; + active_keys.insert(key); + } + + let keys = wasm_storage_keys_with_prefix(&(prefix.clone() + "/"))?; + + for key in keys { + if !active_keys.contains(&key) { + wasm_remove_storage_key(&key)?; + } + } + + Ok(values) + } + + #[cfg(not(target_arch = "wasm32"))] + fn build_statistics_meta_path(table_name: &TableName) -> Result { + Ok(require_statistics_base_dir().join(table_name.as_ref())) + } + + #[cfg(target_arch = "wasm32")] + fn build_statistics_meta_prefix(table_name: &TableName) -> Result { + Ok(require_statistics_base_dir() .join(table_name.as_ref()) + .to_string_lossy() + .into_owned()) } } @@ -198,12 +263,13 @@ impl fmt::Display for AnalyzeOperator { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::db::{DataBaseBuilder, ResultIter}; use crate::errors::DatabaseError; - use crate::execution::dml::analyze::{DEFAULT_NUM_OF_BUCKETS, DEFAULT_STATISTICS_META_PATH}; + use crate::execution::dml::analyze::DEFAULT_NUM_OF_BUCKETS; use crate::optimizer::core::statistics_meta::StatisticsMeta; + use crate::paths::require_statistics_base_dir; use crate::storage::rocksdb::RocksTransaction; use std::ffi::OsStr; use std::fs; @@ -219,6 +285,7 @@ mod test { fn test_statistics_meta() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let base_dir = require_statistics_base_dir(); let kite_sql = DataBaseBuilder::path(temp_dir.path()).build()?; kite_sql @@ -234,10 +301,7 @@ mod test { } kite_sql.run("analyze table t1")?.done()?; - let dir_path = dirs::home_dir() - .expect("Your system does not have a Config directory!") - .join(DEFAULT_STATISTICS_META_PATH) - .join("t1"); + let dir_path = base_dir.join("t1"); let mut paths = Vec::new(); @@ -266,6 +330,7 @@ mod test { fn test_clean_expired_index() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let base_dir = require_statistics_base_dir(); let kite_sql = DataBaseBuilder::path(temp_dir.path()).build()?; kite_sql @@ -281,10 +346,7 @@ mod test { } kite_sql.run("analyze table t1")?.done()?; - let dir_path = dirs::home_dir() - .expect("Your system does not have a Config directory!") - .join(DEFAULT_STATISTICS_META_PATH) - .join("t1"); + let dir_path = base_dir.join("t1"); let mut file_names = Vec::new(); diff --git a/src/execution/dml/copy_from_file.rs b/src/execution/dml/copy_from_file.rs index 4dce8e36..3831881c 100644 --- a/src/execution/dml/copy_from_file.rs +++ b/src/execution/dml/copy_from_file.rs @@ -127,7 +127,7 @@ fn return_result(size: usize, tx: Sender) -> Result<(), DatabaseError> { Ok(()) } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; use crate::binder::copy::ExtSource; diff --git a/src/execution/dml/copy_to_file.rs b/src/execution/dml/copy_to_file.rs index 0eddd8dc..6da4f9e4 100644 --- a/src/execution/dml/copy_to_file.rs +++ b/src/execution/dml/copy_to_file.rs @@ -85,7 +85,7 @@ impl CopyToFile { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; use crate::binder::copy::ExtSource; diff --git a/src/execution/dql/aggregate/hash_agg.rs b/src/execution/dql/aggregate/hash_agg.rs index 1ce7b8c4..7d5c745c 100644 --- a/src/execution/dql/aggregate/hash_agg.rs +++ b/src/execution/dql/aggregate/hash_agg.rs @@ -107,7 +107,7 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for HashAggExecutor { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; use crate::errors::DatabaseError; diff --git a/src/execution/dql/join/hash_join.rs b/src/execution/dql/join/hash_join.rs index 3e4db1d3..212d148d 100644 --- a/src/execution/dql/join/hash_join.rs +++ b/src/execution/dql/join/hash_join.rs @@ -252,7 +252,7 @@ impl HashJoin { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; use crate::errors::DatabaseError; diff --git a/src/execution/dql/join/nested_loop_join.rs b/src/execution/dql/join/nested_loop_join.rs index 6161db3e..38588423 100644 --- a/src/execution/dql/join/nested_loop_join.rs +++ b/src/execution/dql/join/nested_loop_join.rs @@ -364,7 +364,7 @@ impl NestedLoopJoin { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use super::*; diff --git a/src/execution/dql/mod.rs b/src/execution/dql/mod.rs index d59b5dd8..747da75f 100644 --- a/src/execution/dql/mod.rs +++ b/src/execution/dql/mod.rs @@ -17,7 +17,7 @@ pub(crate) mod top_k; pub(crate) mod union; pub(crate) mod values; -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) mod test { use crate::types::value::DataValue; use itertools::Itertools; diff --git a/src/execution/dql/sort.rs b/src/execution/dql/sort.rs index 01cef3cd..c89ba794 100644 --- a/src/execution/dql/sort.rs +++ b/src/execution/dql/sort.rs @@ -323,7 +323,7 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Sort { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; use crate::errors::DatabaseError; diff --git a/src/execution/dql/top_k.rs b/src/execution/dql/top_k.rs index 3f5f2ae1..c735fe31 100644 --- a/src/execution/dql/top_k.rs +++ b/src/execution/dql/top_k.rs @@ -154,7 +154,7 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for TopK { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; use crate::errors::DatabaseError; diff --git a/src/execution/mod.rs b/src/execution/mod.rs index 6586ec1e..1517722c 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -242,7 +242,7 @@ pub fn build_write<'a, T: Transaction + 'a>( } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub fn try_collect(executor: Executor) -> Result, DatabaseError> { executor.collect() } diff --git a/src/expression/mod.rs b/src/expression/mod.rs index 2ab903a5..70aa4c38 100644 --- a/src/expression/mod.rs +++ b/src/expression/mod.rs @@ -789,7 +789,7 @@ impl TryFrom for BinaryOperator { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef, ColumnRelation, ColumnSummary}; use crate::db::test::build_table; diff --git a/src/expression/range_detacher.rs b/src/expression/range_detacher.rs index d304fdf0..e815916a 100644 --- a/src/expression/range_detacher.rs +++ b/src/expression/range_detacher.rs @@ -786,7 +786,7 @@ impl fmt::Display for Range { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; diff --git a/src/lib.rs b/src/lib.rs index 4ddf15eb..c0f1d235 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -99,8 +99,11 @@ mod function; pub mod macros; mod optimizer; pub mod parser; +pub mod paths; pub mod planner; pub mod serdes; pub mod storage; pub mod types; pub(crate) mod utils; +#[cfg(target_arch = "wasm32")] +pub mod wasm; diff --git a/src/optimizer/core/cm_sketch.rs b/src/optimizer/core/cm_sketch.rs index 0d2bfa86..09d4bb6e 100644 --- a/src/optimizer/core/cm_sketch.rs +++ b/src/optimizer/core/cm_sketch.rs @@ -3,7 +3,6 @@ use crate::expression::range_detacher::Range; use crate::serdes::{ReferenceSerialization, ReferenceTables}; use crate::storage::{TableCache, Transaction}; use crate::types::value::DataValue; -use rand::random; use siphasher::sip::SipHasher13; use std::borrow::Borrow; use std::hash::{Hash, Hasher}; @@ -138,7 +137,7 @@ impl CountMinSketch { } fn sip_new() -> FastHasher { - FastHasher::new_with_keys(random(), random()) + FastHasher::new_with_keys(0, 1) } fn offset(&self, hashes: &mut [u64; 2], key: &Q, k_i: usize) -> usize @@ -199,7 +198,7 @@ impl ReferenceSerialization for CountMinSketch { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::expression::range_detacher::Range; use crate::optimizer::core::cm_sketch::CountMinSketch; diff --git a/src/optimizer/core/histogram.rs b/src/optimizer/core/histogram.rs index 7c2d3219..fb322829 100644 --- a/src/optimizer/core/histogram.rs +++ b/src/optimizer/core/histogram.rs @@ -509,7 +509,7 @@ impl Bucket { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::errors::DatabaseError; use crate::expression::range_detacher::Range; @@ -848,7 +848,7 @@ mod tests { &sketch, )?; - assert_eq!(count_10, 3); + assert_eq!(count_10, 2); let count_11 = histogram.collect_count( &vec![Range::Scope { diff --git a/src/optimizer/core/memo.rs b/src/optimizer/core/memo.rs index 6f764652..678a0dde 100644 --- a/src/optimizer/core/memo.rs +++ b/src/optimizer/core/memo.rs @@ -78,7 +78,7 @@ impl Memo { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::binder::{Binder, BinderContext}; use crate::db::{DataBaseBuilder, ResultIter}; diff --git a/src/optimizer/core/statistics_meta.rs b/src/optimizer/core/statistics_meta.rs index b3b6a485..2642df10 100644 --- a/src/optimizer/core/statistics_meta.rs +++ b/src/optimizer/core/statistics_meta.rs @@ -7,9 +7,15 @@ use crate::serdes::{ReferenceSerialization, ReferenceTables}; use crate::storage::{StatisticsMetaCache, Transaction}; use crate::types::index::IndexId; use crate::types::value::DataValue; +#[cfg(target_arch = "wasm32")] +use base64::{engine::general_purpose, Engine as _}; use kite_sql_serde_macros::ReferenceSerialization; +#[cfg(not(target_arch = "wasm32"))] use std::fs::OpenOptions; -use std::io::Write; +#[cfg(target_arch = "wasm32")] +use std::io; +use std::io::{Cursor, Read, Write}; +#[cfg(not(target_arch = "wasm32"))] use std::path::Path; use std::slice; @@ -35,9 +41,16 @@ impl<'a, T: Transaction> StatisticMetaLoader<'a, T> { return Ok(Some(statistics_meta)); } if let Some(path) = self.tx.table_meta_path(table_name.as_ref(), index_id)? { - Ok(Some(self.cache.get_or_insert(key, |_| { - StatisticsMeta::from_file::(path) - })?)) + #[cfg(target_arch = "wasm32")] + let statistics_meta = self + .cache + .get_or_insert(key, |_| StatisticsMeta::from_storage_string::(&path))?; + #[cfg(not(target_arch = "wasm32"))] + let statistics_meta = self + .cache + .get_or_insert(key, |_| StatisticsMeta::from_file::(path))?; + + Ok(Some(statistics_meta)) } else { Ok(None) } @@ -78,6 +91,26 @@ impl StatisticsMeta { Ok(count) } + fn encode_into_writer(&self, writer: &mut impl Write) -> Result<(), DatabaseError> { + self.encode(writer, true, &mut ReferenceTables::new()) + } + + fn decode_from_reader(reader: &mut impl Read) -> Result { + Self::decode::(reader, None, &ReferenceTables::new()) + } + + pub fn to_bytes(&self) -> Result, DatabaseError> { + let mut bytes = Vec::new(); + self.encode_into_writer(&mut bytes)?; + Ok(bytes) + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + let mut cursor = Cursor::new(bytes); + Self::decode_from_reader::(&mut cursor) + } + + #[cfg(not(target_arch = "wasm32"))] pub fn to_file(&self, path: impl AsRef) -> Result<(), DatabaseError> { let mut file = OpenOptions::new() .create(true) @@ -85,12 +118,13 @@ impl StatisticsMeta { .read(true) .truncate(false) .open(path)?; - self.encode(&mut file, true, &mut ReferenceTables::new())?; + self.encode_into_writer(&mut file)?; file.flush()?; Ok(()) } + #[cfg(not(target_arch = "wasm32"))] pub fn from_file(path: impl AsRef) -> Result { let mut file = OpenOptions::new() .create(true) @@ -98,11 +132,25 @@ impl StatisticsMeta { .read(true) .truncate(false) .open(path)?; - Self::decode::(&mut file, None, &ReferenceTables::new()) + Self::decode_from_reader::(&mut file) + } + + #[cfg(target_arch = "wasm32")] + pub fn to_storage_string(&self) -> Result { + Ok(general_purpose::STANDARD.encode(self.to_bytes()?)) + } + + #[cfg(target_arch = "wasm32")] + pub fn from_storage_string(value: &str) -> Result { + let bytes = general_purpose::STANDARD + .decode(value) + .map_err(|err| DatabaseError::IO(io::Error::new(io::ErrorKind::InvalidData, err)))?; + + Self::from_bytes::(&bytes) } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::errors::DatabaseError; use crate::optimizer::core::histogram::HistogramBuilder; diff --git a/src/optimizer/heuristic/graph.rs b/src/optimizer/heuristic/graph.rs index e60487ef..ab4ba372 100644 --- a/src/optimizer/heuristic/graph.rs +++ b/src/optimizer/heuristic/graph.rs @@ -229,7 +229,7 @@ impl Iterator for HepGraphIter<'_> { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; diff --git a/src/optimizer/heuristic/matcher.rs b/src/optimizer/heuristic/matcher.rs index 9779178f..f79cdfee 100644 --- a/src/optimizer/heuristic/matcher.rs +++ b/src/optimizer/heuristic/matcher.rs @@ -51,7 +51,7 @@ impl PatternMatcher for HepMatcher<'_, '_> { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; diff --git a/src/optimizer/rule/normalization/column_pruning.rs b/src/optimizer/rule/normalization/column_pruning.rs index b5e8be7a..d6087a97 100644 --- a/src/optimizer/rule/normalization/column_pruning.rs +++ b/src/optimizer/rule/normalization/column_pruning.rs @@ -199,7 +199,7 @@ impl NormalizationRule for ColumnPruning { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; diff --git a/src/optimizer/rule/normalization/combine_operators.rs b/src/optimizer/rule/normalization/combine_operators.rs index 9690112a..6a7a91a1 100644 --- a/src/optimizer/rule/normalization/combine_operators.rs +++ b/src/optimizer/rule/normalization/combine_operators.rs @@ -137,7 +137,7 @@ impl NormalizationRule for CollapseGroupByAgg { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; diff --git a/src/optimizer/rule/normalization/pushdown_limit.rs b/src/optimizer/rule/normalization/pushdown_limit.rs index 452ee34e..30f703f6 100644 --- a/src/optimizer/rule/normalization/pushdown_limit.rs +++ b/src/optimizer/rule/normalization/pushdown_limit.rs @@ -124,7 +124,7 @@ impl NormalizationRule for PushLimitIntoScan { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; diff --git a/src/optimizer/rule/normalization/pushdown_predicates.rs b/src/optimizer/rule/normalization/pushdown_predicates.rs index 49e156e7..b56948e8 100644 --- a/src/optimizer/rule/normalization/pushdown_predicates.rs +++ b/src/optimizer/rule/normalization/pushdown_predicates.rs @@ -296,7 +296,7 @@ impl PushPredicateIntoScan { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; diff --git a/src/optimizer/rule/normalization/simplification.rs b/src/optimizer/rule/normalization/simplification.rs index 0adb7cc3..0d5a7a37 100644 --- a/src/optimizer/rule/normalization/simplification.rs +++ b/src/optimizer/rule/normalization/simplification.rs @@ -114,7 +114,7 @@ impl NormalizationRule for SimplifyFilter { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::binder::test::build_t1_table; use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef, ColumnRelation, ColumnSummary}; diff --git a/src/paths.rs b/src/paths.rs new file mode 100644 index 00000000..b96b0670 --- /dev/null +++ b/src/paths.rs @@ -0,0 +1,123 @@ +use std::path::PathBuf; + +pub const STATISTICS_META_SUBDIR: &str = "kite_sql_statistics_metas"; + +#[cfg(not(target_arch = "wasm32"))] +fn default_statistics_base_dir() -> Option { + dirs::home_dir().map(|path| path.join(STATISTICS_META_SUBDIR)) +} + +#[cfg(target_arch = "wasm32")] +fn default_statistics_base_dir() -> Option { + Some(PathBuf::from(STATISTICS_META_SUBDIR)) +} + +/// Returns the statistics base directory, using a platform default. +pub fn statistics_base_dir() -> Option { + default_statistics_base_dir() +} + +/// Retrieves the statistics base directory, panicking if it cannot be determined. +pub fn require_statistics_base_dir() -> PathBuf { + statistics_base_dir() + .unwrap_or_else(|| panic!("statistics_base_dir is empty and no default is available")) +} + +#[cfg(target_arch = "wasm32")] +mod wasm_storage { + use crate::errors::DatabaseError; + use once_cell::sync::Lazy; + use std::collections::HashMap; + use std::sync::Mutex; + + static MEMORY_STORE: Lazy>> = + Lazy::new(|| Mutex::new(HashMap::new())); + + pub fn storage_keys_with_prefix(prefix: &str) -> Result, DatabaseError> { + let store = MEMORY_STORE.lock().unwrap(); + Ok(store + .keys() + .filter(|k| k.starts_with(prefix)) + .cloned() + .collect()) + } + + pub fn remove_storage_key(key: &str) -> Result<(), DatabaseError> { + MEMORY_STORE.lock().unwrap().remove(key); + Ok(()) + } + + pub fn set_storage_item(key: &str, value: &str) -> Result<(), DatabaseError> { + MEMORY_STORE + .lock() + .unwrap() + .insert(key.to_string(), value.to_string()); + Ok(()) + } + + pub fn get_storage_item(key: &str) -> Result, DatabaseError> { + Ok(MEMORY_STORE.lock().unwrap().get(key).cloned()) + } +} + +#[cfg(target_arch = "wasm32")] +pub use wasm_storage::{ + get_storage_item as wasm_get_storage_item, remove_storage_key as wasm_remove_storage_key, + set_storage_item as wasm_set_storage_item, + storage_keys_with_prefix as wasm_storage_keys_with_prefix, +}; + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + + #[test] + fn uses_default_statistics_base_dir() { + let expected = dirs::home_dir() + .expect("home dir") + .join(STATISTICS_META_SUBDIR); + assert_eq!(statistics_base_dir(), Some(expected)); + } +} + +#[cfg(all(test, target_arch = "wasm32"))] +mod wasm_tests { + use super::*; + use std::path::PathBuf; + use wasm_bindgen_test::*; + + fn clear_prefix(prefix: &str) { + let keys = wasm_storage_keys_with_prefix(prefix).expect("list keys"); + + for key in keys { + wasm_remove_storage_key(&key).expect("remove key"); + } + } + + #[wasm_bindgen_test] + fn base_dir_and_storage_roundtrip() { + let base_prefix = STATISTICS_META_SUBDIR; + let prefix_with_sep = format!("{base_prefix}/"); + + clear_prefix(&prefix_with_sep); + + let configured = require_statistics_base_dir(); + assert_eq!(configured, PathBuf::from(base_prefix)); + + let key = format!("{prefix_with_sep}sample"); + let value = "value"; + + wasm_set_storage_item(&key, value).expect("set item"); + assert_eq!( + wasm_get_storage_item(&key).expect("get item"), + Some(value.to_string()) + ); + + let keys = wasm_storage_keys_with_prefix(&prefix_with_sep).expect("keys"); + assert!(keys.iter().any(|existing| existing == &key)); + + wasm_remove_storage_key(&key).expect("remove key"); + let keys = wasm_storage_keys_with_prefix(&prefix_with_sep).expect("keys after remove"); + assert!(!keys.iter().any(|existing| existing == &key)); + } +} diff --git a/src/serdes/char_length_units.rs b/src/serdes/char_length_units.rs index 226b34bf..5adefdd6 100644 --- a/src/serdes/char_length_units.rs +++ b/src/serdes/char_length_units.rs @@ -36,7 +36,7 @@ impl ReferenceSerialization for CharLengthUnits { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) mod test { use crate::errors::DatabaseError; use crate::serdes::{ReferenceSerialization, ReferenceTables}; diff --git a/src/serdes/column.rs b/src/serdes/column.rs index 5971c5e0..bff5f597 100644 --- a/src/serdes/column.rs +++ b/src/serdes/column.rs @@ -139,7 +139,7 @@ impl ReferenceSerialization for ColumnRelation { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) mod test { use crate::catalog::{ ColumnCatalog, ColumnDesc, ColumnRef, ColumnRelation, ColumnSummary, TableName, diff --git a/src/serdes/data_value.rs b/src/serdes/data_value.rs index c36a37ca..38b1500a 100644 --- a/src/serdes/data_value.rs +++ b/src/serdes/data_value.rs @@ -3,7 +3,7 @@ use crate::types::value::DataValue; implement_serialization_by_bincode!(DataValue); -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) mod test { use crate::errors::DatabaseError; use crate::serdes::{ReferenceSerialization, ReferenceTables}; diff --git a/src/serdes/mod.rs b/src/serdes/mod.rs index 85ccc855..9113e14f 100644 --- a/src/serdes/mod.rs +++ b/src/serdes/mod.rs @@ -130,7 +130,7 @@ impl ReferenceTables { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::serdes::ReferenceTables; use std::io; diff --git a/src/serdes/num.rs b/src/serdes/num.rs index af58a40e..e59f1417 100644 --- a/src/serdes/num.rs +++ b/src/serdes/num.rs @@ -65,7 +65,7 @@ impl ReferenceSerialization for usize { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) mod test { use crate::errors::DatabaseError; use crate::serdes::{ReferenceSerialization, ReferenceTables}; diff --git a/src/serdes/string.rs b/src/serdes/string.rs index bcb25179..1530ccf8 100644 --- a/src/serdes/string.rs +++ b/src/serdes/string.rs @@ -28,7 +28,7 @@ impl ReferenceSerialization for Arc { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) mod test { use crate::errors::DatabaseError; use crate::serdes::{ReferenceSerialization, ReferenceTables}; diff --git a/src/storage/memory.rs b/src/storage/memory.rs new file mode 100644 index 00000000..9056eba6 --- /dev/null +++ b/src/storage/memory.rs @@ -0,0 +1,363 @@ +use crate::errors::DatabaseError; +use crate::storage::table_codec::{BumpBytes, Bytes, TableCodec}; +use crate::storage::{InnerIter, Storage, Transaction}; +use std::cell::RefCell; +use std::collections::{BTreeMap, Bound, VecDeque}; +use std::rc::Rc; + +#[derive(Clone, Default)] +pub struct MemoryStorage { + inner: Rc, Vec>>>, +} + +impl MemoryStorage { + pub fn new() -> Self { + Self::default() + } +} + +impl Storage for MemoryStorage { + type TransactionType<'a> + = MemoryTransaction + where + Self: 'a; + + fn transaction(&self) -> Result, DatabaseError> { + Ok(MemoryTransaction { + inner: self.inner.clone(), + table_codec: Default::default(), + }) + } +} + +pub struct MemoryTransaction { + inner: Rc, Vec>>>, + table_codec: TableCodec, +} + +pub struct MemoryIter { + entries: VecDeque<(Bytes, Bytes)>, +} + +impl InnerIter for MemoryIter { + fn try_next(&mut self) -> Result, DatabaseError> { + Ok(self.entries.pop_front()) + } +} + +impl Transaction for MemoryTransaction { + type IterType<'a> + = MemoryIter + where + Self: 'a; + + fn table_codec(&self) -> *const TableCodec { + &self.table_codec + } + + fn get(&self, key: &[u8]) -> Result, DatabaseError> { + Ok(self.inner.borrow().get(key).cloned()) + } + + fn set(&mut self, key: BumpBytes, value: BumpBytes) -> Result<(), DatabaseError> { + self.inner.borrow_mut().insert(key.to_vec(), value.to_vec()); + Ok(()) + } + + fn remove(&mut self, key: &[u8]) -> Result<(), DatabaseError> { + self.inner.borrow_mut().remove(key); + Ok(()) + } + + fn range<'a>( + &'a self, + min: Bound>, + max: Bound>, + ) -> Result, DatabaseError> { + let map = self.inner.borrow(); + let start = match &min { + Bound::Included(b) => Bound::Included(b.as_ref()), + Bound::Excluded(b) => Bound::Excluded(b.as_ref()), + Bound::Unbounded => Bound::Unbounded, + }; + let end = match &max { + Bound::Included(b) => Bound::Included(b.as_ref()), + Bound::Excluded(b) => Bound::Excluded(b.as_ref()), + Bound::Unbounded => Bound::Unbounded, + }; + + let entries = map + .range::<[u8], _>((start, end)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + + Ok(MemoryIter { entries }) + } + + fn commit(self) -> Result<(), DatabaseError> { + Ok(()) + } +} + +#[cfg(all(test, target_arch = "wasm32"))] +mod wasm_tests { + use super::*; + use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef, TableName}; + use crate::db::{DataBaseBuilder, ResultIter}; + use crate::expression::range_detacher::Range; + use crate::storage::Iter; + use crate::types::tuple::Tuple; + use crate::types::value::DataValue; + use crate::types::LogicalType; + use crate::utils::lru::SharedLruCache; + use itertools::Itertools; + use std::collections::{BTreeMap, Bound}; + use std::hash::RandomState; + use std::sync::Arc; + use wasm_bindgen_test::*; + + #[wasm_bindgen_test] + fn memory_storage_roundtrip() -> Result<(), DatabaseError> { + let storage = MemoryStorage::new(); + let mut transaction = storage.transaction()?; + let table_cache = Arc::new(SharedLruCache::new(4, 1, RandomState::new())?); + let columns = Arc::new(vec![ + ColumnRef::from(ColumnCatalog::new( + "c1".to_string(), + false, + ColumnDesc::new(LogicalType::Integer, Some(0), false, None).unwrap(), + )), + ColumnRef::from(ColumnCatalog::new( + "c2".to_string(), + false, + ColumnDesc::new(LogicalType::Boolean, None, false, None).unwrap(), + )), + ]); + + let source_columns = columns + .iter() + .map(|col_ref| ColumnCatalog::clone(col_ref)) + .collect_vec(); + transaction.create_table( + &table_cache, + "test".to_string().into(), + source_columns, + false, + )?; + + transaction.append_tuple( + &"test".to_string(), + Tuple::new( + Some(DataValue::Int32(1)), + vec![DataValue::Int32(1), DataValue::Boolean(true)], + ), + &[ + LogicalType::Integer.serializable(), + LogicalType::Boolean.serializable(), + ], + false, + )?; + transaction.append_tuple( + &"test".to_string(), + Tuple::new( + Some(DataValue::Int32(2)), + vec![DataValue::Int32(2), DataValue::Boolean(true)], + ), + &[ + LogicalType::Integer.serializable(), + LogicalType::Boolean.serializable(), + ], + false, + )?; + + let mut read_columns = BTreeMap::new(); + read_columns.insert(0, columns[0].clone()); + + let mut iter = transaction.read( + &table_cache, + "test".to_string().into(), + (Some(1), Some(1)), + read_columns, + true, + )?; + + let option_1 = iter.next_tuple()?; + assert_eq!(option_1.unwrap().pk, Some(DataValue::Int32(2))); + + let option_2 = iter.next_tuple()?; + assert_eq!(option_2, None); + + Ok(()) + } + + #[wasm_bindgen_test] + fn memory_storage_read_by_index() -> Result<(), DatabaseError> { + let kite_sql = DataBaseBuilder::path("./memory").build_in_memory()?; + kite_sql + .run("create table t1 (a int primary key, b int)")? + .done()?; + kite_sql + .run("insert into t1 (a, b) values (0, 0), (1, 1), (2, 2), (3, 4)")? + .done()?; + + let transaction = kite_sql.storage.transaction()?; + let table_name: TableName = "t1".to_string().into(); + let table = transaction + .table(kite_sql.state.table_cache(), table_name.clone())? + .unwrap() + .clone(); + let pk_index = table.indexes().next().unwrap().clone(); + let mut iter = transaction.read_by_index( + kite_sql.state.table_cache(), + table_name, + (Some(0), None), + table.columns().cloned().enumerate().collect(), + pk_index, + vec![Range::Scope { + min: Bound::Excluded(DataValue::Int32(0)), + max: Bound::Included(DataValue::Int32(2)), + }], + true, + )?; + + let mut result = Vec::new(); + while let Some(tuple) = iter.next_tuple()? { + result.push(tuple.pk.unwrap()); + } + + assert_eq!(result, vec![DataValue::Int32(1), DataValue::Int32(2)]); + + Ok(()) + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod native_tests { + use super::*; + use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef, TableName}; + use crate::db::{DataBaseBuilder, ResultIter}; + use crate::expression::range_detacher::Range; + use crate::storage::Iter; + use crate::types::tuple::Tuple; + use crate::types::value::DataValue; + use crate::types::LogicalType; + use crate::utils::lru::SharedLruCache; + use itertools::Itertools; + use std::collections::{BTreeMap, Bound}; + use std::hash::RandomState; + use std::sync::Arc; + + #[test] + fn memory_storage_roundtrip() -> Result<(), DatabaseError> { + let storage = MemoryStorage::new(); + let mut transaction = storage.transaction()?; + let table_cache = Arc::new(SharedLruCache::new(4, 1, RandomState::new())?); + let columns = Arc::new(vec![ + ColumnRef::from(ColumnCatalog::new( + "c1".to_string(), + false, + ColumnDesc::new(LogicalType::Integer, Some(0), false, None).unwrap(), + )), + ColumnRef::from(ColumnCatalog::new( + "c2".to_string(), + false, + ColumnDesc::new(LogicalType::Boolean, None, false, None).unwrap(), + )), + ]); + + let source_columns = columns + .iter() + .map(|col_ref| ColumnCatalog::clone(col_ref)) + .collect_vec(); + transaction.create_table( + &table_cache, + "test".to_string().into(), + source_columns, + false, + )?; + + transaction.append_tuple( + &"test".to_string(), + Tuple::new( + Some(DataValue::Int32(1)), + vec![DataValue::Int32(1), DataValue::Boolean(true)], + ), + &[ + LogicalType::Integer.serializable(), + LogicalType::Boolean.serializable(), + ], + false, + )?; + transaction.append_tuple( + &"test".to_string(), + Tuple::new( + Some(DataValue::Int32(2)), + vec![DataValue::Int32(2), DataValue::Boolean(true)], + ), + &[ + LogicalType::Integer.serializable(), + LogicalType::Boolean.serializable(), + ], + false, + )?; + + let mut read_columns = BTreeMap::new(); + read_columns.insert(0, columns[0].clone()); + + let mut iter = transaction.read( + &table_cache, + "test".to_string().into(), + (Some(1), Some(1)), + read_columns, + true, + )?; + + let option_1 = iter.next_tuple()?; + assert_eq!(option_1.unwrap().pk, Some(DataValue::Int32(2))); + + let option_2 = iter.next_tuple()?; + assert_eq!(option_2, None); + + Ok(()) + } + + #[test] + fn memory_storage_read_by_index() -> Result<(), DatabaseError> { + let kite_sql = DataBaseBuilder::path("./memory").build_in_memory()?; + kite_sql + .run("create table t1 (a int primary key, b int)")? + .done()?; + kite_sql + .run("insert into t1 (a, b) values (0, 0), (1, 1), (2, 2), (3, 4)")? + .done()?; + + let transaction = kite_sql.storage.transaction()?; + let table_name: TableName = "t1".to_string().into(); + let table = transaction + .table(kite_sql.state.table_cache(), table_name.clone())? + .unwrap() + .clone(); + let pk_index = table.indexes().next().unwrap().clone(); + let mut iter = transaction.read_by_index( + kite_sql.state.table_cache(), + table_name, + (Some(0), None), + table.columns().cloned().enumerate().collect(), + pk_index, + vec![Range::Scope { + min: Bound::Excluded(DataValue::Int32(0)), + max: Bound::Included(DataValue::Int32(2)), + }], + true, + )?; + + let mut result = Vec::new(); + while let Some(tuple) = iter.next_tuple()? { + result.push(tuple.pk.unwrap()); + } + + assert_eq!(result, vec![DataValue::Int32(1), DataValue::Int32(2)]); + + Ok(()) + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index c515bbb9..1d89d899 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,12 +1,15 @@ +pub mod memory; +#[cfg(not(target_arch = "wasm32"))] pub mod rocksdb; pub(crate) mod table_codec; use crate::catalog::view::View; use crate::catalog::{ColumnCatalog, ColumnRef, TableCatalog, TableMeta, TableName}; use crate::errors::DatabaseError; -use crate::execution::dml::analyze::Analyze; use crate::expression::range_detacher::Range; use crate::optimizer::core::statistics_meta::{StatisticMetaLoader, StatisticsMeta}; +#[cfg(not(target_arch = "wasm32"))] +use crate::paths::require_statistics_base_dir; use crate::serdes::ReferenceTables; use crate::storage::table_codec::{BumpBytes, Bytes, TableCodec}; use crate::types::index::{Index, IndexId, IndexMetaRef, IndexType}; @@ -17,11 +20,13 @@ use crate::types::{ColumnId, LogicalType}; use crate::utils::lru::SharedLruCache; use itertools::Itertools; use std::collections::{BTreeMap, Bound}; +#[cfg(not(target_arch = "wasm32"))] +use std::fs; use std::io::Cursor; +use std::mem; use std::ops::SubAssign; use std::sync::Arc; use std::vec::IntoIter; -use std::{fs, mem}; use ulid::Generator; pub(crate) type StatisticsMetaCache = SharedLruCache<(TableName, IndexId), StatisticsMeta>; @@ -504,7 +509,11 @@ pub trait Transaction: Sized { self.remove(&unsafe { &*self.table_codec() }.encode_root_table_key(table_name.as_ref()))?; table_cache.remove(&table_name); - let _ = fs::remove_dir(Analyze::build_statistics_meta_path(&table_name)); + #[cfg(not(target_arch = "wasm32"))] + { + let path = require_statistics_base_dir().join(table_name.as_ref()); + let _ = fs::remove_dir(path); + } Ok(()) } @@ -1269,7 +1278,7 @@ pub trait Iter { fn next_tuple(&mut self) -> Result, DatabaseError>; } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::binder::test::build_t1_table; use crate::catalog::view::View; diff --git a/src/storage/rocksdb.rs b/src/storage/rocksdb.rs index 2bb60a9c..c705d763 100644 --- a/src/storage/rocksdb.rs +++ b/src/storage/rocksdb.rs @@ -221,7 +221,7 @@ fn next( Ok(Some((Vec::from(key), Vec::from(value)))) } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef, TableName}; use crate::db::{DataBaseBuilder, ResultIter}; diff --git a/src/storage/table_codec.rs b/src/storage/table_codec.rs index cb3aaca5..b46275ac 100644 --- a/src/storage/table_codec.rs +++ b/src/storage/table_codec.rs @@ -533,7 +533,7 @@ impl TableCodec { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::binder::test::build_t1_table; use crate::catalog::view::View; diff --git a/src/types/evaluator/mod.rs b/src/types/evaluator/mod.rs index 3ea8ba33..b1201c2e 100644 --- a/src/types/evaluator/mod.rs +++ b/src/types/evaluator/mod.rs @@ -449,7 +449,7 @@ macro_rules! numeric_binary_evaluator_definition { }; } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::errors::DatabaseError; use crate::expression::BinaryOperator; diff --git a/src/types/mod.rs b/src/types/mod.rs index 1071fb82..ca9376fb 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -526,7 +526,7 @@ impl std::fmt::Display for LogicalType { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) mod test { use crate::errors::DatabaseError; use crate::serdes::{ReferenceSerialization, ReferenceTables}; diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 5cfdc020..54ed5f11 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -134,7 +134,7 @@ pub fn create_table(iter: I) -> Result { Ok(table) } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; use crate::types::tuple::Tuple; diff --git a/src/types/value.rs b/src/types/value.rs index 4aa36986..9a024b78 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -1935,7 +1935,7 @@ impl fmt::Debug for DataValue { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod test { use crate::errors::DatabaseError; use crate::storage::table_codec::BumpBytes; diff --git a/src/utils/lru.rs b/src/utils/lru.rs index 5ebe7b05..87bbf1a3 100644 --- a/src/utils/lru.rs +++ b/src/utils/lru.rs @@ -387,7 +387,7 @@ impl Drop for LruCache { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use crate::utils::lru::{LruCache, SharedLruCache}; use std::collections::hash_map::RandomState; diff --git a/src/wasm.rs b/src/wasm.rs new file mode 100644 index 00000000..c56e040f --- /dev/null +++ b/src/wasm.rs @@ -0,0 +1,115 @@ +#![cfg(target_arch = "wasm32")] + +use crate::db::{DataBaseBuilder, Database, DatabaseIter, ResultIter}; +use crate::storage::memory::MemoryStorage; +use crate::types::tuple::Tuple; +use crate::types::value::DataValue; +use serde::Serialize; +use wasm_bindgen::prelude::*; + +#[derive(Serialize)] +struct WasmRow { + pk: Option, + values: Vec, +} + +fn to_js_err(err: impl ToString) -> JsValue { + js_sys::Error::new(&err.to_string()).into() +} + +fn tuple_to_wasm_row(tuple: Tuple) -> WasmRow { + WasmRow { + pk: tuple.pk, + values: tuple.values, + } +} + +#[wasm_bindgen] +pub struct WasmDatabase { + inner: Database, +} + +#[wasm_bindgen] +pub struct WasmResultIter { + inner: Option>, +} + +impl Drop for WasmResultIter { + fn drop(&mut self) { + if let Some(iter) = self.inner.take() { + let _ = iter.done(); + } + } +} + +#[wasm_bindgen] +impl WasmDatabase { + #[wasm_bindgen(constructor)] + pub fn new() -> Result { + let db = DataBaseBuilder::path(".") + .build() + .map_err(|e| to_js_err(format!("init database failed: {e}")))?; + Ok(WasmDatabase { inner: db }) + } + + /// Stream results with a JS-friendly iterator that exposes `next()`. + pub fn run(&self, sql: &str) -> Result { + let iter = self.inner.run(sql).map_err(to_js_err)?; + // DatabaseIter owns its internal state; lifetime is only in the type. + let iter_static: DatabaseIter<'static, MemoryStorage> = + unsafe { std::mem::transmute(iter) }; + Ok(WasmResultIter { + inner: Some(iter_static), + }) + } + + pub fn execute(&self, sql: &str) -> Result<(), JsValue> { + let iter = self.inner.run(sql).map_err(to_js_err)?; + for tuple in iter { + tuple.map_err(to_js_err)?; + } + Ok(()) + } +} + +#[wasm_bindgen] +impl WasmResultIter { + /// Returns the next row as a JS object, or `undefined` when done. + #[wasm_bindgen(js_name = next)] + pub fn next(&mut self) -> Result { + let iter = self + .inner + .as_mut() + .ok_or_else(|| to_js_err("iterator already consumed"))?; + match iter.next() { + Some(Ok(tuple)) => serde_wasm_bindgen::to_value(&tuple_to_wasm_row(tuple)) + .map_err(|e| to_js_err(format!("serialize row: {e}"))), + Some(Err(err)) => Err(to_js_err(err.to_string())), + None => Ok(JsValue::undefined()), + } + } + + /// Collect all remaining rows into an array and finish the iterator. + #[wasm_bindgen(js_name = rows)] + pub fn rows(&mut self) -> Result { + let mut iter = self + .inner + .take() + .ok_or_else(|| to_js_err("iterator already consumed"))?; + let mut rows = Vec::new(); + for tuple in &mut iter { + let tuple = tuple.map_err(to_js_err)?; + rows.push(tuple_to_wasm_row(tuple)); + } + iter.done().map_err(to_js_err)?; + serde_wasm_bindgen::to_value(&rows).map_err(|e| to_js_err(format!("serialize rows: {e}"))) + } + + /// Finish iteration early and commit any work. + pub fn finish(&mut self) -> Result<(), JsValue> { + if let Some(iter) = self.inner.take() { + iter.done().map_err(to_js_err)?; + } + Ok(()) + } +}