From ed58109d41902ab53cadef82b39f37354d91f356 Mon Sep 17 00:00:00 2001 From: Li Yazhou Date: Fri, 21 Feb 2025 18:28:30 +0800 Subject: [PATCH 1/9] add serde_ignored --- Cargo.lock | 10 ++++++++++ Cargo.toml | 1 + src/query/config/Cargo.toml | 1 + 3 files changed, 12 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 124861c70cd1e..aa994a9196222 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3179,6 +3179,7 @@ dependencies = [ "log", "pretty_assertions", "serde", + "serde_ignored", "serde_with", "serfig", ] @@ -12897,6 +12898,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_ignored" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8e319a36d1b52126a0d608f24e93b2d81297091818cd70625fcf50a15d84ddf" +dependencies = [ + "serde", +] + [[package]] name = "serde_json" version = "1.0.132" diff --git a/Cargo.toml b/Cargo.toml index e0b1899861039..77cff81e26e14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -457,6 +457,7 @@ serde_test = "1.0" serde_urlencoded = "0.7.1" serde_with = { version = "3.8.1" } serfig = "0.1.0" +serde_ignored = "0.1.10" sha1 = "0.10.5" sha2 = "0.10.8" simdutf8 = "0.1.4" diff --git a/src/query/config/Cargo.toml b/src/query/config/Cargo.toml index 2c63cbbca8f74..bf018225c0844 100644 --- a/src/query/config/Cargo.toml +++ b/src/query/config/Cargo.toml @@ -31,6 +31,7 @@ log = { workspace = true } serde = { workspace = true } serde_with = { workspace = true } serfig = { workspace = true } +serde_ignored = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } From 53759533e971e9212e30cae0637eccce7b2b2698 Mon Sep 17 00:00:00 2001 From: Li Yazhou Date: Fri, 21 Feb 2025 18:43:03 +0800 Subject: [PATCH 2/9] add toml --- Cargo.lock | 2 ++ src/query/config/Cargo.toml | 2 ++ src/query/config/src/lib.rs | 1 + src/query/config/src/toml.rs | 37 ++++++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+) create mode 100644 src/query/config/src/toml.rs diff --git a/Cargo.lock b/Cargo.lock index aa994a9196222..4f32c3af4b26e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3167,6 +3167,7 @@ dependencies = [ name = "databend-common-config" version = "0.1.0" dependencies = [ + "anyhow", "chrono-tz 0.8.6", "clap", "databend-common-base", @@ -3182,6 +3183,7 @@ dependencies = [ "serde_ignored", "serde_with", "serfig", + "toml 0.8.19", ] [[package]] diff --git a/src/query/config/Cargo.toml b/src/query/config/Cargo.toml index bf018225c0844..271d92008136a 100644 --- a/src/query/config/Cargo.toml +++ b/src/query/config/Cargo.toml @@ -32,6 +32,8 @@ serde = { workspace = true } serde_with = { workspace = true } serfig = { workspace = true } serde_ignored = { workspace = true } +toml = { workspace = true } +anyhow = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/src/query/config/src/lib.rs b/src/query/config/src/lib.rs index 039390dba1e4e..4d47119d50b71 100644 --- a/src/query/config/src/lib.rs +++ b/src/query/config/src/lib.rs @@ -34,6 +34,7 @@ mod global; mod inner; mod mask; mod obsolete; +mod toml; pub use builtin::*; pub use config::CacheStorageTypeConfig; pub use config::Commands; diff --git a/src/query/config/src/toml.rs b/src/query/config/src/toml.rs new file mode 100644 index 0000000000000..ea7cbdbb35b13 --- /dev/null +++ b/src/query/config/src/toml.rs @@ -0,0 +1,37 @@ +use anyhow::anyhow; +use anyhow::Result; +use serde::de::DeserializeOwned; +use serfig::Parser; + +/// Toml parser which used with serfig. +/// +/// This parser will ignore unknown fields and call the handler with the path of the unknown field. +pub struct TomlIgnored { + handler: TomlUnknownFieldHandler, +} + +type TomlUnknownFieldHandler = Box; + +impl TomlIgnored { + pub fn new(handler: TomlUnknownFieldHandler) -> Self { + Self { handler } + } +} + +impl std::fmt::Debug for TomlIgnored { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TomlIgnored").finish() + } +} + +impl Parser for TomlIgnored { + fn parse(&mut self, bs: &[u8]) -> Result { + let s = std::str::from_utf8(bs) + .map_err(|err| anyhow!("input value is not valid utf-8: {err:?}"))?; + let de = toml::de::Deserializer::new(s); + let handler = &self.handler; + Ok(serde_ignored::deserialize(de, move |path| { + handler(path.to_string().as_str()); + })?) + } +} From 762329856f6c3b0f7c8ee1989bcdb72f132712cf Mon Sep 17 00:00:00 2001 From: Li Yazhou Date: Fri, 21 Feb 2025 18:55:05 +0800 Subject: [PATCH 3/9] use TomlIgnored --- src/query/config/src/config.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index 5a98a868c52a6..a33ad5881aa0a 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -73,6 +73,7 @@ use crate::background_config::BackgroundConfig; use crate::builtin::BuiltInConfig; use crate::builtin::UDFConfig; use crate::builtin::UserConfig; +use crate::toml::TomlIgnored; use crate::DATABEND_COMMIT_VERSION; const CATALOG_HIVE: &str = "hive"; @@ -217,7 +218,10 @@ impl Config { }; if !config_file.is_empty() { - builder = builder.collect(from_file(Toml, &config_file)); + let toml = TomlIgnored::new(Box::new(|path| { + log::warn!("unknown field in config: {}", &path); + })); + builder = builder.collect(from_file(toml, &config_file)); } } From 70de3bd5b1920e689981ace4ac27bcdac8baca4c Mon Sep 17 00:00:00 2001 From: Li Yazhou Date: Fri, 21 Feb 2025 18:55:58 +0800 Subject: [PATCH 4/9] remove deny_unkown_fields --- src/query/config/src/config.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index a33ad5881aa0a..eb95cfdd11f97 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -1398,7 +1398,7 @@ impl serde::de::Visitor<'_> for SettingVisitor { /// Query config group. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] -#[serde(default, deny_unknown_fields)] +#[serde(default)] pub struct QueryConfig { /// Tenant id for get the information from the MetaSrv. #[clap(long, value_name = "VALUE", default_value = "admin")] @@ -2514,10 +2514,9 @@ impl From for OTLPEndpointConfig { } /// Meta config group. -/// deny_unknown_fields to check unknown field, like the deprecated `address`. /// TODO(xuanwo): All meta_xxx should be rename to xxx. #[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Args)] -#[serde(default, deny_unknown_fields)] +#[serde(default)] pub struct MetaConfig { /// The dir to store persisted meta state for a embedded meta store #[clap(long = "meta-embedded-dir", value_name = "VALUE", default_value_t)] @@ -2723,7 +2722,7 @@ impl TryInto for LocalConfig { } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] -#[serde(default, deny_unknown_fields)] +#[serde(default)] pub struct CacheConfig { /// Enable table meta cache. Default is enabled. Set it to false to disable all the table meta caches #[clap(long = "cache-enable-table-meta-cache", default_value = "true")] @@ -2961,7 +2960,7 @@ impl Default for DiskCacheKeyReloadPolicy { } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] -#[serde(default, deny_unknown_fields)] +#[serde(default)] pub struct DiskCacheConfig { /// Max bytes of cached raw table data. Default 20GB, set it to 0 to disable it. #[clap( @@ -2999,7 +2998,7 @@ impl Default for DiskCacheConfig { } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] -#[serde(default, deny_unknown_fields)] +#[serde(default)] pub struct SpillConfig { /// Path of spill to local disk. disable if it's empty. #[clap(long, value_name = "VALUE", default_value = "")] @@ -3025,7 +3024,7 @@ impl Default for SpillConfig { } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args, Default)] -#[serde(default, deny_unknown_fields)] +#[serde(default)] pub struct ResourcesManagementConfig { #[clap(long = "type", value_name = "VALUE", default_value = "self_managed")] #[serde(rename = "type")] From 2b56477f611b8e07945ed43bcc2d987b27d2c667 Mon Sep 17 00:00:00 2001 From: Li Yazhou Date: Fri, 21 Feb 2025 20:29:42 +0800 Subject: [PATCH 5/9] fix test --- Cargo.lock | 1 + Cargo.toml | 2 +- src/query/config/Cargo.toml | 1 + src/query/config/src/config.rs | 11 ++++++----- src/query/config/src/inner.rs | 4 ++-- src/query/config/src/toml.rs | 2 +- src/query/config/tests/it/config.rs | 18 ++++++++++++++++++ src/query/service/Cargo.toml | 2 +- src/query/service/src/local/mod.rs | 2 +- 9 files changed, 32 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f32c3af4b26e..689dc7416bd5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3183,6 +3183,7 @@ dependencies = [ "serde_ignored", "serde_with", "serfig", + "tempfile", "toml 0.8.19", ] diff --git a/Cargo.toml b/Cargo.toml index 77cff81e26e14..726dc5de0e213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -493,7 +493,7 @@ tikv-jemalloc-ctl = { version = "0.6.0", features = ["use_std", "stats"] } tikv-jemalloc-sys = "0.6.0" tokio = { version = "1.35.0", features = ["full"] } tokio-stream = "0.1.11" -toml = { version = "0.8", default-features = false } +toml = { version = "0.8", features = ["parse"] } tonic = { version = "0.12.3", features = ["transport", "codegen", "prost", "tls-roots", "tls"] } tonic-build = { version = "0.12.3" } tonic-reflection = { version = "0.12.3" } diff --git a/src/query/config/Cargo.toml b/src/query/config/Cargo.toml index 271d92008136a..8669d6e02e43f 100644 --- a/src/query/config/Cargo.toml +++ b/src/query/config/Cargo.toml @@ -37,6 +37,7 @@ anyhow = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } +tempfile = { workspace = true } [build-dependencies] databend-common-building = { workspace = true } diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index eb95cfdd11f97..c6d8b91f8a4e2 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -60,7 +60,6 @@ use serde_with::with_prefix; use serfig::collectors::from_env; use serfig::collectors::from_file; use serfig::collectors::from_self; -use serfig::parsers::Toml; use super::inner; use super::inner::CatalogConfig as InnerCatalogConfig; @@ -186,7 +185,7 @@ impl Config { /// We should set this to false during tests because we don't want /// our test binary to parse cargo's args. #[no_sanitize(address)] - pub fn load(with_args: bool) -> Result { + pub fn load(config_file: Option, with_args: bool) -> Result { let mut arg_conf = Self::default(); if with_args { @@ -205,7 +204,9 @@ impl Config { // Load from config file first. { - let config_file = if !arg_conf.config_file.is_empty() { + let final_config_file = if let Some(config_file) = config_file { + config_file + } else if !arg_conf.config_file.is_empty() { // TODO: remove this `allow(clippy::redundant_clone)` // as soon as this issue is fixed: // https://github.com/rust-lang/rust-clippy/issues/10940 @@ -217,11 +218,11 @@ impl Config { "".to_string() }; - if !config_file.is_empty() { + if !final_config_file.is_empty() { let toml = TomlIgnored::new(Box::new(|path| { log::warn!("unknown field in config: {}", &path); })); - builder = builder.collect(from_file(toml, &config_file)); + builder = builder.collect(from_file(toml, &final_config_file)); } } diff --git a/src/query/config/src/inner.rs b/src/query/config/src/inner.rs index 424caece170c7..5546be1d9aaae 100644 --- a/src/query/config/src/inner.rs +++ b/src/query/config/src/inner.rs @@ -80,7 +80,7 @@ impl InnerConfig { /// /// In the future, we could have `ConfigV1` and `ConfigV2`. pub async fn load() -> Result { - let mut cfg: Self = Config::load(true)?.try_into()?; + let mut cfg: Self = Config::load(None, true)?.try_into()?; // Handle the node_id and node_secret for query node. cfg.query.node_id = GlobalUniqName::unique(); @@ -100,7 +100,7 @@ impl InnerConfig { /// /// This function is served for tests only. pub fn load_for_test() -> Result { - let cfg: Self = Config::load(false)?.try_into()?; + let cfg: Self = Config::load(None, false)?.try_into()?; Ok(cfg) } diff --git a/src/query/config/src/toml.rs b/src/query/config/src/toml.rs index ea7cbdbb35b13..7f126455128b8 100644 --- a/src/query/config/src/toml.rs +++ b/src/query/config/src/toml.rs @@ -28,7 +28,7 @@ impl Parser for TomlIgnored { fn parse(&mut self, bs: &[u8]) -> Result { let s = std::str::from_utf8(bs) .map_err(|err| anyhow!("input value is not valid utf-8: {err:?}"))?; - let de = toml::de::Deserializer::new(s); + let de = toml::Deserializer::new(s); let handler = &self.handler; Ok(serde_ignored::deserialize(de, move |path| { handler(path.to_string().as_str()); diff --git a/src/query/config/tests/it/config.rs b/src/query/config/tests/it/config.rs index 745cdf284d7c8..a5dbf4f74280b 100644 --- a/src/query/config/tests/it/config.rs +++ b/src/query/config/tests/it/config.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::ffi::OsString; +use std::io::Write; use clap::Parser; use databend_common_config::Config; @@ -32,3 +33,20 @@ fn test_config_default() { "default setting is different from default config, please check again" ) } + +#[test] +fn test_load_config() { + let conf = " +[query] +max_active_sessions = 256 +shutdown_wait_timeout_ms = 5000 +unknown_field = \"123\" + "; + let mut temp_file = tempfile::NamedTempFile::new().unwrap(); + temp_file.write_all(conf.as_bytes()).unwrap(); + let temp_path = temp_file.path().to_str().unwrap().to_string(); + + let config = Config::load(Some(temp_path), false).unwrap(); + assert_eq!(config.query.max_active_sessions, 256); + assert_eq!(config.query.shutdown_wait_timeout_ms, 5000); +} diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index 43d0205bb7e38..973d3a085ed30 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -172,7 +172,7 @@ sysinfo = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true, features = ["net"] } -toml = { workspace = true, default-features = false } +toml = { workspace = true, features = ["parse"] } tonic = { workspace = true } typetag = { workspace = true } url = { workspace = true } diff --git a/src/query/service/src/local/mod.rs b/src/query/service/src/local/mod.rs index 0a64768e7e193..f0b9ff93273c8 100644 --- a/src/query/service/src/local/mod.rs +++ b/src/query/service/src/local/mod.rs @@ -42,7 +42,7 @@ pub async fn query_local(query_sql: &str, output_format: &str) -> Result<()> { }; env::set_var("META_EMBEDDED_DIR", path.join("_meta")); - let mut conf: InnerConfig = Config::load(true).unwrap().try_into().unwrap(); + let mut conf: InnerConfig = Config::load(None, true).unwrap().try_into().unwrap(); conf.storage.allow_insecure = true; conf.storage.params = StorageParams::Fs(StorageFsConfig { root: path.join("_data").to_str().unwrap().to_owned(), From 60af507b0e4f39dfe06b132d2b6c37912592736d Mon Sep 17 00:00:00 2001 From: Li Yazhou Date: Fri, 21 Feb 2025 20:38:53 +0800 Subject: [PATCH 6/9] fix license --- src/query/config/src/toml.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/query/config/src/toml.rs b/src/query/config/src/toml.rs index 7f126455128b8..652f64aceb248 100644 --- a/src/query/config/src/toml.rs +++ b/src/query/config/src/toml.rs @@ -1,3 +1,17 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use anyhow::anyhow; use anyhow::Result; use serde::de::DeserializeOwned; From dd2f39635e1c8f1b0a5f6e0ff4b67659362eb233 Mon Sep 17 00:00:00 2001 From: Li Yazhou Date: Fri, 21 Feb 2025 20:46:25 +0800 Subject: [PATCH 7/9] fix format --- Cargo.toml | 2 +- src/query/config/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 726dc5de0e213..4e62487039e0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -450,6 +450,7 @@ scroll = "0.12.0" semver = "1.0.14" serde = { version = "1.0.164", features = ["derive", "rc"] } serde_derive = "1" +serde_ignored = "0.1.10" serde_json = { version = "1.0.85", default-features = false, features = ["preserve_order", "unbounded_depth"] } serde_repr = "0.1.9" serde_stacker = { version = "0.1" } @@ -457,7 +458,6 @@ serde_test = "1.0" serde_urlencoded = "0.7.1" serde_with = { version = "3.8.1" } serfig = "0.1.0" -serde_ignored = "0.1.10" sha1 = "0.10.5" sha2 = "0.10.8" simdutf8 = "0.1.4" diff --git a/src/query/config/Cargo.toml b/src/query/config/Cargo.toml index 8669d6e02e43f..e6b1ddc288196 100644 --- a/src/query/config/Cargo.toml +++ b/src/query/config/Cargo.toml @@ -19,6 +19,7 @@ storage-hdfs = ["databend-common-storage/storage-hdfs"] ignored = ["strum"] [dependencies] +anyhow = { workspace = true } chrono-tz = { workspace = true } clap = { workspace = true } databend-common-base = { workspace = true } @@ -29,11 +30,10 @@ databend-common-storage = { workspace = true } databend-common-tracing = { workspace = true } log = { workspace = true } serde = { workspace = true } +serde_ignored = { workspace = true } serde_with = { workspace = true } serfig = { workspace = true } -serde_ignored = { workspace = true } toml = { workspace = true } -anyhow = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } From f19fbcb8f78a1e5f40fea0caba71455745cbb6f6 Mon Sep 17 00:00:00 2001 From: BohuTANG Date: Sat, 29 Mar 2025 13:26:37 +0800 Subject: [PATCH 8/9] keep config load as origin and add more unit test --- src/query/config/src/config.rs | 6 +-- src/query/config/src/inner.rs | 4 +- src/query/config/tests/it/config.rs | 65 +++++++++++++++++++++++++---- src/query/service/src/local/mod.rs | 2 +- 4 files changed, 63 insertions(+), 14 deletions(-) diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index 33243f659a80a..493215f0709be 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -181,7 +181,7 @@ impl Config { /// We should set this to false during tests because we don't want /// our test binary to parse cargo's args. #[no_sanitize(address)] - pub fn load(config_file: Option, with_args: bool) -> Result { + pub fn load(with_args: bool) -> Result { let mut arg_conf = Self::default(); if with_args { @@ -200,9 +200,7 @@ impl Config { // Load from config file first. { - let final_config_file = if let Some(config_file) = config_file { - config_file - } else if !arg_conf.config_file.is_empty() { + let final_config_file = if !arg_conf.config_file.is_empty() { // TODO: remove this `allow(clippy::redundant_clone)` // as soon as this issue is fixed: // https://github.com/rust-lang/rust-clippy/issues/10940 diff --git a/src/query/config/src/inner.rs b/src/query/config/src/inner.rs index 9bd70b1d9304b..813986a3bdbb6 100644 --- a/src/query/config/src/inner.rs +++ b/src/query/config/src/inner.rs @@ -76,7 +76,7 @@ impl InnerConfig { /// /// In the future, we could have `ConfigV1` and `ConfigV2`. pub async fn load() -> Result { - let mut cfg: Self = Config::load(None, true)?.try_into()?; + let mut cfg: Self = Config::load(true)?.try_into()?; // Handle the node_id and node_secret for query node. cfg.query.node_id = GlobalUniqName::unique(); @@ -96,7 +96,7 @@ impl InnerConfig { /// /// This function is served for tests only. pub fn load_for_test() -> Result { - let cfg: Self = Config::load(None, false)?.try_into()?; + let cfg: Self = Config::load(false)?.try_into()?; Ok(cfg) } diff --git a/src/query/config/tests/it/config.rs b/src/query/config/tests/it/config.rs index a5dbf4f74280b..ed8b6bf33171d 100644 --- a/src/query/config/tests/it/config.rs +++ b/src/query/config/tests/it/config.rs @@ -36,17 +36,68 @@ fn test_config_default() { #[test] fn test_load_config() { - let conf = " + // Create a comprehensive test configuration with multiple sections and data types + let conf = r#" +# Query configuration section [query] max_active_sessions = 256 shutdown_wait_timeout_ms = 5000 -unknown_field = \"123\" - "; +flight_record_quota_size = 1048576 +unknown_query_field = "should be ignored" + +# Log configuration section +[log] +file.level = "INFO" +file.dir = "/tmp/databend/logs" +unknown_log_field = 123 + +# Storage configuration section +[storage] +type = "fs" + +[storage.fs] +data_path = "/tmp/databend/data" + +# Meta configuration section +[meta] +endpoint = "localhost:9191" +username = "databend" +password = "databend123" +"#; + let mut temp_file = tempfile::NamedTempFile::new().unwrap(); temp_file.write_all(conf.as_bytes()).unwrap(); let temp_path = temp_file.path().to_str().unwrap().to_string(); - - let config = Config::load(Some(temp_path), false).unwrap(); - assert_eq!(config.query.max_active_sessions, 256); - assert_eq!(config.query.shutdown_wait_timeout_ms, 5000); + + // Save the original environment variable (if it exists) + let original_env = std::env::var("CONFIG_FILE").ok(); + + // Set the environment variable to our test config file + std::env::set_var("CONFIG_FILE", &temp_path); + + // Use the original load function (without a config_file parameter) + let config = Config::load(false).unwrap(); + + // Restore the original environment variable + match original_env { + Some(val) => std::env::set_var("CONFIG_FILE", val), + None => std::env::remove_var("CONFIG_FILE"), + } + + // Test query configuration values + assert_eq!(config.query.max_active_sessions, 256, "max_active_sessions should be 256"); + assert_eq!(config.query.shutdown_wait_timeout_ms, 5000, "shutdown_wait_timeout_ms should be 5000"); + assert_eq!(config.query.flight_record_quota_size, 1048576, "flight_record_quota_size should be 1048576"); + + // Test log configuration values + assert_eq!(config.log.file.level, "INFO", "log.file.level should be INFO"); + assert_eq!(config.log.file.dir, "/tmp/databend/logs", "log.file.dir should be /tmp/databend/logs"); + + // Test storage configuration values + assert_eq!(config.storage.fs.data_path, "/tmp/databend/data", "storage.fs.data_path should be /tmp/databend/data"); + + // Test meta configuration values + assert_eq!(config.meta.endpoint, "localhost:9191", "meta.endpoint should be localhost:9191"); + assert_eq!(config.meta.username, "databend", "meta.username should be databend"); + assert_eq!(config.meta.password, "databend123", "meta.password should be databend123"); } diff --git a/src/query/service/src/local/mod.rs b/src/query/service/src/local/mod.rs index f0b9ff93273c8..0a64768e7e193 100644 --- a/src/query/service/src/local/mod.rs +++ b/src/query/service/src/local/mod.rs @@ -42,7 +42,7 @@ pub async fn query_local(query_sql: &str, output_format: &str) -> Result<()> { }; env::set_var("META_EMBEDDED_DIR", path.join("_meta")); - let mut conf: InnerConfig = Config::load(None, true).unwrap().try_into().unwrap(); + let mut conf: InnerConfig = Config::load(true).unwrap().try_into().unwrap(); conf.storage.allow_insecure = true; conf.storage.params = StorageParams::Fs(StorageFsConfig { root: path.join("_data").to_str().unwrap().to_owned(), From ce52ce4860762106c1ec215650185aab8984ee69 Mon Sep 17 00:00:00 2001 From: Bohu Date: Sat, 29 Mar 2025 13:52:02 +0800 Subject: [PATCH 9/9] Update read_cache_content.rs --- src/query/storages/common/cache/benches/read_cache_content.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/storages/common/cache/benches/read_cache_content.rs b/src/query/storages/common/cache/benches/read_cache_content.rs index aed672363d85d..65992cb2e347a 100644 --- a/src/query/storages/common/cache/benches/read_cache_content.rs +++ b/src/query/storages/common/cache/benches/read_cache_content.rs @@ -20,6 +20,7 @@ use criterion::black_box; use criterion::criterion_group; use criterion::criterion_main; use criterion::Criterion; +use databend_storages_common_cache::read_cache_content; use tempfile::TempDir; fn create_test_file(dir: &TempDir, name: &str, size: usize) -> PathBuf {