Skip to content

Commit 3d03417

Browse files
youngsofunyoungsofun
authored
feat: deprecate prefix_filter and change default level of logging. (#17444)
* refactor: code reuse. * feat: change default log level. * feat: deprecate prefix_filter. * update tests.
1 parent d0d7a31 commit 3d03417

File tree

16 files changed

+69
-128
lines changed

16 files changed

+69
-128
lines changed

‎scripts/ci/deploy/config/databend-query-node-1.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ join_spilling_memory_ratio = 60
8585
level = "DEBUG"
8686
format = "text"
8787
dir = "./.databend/logs_1"
88-
prefix_filter = ""
8988

9089
[log.query]
9190
on = true

‎scripts/ci/deploy/config/databend-query-node-2.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLE
5757
level = "INFO"
5858
format = "text"
5959
dir = "./.databend/logs_2"
60-
prefix_filter = ""
6160

6261
[log.structlog]
6362
on = true

‎scripts/ci/deploy/config/databend-query-node-3.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLE
5858
level = "INFO"
5959
format = "text"
6060
dir = "./.databend/logs_3"
61-
prefix_filter = ""
6261

6362
[log.structlog]
6463
on = true

‎scripts/ci/deploy/config/databend-query-node-system-managed.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ node_group
6060
level = "INFO"
6161
format = "text"
6262
dir = "./.databend/query_logs"
63-
prefix_filter = ""
6463

6564
[meta]
6665
# It is a list of `grpc_api_advertise_host:<grpc-api-port>` of databend-meta config

‎src/common/tracing/src/config.rs‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@ use std::collections::BTreeMap;
1616
use std::fmt::Display;
1717
use std::fmt::Formatter;
1818

19+
// use the uncommon usage of Pascal Case level name
20+
// to partially avoid the feature of serfig that can not override with the default value.
21+
// see https://github.com/Xuanwo/serfig/issues/23 for detail.
22+
pub const CONFIG_DEFAULT_LOG_LEVEL: &str = "Warn,databend_=Info,openraft=Info";
23+
1924
/// Config for logging.
2025
#[derive(Clone, Debug, PartialEq, Eq, Default, serde::Serialize)]
2126
pub struct Config {
@@ -34,11 +39,10 @@ impl Config {
3439
Self {
3540
file: FileConfig {
3641
on: true,
37-
level: "DEBUG".to_string(),
42+
level: "WARN,databend=DEBUG,openraft=DEBUG".to_string(),
3843
dir: "./.databend/logs".to_string(),
3944
format: "text".to_string(),
4045
limit: 48,
41-
prefix_filter: "databend_,openraft".to_string(),
4246
},
4347
stderr: StderrConfig {
4448
on: true,
@@ -57,15 +61,14 @@ pub struct FileConfig {
5761
pub dir: String,
5862
pub format: String,
5963
pub limit: usize,
60-
pub prefix_filter: String,
6164
}
6265

6366
impl Display for FileConfig {
6467
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
6568
write!(
6669
f,
67-
"enabled={}, level={}, dir={}, format={}, limit={}, prefix_filter={}",
68-
self.on, self.level, self.dir, self.format, self.limit, self.prefix_filter
70+
"enabled={}, level='{}', dir={}, format={}, limit={}",
71+
self.on, self.level, self.dir, self.format, self.limit,
6972
)
7073
}
7174
}
@@ -74,11 +77,10 @@ impl Default for FileConfig {
7477
fn default() -> Self {
7578
Self {
7679
on: true,
77-
level: "INFO".to_string(),
80+
level: CONFIG_DEFAULT_LOG_LEVEL.to_string(),
7881
dir: "./.databend/logs".to_string(),
7982
format: "json".to_string(),
8083
limit: 48,
81-
prefix_filter: "databend_,openraft".to_string(),
8284
}
8385
}
8486
}
@@ -111,7 +113,7 @@ impl Default for StderrConfig {
111113
fn default() -> Self {
112114
Self {
113115
on: false,
114-
level: "INFO".to_string(),
116+
level: CONFIG_DEFAULT_LOG_LEVEL.to_string(),
115117
format: "text".to_string(),
116118
}
117119
}
@@ -138,7 +140,7 @@ impl Default for OTLPConfig {
138140
fn default() -> Self {
139141
Self {
140142
on: false,
141-
level: "INFO".to_string(),
143+
level: CONFIG_DEFAULT_LOG_LEVEL.to_string(),
142144
endpoint: OTLPEndpointConfig::default(),
143145
}
144146
}
@@ -240,7 +242,7 @@ impl Default for TracingConfig {
240242
fn default() -> Self {
241243
Self {
242244
on: false,
243-
capture_log_level: "INFO".to_string(),
245+
capture_log_level: CONFIG_DEFAULT_LOG_LEVEL.to_string(),
244246
otlp: OTLPEndpointConfig::default(),
245247
}
246248
}

‎src/common/tracing/src/init.rs‎

Lines changed: 14 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,8 @@ use databend_common_base::base::GlobalInstance;
2121
use databend_common_base::runtime::Thread;
2222
use fastrace::prelude::*;
2323
use log::LevelFilter;
24-
use log::Metadata;
2524
use logforth::filter::env::EnvFilterBuilder;
26-
use logforth::filter::CustomFilter;
2725
use logforth::filter::EnvFilter;
28-
use logforth::filter::FilterResult;
2926
use logforth::Dispatch;
3027
use logforth::Logger;
3128
use opentelemetry_otlp::WithExportConfig;
@@ -72,6 +69,16 @@ pub fn inject_span_to_tonic_request<T>(msg: impl tonic::IntoRequest<T>) -> tonic
7269
request
7370
}
7471

72+
fn env_filter(level: &str) -> EnvFilter {
73+
EnvFilter::new(
74+
EnvFilterBuilder::new()
75+
.filter(Some("databend::log::query"), LevelFilter::Off)
76+
.filter(Some("databend::log::profile"), LevelFilter::Off)
77+
.filter(Some("databend::log::structlog"), LevelFilter::Off)
78+
.parse(level),
79+
)
80+
}
81+
7582
pub fn init_logging(
7683
log_name: &str,
7784
cfg: &Config,
@@ -180,28 +187,15 @@ pub fn init_logging(
180187
_drop_guards.push(flush_guard);
181188

182189
let dispatch = Dispatch::new()
183-
.filter(EnvFilter::new(
184-
EnvFilterBuilder::new()
185-
.filter(Some("databend::log::query"), LevelFilter::Off)
186-
.filter(Some("databend::log::profile"), LevelFilter::Off)
187-
.filter(Some("databend::log::structlog"), LevelFilter::Off)
188-
.parse(&cfg.file.level),
189-
))
190-
.filter(make_log_filter(&cfg.file.prefix_filter))
190+
.filter(env_filter(&cfg.file.level))
191191
.append(normal_log_file.with_layout(get_layout(&cfg.file.format)));
192192
logger = logger.dispatch(dispatch);
193193
}
194194

195195
// console logger
196196
if cfg.stderr.on {
197197
let dispatch = Dispatch::new()
198-
.filter(EnvFilter::new(
199-
EnvFilterBuilder::new()
200-
.filter(Some("databend::log::query"), LevelFilter::Off)
201-
.filter(Some("databend::log::profile"), LevelFilter::Off)
202-
.filter(Some("databend::log::structlog"), LevelFilter::Off)
203-
.parse(&cfg.stderr.level),
204-
))
198+
.filter(env_filter(&cfg.stderr.level))
205199
.append(
206200
logforth::append::Stderr::default().with_layout(get_layout(&cfg.stderr.format)),
207201
);
@@ -227,33 +221,15 @@ pub fn init_logging(
227221
.build()
228222
.expect("initialize opentelemetry logger");
229223
let dispatch = Dispatch::new()
230-
.filter(EnvFilter::new(
231-
EnvFilterBuilder::new()
232-
.filter(Some("databend::log::query"), LevelFilter::Off)
233-
.filter(Some("databend::log::profile"), LevelFilter::Off)
234-
.filter(Some("databend::log::structlog"), LevelFilter::Off)
235-
.parse(&cfg.otlp.level),
236-
))
224+
.filter(env_filter(&cfg.otlp.level))
237225
.append(otel);
238226
logger = logger.dispatch(dispatch);
239227
}
240228

241229
// log to fastrace
242230
if cfg.tracing.on || cfg.structlog.on {
243-
let level = cfg
244-
.tracing
245-
.capture_log_level
246-
.parse()
247-
.ok()
248-
.unwrap_or(LevelFilter::Info);
249231
let dispatch = Dispatch::new()
250-
.filter(EnvFilter::new(
251-
EnvFilterBuilder::new()
252-
.filter(Some("databend::log::query"), LevelFilter::Off)
253-
.filter(Some("databend::log::profile"), LevelFilter::Off)
254-
.filter(Some("databend::log::structlog"), LevelFilter::Off),
255-
))
256-
.filter(level)
232+
.filter(env_filter(&cfg.tracing.capture_log_level))
257233
.append(logforth::append::FastraceEvent::default());
258234
logger = logger.dispatch(dispatch);
259235
}
@@ -365,35 +341,3 @@ pub fn init_logging(
365341

366342
_drop_guards
367343
}
368-
369-
/// Creates a log filter that matches log entries based on specified target prefixes or severity.
370-
fn make_log_filter(prefix_filter: &str) -> CustomFilter {
371-
let prefixes = prefix_filter
372-
.split(',')
373-
.map(|x| x.to_string())
374-
.collect::<Vec<_>>();
375-
376-
CustomFilter::new(move |meta| match_prefix(meta, &prefixes))
377-
}
378-
379-
fn match_prefix(meta: &Metadata, prefixes: &[String]) -> FilterResult {
380-
if is_severe(meta) {
381-
return FilterResult::Neutral;
382-
}
383-
384-
for p in prefixes {
385-
if meta.target().starts_with(p) {
386-
return FilterResult::Accept;
387-
}
388-
}
389-
390-
FilterResult::Reject
391-
}
392-
393-
/// Return true if the log level is considered severe.
394-
///
395-
/// Severe logs ignores the prefix filter.
396-
fn is_severe(meta: &Metadata) -> bool {
397-
// For other component, output logs with level <= WARN
398-
meta.level() <= LevelFilter::Warn
399-
}

‎src/common/tracing/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ pub use crate::config::QueryLogConfig;
3737
pub use crate::config::StderrConfig;
3838
pub use crate::config::StructLogConfig;
3939
pub use crate::config::TracingConfig;
40+
pub use crate::config::CONFIG_DEFAULT_LOG_LEVEL;
4041
pub use crate::crash_hook::set_crash_hook;
4142
pub use crate::init::init_logging;
4243
pub use crate::init::inject_span_to_tonic_request;

‎src/meta/binaries/metabench/main.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ struct Config {
6565
#[clap(long, default_value = "10000")]
6666
pub number: u64,
6767

68-
#[clap(long, default_value = "INFO")]
68+
#[clap(long, default_value = "warn,databend=info")]
6969
pub log_level: String,
7070

7171
#[clap(long, env = "METASRV_GRPC_API_ADDRESS", default_value = "")]
@@ -92,7 +92,6 @@ async fn main() {
9292
dir: "./.databend/logs".to_string(),
9393
format: "text".to_string(),
9494
limit: 48,
95-
prefix_filter: "databend_".to_string(),
9695
},
9796
stderr: StderrConfig {
9897
on: true,

‎src/meta/binaries/metactl/main.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,13 @@ use databend_common_meta_raft_store::config::RaftConfig;
3737
use databend_common_tracing::init_logging;
3838
use databend_common_tracing::Config as LogConfig;
3939
use databend_common_tracing::FileConfig;
40+
use databend_common_tracing::CONFIG_DEFAULT_LOG_LEVEL;
4041
use databend_meta::version::METASRV_COMMIT_VERSION;
4142
use serde::Deserialize;
4243

4344
#[derive(Debug, Clone, Deserialize, Args)]
4445
pub struct GlobalArgs {
45-
#[clap(long, default_value = "INFO")]
46+
#[clap(long, default_value = CONFIG_DEFAULT_LOG_LEVEL)]
4647
pub log_level: String,
4748

4849
/// DEPRECATE: use subcommand instead.
@@ -371,7 +372,6 @@ async fn main() -> anyhow::Result<()> {
371372
dir: ".databend/logs".to_string(),
372373
format: "text".to_string(),
373374
limit: 48,
374-
prefix_filter: "databend_".to_string(),
375375
},
376376
..Default::default()
377377
};

‎src/meta/binaries/metaverifier/main.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ struct Config {
6666
#[clap(long, default_value = "10000")]
6767
pub number: u64,
6868

69-
#[clap(long, default_value = "INFO")]
69+
#[clap(long, default_value = "warn,databend_=info")]
7070
pub log_level: String,
7171

7272
#[clap(long, env = "METASRV_GRPC_API_ADDRESS", default_value = "")]
@@ -84,7 +84,6 @@ async fn main() -> Result<()> {
8484
dir: "./.databend/logs".to_string(),
8585
format: "text".to_string(),
8686
limit: 48,
87-
prefix_filter: "databend_".to_string(),
8887
},
8988
stderr: StderrConfig {
9089
on: true,

0 commit comments

Comments
 (0)