Skip to content

Commit aac746d

Browse files
XuanwoBohuTANG
Xuanwo
and
BohuTANG
authored
feat: Bump OpenDAL to 0.50.1 (#16661)
* Save work Signed-off-by: Xuanwo <[email protected]> * feat: Bump OpenDAL to 0.50.1 Signed-off-by: Xuanwo <[email protected]> * Fix cargo.lock Signed-off-by: Xuanwo <[email protected]> * Refactor metrics Signed-off-by: Xuanwo <[email protected]> * Fix build Signed-off-by: Xuanwo <[email protected]> * Fix header Signed-off-by: Xuanwo <[email protected]> * Fix build Signed-off-by: Xuanwo <[email protected]> * Format toml Signed-off-by: Xuanwo <[email protected]> * Update cargo lock Signed-off-by: Xuanwo <[email protected]> * Fix services for opendal Signed-off-by: Xuanwo <[email protected]> * Fix opendal 0.49 Signed-off-by: Xuanwo <[email protected]> * Fix opendal compat Signed-off-by: Xuanwo <[email protected]> * Fix version mismatch Signed-off-by: Xuanwo <[email protected]> * Fix patch Signed-off-by: Xuanwo <[email protected]> * Add ping support in redis mock Signed-off-by: Xuanwo <[email protected]> * Fix hang on auto detect Signed-off-by: Xuanwo <[email protected]> * Ignore other errors Signed-off-by: Xuanwo <[email protected]> * Remove unexpected check Signed-off-by: Xuanwo <[email protected]> * Fix compat Signed-off-by: Xuanwo <[email protected]> * Format toml Signed-off-by: Xuanwo <[email protected]> * Use main branch of opendal_compat Signed-off-by: Xuanwo <[email protected]> * ignore the dir itself Signed-off-by: Xuanwo <[email protected]> * Fix path handling Signed-off-by: Xuanwo <[email protected]> * Try make sure location always end with / Signed-off-by: Xuanwo <[email protected]> * Revert "Try make sure location always end with /" This reverts commit dc7eea8. * Fix hive may return same files Signed-off-by: Xuanwo <[email protected]> --------- Signed-off-by: Xuanwo <[email protected]> Co-authored-by: Bohu <[email protected]>
1 parent 4a80ca2 commit aac746d

File tree

20 files changed

+495
-681
lines changed

20 files changed

+495
-681
lines changed

‎Cargo.lock‎

Lines changed: 163 additions & 50 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ criterion = "0.5"
229229
ctor = "0.2"
230230
dashmap = "5.4.0"
231231
deepsize = { version = "0.2.0" }
232-
deltalake = "0.18"
232+
deltalake = "0.20"
233233
derive-visitor = { version = "0.4.0", features = ["std-types-drive"] }
234234
derive_more = "0.99.17"
235235
enumflags2 = { version = "0.7.7", features = ["serde"] }
@@ -256,7 +256,7 @@ mysql_async = { version = "0.34", default-features = false, features = ["native-
256256
object_store_opendal = "0.46"
257257
once_cell = "1.15.0"
258258
openai_api_rust = "0.1"
259-
opendal = { version = "0.49.0", features = [
259+
opendal = { version = "0.50.1", features = [
260260
"layers-fastrace",
261261
"layers-prometheus-client",
262262
"layers-async-backtrace",
@@ -275,6 +275,7 @@ opendal = { version = "0.49.0", features = [
275275
"services-huggingface",
276276
"services-redis",
277277
] }
278+
opendal_compat = { version = "1" }
278279
openraft = { git = "https://github.com/drmingdrmer/openraft", tag = "v0.10.0-alpha.6", features = [
279280
"serde",
280281
"tracing-log",
@@ -413,14 +414,18 @@ backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "7226
413414
"serialize-serde",
414415
] }
415416
color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
416-
deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "57795da" }
417417
ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" }
418418
jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "ada713c" }
419419
openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
420+
opendal_compat = { git = "https://github.com/apache/opendal", rev = "f6e60f6" }
420421
orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
421422
recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }
422423
sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
423424
tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370" }
424425
tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370", package = "tantivy-common" }
425426
tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "0e300e9" }
426427
xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }
428+
429+
# This patch makes sure opendal_compat also used the same version of opendal instead of git one.
430+
[patch.'https://github.com/apache/opendal']
431+
opendal = { version = "0.50.1" }

‎src/common/base/Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ prometheus-parse = "0.2.3"
5050
rand = { workspace = true, features = ["serde1"] }
5151
regex = { workspace = true }
5252
replace_with = "0.1.7"
53+
reqwest = { workspace = true }
54+
reqwest-hickory-resolver = { workspace = true }
5355
rustix = "0.38.37"
5456
semver = { workspace = true }
5557
serde = { workspace = true }

‎src/common/base/src/http_client.rs‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
// Copyright 2021 Datafuse Labs
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
use std::env;
16+
use std::sync::Arc;
17+
use std::sync::LazyLock;
18+
use std::time::Duration;
19+
20+
use reqwest_hickory_resolver::HickoryResolver;
21+
22+
/// Global shared hickory resolver.
23+
static GLOBAL_HICKORY_RESOLVER: LazyLock<Arc<HickoryResolver>> =
24+
LazyLock::new(|| Arc::new(HickoryResolver::default()));
25+
26+
/// Global shared http client.
27+
///
28+
/// Please create your own http client if you want dedicated http connection pool.
29+
pub static GLOBAL_HTTP_CLIENT: LazyLock<HttpClient> = LazyLock::new(HttpClient::new);
30+
31+
/// HttpClient that used by databend.
32+
pub struct HttpClient {
33+
client: reqwest::Client,
34+
}
35+
36+
impl Default for HttpClient {
37+
fn default() -> Self {
38+
Self::new()
39+
}
40+
}
41+
42+
impl HttpClient {
43+
/// Create a new http client.
44+
///
45+
/// # Notes
46+
///
47+
/// This client is optimized for interact with storage services.
48+
/// Please tune the settings if you want to use it for other purposes.
49+
pub fn new() -> Self {
50+
let mut builder = reqwest::ClientBuilder::new();
51+
52+
// Disable http2 for better performance.
53+
builder = builder.http1_only();
54+
55+
// Set dns resolver.
56+
builder = builder.dns_resolver(GLOBAL_HICKORY_RESOLVER.clone());
57+
58+
// Pool max idle per host controls connection pool size.
59+
// Default to no limit, set to `0` for disable it.
60+
let pool_max_idle_per_host = env::var("_DATABEND_INTERNAL_POOL_MAX_IDLE_PER_HOST")
61+
.ok()
62+
.and_then(|v| v.parse::<usize>().ok())
63+
.unwrap_or(usize::MAX);
64+
builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
65+
66+
// Connect timeout default to 30s.
67+
let connect_timeout = env::var("_DATABEND_INTERNAL_CONNECT_TIMEOUT")
68+
.ok()
69+
.and_then(|v| v.parse::<u64>().ok())
70+
.unwrap_or(30);
71+
builder = builder.connect_timeout(Duration::from_secs(connect_timeout));
72+
73+
// Enable TCP keepalive if set.
74+
if let Ok(v) = env::var("_DATABEND_INTERNAL_TCP_KEEPALIVE") {
75+
if let Ok(v) = v.parse::<u64>() {
76+
builder = builder.tcp_keepalive(Duration::from_secs(v));
77+
}
78+
}
79+
80+
let client = builder.build().expect("http client must be created");
81+
HttpClient { client }
82+
}
83+
84+
/// Get the inner reqwest client.
85+
pub fn inner(&self) -> reqwest::Client {
86+
self.client.clone()
87+
}
88+
}

‎src/common/base/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ pub mod containers;
3131
pub mod display;
3232
pub mod future;
3333
pub mod headers;
34+
pub mod http_client;
3435
pub mod mem_allocator;
3536
pub mod rangemap;
3637
pub mod runtime;

‎src/common/storage/Cargo.toml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@ opendal = { workspace = true }
2929
parquet = { workspace = true }
3030
prometheus-client = { workspace = true }
3131
regex = { workspace = true }
32-
reqwest = { workspace = true }
33-
reqwest-hickory-resolver = { workspace = true }
3432
serde = { workspace = true }
3533
thiserror = { workspace = true }
3634

0 commit comments

Comments
 (0)