Skip to content

Commit e6e3aec

Browse files
authored
fix: use full path as cache key for parquet meta. (#17313)
1 parent 31c243b commit e6e3aec

File tree

1 file changed

+11
-12
lines changed
  • src/query/storages/parquet/src/parquet_rs

1 file changed

+11
-12
lines changed

src/query/storages/parquet/src/parquet_rs/meta.rs

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,11 @@ pub async fn read_metadata_async_cached(
3737
operator: &Operator,
3838
file_size: Option<u64>,
3939
) -> Result<Arc<ParquetMetaData>> {
40-
let reader = MetaReader::meta_data_reader(operator.clone());
40+
let info = operator.info();
41+
let location = format!("{}/{}/{}", info.name(), info.root(), path);
42+
let reader = MetaReader::meta_data_reader(operator.clone(), location.len() - path.len());
4143
let load_params = LoadParams {
42-
location: path.to_owned(),
44+
location,
4345
len_hint: file_size,
4446
ver: 0,
4547
put_cache: true,
@@ -246,15 +248,15 @@ fn check_memory_usage(max_memory_usage: u64) -> Result<()> {
246248
Ok(())
247249
}
248250

249-
pub struct LoaderWrapper<T>(T);
251+
pub struct LoaderWrapper<T>(T, usize);
250252
pub type ParquetMetaReader = InMemoryItemCacheReader<ParquetMetaData, LoaderWrapper<Operator>>;
251253

252254
pub struct MetaReader;
253255
impl MetaReader {
254-
pub fn meta_data_reader(dal: Operator) -> ParquetMetaReader {
256+
pub fn meta_data_reader(dal: Operator, prefix_len: usize) -> ParquetMetaReader {
255257
ParquetMetaReader::new(
256258
CacheManager::instance().get_parquet_meta_data_cache(),
257-
LoaderWrapper(dal),
259+
LoaderWrapper(dal, prefix_len),
258260
)
259261
}
260262
}
@@ -263,15 +265,12 @@ impl MetaReader {
263265
impl Loader<ParquetMetaData> for LoaderWrapper<Operator> {
264266
#[async_backtrace::framed]
265267
async fn load(&self, params: &LoadParams) -> Result<ParquetMetaData> {
268+
let location = &params.location[self.1..];
266269
let size = match params.len_hint {
267270
Some(v) => v,
268-
None => self.0.stat(&params.location).await?.content_length(),
271+
None => self.0.stat(location).await?.content_length(),
269272
};
270-
databend_common_storage::parquet_rs::read_metadata_async(
271-
&params.location,
272-
&self.0,
273-
Some(size),
274-
)
275-
.await
273+
databend_common_storage::parquet_rs::read_metadata_async(location, &self.0, Some(size))
274+
.await
276275
}
277276
}

0 commit comments

Comments
 (0)