From 9ab7e8ac47451f30778dd1787a508585fd679f1b Mon Sep 17 00:00:00 2001 From: Dousir9 <736191200@qq.com> Date: Fri, 13 Dec 2024 09:48:21 +0800 Subject: [PATCH 1/7] chore(binder): fix bind materialized cte --- .../sql/src/planner/binder/bind_query/bind.rs | 8 ++- .../binder/bind_table_reference/bind_table.rs | 68 ++++++++++++------- .../bind_table_function.rs | 2 + src/query/sql/src/planner/binder/ddl/table.rs | 49 ++++++++----- src/query/sql/src/planner/binder/table.rs | 2 + src/query/sql/src/planner/dataframe.rs | 1 + .../sql/src/planner/expression_parser.rs | 1 + src/query/sql/src/planner/metadata.rs | 11 +++ src/query/sql/src/planner/plans/plan.rs | 8 +++ tests/sqllogictests/suites/query/cte/cte.test | 15 ++++ 10 files changed, 121 insertions(+), 44 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 94386ce8b817c..4ae20de763154 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -179,15 +179,17 @@ impl Binder { )) } - // The return value is temp_table name` fn m_cte_to_temp_table(&self, cte: &CTE) -> Result<()> { let engine = if self.ctx.get_settings().get_persist_materialized_cte()? { Engine::Fuse } else { Engine::Memory }; + let query_id = self.ctx.get_id(); let database = self.ctx.get_current_database(); - let table_name = normalize_identifier(&cte.alias.name, &self.name_resolution_ctx).name; + let mut table_identifier = cte.alias.name.clone(); + table_identifier.name = format!("{}_{}", table_identifier.name, query_id.replace("-", "_")); + let table_name = normalize_identifier(&table_identifier, &self.name_resolution_ctx).name; if self .ctx .is_temp_table(CATALOG_DEFAULT, &database, &table_name) @@ -201,7 +203,7 @@ impl Binder { create_option: CreateOption::Create, catalog: Some(Identifier::from_name(Span::None, CATALOG_DEFAULT)), database: Some(Identifier::from_name(Span::None, database.clone())), - table: cte.alias.name.clone(), + table: table_identifier, source: None, engine: Some(engine), uri_location: None, diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 93bb3bed0fc0e..e4b2d4e5a0069 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -70,37 +70,51 @@ impl Binder { }; // Check and bind common table expression + let mut is_materialized_cte_table = false; let cte_map = bind_context.cte_context.cte_map.clone(); - if let Some(cte_info) = cte_map.get(&table_name) - && !cte_info.materialized - { - if self - .metadata - .read() - .get_table_index(Some(&database), &table_name) - .is_some() - { - return Err(ErrorCode::SyntaxException(format!( - "Table name `{}` is misleading, please distinguish it.", - table_name - )) - .set_span(*span)); - } - return if cte_info.recursive { - if self.bind_recursive_cte { - self.bind_r_cte_scan(bind_context, cte_info, &table_name, alias) - } else { - self.bind_r_cte(*span, bind_context, cte_info, &table_name, alias) - } + if let Some(cte_info) = cte_map.get(&table_name) { + if cte_info.materialized { + is_materialized_cte_table = true; } else { - self.bind_cte(*span, bind_context, &table_name, alias, cte_info) - }; + if self + .metadata + .read() + .get_table_index(Some(&database), &table_name) + .is_some() + { + return Err(ErrorCode::SyntaxException(format!( + "Table name `{}` is misleading, please distinguish it.", + table_name + )) + .set_span(*span)); + } + return if cte_info.recursive { + if self.bind_recursive_cte { + self.bind_r_cte_scan(bind_context, cte_info, &table_name, alias) + } else { + self.bind_r_cte(*span, bind_context, cte_info, &table_name, alias) + } + } else { + self.bind_cte(*span, bind_context, &table_name, alias, cte_info) + }; + } } let navigation = self.resolve_temporal_clause(bind_context, temporal)?; + let cte_suffix_name = if is_materialized_cte_table { + Some(self.ctx.get_id().replace("-", "_")) + } else { + None + }; + // Resolve table with catalog let table_meta = { + let table_name = if let Some(cte_suffix_name) = cte_suffix_name.as_ref() { + format!("{}_{}", &table_name, cte_suffix_name) + } else { + table_name.clone() + }; match self.resolve_data_source( catalog.as_str(), database.as_str(), @@ -154,6 +168,7 @@ impl Binder { bind_context.planning_agg_index, false, consume, + None, ); let (s_expr, mut bind_context) = self.bind_base_table( bind_context, @@ -232,6 +247,7 @@ impl Binder { false, false, false, + None, ); let (s_expr, mut new_bind_context) = self.bind_query(&mut new_bind_context, query)?; @@ -255,6 +271,11 @@ impl Binder { } } _ => { + let cte_suffix_name = if is_materialized_cte_table { + Some(self.ctx.get_id().replace("-", "_")) + } else { + None + }; let table_index = self.metadata.write().add_table( catalog, database.clone(), @@ -264,6 +285,7 @@ impl Binder { bind_context.planning_agg_index, false, false, + cte_suffix_name, ); let (s_expr, mut bind_context) = self.bind_base_table( diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs index 30eb9e5e889ae..d3cf4a7064be1 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs @@ -147,6 +147,7 @@ impl Binder { false, false, false, + None, ); let (s_expr, mut bind_context) = @@ -209,6 +210,7 @@ impl Binder { false, false, false, + None, ); let (s_expr, mut bind_context) = diff --git a/src/query/sql/src/planner/binder/ddl/table.rs b/src/query/sql/src/planner/binder/ddl/table.rs index 656b25e74e687..6b9cf4a7db9c1 100644 --- a/src/query/sql/src/planner/binder/ddl/table.rs +++ b/src/query/sql/src/planner/binder/ddl/table.rs @@ -37,6 +37,7 @@ use databend_common_ast::ast::InvertedIndexDefinition; use databend_common_ast::ast::ModifyColumnAction; use databend_common_ast::ast::OptimizeTableAction as AstOptimizeTableAction; use databend_common_ast::ast::OptimizeTableStmt; +use databend_common_ast::ast::Query; use databend_common_ast::ast::RenameTableStmt; use databend_common_ast::ast::ShowCreateTableStmt; use databend_common_ast::ast::ShowDropTablesStmt; @@ -414,6 +415,12 @@ impl Binder { } } + async fn as_query_plan(&mut self, query: &Box) -> Result { + let stmt = Statement::Query(Box::new(*query.clone())); + let mut bind_context = BindContext::new(); + self.bind_statement(&mut bind_context, &stmt).await + } + #[async_backtrace::framed] pub(in crate::planner::binder) async fn bind_create_table( &mut self, @@ -513,15 +520,17 @@ impl Binder { } // Build table schema - let (schema, field_comments, inverted_indexes) = match (&source, &as_query) { + let (schema, field_comments, inverted_indexes, as_query_plan) = match (&source, &as_query) { (Some(source), None) => { // `CREATE TABLE` without `AS SELECT ...` - self.analyze_create_table_schema(source).await? + let (schema, field_comments, inverted_indexes) = + self.analyze_create_table_schema(source).await?; + (schema, field_comments, inverted_indexes, None) } (None, Some(query)) => { // `CREATE TABLE AS SELECT ...` without column definitions - let mut init_bind_context = BindContext::new(); - let (_, bind_context) = self.bind_query(&mut init_bind_context, query)?; + let as_query_plan = self.as_query_plan(query).await?; + let bind_context = as_query_plan.bind_context().unwrap(); let fields = bind_context .columns .iter() @@ -534,14 +543,14 @@ impl Binder { .collect::>>()?; let schema = TableSchemaRefExt::create(fields); Self::validate_create_table_schema(&schema)?; - (schema, vec![], None) + (schema, vec![], None, Some(Box::new(as_query_plan))) } (Some(source), Some(query)) => { // e.g. `CREATE TABLE t (i INT) AS SELECT * from old_t` with columns specified let (source_schema, source_comments, inverted_indexes) = self.analyze_create_table_schema(source).await?; - let mut init_bind_context = BindContext::new(); - let (_, bind_context) = self.bind_query(&mut init_bind_context, query)?; + let as_query_plan = self.as_query_plan(query).await?; + let bind_context = as_query_plan.bind_context().unwrap(); let query_fields: Vec = bind_context .columns .iter() @@ -556,9 +565,20 @@ impl Binder { return Err(ErrorCode::BadArguments("Number of columns does not match")); } Self::validate_create_table_schema(&source_schema)?; - (source_schema, source_comments, inverted_indexes) + ( + source_schema, + source_comments, + inverted_indexes, + Some(Box::new(as_query_plan)), + ) } _ => { + let as_query_plan = if let Some(query) = as_query { + let as_query_plan = self.as_query_plan(query).await?; + Some(Box::new(as_query_plan)) + } else { + None + }; match engine { Engine::Iceberg => { let sp = @@ -569,7 +589,7 @@ impl Binder { // since we get it from table options location and connection when load table each time. // we do this in case we change this idea. storage_params = Some(sp); - (Arc::new(table_schema), vec![], None) + (Arc::new(table_schema), vec![], None, as_query_plan) } Engine::Delta => { let sp = @@ -581,7 +601,7 @@ impl Binder { // we do this in case we change this idea. storage_params = Some(sp); engine_options.insert(OPT_KEY_ENGINE_META.to_lowercase().to_string(), meta); - (Arc::new(table_schema), vec![], None) + (Arc::new(table_schema), vec![], None, as_query_plan) } _ => Err(ErrorCode::BadArguments( "Incorrect CREATE query: required list of column descriptions or AS section or SELECT or ICEBERG/DELTA table engine", @@ -689,14 +709,7 @@ impl Binder { options, field_comments, cluster_key, - as_select: if let Some(query) = as_query { - let mut bind_context = BindContext::new(); - let stmt = Statement::Query(Box::new(*query.clone())); - let select_plan = self.bind_statement(&mut bind_context, &stmt).await?; - Some(Box::new(select_plan)) - } else { - None - }, + as_select: as_query_plan, inverted_indexes, }; Ok(Plan::CreateTable(Box::new(plan))) diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index fb2443ee870e2..57866a0743154 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -141,6 +141,7 @@ impl Binder { false, true, false, + None, ); let (s_expr, mut bind_context) = @@ -193,6 +194,7 @@ impl Binder { new_bind_context.cte_context.cte_name = Some(table_name.to_string()); + dbg!("&cte_info.query = {:?}", &cte_info.query); let (s_expr, mut res_bind_context) = self.bind_query(&mut new_bind_context, &cte_info.query)?; let mut cols_alias = cte_info.columns_alias.clone(); diff --git a/src/query/sql/src/planner/dataframe.rs b/src/query/sql/src/planner/dataframe.rs index a590b9bbad8df..8a246eab706c4 100644 --- a/src/query/sql/src/planner/dataframe.rs +++ b/src/query/sql/src/planner/dataframe.rs @@ -102,6 +102,7 @@ impl Dataframe { false, false, false, + None, ); binder.bind_base_table(&bind_context, database, table_index, None, &None) diff --git a/src/query/sql/src/planner/expression_parser.rs b/src/query/sql/src/planner/expression_parser.rs index 9e16429a6d001..a7020fa7b4922 100644 --- a/src/query/sql/src/planner/expression_parser.rs +++ b/src/query/sql/src/planner/expression_parser.rs @@ -72,6 +72,7 @@ pub fn bind_table(table_meta: Arc) -> Result<(BindContext, MetadataRe false, false, false, + None, ); let columns = metadata.read().columns_by_table_index(table_index); diff --git a/src/query/sql/src/planner/metadata.rs b/src/query/sql/src/planner/metadata.rs index 042766c848d55..90566f165ff7a 100644 --- a/src/query/sql/src/planner/metadata.rs +++ b/src/query/sql/src/planner/metadata.rs @@ -354,8 +354,10 @@ impl Metadata { source_of_index: bool, source_of_stage: bool, consume: bool, + cte_suffix_name: Option, ) -> IndexType { let table_name = table_meta.name().to_string(); + let table_name = Self::remove_cte_suffix(table_name, cte_suffix_name); let table_index = self.tables.len(); // If exists table alias name, use it instead of origin name @@ -485,6 +487,15 @@ impl Metadata { pub fn base_column_scan_id(&self, column_index: usize) -> Option { self.base_column_scan_id.get(&column_index).cloned() } + + fn remove_cte_suffix(mut table_name: String, cte_suffix_name: Option) -> String { + if let Some(suffix) = cte_suffix_name { + if table_name.ends_with(&suffix) { + table_name.truncate(table_name.len() - suffix.len() - 1); + } + } + table_name + } } #[derive(Clone)] diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index dba3003b023be..121c7177ea7f9 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -528,4 +528,12 @@ impl Plan { } self.clone() } + + pub fn bind_context(&self) -> Option { + if let Plan::Query { bind_context, .. } = self { + Some(*bind_context.clone()) + } else { + None + } + } } diff --git a/tests/sqllogictests/suites/query/cte/cte.test b/tests/sqllogictests/suites/query/cte/cte.test index fec1749d5921c..f540f6e591e90 100644 --- a/tests/sqllogictests/suites/query/cte/cte.test +++ b/tests/sqllogictests/suites/query/cte/cte.test @@ -543,3 +543,18 @@ drop table products statement ok drop table sales + +statement ok +create or replace table t1(a int, b int); + +statement ok +create or replace table t2(a int, b int); + +statement ok +CREATE or replace TABLE t3 AS with x as MATERIALIZED (select t1.a as t1_a, t2.a as t2_a from t1 inner join t2 on t1.a = t2.a) (select * from (select * from x)); + +statement ok +drop table if exists t1; + +statement ok +drop table if exists t2; \ No newline at end of file From 378c757936689b9923905889f938dcc808d471ff Mon Sep 17 00:00:00 2001 From: Dousir9 <736191200@qq.com> Date: Fri, 13 Dec 2024 09:50:13 +0800 Subject: [PATCH 2/7] chore(code): remove unused code --- src/query/sql/src/planner/binder/table.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index 57866a0743154..dc7bc681be7f8 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -194,7 +194,6 @@ impl Binder { new_bind_context.cte_context.cte_name = Some(table_name.to_string()); - dbg!("&cte_info.query = {:?}", &cte_info.query); let (s_expr, mut res_bind_context) = self.bind_query(&mut new_bind_context, &cte_info.query)?; let mut cols_alias = cte_info.columns_alias.clone(); From f1edc92e64fac23e305919fb157f6e98fd587660 Mon Sep 17 00:00:00 2001 From: Dousir9 <736191200@qq.com> Date: Fri, 13 Dec 2024 10:07:48 +0800 Subject: [PATCH 3/7] chore(code): refine code --- src/query/sql/src/planner/metadata.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/sql/src/planner/metadata.rs b/src/query/sql/src/planner/metadata.rs index 90566f165ff7a..4f77eb8063700 100644 --- a/src/query/sql/src/planner/metadata.rs +++ b/src/query/sql/src/planner/metadata.rs @@ -495,7 +495,7 @@ impl Metadata { } } table_name - } + } } #[derive(Clone)] From 0ed0ae4edd8fb0b8c3d018ee5aaa1c6336bcc880 Mon Sep 17 00:00:00 2001 From: Dousir9 <736191200@qq.com> Date: Fri, 13 Dec 2024 10:39:52 +0800 Subject: [PATCH 4/7] chore(binder): refine code --- .../binder/bind_table_reference/bind_table.rs | 15 ++------------- src/query/sql/src/planner/binder/ddl/table.rs | 2 +- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index e4b2d4e5a0069..a95132d04ec14 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -70,11 +70,11 @@ impl Binder { }; // Check and bind common table expression - let mut is_materialized_cte_table = false; + let mut cte_suffix_name = None; let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - is_materialized_cte_table = true; + cte_suffix_name = Some(self.ctx.get_id().replace("-", "_")); } else { if self .metadata @@ -102,12 +102,6 @@ impl Binder { let navigation = self.resolve_temporal_clause(bind_context, temporal)?; - let cte_suffix_name = if is_materialized_cte_table { - Some(self.ctx.get_id().replace("-", "_")) - } else { - None - }; - // Resolve table with catalog let table_meta = { let table_name = if let Some(cte_suffix_name) = cte_suffix_name.as_ref() { @@ -271,11 +265,6 @@ impl Binder { } } _ => { - let cte_suffix_name = if is_materialized_cte_table { - Some(self.ctx.get_id().replace("-", "_")) - } else { - None - }; let table_index = self.metadata.write().add_table( catalog, database.clone(), diff --git a/src/query/sql/src/planner/binder/ddl/table.rs b/src/query/sql/src/planner/binder/ddl/table.rs index 6b9cf4a7db9c1..ec5d0b1d7adf6 100644 --- a/src/query/sql/src/planner/binder/ddl/table.rs +++ b/src/query/sql/src/planner/binder/ddl/table.rs @@ -415,7 +415,7 @@ impl Binder { } } - async fn as_query_plan(&mut self, query: &Box) -> Result { + async fn as_query_plan(&mut self, query: &Query) -> Result { let stmt = Statement::Query(Box::new(*query.clone())); let mut bind_context = BindContext::new(); self.bind_statement(&mut bind_context, &stmt).await From 626ba747c5acad32ca55346101987a989385d7a2 Mon Sep 17 00:00:00 2001 From: Dousir9 <736191200@qq.com> Date: Fri, 13 Dec 2024 10:51:00 +0800 Subject: [PATCH 5/7] chore(binder): fix code --- src/query/sql/src/planner/binder/ddl/table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/sql/src/planner/binder/ddl/table.rs b/src/query/sql/src/planner/binder/ddl/table.rs index ec5d0b1d7adf6..6b6ade0b96df1 100644 --- a/src/query/sql/src/planner/binder/ddl/table.rs +++ b/src/query/sql/src/planner/binder/ddl/table.rs @@ -416,7 +416,7 @@ impl Binder { } async fn as_query_plan(&mut self, query: &Query) -> Result { - let stmt = Statement::Query(Box::new(*query.clone())); + let stmt = Statement::Query(Box::new(query.clone())); let mut bind_context = BindContext::new(); self.bind_statement(&mut bind_context, &stmt).await } From 61ffee8cf17867aad5ff15ee0678556b09577214 Mon Sep 17 00:00:00 2001 From: Dousir9 <736191200@qq.com> Date: Tue, 17 Dec 2024 02:21:29 +0800 Subject: [PATCH 6/7] chore(query): replace m_cte table name --- src/query/ast/src/ast/expr.rs | 446 ++++++++++++++++++ src/query/ast/src/lib.rs | 2 + .../sql/src/planner/binder/bind_query/bind.rs | 14 +- src/query/sql/src/planner/binder/binder.rs | 2 + 4 files changed, 462 insertions(+), 2 deletions(-) diff --git a/src/query/ast/src/ast/expr.rs b/src/query/ast/src/ast/expr.rs index d19f00b2278b2..5a1abfb6c4427 100644 --- a/src/query/ast/src/ast/expr.rs +++ b/src/query/ast/src/ast/expr.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::fmt::Display; use std::fmt::Formatter; @@ -22,13 +23,23 @@ use ethnum::i256; use pratt::Affix; use pratt::Associativity; +use super::ColumnFilter; use super::ColumnRef; +use super::GroupBy; +use super::JoinCondition; use super::OrderByExpr; +use super::Pivot; +use super::PivotValues; +use super::SelectTarget; +use super::TableReference; +use super::TemporalClause; +use super::TimeTravelPoint; use crate::ast::display_decimal_256; use crate::ast::quote::QuotedString; use crate::ast::write_comma_separated_list; use crate::ast::Identifier; use crate::ast::Query; +use crate::ast::SetExpr; use crate::span::merge_span; use crate::ParseError; use crate::Result; @@ -1658,3 +1669,438 @@ pub fn split_equivalent_predicate_expr(expr: &Expr) -> Option<(Expr, Expr)> { _ => None, } } + +pub struct ExprReplacer { + database: String, + new_name: HashMap, +} + +impl ExprReplacer { + pub fn new(database: String, new_name: HashMap) -> Self { + Self { database, new_name } + } + + #[recursive::recursive] + pub fn replace_query(&self, query: &mut Query) { + if let Some(with) = &mut query.with { + for cte in with.ctes.iter_mut() { + self.replace_query(&mut cte.query); + } + } + + self.replace_set_expr(&mut query.body); + + for order_by in query.order_by.iter_mut() { + self.replace_expr(&mut order_by.expr); + } + + for limit in query.limit.iter_mut() { + self.replace_expr(limit); + } + + if let Some(offset) = &mut query.offset { + self.replace_expr(offset); + } + } + + fn replace_identifier(&self, identifier: &mut Identifier) { + if let Some(new_name) = self.new_name.get(&identifier.name) { + identifier.name = new_name.clone(); + } + } + + fn replace_time_travel_point(&self, time_travel_point: &mut TimeTravelPoint) { + match time_travel_point { + TimeTravelPoint::Timestamp(timestamp) => { + self.replace_expr(timestamp); + } + TimeTravelPoint::Offset(offset) => { + self.replace_expr(offset); + } + _ => (), + } + } + + fn replace_pivot(&self, pivot: &mut Pivot) { + self.replace_expr(&mut pivot.aggregate); + match &mut pivot.values { + PivotValues::ColumnValues(exprs) => { + for expr in exprs.iter_mut() { + self.replace_expr(expr); + } + } + PivotValues::Subquery(subquery) => { + self.replace_query(subquery); + } + } + } + + #[recursive::recursive] + fn replace_table_table_reference(&self, table_reference: &mut TableReference) { + match table_reference { + TableReference::Table { + database, + table, + temporal, + pivot, + .. + } => { + if database.is_none() || database.as_ref().unwrap().name == self.database { + self.replace_identifier(table); + } + if let Some(temporal) = temporal { + match temporal { + TemporalClause::TimeTravel(time_travel) => { + self.replace_time_travel_point(time_travel); + } + TemporalClause::Changes(changes) => { + self.replace_time_travel_point(&mut changes.at_point); + if let Some(end_point) = &mut changes.end_point { + self.replace_time_travel_point(end_point); + } + } + } + } + if let Some(pivot) = pivot { + self.replace_pivot(pivot); + } + } + TableReference::TableFunction { + params, + named_params, + .. + } => { + for param in params.iter_mut() { + self.replace_expr(param); + } + for named_param in named_params.iter_mut() { + self.replace_expr(&mut named_param.1); + } + } + TableReference::Subquery { + subquery, pivot, .. + } => { + self.replace_query(subquery); + if let Some(pivot) = pivot { + self.replace_pivot(pivot); + } + } + TableReference::Join { join, .. } => { + if let JoinCondition::On(expr) = &mut join.condition { + self.replace_expr(expr); + } + self.replace_table_table_reference(&mut join.left); + self.replace_table_table_reference(&mut join.right); + } + TableReference::Location { .. } => (), + } + } + + #[recursive::recursive] + fn replace_group_by(&self, group_by: &mut GroupBy) { + match group_by { + GroupBy::Normal(exprs) | GroupBy::Cube(exprs) | GroupBy::Rollup(exprs) => { + for expr in exprs.iter_mut() { + self.replace_expr(expr); + } + } + GroupBy::GroupingSets(expr_sets) => { + for exprs in expr_sets.iter_mut() { + for expr in exprs.iter_mut() { + self.replace_expr(expr); + } + } + } + GroupBy::Combined(groups) => { + for group_by in groups.iter_mut() { + self.replace_group_by(group_by); + } + } + _ => (), + } + } + + fn replace_window_spec(&self, window_spec: &mut WindowSpec) { + for partition_expr in window_spec.partition_by.iter_mut() { + self.replace_expr(partition_expr); + } + for order_by in window_spec.order_by.iter_mut() { + self.replace_expr(&mut order_by.expr); + } + if let Some(window_frame) = &mut window_spec.window_frame { + if let WindowFrameBound::Preceding(expr) | WindowFrameBound::Following(expr) = + &mut window_frame.start_bound + { + if let Some(expr) = expr { + self.replace_expr(expr); + } + } + if let WindowFrameBound::Preceding(expr) | WindowFrameBound::Following(expr) = + &mut window_frame.end_bound + { + if let Some(expr) = expr { + self.replace_expr(expr); + } + } + } + } + + #[recursive::recursive] + fn replace_set_expr(&self, set_expr: &mut SetExpr) { + match set_expr { + SetExpr::Query(query) => { + self.replace_query(query); + } + SetExpr::Select(select) => { + if let Some(hints) = &mut select.hints { + for hint in hints.hints_list.iter_mut() { + self.replace_expr(&mut hint.expr); + } + } + + for select_list_item in select.select_list.iter_mut() { + match select_list_item { + SelectTarget::AliasedExpr { expr, .. } => { + self.replace_expr(expr); + } + SelectTarget::StarColumns { column_filter, .. } => { + if let Some(column_filter) = column_filter + && let ColumnFilter::Lambda(lambda) = column_filter + { + self.replace_expr(&mut lambda.expr); + } + } + } + } + + for table_reference in select.from.iter_mut() { + self.replace_table_table_reference(table_reference); + } + + if let Some(selection) = &mut select.selection { + self.replace_expr(selection); + } + + if let Some(group_by) = &mut select.group_by { + self.replace_group_by(group_by); + } + + if let Some(having) = &mut select.having { + self.replace_expr(having); + } + + if let Some(window_list) = &mut select.window_list { + for window in window_list.iter_mut() { + self.replace_window_spec(&mut window.spec); + } + } + + if let Some(qualify) = &mut select.qualify { + self.replace_expr(qualify); + } + } + SetExpr::SetOperation(set_operation) => { + self.replace_set_expr(&mut set_operation.left); + self.replace_set_expr(&mut set_operation.right); + } + SetExpr::Values { values, .. } => { + for value in values.iter_mut() { + for expr in value.iter_mut() { + self.replace_expr(expr); + } + } + } + } + } + + #[recursive::recursive] + fn replace_expr(&self, expr: &mut Expr) { + match expr { + Expr::ColumnRef { column, .. } => { + if column.database.is_none() + || column.database.as_ref().unwrap().name == self.database + { + if let Some(table_identifier) = &mut column.table { + self.replace_identifier(table_identifier); + } + } + } + Expr::IsNull { expr, .. } => { + self.replace_expr(expr); + } + Expr::IsDistinctFrom { left, right, .. } => { + self.replace_expr(left); + self.replace_expr(right); + } + + Expr::InList { expr, list, .. } => { + self.replace_expr(expr); + for expr in list.iter_mut() { + self.replace_expr(expr); + } + } + Expr::InSubquery { expr, subquery, .. } => { + self.replace_expr(expr); + self.replace_query(subquery); + } + Expr::Between { + expr, low, high, .. + } => { + self.replace_expr(expr); + self.replace_expr(low); + self.replace_expr(high); + } + Expr::UnaryOp { expr, .. } => { + self.replace_expr(expr); + } + Expr::BinaryOp { left, right, .. } => { + self.replace_expr(left); + self.replace_expr(right); + } + Expr::JsonOp { left, right, .. } => { + self.replace_expr(left); + self.replace_expr(right); + } + Expr::Cast { expr, .. } => { + self.replace_expr(expr); + } + Expr::TryCast { expr, .. } => { + self.replace_expr(expr); + } + Expr::Extract { expr, .. } => { + self.replace_expr(expr); + } + Expr::DatePart { expr, .. } => { + self.replace_expr(expr); + } + Expr::Position { + substr_expr, + str_expr, + .. + } => { + self.replace_expr(substr_expr); + self.replace_expr(str_expr); + } + Expr::Substring { + expr, + substring_from, + substring_for, + .. + } => { + self.replace_expr(expr); + self.replace_expr(substring_from); + if let Some(substring_for) = substring_for { + self.replace_expr(substring_for); + } + } + Expr::Trim { + expr, trim_where, .. + } => { + self.replace_expr(expr); + if let Some((_, trim_str)) = trim_where { + self.replace_expr(trim_str); + } + } + Expr::CountAll { window, .. } => { + if let Some(window) = window + && let Window::WindowSpec(window_spec) = window + { + self.replace_window_spec(window_spec); + } + } + Expr::Tuple { exprs, .. } => { + for expr in exprs.iter_mut() { + self.replace_expr(expr); + } + } + Expr::FunctionCall { func, .. } => { + for arg in func.args.iter_mut() { + self.replace_expr(arg); + } + for param in func.params.iter_mut() { + self.replace_expr(param); + } + if let Some(window_desc) = &mut func.window + && let Window::WindowSpec(window_spec) = &mut window_desc.window + { + self.replace_window_spec(window_spec); + } + if let Some(lambda) = &mut func.lambda { + self.replace_expr(&mut lambda.expr); + } + } + Expr::Case { + operand, + conditions, + results, + else_result, + .. + } => { + if let Some(op) = operand { + self.replace_expr(op); + } + for (cond, res) in conditions.iter_mut().zip(results.iter_mut()) { + self.replace_expr(cond); + self.replace_expr(res); + } + if let Some(el) = else_result { + self.replace_expr(el); + } + } + Expr::Exists { subquery, .. } => { + self.replace_query(subquery); + } + Expr::Subquery { subquery, .. } => { + self.replace_query(subquery); + } + Expr::MapAccess { expr, accessor, .. } => { + self.replace_expr(expr); + if let MapAccessor::Bracket { key } = accessor { + self.replace_expr(key); + } + } + Expr::Array { exprs, .. } => { + for expr in exprs.iter_mut() { + self.replace_expr(expr); + } + } + Expr::Map { kvs, .. } => { + for (_, v) in kvs.iter_mut() { + self.replace_expr(v); + } + } + Expr::Interval { expr, .. } => { + self.replace_expr(expr); + } + Expr::DateAdd { interval, date, .. } => { + self.replace_expr(interval); + self.replace_expr(date); + } + Expr::DateDiff { + date_start, + date_end, + .. + } => { + self.replace_expr(date_start); + self.replace_expr(date_end); + } + Expr::DateSub { interval, date, .. } => { + self.replace_expr(interval); + self.replace_expr(date); + } + Expr::DateTrunc { date, .. } => { + self.replace_expr(date); + } + Expr::LastDay { date, .. } => { + self.replace_expr(date); + } + Expr::PreviousDay { date, .. } => { + self.replace_expr(date); + } + Expr::NextDay { date, .. } => { + self.replace_expr(date); + } + Expr::Literal { .. } | Expr::Hole { .. } => (), + } + } +} diff --git a/src/query/ast/src/lib.rs b/src/query/ast/src/lib.rs index 000419e3f31b1..697125c05768c 100644 --- a/src/query/ast/src/lib.rs +++ b/src/query/ast/src/lib.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(let_chains)] + // TODO(xuanwo): Add crate level documents here. pub mod ast; mod error; diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 4ae20de763154..250950f37b4c2 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -18,6 +18,7 @@ use databend_common_ast::ast::CreateOption; use databend_common_ast::ast::CreateTableStmt; use databend_common_ast::ast::Engine; use databend_common_ast::ast::Expr; +use databend_common_ast::ast::ExprReplacer; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::Query; use databend_common_ast::ast::SetExpr; @@ -179,7 +180,7 @@ impl Binder { )) } - fn m_cte_to_temp_table(&self, cte: &CTE) -> Result<()> { + fn m_cte_to_temp_table(&mut self, cte: &CTE) -> Result<()> { let engine = if self.ctx.get_settings().get_persist_materialized_cte()? { Engine::Fuse } else { @@ -190,6 +191,10 @@ impl Binder { let mut table_identifier = cte.alias.name.clone(); table_identifier.name = format!("{}_{}", table_identifier.name, query_id.replace("-", "_")); let table_name = normalize_identifier(&table_identifier, &self.name_resolution_ctx).name; + self.m_cte_table_name.insert( + normalize_identifier(&cte.alias.name, &self.name_resolution_ctx).name, + table_name.clone(), + ); if self .ctx .is_temp_table(CATALOG_DEFAULT, &database, &table_name) @@ -199,6 +204,11 @@ impl Binder { table_name ))); } + + let expr_replacer = ExprReplacer::new(database.clone(), self.m_cte_table_name.clone()); + let mut as_query = cte.query.clone(); + expr_replacer.replace_query(&mut as_query); + let create_table_stmt = CreateTableStmt { create_option: CreateOption::Create, catalog: Some(Identifier::from_name(Span::None, CATALOG_DEFAULT)), @@ -209,7 +219,7 @@ impl Binder { uri_location: None, cluster_by: None, table_options: Default::default(), - as_query: Some(cte.query.clone()), + as_query: Some(as_query), table_type: TableType::Temporary, }; diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index 9c1d566647f0f..e90f9a7bb837d 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -106,6 +106,7 @@ pub struct Binder { /// For the recursive cte, the cte table name occurs in the recursive cte definition and main query /// if meet recursive cte table name in cte definition, set `bind_recursive_cte` true and treat it as `CteScan`. pub bind_recursive_cte: bool, + pub m_cte_table_name: HashMap, pub enable_result_cache: bool, @@ -132,6 +133,7 @@ impl<'a> Binder { metadata, expression_scan_context: ExpressionScanContext::new(), bind_recursive_cte: false, + m_cte_table_name: HashMap::new(), enable_result_cache, subquery_executor: None, } From 804869d5cbd6f7524a825bffa1d54fa46d649302 Mon Sep 17 00:00:00 2001 From: Dousir9 <736191200@qq.com> Date: Tue, 17 Dec 2024 09:10:03 +0800 Subject: [PATCH 7/7] chore(test): update sqllogictest --- .../suites/mode/cluster/create_table.test | 18 +++++++++--------- .../mode/standalone/explain/explain_ddl.test | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/sqllogictests/suites/mode/cluster/create_table.test b/tests/sqllogictests/suites/mode/cluster/create_table.test index f96f973831b6b..5d8ba4d540a8d 100644 --- a/tests/sqllogictests/suites/mode/cluster/create_table.test +++ b/tests/sqllogictests/suites/mode/cluster/create_table.test @@ -4,32 +4,32 @@ explain create or replace table t2 as select number % 400 d, max(number) from n CreateTableAsSelect: (empty) EvalScalar -├── output columns: [max(number) (#4), d (#5)] -├── expressions: [numbers.number (#3) % 400] +├── output columns: [max(number) (#1), d (#2)] +├── expressions: [numbers.number (#0) % 400] ├── estimated rows: 3.00 └── Limit - ├── output columns: [max(number) (#4), numbers.number (#3)] + ├── output columns: [max(number) (#1), numbers.number (#0)] ├── limit: 3 ├── offset: 0 ├── estimated rows: 3.00 └── Sort - ├── output columns: [max(number) (#4), numbers.number (#3)] + ├── output columns: [max(number) (#1), numbers.number (#0)] ├── sort keys: [number ASC NULLS LAST] ├── estimated rows: 10000000.00 └── Exchange - ├── output columns: [max(number) (#4), numbers.number (#3), #_order_col] + ├── output columns: [max(number) (#1), numbers.number (#0), #_order_col] ├── exchange type: Merge └── Sort - ├── output columns: [max(number) (#4), numbers.number (#3), #_order_col] + ├── output columns: [max(number) (#1), numbers.number (#0), #_order_col] ├── sort keys: [number ASC NULLS LAST] ├── estimated rows: 10000000.00 └── AggregateFinal - ├── output columns: [max(number) (#4), numbers.number (#3)] + ├── output columns: [max(number) (#1), numbers.number (#0)] ├── group by: [number] ├── aggregate functions: [max(number)] ├── estimated rows: 10000000.00 └── Exchange - ├── output columns: [max(number) (#4), numbers.number (#3)] + ├── output columns: [max(number) (#1), numbers.number (#0)] ├── exchange type: Hash(0) └── AggregatePartial ├── group by: [number] @@ -38,7 +38,7 @@ EvalScalar ├── rank limit: 3 └── TableScan ├── table: default.system.numbers - ├── output columns: [number (#3)] + ├── output columns: [number (#0)] ├── read rows: 10000000 ├── read size: 76.29 MiB ├── partitions total: 153 diff --git a/tests/sqllogictests/suites/mode/standalone/explain/explain_ddl.test b/tests/sqllogictests/suites/mode/standalone/explain/explain_ddl.test index 9f9fb73bbbb4b..d32ffa17e6856 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/explain_ddl.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/explain_ddl.test @@ -4,17 +4,17 @@ explain create table if not exists t as select * from numbers(10) where number > CreateTableAsSelect: (empty) Filter -├── output columns: [numbers.number (#1)] -├── filters: [numbers.number (#1) > 5] +├── output columns: [numbers.number (#0)] +├── filters: [numbers.number (#0) > 5] ├── estimated rows: 0.00 └── TableScan ├── table: default.system.numbers - ├── output columns: [number (#1)] + ├── output columns: [number (#0)] ├── read rows: 10 ├── read size: < 1 KiB ├── partitions total: 1 ├── partitions scanned: 1 - ├── push downs: [filters: [numbers.number (#1) > 5], limit: NONE] + ├── push downs: [filters: [numbers.number (#0) > 5], limit: NONE] └── estimated rows: 10.00 query T