Skip to content

Commit 0ab6572

Browse files
dantengskyxudong963Dousir9youngsofunDragonliu2018
authored
chore: backports PR #16730 and PR #16938 to release/v1.2.636-rc6 (#16960)
* feat: implement `is_not_null` selectivity based on null count in stats (#16730) * feat: implement is_not_null selectivity based on null count in stats * fix test * chore(planner): improve cardinality estimation (#16938) * chore(planner): improve cardinality estimation * chore(planner): improve histogram cardinality estimation * chore(planner): improve join cardinality * chore(test): update sqllogictest * chore(test): update sqllogictest * chore(code): refine code * chore(test): update sqllogictest * chore(test): test ci tpch * chore(code): fix typos * chore(test): remove accurate_his test * chore(test): fix sqllogictest * chore(query): fix sub overflow * chore(planner): refine scan histogram * chore(test): update sqllogictest * chore(test): update sqllogictest * ci: fix flaky test (#16945) * ci: fix flaky test #16935 * ci: update error format of Bendsql. * feat: filter null value before join (#16722) * feat: filter null value before join * fix lint * add annotations and process possible crash * dedup filters and fix tests (also need to fix native explain test) * fix test * support semi join * fix test for semi join * adjust threshold and enable only distribution * chore(planner): resolve conflicts * fix(query): support subquery in pivot (#16631) * fix(query): support subquery in pivot * add pivot and unpivot sqllogictests, fix unit-test * code format * chore(code): resolve conflicts * chore(test): update sqllogictest * chore(test): update sqllogictest * Revert "ci: fix flaky test (#16945)" This reverts commit efcbac3. * chore: add extra bracket for `and` and `or` to make explain clear (#16494) * fix: add extra bracket for and or * add task test * chore(test): update sqllogictest * Revert "Revert "ci: fix flaky test (#16945)"" This reverts commit 49ea151. * fix(query): forbid explain explain statement (#16654) fix(query): forbiden explain explain statement * fix(ci): flaky test (#16933) * flaky test * fix * fix test * chore(code): resolve conflicts * chore(test): update test --------- Co-authored-by: xudong.w <[email protected]> Co-authored-by: Jk Xu <[email protected]> Co-authored-by: Yang Xiufeng <[email protected]> Co-authored-by: Liu Zhenlong <[email protected]> Co-authored-by: Dousir9 <[email protected]> Co-authored-by: TCeason <[email protected]> Co-authored-by: zhya <[email protected]>
1 parent 03f56f1 commit 0ab6572

File tree

103 files changed

+2653
-2006
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

103 files changed

+2653
-2006
lines changed

โ€Žsrc/query/ast/src/ast/format/syntax/query.rsโ€Ž

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,8 @@ pub(crate) fn pretty_table(table: TableReference) -> RcDoc<'static> {
369369
lateral,
370370
subquery,
371371
alias,
372+
pivot,
373+
unpivot,
372374
} => (if lateral {
373375
RcDoc::text("LATERAL")
374376
} else {
@@ -379,6 +381,16 @@ pub(crate) fn pretty_table(table: TableReference) -> RcDoc<'static> {
379381
RcDoc::text(format!(" AS {alias}"))
380382
} else {
381383
RcDoc::nil()
384+
})
385+
.append(if let Some(pivot) = pivot {
386+
RcDoc::text(format!(" {pivot}"))
387+
} else {
388+
RcDoc::nil()
389+
})
390+
.append(if let Some(unpivot) = unpivot {
391+
RcDoc::text(format!(" {unpivot}"))
392+
} else {
393+
RcDoc::nil()
382394
}),
383395
TableReference::TableFunction {
384396
span: _,

โ€Žsrc/query/ast/src/ast/query.rsโ€Ž

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -531,17 +531,30 @@ impl Display for TimeTravelPoint {
531531
}
532532
}
533533

534+
#[derive(Debug, Clone, PartialEq, Drive, DriveMut)]
535+
pub enum PivotValues {
536+
ColumnValues(Vec<Expr>),
537+
Subquery(Box<Query>),
538+
}
539+
534540
#[derive(Debug, Clone, PartialEq, Drive, DriveMut)]
535541
pub struct Pivot {
536542
pub aggregate: Expr,
537543
pub value_column: Identifier,
538-
pub values: Vec<Expr>,
544+
pub values: PivotValues,
539545
}
540546

541547
impl Display for Pivot {
542548
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
543549
write!(f, "PIVOT({} FOR {} IN (", self.aggregate, self.value_column)?;
544-
write_comma_separated_list(f, &self.values)?;
550+
match &self.values {
551+
PivotValues::ColumnValues(column_values) => {
552+
write_comma_separated_list(f, column_values)?;
553+
}
554+
PivotValues::Subquery(subquery) => {
555+
write!(f, "{}", subquery)?;
556+
}
557+
}
545558
write!(f, "))")?;
546559
Ok(())
547560
}
@@ -712,6 +725,8 @@ pub enum TableReference {
712725
lateral: bool,
713726
subquery: Box<Query>,
714727
alias: Option<TableAlias>,
728+
pivot: Option<Box<Pivot>>,
729+
unpivot: Option<Box<Unpivot>>,
715730
},
716731
Join {
717732
span: Span,
@@ -729,13 +744,15 @@ impl TableReference {
729744
pub fn pivot(&self) -> Option<&Pivot> {
730745
match self {
731746
TableReference::Table { pivot, .. } => pivot.as_ref().map(|b| b.as_ref()),
747+
TableReference::Subquery { pivot, .. } => pivot.as_ref().map(|b| b.as_ref()),
732748
_ => None,
733749
}
734750
}
735751

736752
pub fn unpivot(&self) -> Option<&Unpivot> {
737753
match self {
738754
TableReference::Table { unpivot, .. } => unpivot.as_ref().map(|b| b.as_ref()),
755+
TableReference::Subquery { unpivot, .. } => unpivot.as_ref().map(|b| b.as_ref()),
739756
_ => None,
740757
}
741758
}
@@ -834,6 +851,8 @@ impl Display for TableReference {
834851
lateral,
835852
subquery,
836853
alias,
854+
pivot,
855+
unpivot,
837856
} => {
838857
if *lateral {
839858
write!(f, "LATERAL ")?;
@@ -842,6 +861,14 @@ impl Display for TableReference {
842861
if let Some(alias) = alias {
843862
write!(f, " AS {alias}")?;
844863
}
864+
865+
if let Some(pivot) = pivot {
866+
write!(f, " {pivot}")?;
867+
}
868+
869+
if let Some(unpivot) = unpivot {
870+
write!(f, " {unpivot}")?;
871+
}
845872
}
846873
TableReference::Join { span: _, join } => {
847874
write!(f, "{}", join.left)?;

โ€Žsrc/query/ast/src/ast/statements/merge_into.rsโ€Ž

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,8 @@ impl MergeSource {
204204
lateral: false,
205205
subquery: query.clone(),
206206
alias: Some(source_alias.clone()),
207+
pivot: None,
208+
unpivot: None,
207209
},
208210
Self::Table {
209211
catalog,

โ€Žsrc/query/ast/src/parser/query.rsโ€Ž

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -719,6 +719,8 @@ pub enum TableReferenceElement {
719719
lateral: bool,
720720
subquery: Box<Query>,
721721
alias: Option<TableAlias>,
722+
pivot: Option<Box<Pivot>>,
723+
unpivot: Option<Box<Unpivot>>,
722724
},
723725
// [NATURAL] [INNER|OUTER|CROSS|...] JOIN
724726
Join {
@@ -736,28 +738,6 @@ pub enum TableReferenceElement {
736738
}
737739

738740
pub fn table_reference_element(i: Input) -> IResult<WithSpan<TableReferenceElement>> {
739-
// PIVOT(expr FOR col IN (ident, ...))
740-
let pivot = map(
741-
rule! {
742-
PIVOT ~ "(" ~ #expr ~ FOR ~ #ident ~ IN ~ "(" ~ #comma_separated_list1(expr) ~ ")" ~ ")"
743-
},
744-
|(_pivot, _, aggregate, _for, value_column, _in, _, values, _, _)| Pivot {
745-
aggregate,
746-
value_column,
747-
values,
748-
},
749-
);
750-
// UNPIVOT(ident for ident IN (ident, ...))
751-
let unpivot = map(
752-
rule! {
753-
UNPIVOT ~ "(" ~ #ident ~ FOR ~ #ident ~ IN ~ "(" ~ #comma_separated_list1(ident) ~ ")" ~ ")"
754-
},
755-
|(_unpivot, _, value_column, _for, column_name, _in, _, names, _, _)| Unpivot {
756-
value_column,
757-
column_name,
758-
names,
759-
},
760-
);
761741
let aliased_table = map(
762742
rule! {
763743
#dot_separated_idents_1_to_3 ~ #temporal_clause? ~ #with_options? ~ #table_alias? ~ #pivot? ~ #unpivot? ~ SAMPLE? ~ (ROW | BLOCK)? ~ ("(" ~ #expr ~ ROWS? ~ ")")?
@@ -825,12 +805,14 @@ pub fn table_reference_element(i: Input) -> IResult<WithSpan<TableReferenceEleme
825805
);
826806
let subquery = map(
827807
rule! {
828-
LATERAL? ~ "(" ~ #query ~ ")" ~ #table_alias?
808+
LATERAL? ~ "(" ~ #query ~ ")" ~ #table_alias? ~ #pivot? ~ #unpivot?
829809
},
830-
|(lateral, _, subquery, _, alias)| TableReferenceElement::Subquery {
810+
|(lateral, _, subquery, _, alias, pivot, unpivot)| TableReferenceElement::Subquery {
831811
lateral: lateral.is_some(),
832812
subquery: Box::new(subquery),
833813
alias,
814+
pivot: pivot.map(Box::new),
815+
unpivot: unpivot.map(Box::new),
834816
},
835817
);
836818

@@ -869,6 +851,41 @@ pub fn table_reference_element(i: Input) -> IResult<WithSpan<TableReferenceEleme
869851
Ok((rest, WithSpan { span, elem }))
870852
}
871853

854+
// PIVOT(expr FOR col IN (ident, ... | subquery))
855+
fn pivot(i: Input) -> IResult<Pivot> {
856+
map(
857+
rule! {
858+
PIVOT ~ "(" ~ #expr ~ FOR ~ #ident ~ IN ~ "(" ~ #pivot_values ~ ")" ~ ")"
859+
},
860+
|(_pivot, _, aggregate, _for, value_column, _in, _, values, _, _)| Pivot {
861+
aggregate,
862+
value_column,
863+
values,
864+
},
865+
)(i)
866+
}
867+
868+
// UNPIVOT(ident for ident IN (ident, ...))
869+
fn unpivot(i: Input) -> IResult<Unpivot> {
870+
map(
871+
rule! {
872+
UNPIVOT ~ "(" ~ #ident ~ FOR ~ #ident ~ IN ~ "(" ~ #comma_separated_list1(ident) ~ ")" ~ ")"
873+
},
874+
|(_unpivot, _, value_column, _for, column_name, _in, _, names, _, _)| Unpivot {
875+
value_column,
876+
column_name,
877+
names,
878+
},
879+
)(i)
880+
}
881+
882+
fn pivot_values(i: Input) -> IResult<PivotValues> {
883+
alt((
884+
map(comma_separated_list1(expr), PivotValues::ColumnValues),
885+
map(query, |q| PivotValues::Subquery(Box::new(q))),
886+
))(i)
887+
}
888+
872889
fn get_table_sample(
873890
sample: Option<&Token>,
874891
level: Option<&Token>,
@@ -979,11 +996,15 @@ impl<'a, I: Iterator<Item = WithSpan<'a, TableReferenceElement>>> PrattParser<I>
979996
lateral,
980997
subquery,
981998
alias,
999+
pivot,
1000+
unpivot,
9821001
} => TableReference::Subquery {
9831002
span: transform_span(input.span.tokens),
9841003
lateral,
9851004
subquery,
9861005
alias,
1006+
pivot,
1007+
unpivot,
9871008
},
9881009
TableReferenceElement::Stage {
9891010
location,

โ€Žsrc/query/ast/tests/it/parser.rsโ€Ž

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1078,7 +1078,11 @@ fn test_query() {
10781078
r#"SELECT * FROM (((SELECT *) EXCEPT (SELECT *))) foo"#,
10791079
r#"SELECT * FROM (SELECT * FROM xyu ORDER BY x, y) AS xyu"#,
10801080
r#"select * from monthly_sales pivot(sum(amount) for month in ('JAN', 'FEB', 'MAR', 'APR')) order by empid"#,
1081+
r#"select * from (select * from monthly_sales) pivot(sum(amount) for month in ('JAN', 'FEB', 'MAR', 'APR')) order by empid"#,
1082+
r#"select * from monthly_sales pivot(sum(amount) for month in (select distinct month from monthly_sales)) order by empid"#,
1083+
r#"select * from (select * from monthly_sales) pivot(sum(amount) for month in ((select distinct month from monthly_sales))) order by empid"#,
10811084
r#"select * from monthly_sales_1 unpivot(sales for month in (jan, feb, mar, april)) order by empid"#,
1085+
r#"select * from (select * from monthly_sales_1) unpivot(sales for month in (jan, feb, mar, april)) order by empid"#,
10821086
r#"select * from range(1, 2)"#,
10831087
r#"select sum(a) over w from customer window w as (partition by a order by b)"#,
10841088
r#"select a, sum(a) over w, sum(a) over w1, sum(a) over w2 from t1 window w as (partition by a), w2 as (w1 rows current row), w1 as (w order by a) order by a"#,

0 commit comments

Comments
ย (0)