fix(query): fix order by derived column with limit return wrong values (#17457)

Author: b41sh

src/query/sql/src/executor/physical_plans/physical_table_scan.rs

@@ -497,12 +497,7 @@ impl PhysicalPlanBuilder {
                             internal_column.column_name().to_owned(),
                             internal_column.data_type(),
                         ),
-                        ColumnEntry::VirtualColumn(VirtualColumn {
-                            column_name,
-                            data_type,
-                            ..
-                        }) => (column_name.clone(), DataType::from(data_type)),
-                        ColumnEntry::DerivedColumn(_) => {
+                        ColumnEntry::VirtualColumn(_) | ColumnEntry::DerivedColumn(_) => {
                             return None;
                         }
                     };
@@ -520,6 +515,16 @@ impl PhysicalPlanBuilder {
                 .collect::<Vec<_>>()
         });
 
+        let order_by = order_by.unwrap_or_default();
+        let mut limit = scan.limit;
+        if let Some(scan_order_by) = &scan.order_by {
+            // If some order by columns can't be pushed down, then the limit can't be pushed down either,
+            // as this may cause some blocks are pruned by the limit pruner.
+            if scan_order_by.len() != order_by.len() {
+                limit = None;
+            }
+        }
+
         let virtual_column = self.build_virtual_column(&scan.columns);
 
         Ok(PushDownInfo {
@@ -528,8 +533,8 @@ impl PhysicalPlanBuilder {
             filters: push_down_filter,
             is_deterministic,
             prewhere: prewhere_info,
-            limit: scan.limit,
-            order_by: order_by.unwrap_or_default(),
+            limit,
+            order_by,
             virtual_column,
             lazy_materialization: !metadata.lazy_columns().is_empty(),
             agg_index: None,

src/query/sql/src/planner/binder/select.rs

@@ -520,6 +520,18 @@ impl Binder {
             return Ok(());
         }
 
+        for order_by_item in order_by {
+            let column = metadata.column(order_by_item.index);
+            // If order by contains derived column or virtual column,
+            // limit can't be pushed down, so there's no need row fetcher.
+            if matches!(
+                column,
+                ColumnEntry::DerivedColumn(_) | ColumnEntry::VirtualColumn(_)
+            ) {
+                return Ok(());
+            }
+        }
+
         // As we don't if this is subquery, we need add required cols to metadata's non_lazy_columns,
         // so if the inner query not match the lazy materialized but outer query matched, we can prevent
         // the cols that inner query required not be pruned when analyze outer query.

tests/sqllogictests/suites/base/03_common/03_0004_select_order_by.test

@@ -59,6 +59,16 @@ select * from t1 order by id desc limit 3,3
 5
 4
 
+statement ok
+insert into t1 select number as id from numbers(10)
+
+query I
+select * from t1 order by id::string desc limit 3
+----
+9
+9
+8
+
 statement ok
 drop table t1
 
@@ -188,4 +198,4 @@ SELECT * from t1 order by '1', 1;
 3 2
 
 statement ok
-DROP TABLE if EXISTS t1
+DROP TABLE if EXISTS t1

tests/sqllogictests/suites/mode/standalone/explain/sort.test

@@ -64,6 +64,32 @@ Sort
         ├── push downs: [filters: [is_true(t1.a (#0) > 1)], limit: NONE]
         └── estimated rows: 0.00
 
+query T
+explain select a from t1 order by a::string limit 1;
+----
+Limit
+├── output columns: [t1.a (#0), a::STRING (#2)]
+├── limit: 1
+├── offset: 0
+├── estimated rows: 0.00
+└── Sort
+    ├── output columns: [t1.a (#0), a::STRING (#2)]
+    ├── sort keys: [a::STRING ASC NULLS LAST]
+    ├── estimated rows: 0.00
+    └── EvalScalar
+        ├── output columns: [t1.a (#0), a::STRING (#2)]
+        ├── expressions: [CAST(t1.a (#0) AS String NULL)]
+        ├── estimated rows: 0.00
+        └── TableScan
+            ├── table: default.default.t1
+            ├── output columns: [a (#0)]
+            ├── read rows: 0
+            ├── read size: 0
+            ├── partitions total: 0
+            ├── partitions scanned: 0
+            ├── push downs: [filters: [], limit: NONE]
+            └── estimated rows: 0.00
+
 statement ok
 set max_threads = 4;