chore(query): testing tpch/tpcds using 1G scale factor (#17449)

* update

* update

* update

Author: sundy-li

.github/actions/setup_test/action.yml

@@ -33,6 +33,13 @@ runs:
       with:
         python-version: "3.12"
 
+    - name: Install Python packages
+      shell: bash
+      run: |
+        pip install rich
+        pip install databend_driver
+        pip install duckdb
+
     - uses: actions/setup-java@v4
       with:
         distribution: temurin

benchmark/tpcds/shell_env.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-export MYSQL_DATABASE=${MYSQL_DATABASE:="tpcds"}
+export QUERY_DATABASE=${MYSQL_DATABASE:="tpcds"}
 export QUERY_MYSQL_HANDLER_HOST=${QUERY_MYSQL_HANDLER_HOST:="127.0.0.1"}
 export QUERY_MYSQL_HANDLER_PORT=${QUERY_MYSQL_HANDLER_PORT:="3307"}
 export QUERY_HTTP_HANDLER_PORT=${QUERY_HTTP_HANDLER_PORT:="8000"}

benchmark/tpch/shell_env.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-export MYSQL_DATABASE=${MYSQL_DATABASE:="tpch"}
+export QUERY_DATABASE=${MYSQL_DATABASE:="tpch"}
 export QUERY_MYSQL_HANDLER_HOST=${QUERY_MYSQL_HANDLER_HOST:="127.0.0.1"}
 export QUERY_MYSQL_HANDLER_PORT=${QUERY_MYSQL_HANDLER_PORT:="3307"}
 export QUERY_HTTP_HANDLER_PORT=${QUERY_HTTP_HANDLER_PORT:="8000"}

benchmark/tpch/tpch.sh

@@ -4,7 +4,7 @@
 echo """
 INSTALL tpch;
 LOAD tpch;
-SELECT * FROM dsdgen(sf=1); -- sf can be other values, such as 0.1, 1, 10, ...
+SELECT * FROM dbgen(sf=1); -- sf can be other values, such as 0.1, 1, 10, ...
 EXPORT DATABASE '/tmp/tpch_1/' (FORMAT CSV, DELIMITER '|');
 """ | duckdb
 

src/query/sql/src/planner/binder/sort.rs

@@ -129,7 +129,7 @@ impl Binder {
                                 .to_string(),
                         ));
                     } else {
-                        let rewrite_scalar = self
+                        let mut rewrite_scalar = self
                             .rewrite_scalar_with_replacement(
                                 bind_context,
                                 &bound_expr,
@@ -148,6 +148,10 @@ impl Binder {
                             )
                             .map_err(|e| ErrorCode::SemanticError(e.message()))?;
 
+                        let mut rewriter =
+                            AggregateRewriter::new(bind_context, self.metadata.clone());
+                        rewriter.visit(&mut rewrite_scalar)?;
+
                         if let ScalarExpr::ConstantExpr(..) = rewrite_scalar {
                             continue;
                         }

tests/shell_env.sh

@@ -6,7 +6,7 @@ export AWS_ACCESS_KEY_ID=minioadmin
 export AWS_SECRET_ACCESS_KEY=minioadmin
 export AWS_EC2_METADATA_DISABLED=true
 
-export MYSQL_DATABASE=${MYSQL_DATABASE:="default"}
+export QUERY_DATABASE=${QUERY_DATABASE:="default"}
 export QUERY_MYSQL_HANDLER_HOST=${QUERY_MYSQL_HANDLER_HOST:="127.0.0.1"}
 export QUERY_MYSQL_HANDLER_PORT=${QUERY_MYSQL_HANDLER_PORT:="3307"}
 export QUERY_HTTP_HANDLER_PORT=${QUERY_HTTP_HANDLER_PORT:="8000"}

tests/sqllogictests/scripts/prepare_duckdb_tpcds_data.py

@@ -0,0 +1,15 @@
+import duckdb, sys
+
+sf = sys.argv[1]
+
+# Initialize a DuckDB instance
+con = duckdb.connect(":memory:")
+
+con.install_extension("tpcds")
+con.load_extension("tpcds")
+# Execute the commands
+con.execute(f"CALL dsdgen(sf={sf})")
+con.execute(f"EXPORT DATABASE '/tmp/tpcds_{sf}/' (FORMAT CSV, DELIMITER '|')")
+
+# Close the connection
+con.close()

tests/sqllogictests/scripts/prepare_duckdb_tpch_data.py

@@ -0,0 +1,15 @@
+import duckdb, sys
+
+sf = sys.argv[1]
+
+# Initialize a DuckDB instance
+con = duckdb.connect(":memory:")
+
+con.install_extension("tpch")
+con.load_extension("tpch")
+# Execute the commands
+con.execute(f"CALL dbgen(sf={sf})")
+con.execute(f"EXPORT DATABASE '/tmp/tpch_{sf}/' (FORMAT CSV, DELIMITER '|')")
+
+# Close the connection
+con.close()

tests/sqllogictests/scripts/prepare_spill_data.sh

@@ -5,7 +5,8 @@
 # shellcheck disable=SC2034
 target_dir="tests/sqllogictests/"
 
-db="tpcds"
+db=${1:-"tpcds"}
+
 
 tables=(
     call_center
@@ -34,39 +35,30 @@ tables=(
     web_site
 )
 
-# Clear Data
-# shellcheck disable=SC2068
-for t in ${tables[@]}; do
-    echo "DROP TABLE IF EXISTS ${db}.$t" | $BENDSQL_CLIENT_CONNECT
-done
 
-echo "CREATE DATABASE IF NOT EXISTS tpcds" | $BENDSQL_CLIENT_CONNECT
+force=${2:-"1"}
+if [ "$force" == "0" ]; then
+    res=`echo "SELECT COUNT() from ${db}.call_center" | $BENDSQL_CLIENT_CONNECT`
+    if [ "$res" != "0" -a "$res" != "" ]; then
+        echo "Table $db.call_center already exists and is not empty, size: ${res}. Use force=1 to override it."
+        exit 0
+    fi
+fi
+
+echo "CREATE OR REPLACE DATABASE tpcds" | $BENDSQL_CLIENT_CONNECT
 
 # Create Tables;
 # shellcheck disable=SC2002
 cat ${target_dir}/scripts/tpcds.sql | $BENDSQL_CLIENT_CONNECT
-
-# download data
-mkdir -p ${target_dir}/data/
-if [ ! -d ${target_dir}/data/tpcds.tar.gz ]; then
-    curl -s -o ${target_dir}/data/tpcds.tar.gz https://ci.databend.com/dataset/stateful/tpcds.tar.gz
-fi
-
-tar -zxf ${target_dir}/data/tpcds.tar.gz -C ${target_dir}/data
-
-# insert data to tables
-# shellcheck disable=SC2068
+python ${target_dir}/scripts/prepare_duckdb_tpcds_data.py 1
 
 stmt "drop stage if exists s1"
-stmt "create stage s1 url='fs://${PWD}/${target_dir}/'"
+stmt "create stage s1 url='fs:///tmp/tpcds_1/'"
 
 for t in ${tables[@]}; do
     echo "$t"
-    sub_path="data/data/$t.csv"
-   	query "copy into ${db}.${t} from @s1/${sub_path} file_format = (type = CSV skip_header = 0 field_delimiter = '|' record_delimiter = '\n')"
+   	query "copy into ${db}.${t} from @s1/${t}.csv file_format = (type = CSV skip_header = 1 field_delimiter = '|' record_delimiter = '\n')"
     query "analyze table $db.$t"
 done
 
-if [ -d "tests/sqllogictests/data" ]; then
-    rm -rf tests/sqllogictests/data
-fi
+# rm -rf /tmp/tpcds_1