merge origin/main

Author: flaneur2020

.github/actions/build_linux/action.yml

@@ -43,10 +43,10 @@ runs:
       run: |
         case ${{ inputs.target }} in
           x86_64-unknown-linux-gnu)
-            flags="-C target-feature=+sse4.2"
+            flags="-C target-feature=+sse4.2,+avx2"
             ;;
           x86_64-unknown-linux-musl)
-            flags="-C target-feature=+sse4.2"
+            flags="-C target-feature=+sse4.2,+avx2"
             ;;
           aarch64-unknown-linux-gnu)
             flags=""
@@ -68,7 +68,7 @@ runs:
       if: inputs.artifacts == 'all'
       shell: bash
       run: |
-        artifacts="meta,metactl,metabench,query,sqllogictests"
+        artifacts="meta,metactl,metabench,query,sqllogictests,bendsave"
         binaries=""
         for artifact in ${artifacts//,/ }; do
           binaries="${binaries} --bin databend-$artifact"

.github/actions/pack_binaries/action.yml

@@ -5,7 +5,7 @@ inputs:
     description: "Release target"
     required: true
   category:
-    description: "Release default/hdfs/udf/testsuite"
+    description: "Release default/hdfs/udf/testsuite/dbg"
     required: false
     default: default
   version:
@@ -15,14 +15,24 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Download artifact
+    - name: Download artifact for dbg
+      if: inputs.category == 'dbg'
+      uses: ./.github/actions/artifact_download
+      with:
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+        category: default
+        path: distro/bin
+        artifacts: metactl,meta,query,query.debug,bendsave
+    - name: Download artifact for others
+      if: inputs.category != 'dbg'
       uses: ./.github/actions/artifact_download
       with:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
         category: ${{ inputs.category }}
         path: distro/bin
-        artifacts: metactl,meta,query,query.debug
+        artifacts: metactl,meta,query,bendsave
     - name: Get Latest BendSQL
       id: bendsql
       uses: pozetroninc/github-action-get-latest-release@master
@@ -53,7 +63,7 @@ runs:
         cp ./scripts/distribution/configs/databend-* distro/configs/
         cp ./scripts/distribution/release-readme.txt distro/readme.txt
         cp -r ./scripts/distribution/package-scripts/* distro/scripts/
-        tar -C ./distro --exclude='*.debug' -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
+        tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
         sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
         echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
     - name: post sha256
@@ -68,37 +78,3 @@ runs:
         name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
         path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
         retention-days: 1
-    - name: Pack DBG Binaries
-      id: pack_dbg_binaries
-      shell: bash
-      run: |
-        target=${{ inputs.target }}
-        version=${{ inputs.version }}
-        case ${{ inputs.category }} in
-          default)
-            pkg_name="databend-dbg-${version}-${target}"
-            ;;
-          *)
-          pkg_name="databend-dbg-${{ inputs.category }}-${version}-${target}"
-            ;;
-        esac
-        mkdir -p distro/{bin,configs,systemd,scripts}
-        cp ./scripts/distribution/systemd/databend-* distro/systemd/
-        cp ./scripts/distribution/configs/databend-* distro/configs/
-        cp ./scripts/distribution/release-readme.txt distro/readme.txt
-        cp -r ./scripts/distribution/package-scripts/* distro/scripts/
-        tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
-        sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
-        echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
-    - name: post dbg sha256
-      uses: actions/upload-artifact@v4
-      with:
-        name: sha256sums-${{ inputs.category }}-${{ inputs.target }}-gdb
-        path: sha256-${{ steps.pack_dbg_binaries.outputs.pkg_name }}.txt
-        retention-days: 1
-    - name: post dbg binaries
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
-        path: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
-        retention-days: 1

.github/actions/pack_deb/action.yml

@@ -64,7 +64,7 @@ runs:
         export deb_version="${{ steps.info.outputs.deb_version }}"
         export deb_arch="${{ steps.info.outputs.deb_arch }}"
         pkg_name="databend-query_${deb_version}_${deb_arch}.deb"
-        nfpm pkg --packager deb -t "$pkg_name" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query.yaml)
+        nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query.yaml)
 
     - name: Build Databend Meta Package
       shell: bash
@@ -74,7 +74,7 @@ runs:
         export deb_version="${{ steps.info.outputs.deb_version }}"
         export deb_arch="${{ steps.info.outputs.deb_arch }}"
         pkg_name="databend-meta_${deb_version}_${deb_arch}.deb"
-        nfpm pkg --packager deb -t "$pkg_name" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-meta.yaml)
+        nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-meta.yaml)
 
     - name: Build Databend Debug Package
       shell: bash
@@ -84,7 +84,7 @@ runs:
         export deb_version="${{ steps.info.outputs.deb_version }}"
         export deb_arch="${{ steps.info.outputs.deb_arch }}"
         pkg_name="databend-query-dbg_${deb_version}_${deb_arch}.deb"
-        nfpm pkg --packager deb -t "$pkg_name" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query-dbg.yaml)
+        nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query-dbg.yaml)
 
     - name: Update release to github
       shell: bash

.github/actions/publish_binary/action.yml

@@ -23,7 +23,7 @@ runs:
           default)
             publish_name="databend-${{ inputs.version }}-${{ inputs.target }}"
             ;;
-          hdfs|udf|testsuite)
+          hdfs|udf|testsuite|dbg)
             publish_name="databend-${{ inputs.category }}-${{ inputs.version }}-${{ inputs.target }}"
             ;;
           *)
@@ -38,18 +38,12 @@ runs:
       # Reference: https://cli.github.com/manual/gh_release_upload
       run: |
         gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.tar.gz --clobber
-        if [ -f ${{ steps.name.outputs.name }}-dbg.tar.gz ]; then
-          gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}-dbg.tar.gz --clobber
-        fi
 
     - name: Sync normal release to R2
       shell: bash
       if: inputs.category == 'default'
       run: |
         aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress --checksum-algorithm=CRC32
-        if [ -f ${{ steps.name.outputs.name }}-dbg.tar.gz ]; then
-          aws s3 cp ${{ steps.name.outputs.name }}-dbg.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}-dbg.tar.gz --no-progress --checksum-algorithm=CRC32
-        fi
         gh api /repos/databendlabs/databend/tags > tags.json
         aws s3 cp ./tags.json s3://repo/databend/tags.json --no-progress --checksum-algorithm=CRC32
         gh api /repos/databendlabs/databend/releases > releases.json

.github/actions/test_sqllogic_cluster_linux/action.yml

@@ -9,6 +9,11 @@ inputs:
     description: "logic test handlers, mysql,http,clickhouse"
     required: false
     default: ""
+  parallel:
+    description: "logic test parallel"
+    required: false
+    default: ""
+
 runs:
   using: "composite"
   steps:
@@ -20,4 +25,6 @@ runs:
       shell: bash
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}
+        TEST_PARALLEL: ${{ inputs.parallel }}
+        TEST_EXT_ARGS: '--skip_file tpcds_spill_1.test,tpcds_spill_2.test,tpcds_spill_3.test'
       run: bash ./scripts/ci/ci-run-sqllogic-tests-cluster.sh ${{ inputs.dirs }}

.github/actions/test_sqllogic_iceberg_tpch/action.yml

@@ -32,6 +32,7 @@ runs:
         pip install pyspark
         python3 tests/sqllogictests/scripts/prepare_iceberg_tpch_data.py
 
+
     - name: Run sqllogic Tests with Standalone lib
       shell: bash
       env:

.github/actions/test_sqllogic_stage/action.yml

@@ -13,7 +13,7 @@ inputs:
     description: "storage backend for stage, choices: s3,fs"
     required: true
     default: ""
-  deducp:
+  dedup:
     description: "path type for dedup when copy, choices: full_path,sub_path"
     required: true
     default: ""

.github/actions/test_sqllogic_standalone_linux/action.yml

@@ -29,12 +29,20 @@ runs:
       with:
         artifacts: sqllogictests,meta,query
 
+    - uses: actions/github-script@v7
+      id: ext-args
+      env:
+        DIRS: ${{ inputs.dirs }}
+      with:
+        script: require('.github/actions/test_sqllogic_standalone_linux/script.js')(core)
+
     - name: Run sqllogic Tests with Standalone mode
       if: inputs.storage-format == 'all' || inputs.storage-format == 'parquet'
       shell: bash
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}
         TEST_PARALLEL: ${{ inputs.parallel }}
+        TEST_EXT_ARGS: ${{ steps.ext-args.outputs.parquet }}
         CACHE_ENABLE_TABLE_META_CACHE: ${{ inputs.enable_table_meta_cache}}
       run: bash ./scripts/ci/ci-run-sqllogic-tests.sh ${{ inputs.dirs }}
 
@@ -44,5 +52,6 @@ runs:
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}
         TEST_PARALLEL: ${{ inputs.parallel }}
+        TEST_EXT_ARGS: '--skip_file tpcds_spill_1.test,tpcds_spill_2.test,tpcds_spill_3.test'
         CACHE_ENABLE_TABLE_META_CACHE: ${{ inputs.enable_table_meta_cache}}
       run: bash ./scripts/ci/ci-run-sqllogic-tests-native.sh ${{ inputs.dirs }}

.github/actions/test_sqllogic_standalone_linux/script.js

@@ -0,0 +1,12 @@
+module.exports = (core) => {
+    switch (process.env.DIRS) {
+        case 'tpcds':
+            const parquet = [
+                '--skip_file tpcds_spill_2.test,tpcds_spill_3.test',
+                '--skip_file tpcds_spill_1.test,tpcds_spill_3.test',
+                '--skip_file tpcds_spill_1.test,tpcds_spill_2.test',
+            ][Date.now() % 3];
+            core.setOutput('parquet', parquet)
+            return
+    }
+}

.github/actions/test_sqllogic_standalone_linux_minio_with_bendsave/action.yml

@@ -0,0 +1,93 @@
+name: "Test sqllogic Standalone(minio) with bendsave"
+description: "Running sqllogic tests in standalone mode"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/actions/setup_test
+      with:
+        artifacts: sqllogictests,meta,query,bendsave
+
+    - name: Minio Setup for (ubuntu-latest only)
+      shell: bash
+      run: |
+        docker run -d --network host --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -e "MINIO_ADDRESS=:9900" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                minio/minio server /data
+
+        export AWS_ACCESS_KEY_ID=minioadmin
+        export AWS_SECRET_ACCESS_KEY=minioadmin
+        export AWS_EC2_METADATA_DISABLED=true
+
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://backupbucket
+
+    - name: Run sqllogic Tests with Standalone mode
+      shell: bash
+      env:
+        TEST_HANDLERS: "http"
+      run: bash ./scripts/ci/ci-run-sqllogic-tests-minio-with-bendsave.sh tpch
+
+    - name: Run bendsave backup
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: minioadmin
+        AWS_SECRET_ACCESS_KEY: minioadmin
+        AWS_EC2_METADATA_DISABLED: true
+        AWS_REGION: us-west-2
+      run: |
+        export STORAGE_TYPE=s3
+        export STORAGE_S3_BUCKET=testbucket
+        export STORAGE_S3_ROOT=admin
+        export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
+        export STORAGE_S3_ACCESS_KEY_ID=minioadmin
+        export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
+        export STORAGE_ALLOW_INSECURE=true
+
+        ./target/${{ env.BUILD_PROFILE }}/databend-bendsave backup --from ./scripts/ci/deploy/config/databend-query-node-1.toml --to s3://backupbucket?endpoint=http://127.0.0.1:9900/
+
+        aws --endpoint-url http://127.0.0.1:9900/ s3 ls s3://backupbucket --recursive
+
+    - name: Destroy the existing services.
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: minioadmin
+        AWS_SECRET_ACCESS_KEY: minioadmin
+        AWS_EC2_METADATA_DISABLED: true
+        AWS_REGION: us-west-2
+      run: |
+        # kill all services
+        pkill databend-query
+        pkill databend-meta
+        # destory databend query
+        aws --endpoint-url http://127.0.0.1:9900/ s3 rm s3://testbucket --recursive
+        # destory databend meta
+        rm -rf /tmp/databend/meta1
+
+    - name: Run bendsave restore
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: minioadmin
+        AWS_SECRET_ACCESS_KEY: minioadmin
+        AWS_EC2_METADATA_DISABLED: true
+        AWS_REGION: us-west-2
+      run: |
+        export STORAGE_TYPE=s3
+        export STORAGE_S3_BUCKET=testbucket
+        export STORAGE_S3_ROOT=admin
+        export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
+        export STORAGE_S3_ACCESS_KEY_ID=minioadmin
+        export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
+        export STORAGE_ALLOW_INSECURE=true
+
+        ./target/${{ env.BUILD_PROFILE }}/databend-bendsave restore --from s3://backupbucket?endpoint=http://127.0.0.1:9900/ --to-query ./scripts/ci/deploy/config/databend-query-node-1.toml --to-meta ./scripts/ci/deploy/config/databend-meta-node-for-bendsave.toml --confirm
+
+    - name: Run sqllogic Tests with Standalone mode again for testing
+      shell: bash
+      env:
+        TEST_HANDLERS: "http"
+      run: bash ./scripts/ci/ci-run-sqllogic-tests-minio-with-bendsave.sh tpch