chore: (backport) enable overflow-checks in ci profile (#16895) (#16931)

* chore: sync ci workflow & actions

* chore(ci): enable overflow-checks  in ci profile (#16895)

* chore(ci): enable overflow-checks  in ci profile

* update

* update

* update

* update

* update

* update

* update

* update

* chore: remove bench (not supported on br v1.2.636-rc5)

---------

Co-authored-by: sundyli <[email protected]>

Author: dantengsky

.cargo/audit.toml

@@ -49,4 +49,6 @@ ignore = [
     "RUSTSEC-2024-0371",
     # Remotely exploitable Denial of Service in Tonic, ignored temporarily 
     "RUSTSEC-2024-0376",
+    #rustls network-reachable panic in `Acceptor::accept`
+    "RUSTSEC-2024-0399",
 ]

.github/actions/build_linux/action.yml

@@ -30,7 +30,7 @@ runs:
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}
-        bypass_env_vars: RUSTFLAGS,RUST_LOG
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY
 
     - name: Cross setup
       if: startsWith(inputs.target, 'aarch64-')
@@ -58,53 +58,31 @@ runs:
             flags=""
             ;;
         esac
-        echo "RUSTFLAGS=${flags} -C link-arg=-Wl,--compress-debug-sections=zlib" >> $GITHUB_ENV
+        if [[ ! -z "${flags}" ]]; then
+          echo "RUSTFLAGS=${flags}" >> $GITHUB_ENV
+        fi
         target=${{ inputs.target }}
         echo "BUILD_ARCH=${target/-unknown-linux-*}" >> $GITHUB_ENV
 
-    # build all binaries for debug
-    - name: Build Debug
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts == 'all'
-      shell: bash
-      run: |
-        artifacts="meta,metactl,metabench,query,sqllogictests"
-        for artifact in ${artifacts//,/ }; do
-          echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
-        done
-        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }}
-        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-*
-
-    - name: Build Debug for specific artifacts
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts != 'all'
-      shell: bash
-      run: |
-        artifacts="${{ inputs.artifacts }}"
-        for artifact in ${artifacts//,/ }; do
-          echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
-        done
-        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
-
     - name: Build Release
-      if: env.BUILD_PROFILE == 'release' && inputs.artifacts == 'all'
+      if: inputs.artifacts == 'all'
       shell: bash
       run: |
         artifacts="meta,metactl,metabench,query,sqllogictests"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
         done
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 
     - name: Build Release for specific artifacts
-      if: env.BUILD_PROFILE == 'release' && inputs.artifacts != 'all'
+      if: inputs.artifacts != 'all'
       shell: bash
       run: |
         artifacts="${{ inputs.artifacts }}"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
         done
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 

.github/workflows/dev.yml

@@ -50,7 +50,7 @@ jobs:
     uses: ./.github/workflows/reuse.linux.yml
     secrets: inherit
     with:
-      build_profile: debug
+      build_profile: ci
       runner_provider: aws
       license_type: trial
 
@@ -60,7 +60,7 @@ jobs:
     uses: ./.github/workflows/reuse.linux.hive.yml
     secrets: inherit
     with:
-      build_profile: debug
+      build_profile: ci
       runner_provider: aws
 
   ready:

.github/workflows/reuse.linux.yml

@@ -25,7 +25,7 @@ env:
 
 jobs:
   check:
-    runs-on: [self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}" ]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -115,7 +115,7 @@ jobs:
   #         artifacts: query
 
   test_unit:
-    runs-on: [self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}" ]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -125,64 +125,64 @@ jobs:
         timeout-minutes: 60
 
   test_metactl:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_metactl
         timeout-minutes: 10
 
   test_compat_meta_query:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_compat_meta_query
         timeout-minutes: 10
 
   test_compat_fuse:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_compat_fuse
-        timeout-minutes: 20
+        timeout-minutes: 10
 
   test_compat_meta_meta:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_compat_meta_meta
         timeout-minutes: 20
 
   test_logs:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_logs
         timeout-minutes: 20
 
   test_meta_cluster:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_meta_cluster
         timeout-minutes: 10
 
   test_stateless_standalone:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateless_standalone_linux
         timeout-minutes: 15
 
   test_stateless_cluster:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_license
@@ -193,8 +193,8 @@ jobs:
         timeout-minutes: 15
 
   test_stateful_standalone:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateful_standalone_linux
@@ -206,8 +206,8 @@ jobs:
           name: test-stateful-standalone-linux
 
   test_stateful_cluster:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_license
@@ -224,16 +224,16 @@ jobs:
 
   test_stateful_large_data:
     if: contains(github.event.pull_request.labels.*.name, 'ci-largedata')
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateful_large_data
         timeout-minutes: 60
 
   test_stateful_iceberg_rest:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    needs: [ build, check ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateful_iceberg_rest_standalone
@@ -254,8 +254,8 @@ jobs:
   #       continue-on-error: true
 
   test_ee_standalone:
-    needs: [build, check]
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
+    needs: [ build, check ]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_license
@@ -271,8 +271,8 @@ jobs:
           name: test-stateful-standalone-linux
 
   test_ee_standalone_background:
-    needs: [build, check]
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
+    needs: [ build, check ]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_bendsql
@@ -311,8 +311,8 @@ jobs:
   #          name: test-stateful-standalone-fake-time-linux
 
   test_ee_management_mode:
-    needs: [build, check]
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
+    needs: [ build, check ]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_bendsql
@@ -329,7 +329,7 @@ jobs:
           name: test-ee-management-mode-linux
 
   sqllogic:
-    needs: [build, check]
+    needs: [ build, check ]
     uses: ./.github/workflows/reuse.sqllogic.yml
     secrets: inherit
     with:

Cargo.toml

@@ -353,22 +353,24 @@ result_large_err = "allow"
 debug = 1
 lto = "thin"
 overflow-checks = false
-incremental = false
-opt-level = "s"
+opt-level = "s"         ## defaults to be 3
+incremental = true
 
-# codegen-units = 1 # Reduce number of codegen units to increase optimizations.
+[profile.ci]
+inherits = "release"
+overflow-checks = true
+incremental = false
+debug-assertions = true
 
 # [profile.release.package]
-# arrow2 = { codegen-units = 4 }
-# common-functions = { codegen-units = 16 }
 # databend-query = { codegen-units = 4 }
 # databend-binaries = { codegen-units = 4 }
 
 [profile.dev]
 split-debuginfo = "unpacked"
-overflow-checks = false
-# wait until https://github.com/rust-lang/rust/issues/100142 fixed
-incremental = false
+overflow-checks = true
+# Report to https://github.com/rust-lang/rust/issues/100142 if incremental works well
+incremental = true
 
 [profile.dev.package]
 addr2line = { opt-level = 3 }

src/common/arrow/src/arrow/buffer/iterator.rs

@@ -19,20 +19,20 @@ use crate::arrow::trusted_len::TrustedLen;
 /// This crates' equivalent of [`std::vec::IntoIter`] for [`Buffer`].
 #[derive(Debug, Clone)]
 pub struct IntoIter<T: Copy> {
-    values: Buffer<T>,
-    index: usize,
-    end: usize,
+    buffer: Buffer<T>,
+    current: *const T,
+    end: *const T,
 }
 
 impl<T: Copy> IntoIter<T> {
-    /// Creates a new [`Buffer`]
     #[inline]
-    pub fn new(values: Buffer<T>) -> Self {
-        let end = values.len();
+    pub fn new(buffer: Buffer<T>) -> Self {
+        let ptr = buffer.as_ptr();
+        let len = buffer.len();
         Self {
-            values,
-            index: 0,
-            end,
+            current: ptr,
+            end: unsafe { ptr.add(len) },
+            buffer,
         }
     }
 }
@@ -42,40 +42,30 @@ impl<T: Copy> Iterator for IntoIter<T> {
 
     #[inline]
     fn next(&mut self) -> Option<Self::Item> {
-        if self.index == self.end {
-            return None;
+        if self.current == self.end {
+            None
+        } else {
+            let value = unsafe { *self.current };
+            self.current = unsafe { self.current.add(1) };
+            Some(value)
         }
-        let old = self.index;
-        self.index += 1;
-        Some(*unsafe { self.values.get_unchecked(old) })
     }
 
     #[inline]
     fn size_hint(&self) -> (usize, Option<usize>) {
-        (self.end - self.index, Some(self.end - self.index))
-    }
-
-    #[inline]
-    fn nth(&mut self, n: usize) -> Option<Self::Item> {
-        let new_index = self.index + n;
-        if new_index > self.end {
-            self.index = self.end;
-            None
-        } else {
-            self.index = new_index;
-            self.next()
-        }
+        let len = unsafe { self.end.offset_from(self.current) } as usize;
+        (len, Some(len))
     }
 }
 
 impl<T: Copy> DoubleEndedIterator for IntoIter<T> {
     #[inline]
     fn next_back(&mut self) -> Option<Self::Item> {
-        if self.index == self.end {
+        if self.current == self.end {
             None
         } else {
-            self.end -= 1;
-            Some(*unsafe { self.values.get_unchecked(self.end) })
+            self.end = unsafe { self.end.sub(1) };
+            Some(unsafe { *self.end })
         }
     }
 }