Skip to content

Commit 7b98a13

Browse files
zhang2014zhang2014
authored
chore(query): spilt binary symbol (#16861)
* chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * Update action.yml * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol * chore(query): spilt binary symbol
1 parent c9b1a82 commit 7b98a13

File tree

7 files changed

+252
-80
lines changed

7 files changed

+252
-80
lines changed

‎.github/actions/build_linux/action.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,14 @@ runs:
9292
readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
9393
ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
9494
95+
- name: Spilt Binary Symbols
96+
shell: bash
97+
run: |
98+
objcopy --only-keep-debug ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug
99+
chmod 0644 ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug
100+
strip --strip-debug --remove-section=.comment --remove-section=.note ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query
101+
pushd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }} && objcopy --add-gnu-debuglink databend-query.debug databend-query && popd
102+
95103
# - name: Compress Binaries with UPX
96104
# if: env.BUILD_PROFILE == 'debug'
97105
# uses: crazy-max/ghaction-upx@v2

‎.github/actions/pack_binaries/action.yml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: "Pack Binary"
2+
description: "Pack releases binaries"
3+
inputs:
4+
target:
5+
description: "Release target"
6+
required: true
7+
category:
8+
description: "Release default/hdfs/udf/testsuite"
9+
required: false
10+
default: default
11+
version:
12+
description: "Release version"
13+
required: true
14+
15+
runs:
16+
using: "composite"
17+
steps:
18+
- name: Download artifact
19+
uses: ./.github/actions/artifact_download
20+
with:
21+
sha: ${{ github.sha }}
22+
target: ${{ inputs.target }}
23+
category: ${{ inputs.category }}
24+
path: distro/bin
25+
artifacts: metactl,meta,query,query.debug
26+
- name: Pack Binaries
27+
id: pack_binaries
28+
shell: bash
29+
run: |
30+
target=${{ inputs.target }}
31+
version=${{ inputs.version }}
32+
case ${{ inputs.category }} in
33+
default)
34+
pkg_name="databend-${version}-${target}"
35+
;;
36+
*)
37+
pkg_name="databend-${{ inputs.category }}-${version}-${target}"
38+
;;
39+
esac
40+
mkdir -p distro/{bin,configs,systemd,scripts}
41+
cp ./scripts/distribution/systemd/databend-* distro/systemd/
42+
cp ./scripts/distribution/configs/databend-* distro/configs/
43+
cp ./scripts/distribution/release-readme.txt distro/readme.txt
44+
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
45+
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
46+
tar -C ./distro --exclude='*.debug' -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
47+
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
48+
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
49+
- name: post sha256
50+
uses: actions/upload-artifact@v4
51+
with:
52+
name: sha256sums-${{ inputs.category }}-${{ inputs.target }}
53+
path: sha256-${{ steps.pack_binaries.outputs.pkg_name }}.txt
54+
retention-days: 1
55+
- name: post binaries
56+
uses: actions/upload-artifact@v4
57+
with:
58+
name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
59+
path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
60+
retention-days: 1
61+
- name: Pack DBG Binaries
62+
id: pack_dbg_binaries
63+
shell: bash
64+
run: |
65+
target=${{ inputs.target }}
66+
version=${{ inputs.version }}
67+
case ${{ inputs.category }} in
68+
default)
69+
pkg_name="databend-${version}-${target}-dbg"
70+
;;
71+
*)
72+
pkg_name="databend-${{ inputs.category }}-${version}-${target}-dbg"
73+
;;
74+
esac
75+
mkdir -p distro/{bin,configs,systemd,scripts}
76+
cp ./scripts/distribution/systemd/databend-* distro/systemd/
77+
cp ./scripts/distribution/configs/databend-* distro/configs/
78+
cp ./scripts/distribution/release-readme.txt distro/readme.txt
79+
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
80+
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
81+
tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
82+
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
83+
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
84+
- name: post dbg sha256
85+
uses: actions/upload-artifact@v4
86+
with:
87+
name: sha256sums-${{ inputs.category }}-${{ inputs.target }}-gdb
88+
path: sha256-${{ steps.pack_dbg_binaries.outputs.pkg_name }}.txt
89+
retention-days: 1
90+
- name: post dbg binaries
91+
uses: actions/upload-artifact@v4
92+
with:
93+
name: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
94+
path: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
95+
retention-days: 1

‎.github/actions/pack_distribution/action.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: "Pack Deb"
2+
description: "Pack releases deb"
3+
inputs:
4+
arch:
5+
description: "Release arch"
6+
required: true
7+
packager:
8+
description: "Release default/hdfs/udf/testsuite"
9+
required: false
10+
default: default
11+
version:
12+
description: "Release version"
13+
required: true
14+
15+
runs:
16+
using: "composite"
17+
steps:
18+
- name: Install nfpm@latest
19+
shell: bash
20+
run: |
21+
curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
22+
tar xf nfpm.tar.gz
23+
sudo mv nfpm /usr/local/bin
24+
sudo chmod a+x /usr/local/bin/nfpm
25+
rm nfpm.tar.gz
26+
- name: Get target
27+
id: target
28+
shell: bash
29+
run: |
30+
echo 'target=${{ inputs.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
31+
- name: Download artifacts
32+
uses: ./.github/actions/artifact_download
33+
with:
34+
sha: ${{ github.sha }}
35+
target: ${{ steps.target.outputs.target }}
36+
category: default
37+
artifacts: metactl,meta,query,query.debug
38+
path: distro/bin
39+
- name: Build Packages
40+
shell: bash
41+
id: build_packages
42+
run: |
43+
export name="databend"
44+
export version="${{ inputs.version }}"
45+
export path="distro"
46+
case "${{ inputs.arch }}" in
47+
x86_64)
48+
export arch="amd64"
49+
;;
50+
aarch64)
51+
export arch="arm64"
52+
;;
53+
esac
54+
55+
deb_version=${version/-/.}
56+
pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}.${{ inputs.packager }}"
57+
mkdir -p distro/{bin,configs,systemd,scripts}
58+
cp ./scripts/distribution/systemd/databend-* distro/systemd/
59+
cp ./scripts/distribution/configs/databend-* distro/configs/
60+
cp ./scripts/distribution/release-readme.txt distro/readme.txt
61+
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
62+
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
63+
nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
64+
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
65+
- name: Build dbg Packages
66+
shell: bash
67+
id: build_dbg_packages
68+
run: |
69+
export name="databend_dbg"
70+
export version="${{ inputs.version }}"
71+
export path="distro"
72+
case "${{ inputs.arch }}" in
73+
x86_64)
74+
export arch="amd64"
75+
;;
76+
aarch64)
77+
export arch="arm64"
78+
;;
79+
esac
80+
81+
deb_version=${version/-/.}
82+
pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}_dbg.${{ inputs.packager }}"
83+
nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm-dbg.yaml)
84+
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
85+
- name: Update release to github
86+
shell: bash
87+
env:
88+
GH_TOKEN: ${{ github.token }}
89+
# Reference: https://cli.github.com/manual/gh_release_upload
90+
run: |
91+
version="${{ inputs.version }}"
92+
# name looks like: `databend_0.8.144~nightly_amd64.deb`
93+
gh release upload ${version} ${{ steps.build_packages.outputs.pkg_name }} --clobber
94+
gh release upload ${version} ${{ steps.build_dbg_packages.outputs.pkg_name }} --clobber

‎.github/actions/publish_binary/action.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,19 @@ runs:
3737
shell: bash
3838
# Reference: https://cli.github.com/manual/gh_release_upload
3939
run: |
40-
gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.* --clobber
40+
gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.tar.gz --clobber
41+
if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then
42+
gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}-dbg.* --clobber
43+
fi
4144
4245
- name: Sync normal release to R2
4346
shell: bash
4447
if: inputs.category == 'default'
4548
run: |
4649
aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress
50+
if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then
51+
aws s3 cp ${{ steps.name.outputs.name }}-dbg.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}-dbg.tar.gz --no-progress
52+
fi
4753
gh api /repos/databendlabs/databend/tags > tags.json
4854
aws s3 cp ./tags.json s3://repo/databend/tags.json
4955
gh api /repos/databendlabs/databend/releases > releases.json

‎.github/workflows/release.yml‎

Lines changed: 8 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -203,40 +203,12 @@ jobs:
203203
uses: actions/checkout@v4
204204
with:
205205
ref: ${{ needs.create_release.outputs.sha }}
206-
- name: Download artifact
207-
uses: ./.github/actions/artifact_download
206+
- name: Pack Binaries
207+
uses: ./.github/actions/pack_binaries
208208
with:
209-
sha: ${{ github.sha }}
209+
version: ${{ needs.create_release.outputs.version }}
210210
target: ${{ matrix.target }}
211211
category: ${{ matrix.category }}
212-
path: distro/bin
213-
artifacts: metactl,meta,query
214-
- name: Pack Binaries
215-
run: |
216-
target=${{ matrix.target }}
217-
version=${{ needs.create_release.outputs.version }}
218-
case ${{ matrix.category }} in
219-
default)
220-
pkg_name="databend-${version}-${target}"
221-
;;
222-
*)
223-
pkg_name="databend-${{ matrix.category }}-${version}-${target}"
224-
;;
225-
esac
226-
mkdir -p distro/{bin,configs,systemd,scripts}
227-
cp ./scripts/distribution/systemd/databend-* distro/systemd/
228-
cp ./scripts/distribution/configs/databend-* distro/configs/
229-
cp ./scripts/distribution/release-readme.txt distro/readme.txt
230-
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
231-
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
232-
tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
233-
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
234-
- name: post sha256
235-
uses: actions/upload-artifact@v4
236-
with:
237-
name: sha256sums-${{ matrix.category }}-${{ matrix.target }}
238-
path: sha256-*.txt
239-
retention-days: 1
240212
- name: Publish Binaries
241213
uses: ./.github/actions/publish_binary
242214
env:
@@ -455,55 +427,12 @@ jobs:
455427
uses: actions/checkout@v4
456428
with:
457429
ref: ${{ needs.create_release.outputs.sha }}
458-
- name: Install nfpm@latest
459-
run: |
460-
curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
461-
tar xf nfpm.tar.gz
462-
sudo mv nfpm /usr/local/bin
463-
sudo chmod a+x /usr/local/bin/nfpm
464-
rm nfpm.tar.gz
465-
- name: Get target
466-
id: target
467-
run: |
468-
echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
469-
- name: Download artifacts
470-
uses: ./.github/actions/artifact_download
430+
- name: Pack distribution
431+
uses: ./.github/actions/pack_distribution
471432
with:
472-
sha: ${{ github.sha }}
473-
target: ${{ steps.target.outputs.target }}
474-
category: default
475-
artifacts: metactl,meta,query
476-
path: distro/bin
477-
- name: Build Packages
478-
id: build_packages
479-
run: |
480-
export name="databend"
481-
export version="${{ needs.create_release.outputs.version }}"
482-
export path="distro"
483-
case "${{ matrix.arch }}" in
484-
x86_64)
485-
export arch="amd64"
486-
;;
487-
aarch64)
488-
export arch="arm64"
489-
;;
490-
esac
491-
mkdir -p distro/{bin,configs,systemd,scripts}
492-
cp ./scripts/distribution/systemd/databend-* distro/systemd/
493-
cp ./scripts/distribution/configs/databend-* distro/configs/
494-
cp ./scripts/distribution/release-readme.txt distro/readme.txt
495-
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
496-
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
497-
nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
498-
- name: Update release to github
499-
shell: bash
500-
env:
501-
GH_TOKEN: ${{ github.token }}
502-
# Reference: https://cli.github.com/manual/gh_release_upload
503-
run: |
504-
version="${{ needs.create_release.outputs.version }}"
505-
# name looks like: `databend_0.8.144~nightly_amd64.deb`
506-
gh release upload ${version} databend_*.${{ matrix.packager }} --clobber
433+
version: ${{ needs.create_release.outputs.version }}
434+
arch: ${{ matrix.arch }}
435+
packager: ${{ matrix.packager }}
507436

508437
# bindings_python:
509438
# if: inputs.stable

‎scripts/distribution/nfpm-dbg.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
name: "${name}"
2+
arch: "${arch}"
3+
platform: "linux"
4+
version: "${version}"
5+
section: "database"
6+
priority: "extra"
7+
maintainer: "Databend Authors <[email protected]>"
8+
description: |
9+
Databend is a powerful cloud data warehouse. Built for elasticity and efficiency.
10+
Free and open. Also available in the cloud: https://app.databend.com
11+
vendor: "Datafuse Labs"
12+
homepage: "https://databend.com"
13+
license: "Apache-2.0"
14+
contents:
15+
# Binaries
16+
- src: ${path}/bin/databend-query.debug
17+
dst: /usr/bin/databend-query.debug

‎scripts/setup/dev_setup.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,28 @@ function install_libtiff {
393393
esac
394394
}
395395

396+
function install_binutils {
397+
PACKAGE_MANAGER=$1
398+
399+
echo "==> installing binutils..."
400+
401+
case "$PACKAGE_MANAGER" in
402+
apt-get)
403+
install_pkg binutils "$PACKAGE_MANAGER"
404+
;;
405+
yum | dnf)
406+
install_pkg binutils "$PACKAGE_MANAGER"
407+
;;
408+
brew)
409+
# skip
410+
;;
411+
*)
412+
echo "Unable to install binutils with package manager: $PACKAGE_MANAGER"
413+
exit 1
414+
;;
415+
esac
416+
}
417+
396418
function install_rustup {
397419
RUST_TOOLCHAIN=$1
398420

@@ -624,6 +646,7 @@ if [[ "$INSTALL_BUILD_TOOLS" == "true" ]]; then
624646
install_python3 "$PACKAGE_MANAGER"
625647
install_sqlite3 "$PACKAGE_MANAGER"
626648
install_libtiff "$PACKAGE_MANAGER"
649+
install_binutils "$PACKAGE_MANAGER"
627650

628651
# Any call to cargo will make rustup install the correct toolchain
629652
cargo version

0 commit comments

Comments
 (0)