Release #1182
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| on: | |
| schedule: | |
| - cron: "0 22 * * 0" | |
| workflow_dispatch: | |
| inputs: | |
| tag: | |
| description: | | |
| The tags to be released. | |
| Nightly release dose not require a tag. | |
| Stable release requires a nightly version (v1.0.0-nightly). | |
| Patch release requires a stable version (v1.0.0). | |
| required: false | |
| type: string | |
| type: | |
| description: Release type, default to nightly | |
| required: true | |
| type: choice | |
| default: nightly | |
| options: | |
| - nightly | |
| - stable | |
| - patch | |
| permissions: | |
| id-token: write | |
| pull-requests: write | |
| checks: write | |
| statuses: write | |
| contents: write | |
| packages: write | |
| env: | |
| BUILD_PROFILE: release | |
| RUNNER_PROVIDER: aws | |
| jobs: | |
| create_release: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.bump.outputs.tag }} | |
| sha: ${{ steps.bump.outputs.sha }} | |
| type: ${{ steps.bump.outputs.type }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Bump version | |
| id: bump | |
| uses: actions/github-script@v7 | |
| env: | |
| TYPE: "${{ inputs.type }}" | |
| TAG: "${{ inputs.tag }}" | |
| with: | |
| script: | | |
| const script = require('./.github/scripts/bump_version.js') | |
| await script({ github, context, core }) | |
| - name: Create release | |
| env: | |
| # we need workflow:write permission to create release if there were any workflow changes | |
| # which is not possible for github actions token | |
| GH_TOKEN: ${{ secrets.DATABEND_BOT_TOKEN }} | |
| run: | | |
| case "${{ steps.bump.outputs.type }}" in | |
| nightly) | |
| gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag ${{ steps.bump.outputs.previous }} --prerelease --draft | |
| ;; | |
| stable) | |
| gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag ${{ steps.bump.outputs.previous }} --latest --draft | |
| ;; | |
| patch) | |
| gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag ${{ steps.bump.outputs.previous }} --prerelease --draft | |
| ;; | |
| *) | |
| echo "Invalid release type: ${{ steps.bump.outputs.type }}" | |
| exit 1 | |
| esac | |
| changelog: | |
| runs-on: ubuntu-latest | |
| needs: create_release | |
| if: needs.create_release.outputs.type == 'stable' | |
| steps: | |
| - name: Checkout Docs | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: databendlabs/databend-docs | |
| ref: main | |
| - name: Get date | |
| id: date | |
| shell: bash | |
| run: echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| - name: Generate Release Note | |
| uses: actions/github-script@v7 | |
| env: | |
| VERSION: ${{ needs.create_release.outputs.version }} | |
| DATE: ${{ steps.date.outputs.DATE }} | |
| with: | |
| script: | | |
| const script = require('./.github/scripts/generate_release_note.js') | |
| await script({ github, context, core }) | |
| - name: Add Release Note | |
| run: | | |
| git add docs/release-stable | |
| git status | |
| - uses: peter-evans/create-pull-request@v4 | |
| with: | |
| token: ${{ secrets.DATABEND_BOT_TOKEN }} | |
| title: "chore(docs): Update Release Notes - ${{ steps.date.outputs.DATE }}" | |
| base: main | |
| commit-message: "chore(docs): Update Release Notes - ${{ steps.date.outputs.DATE }}" | |
| branch-suffix: random | |
| delete-branch: true | |
| build_default: | |
| runs-on: | |
| - self-hosted | |
| - "${{ matrix.runner }}" | |
| - Linux | |
| - 16c64g | |
| - aws | |
| needs: create_release | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - { target: x86_64-unknown-linux-gnu, runner: X64 } | |
| - { target: aarch64-unknown-linux-gnu, runner: ARM64 } | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| fetch-depth: 0 | |
| - name: Build Release | |
| uses: ./.github/actions/build_linux | |
| env: | |
| DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} | |
| DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} | |
| DATABEND_ENTERPRISE_LICENSE_EMBEDDED: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_RELEASE }} | |
| DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}} | |
| DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}} | |
| DATABEND_TELEMETRY_SOURCE: "production" | |
| with: | |
| sha: ${{ github.sha }} | |
| target: ${{ matrix.target }} | |
| artifacts: sqllogictests,sqlsmith,query,meta,metactl,metaverifier,bendsave | |
| - name: Basic Sqllogic Test | |
| shell: bash | |
| env: | |
| TEST_HANDLERS: http | |
| run: | | |
| mkdir -p ./target/${{ env.BUILD_PROFILE }} | |
| cp ./target/${{ matrix.target }}/${{ env.BUILD_PROFILE }}/databend-* ./target/${{ env.BUILD_PROFILE }}/ | |
| bash ./scripts/ci/ci-run-sqllogic-tests.sh base | |
| build_docker: | |
| runs-on: | |
| - self-hosted | |
| - "${{ matrix.runner }}" | |
| - Linux | |
| - 16c64g | |
| - aws | |
| needs: create_release | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - { target: x86_64-unknown-linux-gnu, runner: X64 } | |
| - { target: aarch64-unknown-linux-gnu, runner: ARM64 } | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| fetch-depth: 0 | |
| - name: Build Release | |
| uses: ./.github/actions/build_linux | |
| env: | |
| DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} | |
| DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} | |
| DATABEND_ENTERPRISE_LICENSE_EMBEDDED: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_RELEASE }} | |
| DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}} | |
| DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}} | |
| DATABEND_TELEMETRY_SOURCE: "docker-release" | |
| with: | |
| sha: ${{ github.sha }} | |
| target: ${{ matrix.target }} | |
| artifacts: sqllogictests,sqlsmith,metactl,meta,query,bendsave | |
| features: python-udf | |
| category: docker | |
| publish: | |
| runs-on: | |
| - self-hosted | |
| - X64 | |
| - Linux | |
| - 2c8g | |
| - aws | |
| needs: | |
| - create_release | |
| - build_default | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: | |
| - x86_64-unknown-linux-gnu | |
| - aarch64-unknown-linux-gnu | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: Pack Binaries | |
| uses: ./.github/actions/pack_binaries | |
| with: | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ matrix.target }} | |
| category: default | |
| - name: Publish Binaries | |
| uses: ./.github/actions/publish_binary | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: auto | |
| AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} | |
| with: | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ matrix.target }} | |
| category: default | |
| publish_testsuite: | |
| runs-on: | |
| - self-hosted | |
| - ARM64 | |
| - Linux | |
| - 2c8g | |
| - aws | |
| needs: | |
| - create_release | |
| - build_default | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: | |
| - x86_64-unknown-linux-gnu | |
| - aarch64-unknown-linux-gnu | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: Download artifact | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ github.sha }} | |
| target: ${{ matrix.target }} | |
| category: default | |
| artifacts: sqllogictests,sqlsmith | |
| path: distro/bin | |
| - name: Pack Testsuite | |
| run: | | |
| target=${{ matrix.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| pkg_name="databend-testsuite-${version}-${target}" | |
| cp -r ./tests/sqllogictests/suites ./distro/ | |
| tar -C ./distro -czvf ${pkg_name}.tar.gz bin suites | |
| sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt | |
| - name: post sha256 | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sha256sums-testsuite-${{ matrix.target }} | |
| path: sha256-*.txt | |
| retention-days: 1 | |
| - name: Publish Testsuite | |
| uses: ./.github/actions/publish_binary | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| with: | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ matrix.target }} | |
| category: testsuite | |
| publish_dbg: | |
| runs-on: | |
| - self-hosted | |
| - ARM64 | |
| - Linux | |
| - 2c8g | |
| - aws | |
| needs: | |
| - create_release | |
| - build_default | |
| - build_docker | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| category: | |
| - default | |
| - docker | |
| target: | |
| - x86_64-unknown-linux-gnu | |
| - aarch64-unknown-linux-gnu | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: Publish Debug Symbols | |
| uses: ./.github/actions/publish_debug_symbols | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| with: | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ matrix.target }} | |
| category: ${{ matrix.category }} | |
| sha: ${{ github.sha }} | |
| docker_all_in_one: | |
| runs-on: | |
| - self-hosted | |
| - ARM64 | |
| - Linux | |
| - 2c8g | |
| - aws | |
| needs: | |
| - create_release | |
| - build_default | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: Download artifacts for x86_64 | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ github.sha }} | |
| target: x86_64-unknown-linux-gnu | |
| category: default | |
| artifacts: metactl,meta,query,bendsave | |
| path: distro/linux/amd64 | |
| - name: Download artifacts for aarch64 | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ github.sha }} | |
| target: aarch64-unknown-linux-gnu | |
| category: default | |
| artifacts: metactl,meta,query,bendsave | |
| path: distro/linux/arm64 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - uses: ./.github/actions/setup_docker | |
| id: login | |
| with: | |
| repo: databend | |
| ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
| dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} | |
| dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Get Image Tags | |
| id: tags | |
| uses: actions/github-script@v7 | |
| env: | |
| REPO_DOCKERHUB: ${{ steps.login.outputs.dockerhub_repo }} | |
| REPO_ECR: ${{ steps.login.outputs.ecr_repo }} | |
| VERSION: ${{ needs.create_release.outputs.version }} | |
| TYPE: ${{ needs.create_release.outputs.type }} | |
| with: | |
| script: | | |
| const { VERSION, TYPE, REPO_DOCKERHUB, REPO_ECR } = process.env; | |
| const repos = [REPO_DOCKERHUB, REPO_ECR]; | |
| let tags = []; | |
| for (const repo of repos) { | |
| tags.push(`${repo}:${VERSION}`); | |
| if (TYPE === 'stable') { | |
| tags.push(`${repo}:latest`); | |
| } else { | |
| tags.push(`${repo}:nightly`); | |
| } | |
| } | |
| core.setOutput('tags', tags.join(',')); | |
| - name: Build and push | |
| id: docker_build | |
| uses: docker/build-push-action@v5 | |
| with: | |
| push: true | |
| tags: ${{ steps.tags.outputs.tags }} | |
| platforms: linux/amd64,linux/arm64 | |
| context: . | |
| file: ./docker/Dockerfile | |
| - name: Update repo description | |
| continue-on-error: true | |
| uses: peter-evans/dockerhub-description@v4 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| repository: ${{ steps.login.outputs.dockerhub_repo }} | |
| short-description: "A modern cloud data warehouse. Also available in the cloud: https://app.databend.com." | |
| readme-filepath: ./docker/README.md | |
| docker_service: | |
| runs-on: | |
| - self-hosted | |
| - ARM64 | |
| - Linux | |
| - 2c8g | |
| - aws | |
| needs: | |
| - create_release | |
| - build_docker | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| service: | |
| - meta | |
| - query | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: Download artifacts for x86_64 | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ github.sha }} | |
| target: x86_64-unknown-linux-gnu | |
| category: docker | |
| artifacts: ${{ matrix.service }},metactl | |
| path: distro/linux/amd64 | |
| - name: Download artifacts for aarch64 | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ github.sha }} | |
| target: aarch64-unknown-linux-gnu | |
| category: docker | |
| artifacts: ${{ matrix.service }},metactl | |
| path: distro/linux/arm64 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - uses: ./.github/actions/setup_docker | |
| id: login | |
| with: | |
| repo: databend-${{ matrix.service }} | |
| ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
| dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} | |
| dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Get Image Tags | |
| id: tags | |
| uses: actions/github-script@v7 | |
| env: | |
| REPO_DOCKERHUB: ${{ steps.login.outputs.dockerhub_repo }} | |
| REPO_ECR: ${{ steps.login.outputs.ecr_repo }} | |
| VERSION: ${{ needs.create_release.outputs.version }} | |
| TYPE: ${{ needs.create_release.outputs.type }} | |
| with: | |
| script: | | |
| const { VERSION, TYPE, REPO_DOCKERHUB, REPO_ECR } = process.env; | |
| const repos = [REPO_DOCKERHUB, REPO_ECR]; | |
| let tags = []; | |
| for (const repo of repos) { | |
| tags.push(`${repo}:${VERSION}`); | |
| if (TYPE === 'stable') { | |
| tags.push(`${repo}:latest`); | |
| } else { | |
| tags.push(`${repo}:nightly`); | |
| } | |
| } | |
| core.setOutput('tags', tags.join(',')); | |
| - name: push service image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| push: true | |
| tags: ${{ steps.tags.outputs.tags }} | |
| platforms: linux/amd64,linux/arm64 | |
| context: . | |
| file: ./docker/service/${{ matrix.service }}.Dockerfile | |
| distribution: | |
| runs-on: | |
| - self-hosted | |
| - ARM64 | |
| - Linux | |
| - 2c8g | |
| - aws | |
| needs: | |
| - create_release | |
| - build_default | |
| strategy: | |
| matrix: | |
| arch: | |
| - x86_64 | |
| - aarch64 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: Pack deb | |
| uses: ./.github/actions/pack_deb | |
| with: | |
| version: ${{ needs.create_release.outputs.version }} | |
| arch: ${{ matrix.arch }} | |
| # bindings_python: | |
| # # if: needs.create_release.outputs.type == 'stable' | |
| # needs: create_release | |
| # uses: ./.github/workflows/bindings.python.yml | |
| # secrets: inherit | |
| # with: | |
| # version: ${{ needs.create_release.outputs.version }} | |
| notify: | |
| runs-on: ubuntu-latest | |
| if: always() | |
| needs: | |
| - create_release | |
| - build_default | |
| - build_docker | |
| - publish | |
| - docker_all_in_one | |
| - docker_service | |
| - distribution | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: Notify Release | |
| uses: actions/github-script@v7 | |
| env: | |
| JOBS_STATUS: ${{ join(needs.*.result, ',') }} | |
| REPORT_WEBHOOK: ${{ secrets.RELEASE_REPORT_WEBHOOK }} | |
| VERSION: ${{ needs.create_release.outputs.version }} | |
| with: | |
| script: | | |
| const script = require('./.github/scripts/notify_release.js') | |
| await script({context, core}) | |
| - name: Publish release | |
| env: | |
| GH_TOKEN: ${{ secrets.DATABEND_BOT_TOKEN }} | |
| run: | | |
| gh release edit ${{ needs.create_release.outputs.version }} --draft=false | |
| sha256sums: | |
| needs: | |
| - create_release | |
| - publish | |
| - distribution | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: download sha256sums | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: sha256sums-* | |
| merge-multiple: true | |
| - shell: bash | |
| run: | | |
| for file in *.txt | |
| do | |
| cat ${file} >> sha256sums.txt | |
| done | |
| - name: Upload checksums | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| version="${{ needs.create_release.outputs.version }}" | |
| gh release upload ${version} sha256sums.txt --clobber | |
| sqlsmith: | |
| runs-on: | |
| - self-hosted | |
| - ARM64 | |
| - Linux | |
| - 2c8g | |
| - aws | |
| needs: | |
| - create_release | |
| - notify | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: Download artifacts | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ github.sha }} | |
| target: aarch64-unknown-linux-gnu | |
| category: default | |
| artifacts: meta,query,sqlsmith | |
| - name: Run sqlsmith | |
| timeout-minutes: 60 | |
| shell: bash | |
| run: | | |
| bash ./scripts/ci/ci-run-sqlsmith-tests.sh | |
| - name: Upload failure | |
| if: failure() | |
| uses: ./.github/actions/artifact_failure | |
| with: | |
| name: test-sqlsmith | |
| metachaos: | |
| runs-on: | |
| - self-hosted | |
| - X64 | |
| - Linux | |
| - 4c16g | |
| - aws | |
| needs: | |
| - create_release | |
| - notify | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.create_release.outputs.sha }} | |
| - name: Download artifacts | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ github.sha }} | |
| target: x86_64-unknown-linux-gnu | |
| category: default | |
| artifacts: meta,metactl,metaverifier | |
| - uses: ./.github/actions/io_delay_chaos | |
| timeout-minutes: 20 | |
| - name: Notify failure | |
| if: failure() | |
| uses: actions/github-script@v7 | |
| env: | |
| REPORT_WEBHOOK: ${{ secrets.META_REPORT_WEBHOOK }} | |
| TITLE: Meta Service chaos tests failed | |
| with: | |
| script: | | |
| const script = require('./.github/scripts/notify_failure.js') | |
| await script({context, core}) | |
| benchmark: | |
| needs: | |
| - create_release | |
| - docker_service | |
| - notify | |
| uses: ./.github/workflows/reuse.benchmark.yml | |
| secrets: inherit | |
| with: | |
| sha: ${{ github.sha }} | |
| run_id: ${{ github.run_id }} | |
| source: release | |
| source_id: ${{ needs.create_release.outputs.version }} | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: all | |
| deb: | |
| needs: | |
| - create_release | |
| - distribution | |
| - notify | |
| uses: ./.github/workflows/deb.yml | |
| secrets: inherit |