refactor: move aggregation spill logic into partial aggregate #2250
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cloud | |
| on: | |
| pull_request_target: | |
| types: | |
| - labeled | |
| branches: | |
| - main | |
| - backport/* | |
| permissions: | |
| id-token: write | |
| pull-requests: write | |
| contents: read | |
| packages: write | |
| env: | |
| BUILD_PROFILE: release | |
| RUNNER_PROVIDER: aws | |
| jobs: | |
| info: | |
| if: contains(github.event.pull_request.labels.*.name, 'ci-cloud') || contains(github.event.pull_request.labels.*.name, 'ci-benchmark') || contains(github.event.pull_request.labels.*.name, 'ci-benchmark-cloud') || contains(github.event.pull_request.labels.*.name, 'ci-benchmark-load') || contains(github.event.pull_request.labels.*.name, 'ci-benchmark-local') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| sha: ${{ steps.sha.outputs.sha }} | |
| target: ${{ steps.sha.outputs.target }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: "refs/pull/${{ github.event.number }}/merge" | |
| - name: Get SHA | |
| shell: bash | |
| id: sha | |
| run: | | |
| echo "sha=$(git rev-parse --verify HEAD)" > $GITHUB_OUTPUT | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'ci-benchmark') }}; then | |
| echo "target=all" >> $GITHUB_OUTPUT | |
| elif ${{ contains(github.event.pull_request.labels.*.name, 'ci-benchmark-cloud') }}; then | |
| echo "target=cloud" >> $GITHUB_OUTPUT | |
| elif ${{ contains(github.event.pull_request.labels.*.name, 'ci-benchmark-load') }}; then | |
| echo "target=load" >> $GITHUB_OUTPUT | |
| elif ${{ contains(github.event.pull_request.labels.*.name, 'ci-benchmark-local') }}; then | |
| echo "target=local" >> $GITHUB_OUTPUT | |
| fi | |
| build: | |
| needs: info | |
| runs-on: | |
| - self-hosted | |
| - "${{ matrix.runner }}" | |
| - Linux | |
| - 8c32g | |
| - aws | |
| strategy: | |
| matrix: | |
| include: | |
| - { arch: x86_64, runner: X64 } | |
| - { arch: aarch64, runner: ARM64 } | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: "refs/pull/${{ github.event.number }}/merge" | |
| - name: Build Release | |
| uses: ./.github/actions/build_linux | |
| timeout-minutes: 60 | |
| env: | |
| DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} | |
| DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}} | |
| DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}} | |
| DATABEND_TELEMETRY_SOURCE: "pr-test" | |
| with: | |
| sha: ${{ needs.info.outputs.sha }} | |
| target: ${{ matrix.arch }}-unknown-linux-gnu | |
| artifacts: meta,query | |
| category: docker | |
| features: python-udf | |
| docker: | |
| needs: [ info, build ] | |
| timeout-minutes: 10 | |
| runs-on: | |
| - self-hosted | |
| - ARM64 | |
| - Linux | |
| - 2c8g | |
| - aws | |
| outputs: | |
| tag: ${{ steps.prepare.outputs.tag }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download artifact for x86_64 | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ needs.info.outputs.sha }} | |
| target: x86_64-unknown-linux-gnu | |
| path: ./distro/linux/amd64/ | |
| category: docker | |
| artifacts: query | |
| - name: Download artifact for aarch64 | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ needs.info.outputs.sha }} | |
| target: aarch64-unknown-linux-gnu | |
| path: ./distro/linux/arm64/ | |
| category: docker | |
| artifacts: query | |
| - uses: ./.github/actions/setup_docker | |
| id: login | |
| with: | |
| repo: databend-query | |
| ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
| dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} | |
| dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Prepare for docker | |
| id: prepare | |
| run: | | |
| short_sha=$(echo "${{ needs.info.outputs.sha }}" | cut -c1-7) | |
| ts=$(date +%s) | |
| echo "tag=pr-${{ github.event.pull_request.number }}-${short_sha}-${ts}" >> $GITHUB_OUTPUT | |
| - name: Get Image Tags | |
| id: tags | |
| uses: actions/github-script@v7 | |
| env: | |
| REPO_DOCKERHUB: ${{ steps.login.outputs.dockerhub_repo }} | |
| REPO_ECR: ${{ steps.login.outputs.ecr_repo }} | |
| VERSION: ${{ steps.prepare.outputs.tag }} | |
| with: | |
| script: | | |
| const { VERSION, REPO_DOCKERHUB, REPO_ECR } = process.env; | |
| const repos = [REPO_DOCKERHUB, REPO_ECR]; | |
| let tags = []; | |
| for (const repo of repos) { | |
| tags.push(`${repo}:${VERSION}`); | |
| } | |
| core.setOutput('tags', tags.join(',')); | |
| - name: push service image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| push: true | |
| tags: ${{ steps.tags.outputs.tags }} | |
| platforms: linux/amd64,linux/arm64 | |
| context: . | |
| file: ./docker/service/query.Dockerfile | |
| - name: Comment on PR | |
| uses: everpcpc/comment-on-pr-action@v1 | |
| with: | |
| number: ${{ github.event.pull_request.number }} | |
| token: ${{ github.token }} | |
| body: | | |
| ## Docker Image for PR | |
| * **tag**: `${{ steps.prepare.outputs.tag }}` | |
| > note: this image tag is only available for internal use. | |
| explain: | |
| needs: [info, build, docker] | |
| uses: ./.github/workflows/reuse.explain.yml | |
| secrets: inherit | |
| with: | |
| run_id: ${{ github.run_id }} | |
| version: ${{ needs.docker.outputs.tag }} | |
| benchmark: | |
| if: needs.info.outputs.target | |
| needs: [ info, build, docker ] | |
| uses: ./.github/workflows/reuse.benchmark.yml | |
| secrets: inherit | |
| with: | |
| sha: ${{ needs.info.outputs.sha }} | |
| run_id: ${{ github.run_id }} | |
| source: pr | |
| source_id: ${{ github.event.pull_request.number }} | |
| version: ${{ needs.docker.outputs.tag }} | |
| target: ${{ needs.info.outputs.target }} |