Skip to content

[CI] [GHA] Split wheel building job into several parallel ones #3032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
+293 −124
Open

[CI] [GHA] Split wheel building job into several parallel ones #3032

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
57d3e7e
use sccache
akashchi Nov 18, 2025
933df05
use parallel
akashchi Nov 18, 2025
151574f
Revert "use parallel"
akashchi Nov 19, 2025
78500f8
Merge remote-tracking branch 'upstream/master' into ci/gha/speed-up-w…
akashchi Nov 19, 2025
d4ca5db
use ninja
akashchi Nov 19, 2025
81a319c
check sccache stats
akashchi Nov 21, 2025
ca65515
Merge remote-tracking branch 'upstream/master' into ci/gha/speed-up-w…
akashchi Nov 21, 2025
fd3defe
use matrix
akashchi Nov 21, 2025
7fedb35
combine tokenizer and wwb wheel
akashchi Nov 21, 2025
66ce9ac
use 8 core, use the same approach in manylinux
akashchi Nov 21, 2025
4671b36
use approach in win
akashchi Nov 21, 2025
e231ae0
use ninja for building wheels
akashchi Nov 26, 2025
e790d6b
Merge remote-tracking branch 'upstream/master' into ci/gha/speed-up-w…
akashchi Nov 26, 2025
223ce56
Merge branch 'master' into ci/gha/speed-up-wheel-build
akashchi Nov 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 98 additions & 36 deletions .github/workflows/linux.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ jobs:
cmake -DOpenVINODeveloperPackage_DIR=${{ env.OV_INSTALL_DIR }}/developer_package/cmake \
-DENABLE_PYTHON=OFF \
-DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-DCMAKE_C_COMPILER_LAUNCHER=sccache \
-S ${{ env.SRC_DIR }} \
-B ${{ env.BUILD_DIR }}
cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose
Expand All @@ -192,6 +194,8 @@ jobs:
cmake -DOpenVINODeveloperPackage_DIR=${{ env.OV_INSTALL_DIR }}/developer_package/cmake \
-DENABLE_PYTHON=ON \
-DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-DCMAKE_C_COMPILER_LAUNCHER=sccache \
-S ${{ env.SRC_DIR }} \
-B ${{ env.BUILD_DIR }}
echo "Configuring cmake done for python$py_version"
Expand Down Expand Up @@ -240,10 +244,10 @@ jobs:
path: ${{ env.MANIFEST_PATH }}
if-no-files-found: 'error'

genai_build_wheel:
name: Build Wheel
genai_build_wheels:
name: Build Tokenizers & WWB Wheels
needs: [ openvino_download ]
timeout-minutes: 90
timeout-minutes: 30
defaults:
run:
shell: bash
Expand All @@ -255,12 +259,11 @@ jobs:
- ${{ github.workspace }}:${{ github.workspace }}
options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING
env:
CMAKE_GENERATOR: Unix Makefiles
CMAKE_GENERATOR: Ninja
OV_INSTALL_DIR: ${{ github.workspace }}/ov
INSTALL_DIR: ${{ github.workspace }}/install
WHEELS_DIR: ${{ github.workspace }}/install/wheels
SRC_DIR: ${{ github.workspace }}/src
OpenVINODeveloperPackage_DIR: ${{ github.workspace }}/ov/developer_package/cmake

steps:
- name: Clone openvino.genai
Expand All @@ -276,46 +279,98 @@ jobs:
path: ${{ env.OV_INSTALL_DIR }}
merge-multiple: true

- name: Set CI environment
id: create_manifest
uses: openvinotoolkit/openvino/.github/actions/create_manifest@master
with:
repos: ${{ env.SRC_DIR }}
product_type: ${{ env.BASE_PRODUCT_TYPE }}_Release
target_arch: 'x86_64'
build_type: Release
save_to: ${{ github.workspace }}

- name: Build Tokenizers Wheel
run: |
python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \
--config-settings='override=wheel.build_tag="${{ github.run_number }}"' \
--config-settings=override=cross.arch="manylinux_2_31_x86_64" \
--config-settings=cmake.args="-DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache" \
${{ needs.openvino_download.outputs.ov_wheel_source }} \
${{ env.SRC_DIR }}/thirdparty/openvino_tokenizers
working-directory: ${{ env.OV_INSTALL_DIR }}

- name: Build WWB Wheel
run: python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark
working-directory: ${{ env.OV_INSTALL_DIR }}

- name: Upload Wheels
if: ${{ always() }}
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: genai_wheels
path: ${{ env.INSTALL_DIR }}
if-no-files-found: 'error'

genai_build_genai_wheel:
name: Build GenAI Wheel - Python ${{ matrix.python-version }}
needs: [ openvino_download ]
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
python-version: ['3.10', '3.11', '3.12', '3.13']
defaults:
run:
shell: bash
runs-on: aks-linux-8-cores-16gb
container:
image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }}
volumes:
- /mount:/mount
- ${{ github.workspace }}:${{ github.workspace }}
options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING
env:
CMAKE_GENERATOR: Ninja
OV_INSTALL_DIR: ${{ github.workspace }}/ov
INSTALL_DIR: ${{ github.workspace }}/install
WHEELS_DIR: ${{ github.workspace }}/install/wheels
SRC_DIR: ${{ github.workspace }}/src
OpenVINODeveloperPackage_DIR: ${{ github.workspace }}/ov/developer_package/cmake
steps:
- name: Clone openvino.genai
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
path: ${{ env.SRC_DIR }}
submodules: recursive

- name: Download OpenVINO package
uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
with:
name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
path: ${{ env.OV_INSTALL_DIR }}
merge-multiple: true

- name: Set CI environment
id: create_manifest
uses: openvinotoolkit/openvino/.github/actions/create_manifest@master
with:
repos: ${{ env.SRC_DIR }}
product_type: ${{ env.BASE_PRODUCT_TYPE }}_Release
target_arch: 'x86_64'
build_type: Release
save_to: ${{ github.workspace }}

- name: Clean sccache stats
run: ${SCCACHE_PATH} --zero-stats

- name: Build GenAI Wheel
run: |
for py_version in "3.10" "3.11" "3.12" "3.13"
do
python_exec_path=$(python$py_version -c "import sys; print(sys.executable)")
$python_exec_path -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \
--config-settings=override=cross.arch="manylinux_2_31_x86_64" \
${{ needs.openvino_download.outputs.ov_wheel_source }} \
${{ env.SRC_DIR }}
done
python_exec_path=$(python${{ matrix.python-version }} -c "import sys; print(sys.executable)")
$python_exec_path -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \
--config-settings=override=cross.arch="manylinux_2_31_x86_64" \
--config-settings=cmake.args="-DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache" \
${{ needs.openvino_download.outputs.ov_wheel_source }} \
${{ env.SRC_DIR }}
working-directory: ${{ env.OV_INSTALL_DIR }}

- name: Upload Wheels
- name: Show sccache stats
run: ${SCCACHE_PATH} --show-stats

- name: Upload GenAI Wheel
if: ${{ always() }}
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: ${{ env.GENAI_WHEELS_ARTIFACT_NAME }}
name: genai_wheel_python_${{ matrix.python-version }}
path: ${{ env.INSTALL_DIR }}
if-no-files-found: 'error'

Expand All @@ -324,7 +379,7 @@ jobs:
strategy:
matrix:
build-type: [Release]
needs: [openvino_download, genai_build_wheel, genai_build_cmake]
needs: [openvino_download, genai_build_wheels, genai_build_genai_wheel, genai_build_cmake]
timeout-minutes: 10
defaults:
run:
Expand Down Expand Up @@ -355,7 +410,7 @@ jobs:
- name: Download manifest and wheels
uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
with:
pattern: '{${{ env.GENAI_WHEELS_ARTIFACT_NAME }},manifest_${{ matrix.build-type }}}'
pattern: '{genai_wheels,genai_wheel_python_*,manifest_${{ matrix.build-type }}}'
path: ${{ github.workspace }}
merge-multiple: true

Expand Down Expand Up @@ -427,7 +482,10 @@ jobs:
if: ${{ 'Release' != matrix.build-type }}
run: |
source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${{ env.OV_INSTALL_DIR }}/samples/cpp/ -B ${{ env.BUILD_DIR }}
cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-DCMAKE_C_COMPILER_LAUNCHER=sccache \
-S ${{ env.OV_INSTALL_DIR }}/samples/cpp/ -B ${{ env.BUILD_DIR }}
cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc)
cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --component samples_bin --prefix ${{ env.INSTALL_DIR }}

Expand Down Expand Up @@ -486,6 +544,8 @@ jobs:
run: |
source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-DCMAKE_C_COMPILER_LAUNCHER=sccache \
-DENABLE_JS=ON -DCPACK_GENERATOR=NPM \
-DENABLE_PYTHON=OFF -DENABLE_WHEEL=OFF \
-S ${{ env.SRC_DIR }} -B ${{ env.BUILD_DIR }}
Expand All @@ -502,7 +562,7 @@ jobs:

genai_tests_wheel:
name: Python (${{ matrix.test.name}}) Tests (wheel)
needs: [ smart_ci, openvino_download, genai_build_wheel ]
needs: [ smart_ci, openvino_download, genai_build_wheels, genai_build_genai_wheel ]
timeout-minutes: ${{ matrix.test.timeout }}
strategy:
fail-fast: false
Expand Down Expand Up @@ -579,7 +639,7 @@ jobs:
if: ${{ matrix.test.run_condition }}
uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
with:
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_wheels}"
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_wheels,genai_wheel_python_*}"
path: ${{ env.INSTALL_DIR }}
merge-multiple: true

Expand All @@ -598,7 +658,7 @@ jobs:

genai_samples_tests:
name: Samples ${{ matrix.test.name }} (${{ matrix.build-type }})
needs: [ smart_ci, openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_build_nodejs ]
needs: [ smart_ci, openvino_download, genai_build_cmake, genai_build_wheels, genai_build_genai_wheel, genai_build_samples, genai_build_nodejs ]
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -671,7 +731,7 @@ jobs:
if: ${{ matrix.test.run_condition }}
uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
with:
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_archive_${{ matrix.build-type }},genai_samples_${{ matrix.build-type }},genai_wheels}"
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_archive_${{ matrix.build-type }},genai_samples_${{ matrix.build-type }},genai_wheels,genai_wheel_python_*}"
path: ${{ env.INSTALL_DIR }}
merge-multiple: true

Expand Down Expand Up @@ -735,7 +795,7 @@ jobs:
fail-fast: false
matrix:
build-type: [Release]
needs: [ smart_ci, openvino_download, genai_build_cmake, genai_build_wheel ]
needs: [ smart_ci, openvino_download, genai_build_cmake, genai_build_wheels, genai_build_genai_wheel ]
if: ${{ fromJSON(needs.smart_ci.outputs.affected_components).llm_bench || fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching }}
timeout-minutes: 90
defaults:
Expand Down Expand Up @@ -764,7 +824,7 @@ jobs:
- name: Download Build Artifacts
uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
with:
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_archive_${{ matrix.build-type }},genai_tools_${{ matrix.build-type }},genai_tests_${{ matrix.build-type }},genai_wheels}"
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_archive_${{ matrix.build-type }},genai_tools_${{ matrix.build-type }},genai_tests_${{ matrix.build-type }},genai_wheels,genai_wheel_python_*}"
path: ${{ env.INSTALL_DIR }}
merge-multiple: true

Expand Down Expand Up @@ -875,7 +935,7 @@ jobs:

genai_xgrammar_off_tests:
name: Build & Test when ENABLE_XGRAMMAR=OFF
needs: [smart_ci, openvino_download, genai_build_wheel]
needs: [smart_ci, openvino_download, genai_build_wheels]
timeout-minutes: 45
defaults:
run:
Expand Down Expand Up @@ -915,6 +975,8 @@ jobs:
-DENABLE_XGRAMMAR=OFF \
-BUILD_TOKENIZERS=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-DCMAKE_C_COMPILER_LAUNCHER=sccache \
-S ${{ env.SRC_DIR}} \
-B ${{ env.BUILD_DIR }}
cmake --build ${{ env.BUILD_DIR}} --config Release --parallel $(nproc) --target py_openvino_genai --verbose
Expand All @@ -935,7 +997,7 @@ jobs:

Overall_Status:
name: ci/gha_overall_status_linux
needs: [smart_ci, openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_build_nodejs, genai_tests_wheel, genai_samples_tests, genai_tools_tests, genai_nodejs_tests, genai_xgrammar_off_tests]
needs: [smart_ci, openvino_download, genai_build_cmake, genai_build_wheels, genai_build_genai_wheel, genai_build_samples, genai_build_nodejs, genai_tests_wheel, genai_samples_tests, genai_tools_tests, genai_nodejs_tests, genai_xgrammar_off_tests]
if: ${{ always() }}
runs-on: ubuntu-latest
steps:
Expand Down
Loading
Loading