Skip to content

Commit 3004d9f

Browse files
maksleventalmakslevental
committed
[LLVM] add wasm32 wheels
1 parent f346576 commit 3004d9f

File tree

13 files changed

+497
-227
lines changed

13 files changed

+497
-227
lines changed

‎.github/actions/setup_base/action.yml‎

Lines changed: 43 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@ inputs:
1010
os:
1111
required: true
1212
description: ''
13-
arch:
13+
host-arch:
14+
required: true
15+
description: ''
16+
target-arch:
1417
required: true
1518
description: ''
1619
container:
@@ -119,18 +122,18 @@ runs:
119122
dnf install -y sudo ncurses-compat-libs tmate python3-pip git
120123
121124
# install ccache
122-
if [[ "${{ inputs.arch }}" == "x86_64" ]]; then
125+
if [[ "${{ inputs.host-arch }}" == "x86_64" ]]; then
123126
curl -sLO https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz
124127
tar -xf ccache-4.10.2-linux-x86_64.tar.xz
125128
pushd ccache-4.10.2-linux-x86_64 && make install && popd
126-
elif [[ "${{ inputs.arch }}" == "aarch64" ]]; then
129+
elif [[ "${{ inputs.host-arch }}" == "aarch64" ]]; then
127130
dnf install -y ccache
128131
fi
129132
130133
# install gh-cli
131-
if [[ "${{ inputs.arch }}" == "x86_64" ]]; then
134+
if [[ "${{ inputs.host-arch }}" == "x86_64" ]]; then
132135
rpm_arch=amd64
133-
elif [[ "${{ inputs.arch }}" == "aarch64" ]]; then
136+
elif [[ "${{ inputs.host-arch }}" == "aarch64" ]]; then
134137
rpm_arch=arm64
135138
fi
136139
curl -sLO https://github.com/cli/cli/releases/download/v2.61.0/gh_2.61.0_linux_${rpm_arch}.rpm
@@ -235,3 +238,38 @@ runs:
235238
echo MACOSX_DEPLOYMENT_TARGET=11.0 >> $GITHUB_ENV
236239
echo CMAKE_OSX_DEPLOYMENT_TARGET=11.0 >> $GITHUB_ENV
237240
fi
241+
242+
- name: Install pyodide
243+
if: inputs.target-arch == 'wasm32'
244+
shell: bash
245+
run: |
246+
247+
pip install pyodide-build>=0.28.0
248+
echo "EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version)" >> $GITHUB_ENV
249+
250+
- name: Setup Emscripten
251+
if: inputs.target-arch == 'wasm32'
252+
shell: bash
253+
run: |
254+
255+
git clone https://github.com/emscripten-core/emsdk.git
256+
pushd emsdk
257+
./emsdk install ${{ env.EMSCRIPTEN_VERSION }}
258+
./emsdk activate ${{ env.EMSCRIPTEN_VERSION }}
259+
echo "EMSDK=$PWD" >> $GITHUB_ENV
260+
popd
261+
262+
- name: Setup emscripten ccache
263+
if: inputs.target-arch == 'wasm32'
264+
shell: bash
265+
run: |
266+
267+
source "$EMSDK/emsdk_env.sh"
268+
269+
$CMAKE_C_COMPILER_LAUNCHER -z
270+
# https://github.com/juj/ccache/blob/c4284c78e9ff286ee5208f449af4c2aaba062d37/cmake/StandardWarnings.cmake#L5
271+
# https://github.com/juj/ccache/blob/c4284c78e9ff286ee5208f449af4c2aaba062d37/CMakeLists.txt#L61
272+
# error: builtin __has_trivial_copy is deprecated; use __is_trivially_copyable instead [-Werror,-Wdeprecated-builtins]
273+
VERBOSE=1 CCACHE_COMPILERCHECK="string:$(g++ --version | head -n 1)" CC=gcc CXX=g++ emsdk install ccache-git-emscripten-64bit
274+
$CMAKE_C_COMPILER_LAUNCHER -s
275+
emsdk activate ccache-git-emscripten-64bit

‎.github/workflows/build_llvm.yml‎

Lines changed: 133 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -20,23 +20,23 @@ on:
2020
default: false
2121
pull_request:
2222
paths:
23-
- ".github/actions/setup_base"
23+
- ".github/actions/setup_base/action.yml"
2424
- ".github/workflows/build_llvm.yml"
25-
- "scripts/cmake/llvm_cache.cmake"
26-
- "scripts/cmake/llvm_wasm_cache.cmake"
25+
- "scripts/cmake/*"
26+
- "scripts/llvm_wasm/*"
2727
- "scripts/build_llvm.sh"
28-
- "scripts/build_llvm_wasi.sh"
28+
- "scripts/build_mlir_native_tools.sh"
2929
- "third_party/llvm-project"
3030
push:
3131
branches:
3232
- main
3333
paths:
34-
- ".github/actions/setup_base"
34+
- ".github/actions/setup_base/action.yml"
3535
- ".github/workflows/build_llvm.yml"
36-
- "scripts/cmake/llvm_cache.cmake"
37-
- "scripts/cmake/llvm_wasm_cache.cmake"
36+
- "scripts/cmake/*"
37+
- "scripts/llvm_wasm/*"
3838
- "scripts/build_llvm.sh"
39-
- "scripts/build_llvm_wasi.sh"
39+
- "scripts/build_mlir_native_tools.sh"
4040
- "third_party/llvm-project"
4141

4242
concurrency:
@@ -48,6 +48,7 @@ concurrency:
4848
cancel-in-progress: true
4949

5050
jobs:
51+
5152
build:
5253

5354
strategy:
@@ -58,33 +59,24 @@ jobs:
5859
runs-on: "ubuntu-22.04"
5960
container: "quay.io/pypa/manylinux_2_28_x86_64"
6061
os: "almalinux"
61-
arch: x86_64
62+
host-arch: x86_64
63+
target-arch: x86_64
6264
- name: "manylinux_aarch64"
6365
runs-on: "ubuntu-22.04-arm"
6466
container: "quay.io/pypa/manylinux_2_34_aarch64"
6567
os: "almalinux"
66-
arch: aarch64
67-
# - name: "ubuntu_aarch64"
68-
# runs-on: "ubuntu-22.04-arm"
69-
# os: "ubuntu"
70-
# arch: aarch64
71-
# - name: "wasm_wasi"
72-
# runs-on: "ubuntu-22.04"
73-
# container: "quay.io/pypa/manylinux_2_28_x86_64"
74-
# os: "almalinux"
75-
# arch: "wasm32-wasi"
68+
host-arch: "aarch64"
69+
target-arch: "aarch64"
7670
- name: "windows_amd64"
7771
runs-on: "windows-2022"
7872
os: "windows"
79-
arch: amd64
73+
host-arch: amd64
74+
target-arch: amd64
8075
- name: "macos_arm64"
8176
runs-on: "macos-14"
8277
os: "macos"
83-
arch: arm64
84-
# - name: "macos_x86_64"
85-
# runs-on: "macos-13"
86-
# os: "macos"
87-
# arch: x86_64
78+
host-arch: arm64
79+
target-arch: arm64
8880

8981
runs-on: ${{ matrix.runs-on }}
9082

@@ -127,7 +119,8 @@ jobs:
127119
cache-key: ${{ env.cache-key }}
128120
restore-key: "mlir_${{ matrix.name }}_clang"
129121
os: ${{ matrix.os }}
130-
arch: ${{ matrix.arch }}
122+
host-arch: ${{ matrix.host-arch }}
123+
target-arch: ${{ matrix.target-arch }}
131124
container: ${{ matrix.container }}
132125
runs-on: ${{ matrix.runs-on }}
133126

@@ -150,14 +143,11 @@ jobs:
150143
###############################
151144
# Build MLIR/LLVM distro
152145
###############################
153-
if [[ "${{ matrix.arch }}" == "wasm32-wasi" ]]; then
154-
scripts/build_llvm_wasi.sh
155-
else
156-
scripts/build_llvm.sh
157-
echo "*********************** SMOKE TEST *********************************"
158-
"$LLVM_INSTALL_DIR/bin/mlir-tblgen" --version
159-
echo "*********************** SMOKE TEST *********************************"
160-
fi
146+
147+
scripts/build_llvm.sh
148+
echo "*********************** SMOKE TEST *********************************"
149+
"$LLVM_INSTALL_DIR/bin/mlir-tblgen" --version
150+
echo "*********************** SMOKE TEST *********************************"
161151
ccache -s
162152
163153
###############################
@@ -178,7 +168,7 @@ jobs:
178168
elif [ x"${{ matrix.os }}" == x"windows" ]; then
179169
PLAT="win"
180170
fi
181-
PLAT="${PLAT}_${{ matrix.arch }}"
171+
PLAT="${PLAT}_${{ matrix.host-arch }}"
182172
183173
pushd projects/mlir-native-tools
184174
$python3_command setup.py bdist_wheel --dist-dir ../../wheelhouse --plat $PLAT
@@ -256,6 +246,114 @@ jobs:
256246
limit-access-to-actor: true
257247
install-dependencies: ${{ startsWith(matrix.runs-on, 'macos') || startsWith(matrix.runs-on, 'windows') }}
258248

249+
build-wasm:
250+
251+
runs-on: ubuntu-22.04
252+
253+
name: "Build llvm-mlir wasm"
254+
255+
defaults:
256+
run:
257+
shell: bash
258+
259+
permissions:
260+
id-token: write
261+
contents: write
262+
263+
env:
264+
cache-key: mlir_wasm_emscripten_${{ format('{0}-{1}', github.ref_name, github.run_number) }}
265+
266+
steps:
267+
- name: "Check out repository"
268+
uses: actions/[email protected]
269+
with:
270+
submodules: true
271+
272+
- name: "Setup base"
273+
uses: ./.github/actions/setup_base
274+
id: setup_base
275+
with:
276+
cache-key: ${{ env.cache-key }}
277+
restore-key: "mlir_wasm_emscripten"
278+
os: "ubuntu"
279+
host-arch: "x86_64"
280+
target-arch: "wasm32"
281+
282+
- name: Build MLIR and LLVM
283+
run: |
284+
285+
###############################
286+
# Build host mlir-native-tools wheel
287+
###############################
288+
289+
scripts/build_mlir_native_tools.sh
290+
export LLVM_NATIVE_TOOL_DIR="$PWD/mlir-native-tools-install/bin"
291+
export LLVM_NATIVE_TOOL_DIR="$LLVM_NATIVE_TOOL_DIR"
292+
export LLVM_TABLEGEN="$LLVM_NATIVE_TOOL_DIR/llvm-tblgen"
293+
export MLIR_TABLEGEN="$LLVM_NATIVE_TOOL_DIR/mlir-tblgen"
294+
export MLIR_LINALG_ODS_YAML_GEN="$LLVM_NATIVE_TOOL_DIR/mlir-linalg-ods-yaml-gen"
295+
296+
source "$EMSDK/emsdk_env.sh"
297+
298+
export PATH="$EMSDK/ccache/git-emscripten_64bit/bin:$PATH"
299+
export CCACHE_COMPILERCHECK="string:$($EMSDK/upstream/bin/clang --version | head -n 1)"
300+
export CCACHE="$EMSDK/ccache/git-emscripten_64bit/bin/ccache"
301+
export "CMAKE_C_COMPILER_LAUNCHER=$CCACHE"
302+
export "CMAKE_CXX_COMPILER_LAUNCHER=$CCACHE"
303+
304+
export LLVM_SOURCE_DIR="$PWD/third_party/llvm-project"
305+
pushd $LLVM_SOURCE_DIR && LLVM_SHA_SHORT="$(git rev-parse --short HEAD)" && popd
306+
export WHEEL_VERSION="$(date +'%Y%m%d')+$LLVM_SHA_SHORT"
307+
308+
###############################
309+
# Build MLIR/LLVM distro
310+
###############################
311+
312+
$CCACHE -z
313+
pyodide build scripts/llvm_wasm -o wheelhouse --compression-level 10
314+
$CCACHE -s
315+
316+
- name: Upload ccache log
317+
uses: actions/upload-artifact@v4
318+
with:
319+
name: llvm_ccache_log_mlir_wasm32
320+
path: '/tmp/ccache.log'
321+
322+
- name: "Save cache"
323+
uses: actions/cache/save@v3
324+
if: (!cancelled())
325+
# if: (!cancelled() && github.event_name == 'push' && github.ref_name == 'main')
326+
with:
327+
path: ${{ steps.setup_base.outputs.cache-dir }}
328+
key: ${{ env.cache-key }}
329+
330+
- name: Upload LLVM distro artifacts
331+
uses: actions/upload-artifact@v4
332+
with:
333+
name: mlir_wasm_artifact
334+
path: llvm-install
335+
if-no-files-found: warn
336+
337+
- name: Upload mlir-wheel artifact
338+
uses: actions/upload-artifact@v4
339+
with:
340+
name: mlir_wheel_wasm_artifact
341+
path: wheelhouse/mlir_wheel*
342+
if-no-files-found: warn
343+
344+
- name: Release current commit
345+
if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch'
346+
uses: ncipollo/[email protected]
347+
with:
348+
artifacts: "*.tar.gz,wheelhouse/*.whl"
349+
token: "${{ secrets.GITHUB_TOKEN }}"
350+
tag: "llvm"
351+
name: "llvm"
352+
removeArtifacts: false
353+
allowUpdates: true
354+
replacesArtifacts: true
355+
makeLatest: true
356+
259357
call-build-python-bindings:
260358

261359
needs: [build]

0 commit comments

Comments
 (0)