Skip to content

Commit a9cadb3

Browse files
maksleventalmakslevental
authored
[mlir-python-bindings-wasm] add WasmExecutionEngine (#248)
This PR adds runtime functionality for the mlir-python-bindings-wasm wheels. The way it works is roughly the same as [clang-interpreter](http://github.com/llvm/llvm-project/blob/a941e150749650e6a75e948f10d46b0bedcc128b/clang/lib/Interpreter/Wasm.cpp#L120-L126): the MLIR is compiled to wasm, fully linked into a shared object, and then `dlopen`ed (function symbols are accessed using `dlsym`). A couple of things to call out: 1. We're using the `wasm32` target so lowering to LLVM requires using `index-bitwidth=32` in various passes; 2. The various `numpy` -> `memref_descriptor` helpers need to be adjusted to work with 32b instead of 64b; 3. For some reason (🤷) `@main` is not a valid function name (collides with something somewhere in the emscripten runtime). Follow-up PRs will extend `LLVMBackend` in python-extras to work with this.
1 parent 5e7f971 commit a9cadb3

17 files changed

+1455
-47
lines changed

‎.github/actions/setup_base/action.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,8 +214,8 @@ runs:
214214
215215
echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV
216216
echo "CMAKE_MAKE_PROGRAM=Ninja" >> $GITHUB_ENV
217-
echo "CMAKE_C_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV
218-
echo "CMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV
217+
echo "CMAKE_C_COMPILER_LAUNCHER=$(which ccache)" >> $GITHUB_ENV
218+
echo "CMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)" >> $GITHUB_ENV
219219
echo "Python3_EXECUTABLE=$(which $python3_command)" >> $GITHUB_ENV
220220
221221
if [[ "${{ inputs.os }}" == "ubuntu" ]] || [[ "${{ inputs.os }}" == "windows" ]]; then

‎.github/workflows/build_llvm.yml‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -245,8 +245,7 @@ jobs:
245245
id-token: write
246246
pages: write
247247

248-
# DON'T FORGET TO CHANGE THIS IF YOU UPDATE WHEELS.YML
249-
uses: llvm/eudsl/.github/workflows/build_mlir_python_bindings_wheel.yml@main
248+
uses: ./.github/workflows/build_mlir_python_bindings_wheel.yml
250249
secrets: inherit # pass all secrets
251250
with:
252251
wheel_version: ${{ needs.build.outputs.WHEEL_VERSION }}
@@ -262,8 +261,7 @@ jobs:
262261
id-token: write
263262
pages: write
264263

265-
# DON'T FORGET TO CHANGE THIS IF YOU UPDATE WHEELS.YML
266-
uses: llvm/eudsl/.github/workflows/build_test_release_eudsl.yml@main
264+
uses: ./.github/workflows/build_test_release_eudsl.yml
267265
secrets: inherit # pass all secrets
268266
with:
269267
workflow_call: true
@@ -280,6 +278,5 @@ jobs:
280278
id-token: write
281279
pages: write
282280

283-
# DON'T FORGET TO CHANGE THIS IF YOU UPDATE WHEELS.YML
284-
uses: llvm/eudsl/.github/workflows/deploy_pip_page.yml@main
281+
uses: ./.github/workflows/deploy_pip_page.yml
285282
secrets: inherit # pass all secrets

‎.github/workflows/build_mlir_python_bindings_wheel.yml‎

Lines changed: 55 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,33 @@ jobs:
368368
run: |
369369
370370
pip install pyodide-build>=0.28.0
371-
echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV
371+
echo "EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version)" >> $GITHUB_ENV
372+
373+
- name: Download mlir-native-tools artifacts
374+
if: ${{ inputs.workflow_call }}
375+
uses: dawidd6/action-download-artifact@v11
376+
with:
377+
name: mlir_native_tools_manylinux_x86_64_artifact
378+
path: "./"
379+
run_id: ${{ inputs.workflow_caller_run_id }}
380+
381+
- name: Pip download mlir-native-tools
382+
if: ${{ !inputs.workflow_call }}
383+
run: |
384+
385+
pip download mlir_native_tools -f https://llvm.github.io/eudsl
386+
387+
- name: Configure mlir-native-tools
388+
run: |
389+
390+
unzip -o -j mlir_native_tools-*whl -d mlir_native_tools
391+
rm -rf mlir_native_tools-*whl
392+
393+
export LLVM_NATIVE_TOOL_DIR="$PWD/mlir_native_tools"
394+
echo "LLVM_NATIVE_TOOL_DIR=$LLVM_NATIVE_TOOL_DIR" >> $GITHUB_ENV
395+
echo "LLVM_TABLEGEN=$LLVM_NATIVE_TOOL_DIR/llvm-tblgen" >> $GITHUB_ENV
396+
echo "MLIR_TABLEGEN=$LLVM_NATIVE_TOOL_DIR/mlir-tblgen" >> $GITHUB_ENV
397+
echo "MLIR_LINALG_ODS_YAML_GEN=$LLVM_NATIVE_TOOL_DIR/mlir-linalg-ods-yaml-gen" >> $GITHUB_ENV
372398
373399
- name: Wheel version
374400
run: |
@@ -378,27 +404,31 @@ jobs:
378404
popd
379405
380406
DATETIME=$(date +"%Y%m%d%H")
381-
echo WHEEL_VERSION="${DATETIME}+${LLVM_PROJECT_COMMIT}" >> $GITHUB_ENV
407+
echo "WHEEL_VERSION=${DATETIME}+${LLVM_PROJECT_COMMIT}" >> $GITHUB_ENV
382408
383409
- name: Setup Emscripten
384410
uses: mymindstorm/setup-emsdk@v14
385411
with:
386412
version: ${{ env.EMSCRIPTEN_VERSION }}
387413

388-
- name: build wasm python wheel
389-
env:
390-
GH_TOKEN: ${{ github.token }}
414+
- name: Setup ccache
391415
run: |
392416
393-
RELEASE_URL=$(gh release view llvm --json assets -q '.assets |= sort_by(.createdAt) | .assets | map(select(.name | contains("mlir_manylinux_x86_64"))) | .[-1] | .url')
394-
curl -sLO $RELEASE_URL
395-
RELEASE_NAME="$RELEASE_PREFIX*.tar.gz"
396-
tar xf $RELEASE_NAME
417+
# https://github.com/juj/ccache/blob/c4284c78e9ff286ee5208f449af4c2aaba062d37/cmake/StandardWarnings.cmake#L5
418+
# https://github.com/juj/ccache/blob/c4284c78e9ff286ee5208f449af4c2aaba062d37/CMakeLists.txt#L61
419+
# error: builtin __has_trivial_copy is deprecated; use __is_trivially_copyable instead [-Werror,-Wdeprecated-builtins]
420+
CCACHE_COMPILERCHECK="$(g++ --version | head -n 1)" CC=gcc CXX=g++ emsdk install ccache-git-emscripten-64bit
421+
emsdk activate ccache-git-emscripten-64bit
422+
423+
echo "PATH=$EMSDK/ccache/git-emscripten_64bit/bin:$PATH" >> $GITHUB_ENV
424+
echo "CCACHE_COMPILERCHECK=$($EMSDK/upstream/bin/clang --version | head -n 1)" >> $GITHUB_ENV
425+
echo "LLVM_CCACHE_BUILD=ON" >> $GITHUB_ENV
426+
echo "CMAKE_C_COMPILER_LAUNCHER=$EMSDK/ccache/git-emscripten_64bit/bin/ccache" >> $GITHUB_ENV
427+
echo "CMAKE_CXX_COMPILER_LAUNCHER=$EMSDK/ccache/git-emscripten_64bit/bin/ccache" >> $GITHUB_ENV
428+
429+
- name: build wasm python wheel
430+
run: |
397431
398-
export LLVM_NATIVE_TOOL_DIR="$PWD/llvm-install/bin"
399-
export LLVM_TABLEGEN="$LLVM_NATIVE_TOOL_DIR/llvm-tblgen"
400-
export MLIR_TABLEGEN="$LLVM_NATIVE_TOOL_DIR/mlir-tblgen"
401-
export MLIR_LINALG_ODS_YAML_GEN="$LLVM_NATIVE_TOOL_DIR/mlir-linalg-ods-yaml-gen"
402432
pyodide build projects/mlir-python-bindings-wasm -o wheelhouse --compression-level 10
403433
404434
- name: Install pyodide and test
@@ -408,23 +438,31 @@ jobs:
408438
pyodide venv venv
409439
. venv/bin/activate
410440
pip install mlir-python-bindings -f wheelhouse
411-
pushd third_party/llvm-project/mlir/test/python/dialects
412441
442+
pushd third_party/llvm-project/mlir/test/python/dialects
413443
for f in *.py; do
414444
if [[ "$f" == "python_test.py" ]]; then
415445
continue
416446
fi
417447
python $f
418448
done
419-
420449
popd
450+
451+
python projects/mlir-python-bindings-wasm/test_wasm_execution_engine.py
421452
422453
- name: Upload wasm32 wheels
423454
uses: actions/upload-artifact@v4
424455
with:
425456
path: wheelhouse/*.whl
426457
name: build_artifact_python_bindings-ubuntu-wasm
427458

459+
- name: "Save cache"
460+
uses: actions/cache/save@v3
461+
if: (!cancelled() && github.event_name == 'push' && github.ref_name == 'main')
462+
with:
463+
path: ${{ steps.setup_base.outputs.cache-dir }}
464+
key: ${{ env.cache-key }}
465+
428466
- name: Release current commit
429467
if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch'
430468
uses: ncipollo/[email protected]
@@ -448,8 +486,7 @@ jobs:
448486
id-token: write
449487
pages: write
450488

451-
# DON'T FORGET TO CHANGE THIS IF YOU UPDATE WHEELS.YML
452-
uses: llvm/eudsl/.github/workflows/build_test_release_eudsl_python_extras.yml@main
489+
uses: ./.github/workflows/build_test_release_eudsl_python_extras.yml
453490
secrets: inherit # pass all secrets
454491
with:
455492
workflow_call: true
@@ -466,6 +503,5 @@ jobs:
466503
id-token: write
467504
pages: write
468505

469-
# DON'T FORGET TO CHANGE THIS IF YOU UPDATE WHEELS.YML
470-
uses: llvm/eudsl/.github/workflows/deploy_pip_page.yml@main
506+
uses: ./.github/workflows/deploy_pip_page.yml
471507
secrets: inherit # pass all secrets

‎.github/workflows/build_test_release_eudsl.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,5 @@ jobs:
484484
id-token: write
485485
pages: write
486486

487-
# DON'T FORGET TO CHANGE THIS IF YOU UPDATE WHEELS.YML
488-
uses: llvm/eudsl/.github/workflows/deploy_pip_page.yml@main
487+
uses: ./.github/workflows/deploy_pip_page.yml
489488
secrets: inherit # pass all secrets

‎.github/workflows/build_test_release_eudsl_python_extras.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,5 @@ jobs:
266266
id-token: write
267267
pages: write
268268

269-
# DON'T FORGET TO CHANGE THIS IF YOU UPDATE WHEELS.YML
270-
uses: llvm/eudsl/.github/workflows/deploy_pip_page.yml@main
269+
uses: ./.github/workflows/deploy_pip_page.yml
271270
secrets: inherit # pass all secrets

‎projects/eudsl-tblgen/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ before-all = [
8383
"set -x",
8484
"yum install -y clang",
8585
# ccache
86-
"echo $(if [ \"$(arch)\" == \"x86_64\" ]; then curl -sLO https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz && tar -xf ccache-4.10.2-linux-x86_64.tar.xz && pushd ccache-4.10.2-linux-x86_64 && make install && popd; fi)",
86+
"echo $(if [ \"$(arch)\" == \"x86_64\" ]; then curl -sLO https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz && tar -xf ccache-4.10.2-linux-x86_64.tar.xz && pushd ccache-4.10.2-linux-x86_64 && make install && ln -s /usr/local/bin/ccache /usr/bin/ccache && popd; fi)",
8787
"echo $(if [ \"$(arch)\" == \"aarch64\" ]; then dnf install -y ccache; fi)",
8888
"ccache -z"
8989
]

0 commit comments

Comments
 (0)