build bindings using llvm distro wheel

makslevental · makslevental · commit 430ccd86c0a6 · 2025-12-06T21:09:46.000-08:00
diff --git a/.github/workflows/build_llvm.yml b/.github/workflows/build_llvm.yml
@@ -321,8 +321,7 @@ jobs:
 
       - name: "Save cache"
         uses: actions/cache/save@v3
-        if: (!cancelled())
-        # if: (!cancelled() && github.event_name == 'push' && github.ref_name == 'main')
+        if: (!cancelled() && github.event_name == 'push' && github.ref_name == 'main')
         with:
           path: ${{ steps.setup_base.outputs.cache-dir }}
           key: ${{ env.cache-key }}
@@ -356,7 +355,7 @@ jobs:
 
   call-build-python-bindings:
 
-    needs: [build]
+    needs: [build, build-wasm]
 
     permissions:
       contents: write
diff --git a/.github/workflows/build_mlir_python_bindings_wheel.yml b/.github/workflows/build_mlir_python_bindings_wheel.yml
@@ -75,10 +75,6 @@ jobs:
             runs-on: "macos-14"
             os: "macos"
             host-arch: "arm64"
-#          - name: "macos_x86_64"
-#            runs-on: "macos-13"
-#            os: "macos"
-#            host-arch: "x86_64"
 
     runs-on: ${{ matrix.runs-on }}
 
@@ -112,20 +108,15 @@ jobs:
 
       - name: "Download MLIR from releases"
         if: ${{ !inputs.workflow_call }}
-        env:
-          GH_TOKEN: ${{ github.token }}
         run: |
           
-          if [[ "${{ matrix.os }}" == "almalinux" ]] || [[ "${{ matrix.os }}" == "ubuntu" ]]; then
-            export RELEASE_PREFIX="mlir_manylinux_${{ matrix.host-arch }}"
-          else
-            export RELEASE_PREFIX="mlir_${{ matrix.name }}"
-          fi
-          RELEASE_URL=$(gh release view llvm --json assets -q '.assets |= sort_by(.createdAt) | .assets | map(select(.name | contains(env.RELEASE_PREFIX))) | .[-1] | .url')
-          curl -sLO $RELEASE_URL
-          RELEASE_NAME="$RELEASE_PREFIX*.tar.gz"
-          tar xf $RELEASE_NAME
-          echo WHEEL_VERSION="$(echo $RELEASE_NAME | sed "s/${RELEASE_PREFIX}_//g" | sed "s/.tar.gz//g" | sed "s/_/+/g" | sed "s/%2B/+/g" )" >> $GITHUB_ENV
+          pip download mlir-wheel -f https://llvm.github.io/eudsl
+          unzip mlir_wheel*.whl -d llvm-install
+          
+          WHEEL_NAME=$(ls mlir_wheel*)
+          pip install pkginfo
+          WHEEL_VERSION=$($python3_command -c "from pkginfo import Wheel; print(Wheel('$WHEEL_NAME').version)")
+          echo "WHEEL_VERSION=$WHEEL_VERSION" >> $GITHUB_ENV
 
       - name: Download LLVM distro artifacts
         if: ${{ inputs.workflow_call }}
@@ -138,8 +129,7 @@ jobs:
       - name: "Set WHEEL_VERSION env var"
         if: ${{ inputs.workflow_call }}
         run: |
-          WHEEL_VERSION="${{ inputs.wheel_version }}"
-          echo "WHEEL_VERSION=${WHEEL_VERSION/_/+}" >> $GITHUB_ENV
+          echo "WHEEL_VERSION=${{ inputs.wheel_version }}" >> $GITHUB_ENV
 
       - name: "Configure build"
         run: |
@@ -166,10 +156,6 @@ jobs:
           if [[ "${{ matrix.os }}" == "macos" ]]; then
             brew install antlr boost libarchive
           fi
-          if [[ "${{ matrix.runs-on }}" == "macos-13" ]]; then
-            sudo xcode-select -s /Library/Developer/CommandLineTools
-            echo "SDKROOT=$(xcrun --show-sdk-path)" >> $GITHUB_ENV
-          fi
 
       - name: "Build mlir-python-bindings"
         run: |
@@ -202,7 +188,6 @@ jobs:
         runs-on: [
           "ubuntu-22.04",
           "ubuntu-22.04-arm",
-          # "macos-13",
           "macos-14",
           "windows-2022"
         ]
@@ -215,7 +200,6 @@ jobs:
           {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
           {runs-on: "windows-2022", name: "windows_amd64", os: "windows"},
           {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
-#          {runs-on: "macos-13", name: "macos_x86_64", os: "macos"}
         ]
         exclude:
           # <frozen importlib._bootstrap>:491: Warning: Numpy built with MINGW-W64 on Windows 64 bits is experimental, and only available for testing. You are advised not to use it for production.
@@ -225,12 +209,6 @@ jobs:
           - runs-on: windows-2022
             python-version: "3.14t"
 
-          - runs-on: macos-13
-            python-version: "3.13"
-
-          - runs-on: macos-13
-            python-version: "3.14t"
-
           - runs-on: macos-14
             python-version: "3.10"
 
@@ -366,6 +344,33 @@ jobs:
           host-arch: "x86_64"
           target-arch: "wasm32"
 
+      - name: "Pip download MLIR from releases"
+        if: ${{ !inputs.workflow_call }}
+        run: |
+          
+          pip download mlir-wheel --plat pyodide_2024_0_wasm32 --no-deps --python-version 3.12 -f https://llvm.github.io/eudsl
+          unzip mlir_wheel*.whl -d llvm-install
+          echo "CMAKE_PREFIX_PATH=$PWD" >> $GITHUB_ENV
+          
+          WHEEL_NAME=$(ls mlir_wheel*)
+          pip install pkginfo
+          WHEEL_VERSION=$($python3_command -c "from pkginfo import Wheel; print(Wheel('$WHEEL_NAME').version)")
+          echo "WHEEL_VERSION=$WHEEL_VERSION" >> $GITHUB_ENV
+
+      - name: Download LLVM distro artifacts
+        if: ${{ inputs.workflow_call }}
+        uses: dawidd6/action-download-artifact@v11
+        with:
+          name: mlir_${{ matrix.os == 'ubuntu' && 'manylinux' || matrix.os }}_${{ matrix.host-arch }}_artifact
+          path: "./"
+          run_id: ${{ inputs.workflow_caller_run_id }}
+
+      - name: "Set WHEEL_VERSION env var"
+        if: ${{ inputs.workflow_call }}
+        run: |
+          echo "WHEEL_VERSION=${{ inputs.wheel_version }}" >> $GITHUB_ENV
+
+
       - name: Download mlir-native-tools artifacts
         if: ${{ inputs.workflow_call }}
         uses: dawidd6/action-download-artifact@v11
@@ -380,28 +385,22 @@ jobs:
           
           pip download mlir_native_tools -f https://llvm.github.io/eudsl
 
-      - name: Configure mlir-native-tools
+      - name: "Configure build"
         run: |
           
+          echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
+          echo "MLIR_DIR=$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
+          echo "Clang_DIR=$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
+          echo "CMAKE_PREFIX_PATH=$PWD/llvm-install" >> $GITHUB_ENV
+          
           unzip -o -j mlir_native_tools-*whl -d mlir_native_tools
           rm -rf mlir_native_tools-*whl
-          
           export LLVM_NATIVE_TOOL_DIR="$PWD/mlir_native_tools"
           echo "LLVM_NATIVE_TOOL_DIR=$LLVM_NATIVE_TOOL_DIR" >> $GITHUB_ENV
           echo "LLVM_TABLEGEN=$LLVM_NATIVE_TOOL_DIR/llvm-tblgen" >> $GITHUB_ENV
           echo "MLIR_TABLEGEN=$LLVM_NATIVE_TOOL_DIR/mlir-tblgen" >> $GITHUB_ENV
           echo "MLIR_LINALG_ODS_YAML_GEN=$LLVM_NATIVE_TOOL_DIR/mlir-linalg-ods-yaml-gen" >> $GITHUB_ENV
 
-      - name: Wheel version
-        run: |
-          
-          pushd third_party/llvm-project
-          LLVM_PROJECT_COMMIT=$(git rev-parse --short HEAD)
-          popd
-          
-          DATETIME=$(date +"%Y%m%d%H")
-          echo "WHEEL_VERSION=${DATETIME}+${LLVM_PROJECT_COMMIT}" >> $GITHUB_ENV
-
       - name: build wasm python wheel
         run: |
           
diff --git a/.github/workflows/build_test_release_eudsl.yml b/.github/workflows/build_test_release_eudsl.yml
@@ -76,10 +76,6 @@ jobs:
             runs-on: "macos-14"
             os: "macos"
             host-arch: "arm64"
-#          - name: "macos_x86_64"
-#            runs-on: "macos-13"
-#            os: "macos"
-#            host-arch: "x86_64"
 
     runs-on: ${{ matrix.runs-on }}
 
@@ -158,14 +154,6 @@ jobs:
           if [[ "${{ matrix.os }}" == "macos" ]]; then
             brew install antlr boost libarchive
           fi
-          if [[ "${{ matrix.runs-on }}" == "macos-13" ]]; then
-            sudo xcode-select -s /Library/Developer/CommandLineTools
-            echo "SDKROOT=$(xcrun --show-sdk-path)" >> $GITHUB_ENV
-            # https://github.com/pthom/srcml_caller/blob/a6431edf9ae4ab2aa19b78b5a4682ae3510b199c/setup.py#L38
-            # very dumb
-            cibw_dir="$(dirname $($python3_command -c "import cibuildwheel; import inspect; print(inspect.getfile(cibuildwheel))"))"
-            sed -i.bak 's/CIBUILDWHEEL/__CIBUILDWHEEL__/g' "$cibw_dir/__main__.py"
-          fi
           
           $python3_command -m pip install -r requirements-dev.txt -v
 
@@ -274,7 +262,6 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: ["ubuntu-22.04", "ubuntu-22.04-arm",
-                  # "macos-13",
                   "macos-14", "windows-2022"]
         python-version: [
           "3.10", "3.11", "3.12",
@@ -285,12 +272,8 @@ jobs:
           {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
           {runs-on: "windows-2022", name: "windows_amd64", os: "windows"},
           {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
-#          {runs-on: "macos-13", name: "macos_x86_64", os: "macos"}
         ]
         exclude:
-          - runs-on: macos-13
-            python-version: "3.13"
-
           - runs-on: macos-14
             python-version: "3.10"
 
@@ -359,7 +342,6 @@ jobs:
           "ubuntu-22.04",
           "ubuntu-22.04-arm",
           "macos-14",
-          # "macos-13",
           "windows-2022"
         ]
         python-version: [
@@ -374,12 +356,8 @@ jobs:
           {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
           {runs-on: "windows-2022", name: "windows_amd64", os: "windows"},
           {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
-#          {runs-on: "macos-13", name: "macos_x86_64", os: "macos"}
         ]
         exclude:
-          - runs-on: macos-13
-            python-version: "3.13"
-
           - runs-on: macos-14
             python-version: "3.10"
 
diff --git a/.github/workflows/build_test_release_eudsl_python_extras.yml b/.github/workflows/build_test_release_eudsl_python_extras.yml
@@ -91,7 +91,6 @@ jobs:
         runs-on: [
           "ubuntu-22.04",
           "ubuntu-22.04-arm",
-          # "macos-13",
           "macos-14",
           "windows-2022"
         ]
@@ -104,7 +103,6 @@ jobs:
           {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
           {runs-on: "windows-2022", name: "windows_amd64", os: "windows"},
           {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
-#          {runs-on: "macos-13", name: "macos_x86_64", os: "macos"}
         ]
         exclude:
           # <frozen importlib._bootstrap>:491: Warning: Numpy built with MINGW-W64 on Windows 64 bits is experimental, and only available for testing. You are advised not to use it for production.
@@ -114,12 +112,6 @@ jobs:
           - runs-on: windows-2022
             python-version: "3.14t"
 
-          - runs-on: macos-13
-            python-version: "3.14"
-
-          - runs-on: macos-13
-            python-version: "3.14t"
-
           - runs-on: macos-14
             python-version: "3.10"
 
diff --git a/projects/mlir-python-bindings-wasm/CMakeLists.txt b/projects/mlir-python-bindings-wasm/CMakeLists.txt
@@ -3,12 +3,70 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 # Copyright (c) 2025.
 
-include_directories(${LLVM_SOURCE_DIR}/include)
-include_directories(${LLVM_EXTERNAL_MLIR_SOURCE_DIR}/include)
-include_directories(${LLVM_BINARY_DIR}/include)
-include_directories(${LLVM_BINARY_DIR}/tools/mlir/include)
+cmake_minimum_required(VERSION 3.29)
+project(mlir-python-bindings-wasm LANGUAGES CXX C)
 
-add_mlir_upstream_c_api_library(WasmCompilerLinkerLoaderCAPI
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(POLICY CMP0068)
+  cmake_policy(SET CMP0068 NEW)
+  set(CMAKE_BUILD_WITH_INSTALL_NAME_DIR ON)
+endif()
+
+if(POLICY CMP0075)
+  cmake_policy(SET CMP0075 NEW)
+endif()
+
+if(POLICY CMP0077)
+  cmake_policy(SET CMP0077 NEW)
+endif()
+
+if(POLICY CMP0091)
+  cmake_policy(SET CMP0091 NEW)
+endif()
+
+if(POLICY CMP0116)
+  cmake_policy(SET CMP0116 NEW)
+endif()
+
+if(POLICY CMP0135)
+  cmake_policy(SET CMP0116 OLD)
+endif()
+
+find_package(MLIR REQUIRED CONFIG)
+find_package(LLD REQUIRED CONFIG)
+
+message(STATUS "Using MLIRConfig.cmake in: ${MLIR_DIR}")
+message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+message(STATUS "Using LLDConfig.cmake in: ${LLD_DIR}")
+
+set(MLIR_BINDINGS_PYTHON_NB_DOMAIN "mlir" CACHE STRING "")
+set(MLIR_PYTHON_PACKAGE_PREFIX "mlir" CACHE STRING "")
+
+set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_CURRENT_BINARY_DIR}/bin)
+set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_CURRENT_BINARY_DIR}/lib)
+
+list(APPEND CMAKE_MODULE_PATH "${MLIR_CMAKE_DIR}")
+list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
+include(TableGen)
+include(AddLLVM)
+include(AddMLIR)
+include(AddMLIRPython)
+include(MLIRDetectPythonEnv)
+include(HandleLLVMOptions)
+
+include_directories(${LLVM_INCLUDE_DIRS})
+include_directories(${MLIR_INCLUDE_DIRS})
+link_directories(${LLVM_BUILD_LIBRARY_DIR})
+add_definitions(${LLVM_DEFINITIONS})
+
+add_compile_definitions("MLIR_PYTHON_PACKAGE_PREFIX=${MLIR_PYTHON_PACKAGE_PREFIX}.")
+
+mlir_configure_python_dev_packages()
+
+add_mlir_public_c_api_library(WasmCompilerLinkerLoaderCAPI
   WasmCompilerLinkerLoaderCAPI.cpp
 
   PARTIAL_SOURCES_INTENDED
@@ -61,16 +119,12 @@ add_mlir_python_common_capi_library(MLIRPythonWasmCAPI
 # The fully assembled package of modules. This must come last.
 # ##############################################################################
 
-set(_declared_sources MLIRPythonSources MLIRPythonExtension.RegisterEverything)
-if(NOT CMAKE_CROSSCOMPILING)
-  list(APPEND _declared_sources MLIRPythonExtension.Core.type_stub_gen)
-endif()
-
 add_mlir_python_modules(MLIRPythonWasmModules
   ROOT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/mlir"
   INSTALL_PREFIX "mlir"
   DECLARED_SOURCES
-    ${_declared_sources}
+    MLIRPythonSources
+    MLIRPythonExtension.RegisterEverything
   COMMON_CAPI_LINK_LIBS
     MLIRPythonWasmCAPI
 )
@@ -100,3 +154,5 @@ install(
   LIBRARY DESTINATION "mlir/_mlir_libs"
   COMPONENT MLIRPythonWasmModules
 )
+
+message(STATUS "CMAKE_GENERATOR=${CMAKE_GENERATOR}")
diff --git a/projects/mlir-python-bindings-wasm/pyproject.toml b/projects/mlir-python-bindings-wasm/pyproject.toml
diff --git a/projects/mlir-python-bindings/CMakeLists.txt b/projects/mlir-python-bindings/CMakeLists.txt
