ci: Build statically linked bpf-linker in CI

Build LLVM and bpf-linker with statically linked libLLVM for multiple
architectures. Include the resulting build artifacts on the release
pages, so they can be installed with `cargo binstall`

Author: vadorovsky

.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run --package xtask --"

.github/workflows/ci.yml

@@ -48,19 +48,38 @@ jobs:
         run: cargo fmt --all -- --check
 
   build:
-    # We don't use ubuntu-latest because we care about the apt packages available.
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.target.os }}
     strategy:
       fail-fast: false
       matrix:
         rust:
           - stable
           - beta
           - nightly
-        llvm:
-          - 19
-          - source
-    name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }}
+        # We don't use ubuntu-latest because we care about the apt packages available.
+        target:
+          - os: macos-14
+            target: aarch64-apple-darwin
+            build-type: native
+          - os: macos-13
+            target: x86_64-apple-darwin
+            build-type: native
+          - os: ubuntu-22.04
+            target: aarch64-unknown-linux-gnu
+            build-type: cross
+          - os: ubuntu-22.04
+            target: aarch64-unknown-linux-musl
+            build-type: cross
+          - os: ubuntu-22.04
+            target: riscv64gc-unknown-linux-gnu
+            build-type: cross
+          - os: ubuntu-22.04
+            target: x86_64-unknown-linux-gnu
+            build-type: native
+          - os: ubuntu-22.04
+            target: x86_64-unknown-linux-musl
+            build-type: cross
+    name: rustc=${{ matrix.rust }} target=${{ matrix.target.target }}
     needs: llvm
 
     env:
@@ -85,61 +104,48 @@ jobs:
       - uses: Swatinem/rust-cache@v2
 
       - name: Check (default features, no system LLVM)
+        if: matrix.build-type == 'native'
         run: cargo check
 
       - name: Build (default features, no system LLVM)
+        if: matrix.build-type == 'native'
         run: cargo build
 
       - name: Install btfdump
         if: matrix.rust == 'nightly'
         run: cargo install btfdump
 
       - name: Install prerequisites
-        if: matrix.rust == 'nightly'
+        if: matrix.rust == 'nightly' && runner.os == 'Linux' && startsWith(matrix.target.target, 'x86_64')
         # ubuntu-22.04 comes with clang 13-15[0]; support for signed and 64bit
         # enum values was added in clang 15[1] which isn't in `$PATH`.
         #
         # gcc-multilib provides at least <asm/types.h> which is referenced by libbpf.
         #
+        # libelf is a dependency of libbpf.
+        #
         # [0] https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
         #
         # [1] https://github.com/llvm/llvm-project/commit/dc1c43d
         run: |
           set -euxo pipefail
           sudo apt update
-          sudo apt -y install gcc-multilib
+          sudo apt -y install gcc-multilib libelf-dev
           echo /usr/lib/llvm-15/bin >> $GITHUB_PATH
 
-      - name: Install LLVM
-        if: matrix.llvm != 'source'
+      - name: Install prerequisites
+        if: runner.os == 'macOS'
+        # We need system-wide LLVM only for FileCheck.
         run: |
           set -euxo pipefail
-          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.llvm }} main | sudo tee /etc/apt/sources.list.d/llvm.list
-
-          sudo apt update
-          # TODO(vadorovsky): Remove the requirement of libpolly.
-          #
-          # Packages from apt.llvm.org are being built all at once, with one
-          # cmake build with superset of options, then different binaries and
-          # libraries are being included in different packages.
-          #
-          # That results in `llvm-config --libname --link-static` mentioning
-          # libpolly, even if it's not installed. The output of that command is
-          # being used in build.rs of llvm-sys, so building llvm-sys on such
-          # system is complaining about lack of libpolly.
-          #
-          # Hopefully that nightmare goes away once we switch to binstalls and
-          # ditch the system LLVM option.
-          sudo apt -y install llvm-${{ matrix.llvm }}-dev libpolly-${{ matrix.llvm }}-dev
-          echo /usr/lib/llvm-${{ matrix.llvm }}/bin >> $GITHUB_PATH
+          brew install llvm
+          echo $(brew --prefix)/opt/llvm/bin >> $GITHUB_PATH
 
       - name: Restore LLVM
-        if: matrix.llvm == 'source'
         uses: actions/cache/restore@v4
         with:
           path: llvm-install
-          key: ${{ needs.llvm.outputs.cache-key }}
+          key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.target.target)] }}
           fail-on-cache-miss: true
 
       - name: Add LLVM to PATH && LD_LIBRARY_PATH
@@ -160,17 +166,51 @@ jobs:
           cargo clean -p llvm-sys --release
 
       - uses: taiki-e/install-action@cargo-hack
+        if: matrix.build-type == 'native'
+
+      # Run cargo commands with `cargo hack` for native targets.
+      # Run cargo commands with `xtask` for all targets.
 
       - name: Check
+        if: matrix.build-type == 'native'
         run: cargo hack check --feature-powerset
 
+      - name: Check (xtask)
+        run: |
+          cargo xtask check \
+            --container-engine podman \
+            --container-tag initial \
+            --llvm-install-dir "${{ github.workspace }}/llvm-install" \
+            --pull always \
+            --target ${{ matrix.target.target }}
+
       - name: Build
+        if: matrix.build-type == 'native'
         run: cargo hack build --feature-powerset
 
+      - name: Build (xtask)
+        run: |
+          cargo xtask build \
+            --container-engine podman \
+            --container-tag initial \
+            --llvm-install-dir "${{ github.workspace }}/llvm-install" \
+            --pull always \
+            --target ${{ matrix.target.target }}
+
       - name: Test
-        if: matrix.rust == 'nightly'
+        if: matrix.build-type == 'native' && matrix.rust == 'nightly'
         run: cargo hack test --feature-powerset
 
+      - name: Test (xtask)
+        if: matrix.rust == 'nightly'
+        run: |
+          cargo xtask test \
+            --container-engine podman \
+            --container-tag initial \
+            --llvm-install-dir "${{ github.workspace }}/llvm-install" \
+            --pull always \
+            --target ${{ matrix.target.target }}
+
       - uses: actions/checkout@v4
         if: matrix.rust == 'nightly'
         with:
@@ -179,10 +219,16 @@ jobs:
           submodules: recursive
 
       - name: Install
-        if: matrix.rust == 'nightly'
-        run: cargo install --path . --no-default-features
+        if: matrix.rust == 'nightly' && runner.os == 'Linux' && startsWith(matrix.target.target, 'x86_64')
+        run: |
+          cargo xtask install \
+            --container-engine podman \
+            --container-tag initial \
+            --llvm-install-dir "${{ github.workspace }}/llvm-install" \
+            --pull always \
+            --target ${{ matrix.target.target }}
 
       - name: Run aya integration tests
-        if: matrix.rust == 'nightly'
+        if: matrix.rust == 'nightly' && runner.os == 'Linux' && startsWith(matrix.target.target, 'x86_64')
         working-directory: aya
         run: cargo xtask integration-test local

.github/workflows/llvm.yml

@@ -3,99 +3,119 @@ name: LLVM
 on:
   workflow_call:
     outputs:
-      cache-key:
-        value: ${{ jobs.llvm.outputs.cache-key }}
+      cache-key-aarch64-apple-darwin:
+        value: ${{ jobs.llvm.outputs.cache-key-aarch64-apple-darwin }}
+      cache-key-x86_64-apple-darwin:
+        value: ${{ jobs.llvm.outputs.cache-key-x86_64-apple-darwin }}
+      cache-key-aarch64-unknown-linux-gnu:
+        value: ${{ jobs.llvm.outputs.cache-key-aarch64-unknown-linux-gnu }}
+      cache-key-aarch64-unknown-linux-musl:
+        value: ${{ jobs.llvm.outputs.cache-key-aarch64-unknown-linux-musl }}
+      cache-key-riscv64gc-unknown-linux-gnu:
+        value: ${{ jobs.llvm.outputs.cache-key-riscv64gc-unknown-linux-gnu }}
+      cache-key-x86_64-unknown-linux-gnu:
+        value: ${{ jobs.llvm.outputs.cache-key-x86_64-unknown-linux-gnu }}
+      cache-key-x86_64-unknown-linux-musl:
+        value: ${{ jobs.llvm.outputs.cache-key-x86_64-unknown-linux-musl }}
 
 jobs:
   llvm:
-    runs-on: ubuntu-22.04
-    name: llvm
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: macos-14
+            target: aarch64-apple-darwin
+          - os: macos-13
+            target: x86_64-apple-darwin
+          - os: ubuntu-22.04
+            target: aarch64-unknown-linux-gnu
+          - os: ubuntu-22.04
+            target: aarch64-unknown-linux-musl
+          - os: ubuntu-22.04
+            target: riscv64gc-unknown-linux-gnu
+          - os: ubuntu-22.04
+            target: x86_64-unknown-linux-gnu
+          - os: ubuntu-22.04
+            target: x86_64-unknown-linux-musl
+    name: llvm ${{ matrix.target }}
     outputs:
-      cache-key: ${{ steps.cache-key.outputs.cache-key }}
+      cache-key-aarch64-apple-darwin: ${{ steps.cache-key.outputs.cache-key-aarch64-apple-darwin }}
+      cache-key-x86_64-apple-darwin: ${{ steps.cache-key.outputs.cache-key-x86_64-apple-darwin }}
+      cache-key-aarch64-unknown-linux-gnu: ${{ steps.cache-key.outputs.cache-key-aarch64-unknown-linux-gnu }}
+      cache-key-aarch64-unknown-linux-musl: ${{ steps.cache-key.outputs.cache-key-aarch64-unknown-linux-musl }}
+      cache-key-riscv64gc-unknown-linux-gnu: ${{ steps.cache-key.outputs.cache-key-riscv64gc-unknown-linux-gnu }}
+      cache-key-x86_64-unknown-linux-gnu: ${{ steps.cache-key.outputs.cache-key-x86_64-unknown-linux-gnu }}
+      cache-key-x86_64-unknown-linux-musl: ${{ steps.cache-key.outputs.cache-key-x86_64-unknown-linux-musl }}
     steps:
+      - uses: actions/checkout@v4
+
       - id: ls-remote
         run: |
           set -euxo pipefail
           value=$(git ls-remote https://github.com/aya-rs/llvm-project.git refs/heads/rustc/19.1-2024-09-17 | cut -f1)
           echo "sha=$value" >> "$GITHUB_OUTPUT"
 
       - id: cache-key
-        run: echo "cache-key=llvm-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT"
+        run: echo "cache-key-${{ matrix.target }}=llvm-${{ matrix.target }}-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT"
 
       - name: Cache
         id: cache-llvm
         uses: actions/cache@v4
         with:
           path: llvm-install
-          key: ${{ steps.cache-key.outputs.cache-key }}
+          key: ${{ steps.cache-key.outputs[format('cache-key-{0}', matrix.target)] }}
           lookup-only: true
 
-      - name: Install Tools
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
+      - name: Free disk space
+        if: steps.cache-llvm.outputs.cache-hit != 'true' && runner.os == 'Linux'
         run: |
           set -euxo pipefail
-          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
-            gpg --dearmor - | \
-            sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
-          echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' | \
-            sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
-
-          sudo apt update
-          sudo apt -y install cmake ninja-build clang lld
+          sudo apt -y remove \
+            azure-cli \
+            '^aspnetcore-.*' \
+            '^dotnet-.*' \
+            firefox \
+            '^google-cloud-.*' \
+            '^google-chrome-*' \
+            '^imagemagick.*' \
+            java-common \
+            kubectl \
+            '^libmagick.*' \
+            '^libmono.*' \
+            '^libnginx.*' \
+            '^microsoft-edge.*' \
+            '^mongodb-.*' \
+            mono-devel \
+            '^mysql-.*' \
+            '^php.*' \
+            powershell \
+            '^r-.*'
+          sudo rm -rf \
+            /opt/ghc \
+            /opt/hostedtoolcache \
+            /usr/lib/jvm \
+            /usr/local/.ghcup \
+            /usr/local/aws* \
+            /usr/local/bin/{aliyun,azcopy,bicep,helm,minikube,oc,occopy,packer,pulumi*,stack,terraform} \
+            /usr/local/julia* \
+            /usr/local/lib/android \
+            /usr/local/share/{chromium,powershell,vcpkg} \
+            /usr/share/dotnet || true
+          sudo docker image prune --all --force || true
 
-      - name: Checkout LLVM Source
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
-        uses: actions/checkout@v4
-        with:
-          repository: aya-rs/llvm-project
-          ref: ${{ steps.ls-remote.outputs.sha }}
-          path: llvm-project
-
-      - name: Configure LLVM
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
+      - name: Install dependencies
+        if: steps.cache-llvm.outputs.cache-hit != 'true' && runner.os == 'macOS'
         run: |
           set -euxo pipefail
-          cmake \
-            -S llvm-project/llvm \
-            -B llvm-build \
-            -G Ninja \
-            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-            -DCMAKE_C_COMPILER=clang \
-            -DCMAKE_CXX_COMPILER=clang++ \
-            -DCMAKE_INSTALL_PREFIX="${{ github.workspace }}/llvm-install" \
-            -DLLVM_BUILD_LLVM_DYLIB=ON \
-            -DLLVM_ENABLE_ASSERTIONS=ON \
-            -DLLVM_ENABLE_PROJECTS= \
-            -DLLVM_ENABLE_RUNTIMES= \
-            -DLLVM_INSTALL_UTILS=ON \
-            -DLLVM_LINK_LLVM_DYLIB=ON \
-            -DLLVM_TARGETS_TO_BUILD=BPF \
-            -DLLVM_USE_LINKER=lld
-
-      - name: Install LLVM
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
-        env:
-          # Create symlinks rather than copies to conserve disk space. At the time of this writing,
-          # GitHub-hosted runners have 14GB of SSD space
-          # (https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources).
-          #
-          # Since the LLVM build creates a bunch of symlinks (and this setting does not turn those
-          # into symlinks-to-symlinks), use absolute symlinks so we can distinguish the two cases.
-          CMAKE_INSTALL_MODE: ABS_SYMLINK
-        run: cmake --build llvm-build --target install
+          brew install cmake lld llvm ninja
+          echo $(brew --prefix)/opt/llvm/bin >> $GITHUB_PATH
 
-      - name: Rewrite LLVM Symlinks
+      - name: Build LLVM
         if: steps.cache-llvm.outputs.cache-hit != 'true'
-        # Move targets over the symlinks that point to them.
-        #
-        # This whole dance would be simpler if CMake supported CMAKE_INSTALL_MODE=MOVE.
         run: |
           set -euxo pipefail
-          find llvm-install -type l -execdir sh -eux -c '
-            for link in "$@"; do
-              target=$(readlink "$link")
-              case $target in
-                /*) mv "$target" "$link" ;;
-              esac
-            done
-          ' sh {} +
+          cargo xtask build-llvm \
+            --container-tag initial \
+            --llvm-install-dir "${{ github.workspace }}/llvm-install" \
+            --target "${{ matrix.target }}"