Namespace runners (#4988)

This switches our CI to work on runners provided by namespace.so. It's a
little larger and more involved than I wanted because namespace's
caching system works differently than github's, and requires support for
out-of-tree builds.

On the other hand, the resulting configuration is simpler overall, and
apparently everyone wanted out-of-tree builds anyways, and the runners
seem to be faster and cheaper than github's runners. So I'm proposing we
give it a try!

Author: graydon

.github/workflows/build.yml

@@ -18,22 +18,31 @@ jobs:
 
   complete:
     if: always()
-    needs: [fmt, cargo-deny, rust-check-git-rev-deps, build]
-    runs-on: ubuntu-22.04
+    needs: [fmt, cargo-deny, rust-check-git-rev-deps, build-linux, build-mac]
+    runs-on:
+      - namespace-profile-jammy-1-stellar-core
+      - nscloud-cache-exp-do-not-commit
     steps:
     - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
       run: exit 1
 
   fmt:
-    runs-on: ubuntu-22.04
+    runs-on:
+      - namespace-profile-jammy-1-stellar-core
+      - nscloud-cache-exp-do-not-commit
     steps:
-    - uses: actions/checkout@v4
-    - run: rustup component add rustfmt
-    - run: rustup update
-    - run: cargo fmt --all --check
+      - uses: namespacelabs/nscloud-checkout-action@v7
+        with:
+          fetch-depth: 1
+          submodules: recursive
+      - run: rustup component add rustfmt
+      - run: rustup update
+      - run: cargo fmt --all --check
 
   cargo-deny:
-    runs-on: ubuntu-22.04
+    runs-on:
+      - namespace-profile-jammy-1-stellar-core
+      - nscloud-cache-exp-do-not-commit
     strategy:
       matrix:
         checks:
@@ -42,27 +51,31 @@ jobs:
     # Prevent sudden announcement of a new advisory from failing ci:
     continue-on-error: ${{ matrix.checks == 'advisories' }}
     steps:
-    - uses: actions/checkout@v4
-    - uses: EmbarkStudios/cargo-deny-action@8d73959fce1cdc8989f23fdf03bec6ae6a6576ef
-      with:
-        command: check ${{ matrix.checks }}
-        # leave arguments empty so we don't test --all-features
-        # which will see conflicting env versions
-        arguments:
+      - uses: namespacelabs/nscloud-checkout-action@v7
+        with:
+          fetch-depth: 1
+          submodules: recursive
+      - uses: EmbarkStudios/cargo-deny-action@8d73959fce1cdc8989f23fdf03bec6ae6a6576ef
+        with:
+          command: check ${{ matrix.checks }}
+          # leave arguments empty so we don't test --all-features
+          # which will see conflicting env versions
+          arguments:
 
   rust-check-git-rev-deps:
-    runs-on: ubuntu-22.04
+    runs-on:
+      - namespace-profile-jammy-1-stellar-core
+      - nscloud-cache-exp-do-not-commit
     steps:
-    - uses: actions/checkout@v4
-    - uses: stellar/actions/rust-check-git-rev-deps@main
-
-  build:
-    runs-on: ubuntu-jammy-16-cores-amd64
-    env:
-      CACHED_PATHS: |
-        ~/.ccache
-        ~/.cargo
-        target
+      - uses: namespacelabs/nscloud-checkout-action@v7
+        with:
+          fetch-depth: 1
+          submodules: recursive
+      - uses: stellar/actions/rust-check-git-rev-deps@main
+
+  build-linux:
+    runs-on:
+      - namespace-profile-jammy-32-stellar-core;overrides.cache-tag=config-${{ matrix.toolchain }}-${{ matrix.protocol }}
     strategy:
       fail-fast: false
       matrix:
@@ -75,66 +88,32 @@ jobs:
         # using leading to random crashes: https://reviews.llvm.org/D148280
         run: sudo sysctl vm.mmap_rnd_bits=28
 
-      # We start with as cheap as possible a cache probe to see if we have an exact hit on this
-      # git commit ID (for a given OS/toolchain/protocol). If so we can skip all subsequent
-      # steps: we already tested this exact SHA.
-      #
-      # Unfortunately due to the way github actions control flow works, we have to repeat the
-      # step 'if' condition in each step, and cannot actually "exit early" and skip the job.
-      # There are a lot of duplicate bug reports filed on this aspect of github actions, don't
-      # bother filing another.
-      - name: Probe Cache
-        id: cache
-        uses: actions/cache/restore@v4
+      - name: Checkout with Namespace Git mirrors cache
+        uses: namespacelabs/nscloud-checkout-action@v7
         with:
-          path: ${{ env.CACHED_PATHS }}
-          key: ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.protocol }}-${{ github.sha }}
-          lookup-only: true
-
-      # When we have a cache miss on the exact commit SHA, we restore from the prefix without it.
-      # This will restore the most-recently-written cache (github does this date ordering itself).
-      - name: Restore Cache
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: actions/cache/restore@v4
-        with:
-          path: ${{ env.CACHED_PATHS }}
-          key: ${{ steps.cache.outputs.cache-primary-key }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.protocol }}
+          fetch-depth: 1
+          submodules: recursive
 
-      - uses: actions/checkout@v4
-        if: steps.cache.outputs.cache-hit != 'true'
+      - name: Configure Namespace cache volume
+        uses: namespacelabs/nscloud-cache-action@v1
         with:
-           fetch-depth: 200
-           submodules: true
-      - name: install core packages
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install --no-install-recommends apt-utils dialog git iproute2 procps lsb-release
-      - name: install tool chain
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          sudo apt-get -y install libstdc++-10-dev clang-format-12 ccache lldb
-          if test "${{ matrix.toolchain }}" = "gcc" ; then
-            sudo apt-get -y install cpp-10 gcc-10 g++-10
-          else
-            sudo apt-get -y install clang-12 llvm-12
-          fi
+          path: |
+            ~/.cargo
+            build-${{matrix.toolchain}}-${{matrix.protocol}}
+
+      - name: install rustup
+        run: ./install-rust.sh
+
       - name: install rustup components
-        if: steps.cache.outputs.cache-hit != 'true'
         run: rustup component add rustfmt
+
       - name: install cargo-cache
-        if: steps.cache.outputs.cache-hit != 'true'
         run: cargo install --locked cargo-cache --version 0.8.3
+
       - name: install cargo-sweep
-        if: steps.cache.outputs.cache-hit != 'true'
         run: cargo install --locked cargo-sweep --version 0.7.0
-      - name: install dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: sudo apt-get -y install postgresql git build-essential pkg-config autoconf automake libtool bison flex libpq-dev parallel libunwind-dev sed perl
+
       - name: Build
-        if: steps.cache.outputs.cache-hit != 'true'
         run: |
           if test "${{ matrix.toolchain }}" = "gcc" ; then
             export CC='gcc'
@@ -144,26 +123,52 @@ jobs:
             export CXX='clang++'
           fi
           echo Build with $CC and $CXX
-          echo "Running first scenario"
           ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }}
 
-      # We only _save_ to the cache when we had a cache miss and are running on master.
-      - name: Save cache after first build
-        uses: actions/cache/save@v4
-        if: ${{ steps.cache.outputs.cache-hit != 'true' && github.ref_name == 'master' }}
+  build-mac:
+    runs-on:
+      - namespace-profile-macos-sequoia;overrides.cache-tag=config-macos-sequoia
+    steps:
+
+      - name: Checkout with Namespace Git mirrors cache
+        uses: namespacelabs/nscloud-checkout-action@v7
+        with:
+          fetch-depth: 1
+          submodules: recursive
+
+      - name: Configure Namespace cache volume
+        uses: namespacelabs/nscloud-cache-action@v1
         with:
-          path: ${{ env.CACHED_PATHS }}
-          key: ${{ steps.cache.outputs.cache-primary-key }}
+          path: |
+            ~/.cargo
+            build-clang-current
 
-      - name: Build for check-test-tx-meta scenario
-        if: steps.cache.outputs.cache-hit != 'true'
+      - name: install prerequisites and uninstall brew rust, add rustup
         run: |
-          if test "${{ matrix.toolchain }}" = "gcc" ; then
-            export CC='gcc'
-            export CXX='g++'
-          else
-            export CC='clang'
-            export CXX='clang++'
-          fi
-          echo "Running second scenario with --check-test-tx-meta flag"
-          ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }} --check-test-tx-meta
+          brew install libsodium libtool autoconf automake pkg-config libpq openssl parallel ccache bison gnu-sed perl coreutils
+          brew uninstall rust rustup
+          brew install rustup
+
+      - name: install rustup components
+        run: |
+          rustup-init -y
+          rustup component add rustfmt rustc cargo clippy rust-src rust-std
+
+      - name: install cargo-cache
+        run: |
+          cargo install --locked cargo-cache --version 0.8.3
+
+      - name: install cargo-sweep
+        run: |
+          cargo install --locked cargo-sweep --version 0.7.0
+
+      - name: Build
+        run: |
+          export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$(brew --prefix)/opt/libpq/lib/pkgconfig"
+          export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$(brew --prefix)/opt/openssl@3/lib/pkgconfig"
+          export PATH="$(brew --prefix bison)/bin:$PATH"
+          export CC='clang'
+          export CXX='clang++'
+          export CLANG_VERSION=none
+          export SKIP_FORMAT_CHECK=1
+          ./ci-build.sh --disable-postgres --protocol current