CI performance tweaks (#4989)

This adds a new tool to CI described over here
https://github.com/ClementTsang/mtime-travel -- TL;DR it keeps a json
file listing hashes of source files and mtimes and _resets the mtimes_
if the hashes don't change. This makes the thing CI does where it checks
out files with git every time (and clobbers their mtimes) not cause
spuriouos rebuilds. It should speed things up.

I also threw in a small optimization to the second-run testing (tx-meta
checking) to skip running all soroban summodule tests a second time --
we run them in the first pass.

Also an optimization to avoid testing the same rev back-to-back (eg. on
master immediately after testing in a merge queue). This should cut out
a third of all runs since we (pessimistically) run on all 3 of PR, merge
queue and final master integration.

Author: graydon

.gitattributes

@@ -1,4 +1,4 @@
 *.sln    text -merge eol=crlf
 *.vcxproj text -merge eol=crlf
 *.vcxproj.filters text -merge eol=crlf
-*.sh text -merge eol=lf
+*.sh text merge eol=lf

.github/workflows/build.yml

@@ -107,11 +107,11 @@ jobs:
       - name: install rustup components
         run: rustup component add rustfmt
 
-      - name: install cargo-cache
-        run: cargo install --locked cargo-cache --version 0.8.3
-
-      - name: install cargo-sweep
-        run: cargo install --locked cargo-sweep --version 0.7.0
+      - name: cargo install tools
+        run: |
+          cargo install --locked cargo-cache --version 0.8.3
+          cargo install --locked cargo-sweep --version 0.7.0
+          cargo install --locked mtime-travel --version 0.1.0
 
       - name: Build
         run: |
@@ -154,13 +154,11 @@ jobs:
           rustup-init -y
           rustup component add rustfmt rustc cargo clippy rust-src rust-std
 
-      - name: install cargo-cache
+      - name: cargo install tools
         run: |
           cargo install --locked cargo-cache --version 0.8.3
-
-      - name: install cargo-sweep
-        run: |
           cargo install --locked cargo-sweep --version 0.7.0
+          cargo install --locked mtime-travel --version 0.1.0
 
       - name: Build
         run: |

ci-build.sh

@@ -67,6 +67,35 @@ SRC_DIR=$(pwd)
 mkdir -p "build-${CC}-${PROTOCOL}"
 cd "build-${CC}-${PROTOCOL}"
 
+# Check to see if we _just_ tested this rev in
+# a merge queue, and if so don't bother doing
+# it again. Wastes billable CPU time.
+if [ -e prev-pass-rev ]
+then
+   PREV_REV=$(cat prev-pass-rev)
+   CURR_REV=$(git -C "${SRC_DIR}" rev-parse HEAD)
+   if [ "${PREV_REV}" = "${CURR_REV}" ]
+   then
+       exit 0
+   fi
+   rm -f prev-pass-rev
+fi
+
+
+# restore source file mtimes based on content hashes
+if which mtime-travel >/dev/null 2>&1
+then
+    for DIR in src lib
+    do
+	if [ -e mtimes-${DIR}.json ]
+	then
+	    mtime-travel restore -f mtimes-${DIR}.json ${SRC_DIR}/${DIR}
+	fi
+	rm -f mtimes-${DIR}.json
+	mtime-travel save -f mtimes-${DIR}.json ${SRC_DIR}/${DIR}
+    done
+fi
+
 # Try to ensure we're using the real g++ and clang++ versions we want
 mkdir -p bin
 
@@ -135,11 +164,11 @@ time (cd "${SRC_DIR}" && ./autogen.sh)
 time "${SRC_DIR}/configure" $config_flags
 if [ -z "${SKIP_FORMAT_CHECK}" ]; then
     make format
-    d=`git diff | wc -l`
+    d=`git -C "${SRC_DIR}" diff | wc -l`
     if [ $d -ne 0 ]
     then
         echo "clang format must be run as part of the pull request, current diff:"
-        git diff
+        git -C "${SRC_DIR}" diff
         exit 1
     fi
 fi
@@ -190,8 +219,11 @@ time make check
 echo Running fixed check-test-tx-meta tests
 export TEST_SPEC='[tx]'
 export STELLAR_CORE_TEST_PARAMS="--ll fatal -r simple --all-versions --rng-seed 12345 --check-test-tx-meta ${SRC_DIR}/test-tx-meta-baseline-${PROTOCOL}"
+export SKIP_SOROBAN_TESTS=true
 time make check
 
 echo All done
 date
+
+git -C "${SRC_DIR}" rev-parse HEAD >prev-pass-rev
 exit 0

src/Makefile.am

@@ -128,6 +128,7 @@ CARGO=cargo +$(RUST_TOOLCHAIN_CHANNEL)
 # we pass some configuration by environment variable
 # to tests since they can't take command-line arguments.
 export RUST_TOOLCHAIN_CHANNEL
+export SKIP_SOROBAN_TESTS
 export top_srcdir
 export top_builddir
 

src/test/check-sorobans

@@ -17,6 +17,12 @@ then
     exit 1
 fi
 
+if [ -n "${SKIP_SOROBAN_TESTS}" ];
+then
+    echo "skipping soroban tests"
+    exit 0
+fi
+
 # Define tests to skip
 # `test_expected_size` tests the actual memory sizes of types used in memory metering.
 # This test is skipped because memory sizes vary based on Rust compiler version and