Run the build with check-test-tx-meta sequentially to get most cache hits (#4918)

We currently have a matrix scenario to run with and without the flag
check-test-tx-meta. This means they run in parallel jobs and don't take
advantage of caching.
Running the build with check-test-tx-meta sequentially would get the
most cache hits.
The tradeoff is that we would have longer CI run by not putting the
build with check-test-tx-meta flag in a parallel job.

See the outcome in
https://github.com/stellar/stellar-core/actions/runs/17438212726 where
there are 4 parallel CI jobs taking between 26 to 37 minutes.

This PR is making these changes,

1. Run the tests with check-test-tx-meta sequentially after the regular
build and test scenario instead of parallel. This would reduce the cost
of not having to build again and use local filesystem cache. (maybe 16
core runners are more expensive?)
2. Save the build to Github cache as soon as its available
3. Make ci-build.sh idempotent

Author: anupsdf

.github/workflows/build.yml

@@ -64,7 +64,6 @@ jobs:
       matrix:
         toolchain: [ "gcc", "clang"]
         protocol: ["current", "next"]
-        scenario: ["", "--check-test-tx-meta"]
     steps:
       - name: Fix kernel mmap rnd bits
         # Asan in llvm provided in ubuntu 22.04 is incompatible with
@@ -141,11 +140,26 @@ jobs:
             export CXX='clang++'
           fi
           echo Build with $CC and $CXX
-          ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }} ${{ matrix.scenario }}
+          echo "Running first scenario"
+          ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }}
 
-      # We only _save_ to the cache when we had a cache miss, are running on master, and are the non-txmeta scenario.
-      - uses: actions/cache/save@v4
-        if: ${{ steps.cache.outputs.cache-hit != 'true' && github.ref_name == 'master' && matrix.scenario == ''}}
+      # We only _save_ to the cache when we had a cache miss and are running on master.
+      - name: Save cache after first build
+        uses: actions/cache/save@v4
+        if: ${{ steps.cache.outputs.cache-hit != 'true' && github.ref_name == 'master' }}
         with:
           path: ${{ env.CACHED_PATHS }}
-          key: ${{ steps.cache.outputs.cache-primary-key }}
+          key: ${{ steps.cache.outputs.cache-primary-key }}
+
+      - name: Build for check-test-tx-meta scenario
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          if test "${{ matrix.toolchain }}" = "gcc" ; then
+            export CC='gcc'
+            export CXX='g++'
+          else
+            export CC='clang'
+            export CXX='clang++'
+          fi
+          echo "Running second scenario with --check-test-tx-meta flag"
+          ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }} --check-test-tx-meta

ci-build.sh

@@ -69,7 +69,7 @@ echo "Found $NPROCS processors"
 date
 
 # Try to ensure we're using the real g++ and clang++ versions we want
-mkdir bin
+mkdir -p bin
 
 export PATH=`pwd`/bin:$PATH
 echo "PATH is $PATH"
@@ -80,19 +80,19 @@ if test $CXX = 'clang++'; then
     # Use CLANG_VERSION environment variable if set, otherwise default to 12
     CLANG_VER=${CLANG_VERSION:-12}
     which clang-${CLANG_VER}
-    ln -s `which clang-${CLANG_VER}` bin/clang
+    ln -sf `which clang-${CLANG_VER}` bin/clang
     which clang++-${CLANG_VER}
-    ln -s `which clang++-${CLANG_VER}` bin/clang++
+    ln -sf `which clang++-${CLANG_VER}` bin/clang++
     which llvm-symbolizer-${CLANG_VER}
-    ln -s `which llvm-symbolizer-${CLANG_VER}` bin/llvm-symbolizer
+    ln -sf `which llvm-symbolizer-${CLANG_VER}` bin/llvm-symbolizer
     clang -v
     llvm-symbolizer --version || true
 elif test $CXX = 'g++'; then
     RUN_PARTITIONS=$(seq $NPROCS $((2*NPROCS-1)))
     which gcc-10
-    ln -s `which gcc-10` bin/gcc
+    ln -sf `which gcc-10` bin/gcc
     which g++-10
-    ln -s `which g++-10` bin/g++
+    ln -sf `which g++-10` bin/g++
     which g++
     g++ -v
 fi
@@ -165,13 +165,15 @@ if [ $WITH_TESTS -eq 0 ] ; then
 fi
 
 if [ $TEMP_POSTGRES -eq 0 ] ; then
-    # Create postgres databases
+    # Create postgres databases (drop first if they exist to ensure clean state)
     export PGUSER=postgres
+    psql -c "drop database if exists test;" 2>/dev/null || true
     psql -c "create database test;"
     # we run NPROCS jobs in parallel
     for j in $(seq 0 $((NPROCS-1))); do
         base_instance=$((j*50))
         for i in $(seq $base_instance $((base_instance+15))); do
+            psql -c "drop database if exists test$i;" 2>/dev/null || true
             psql -c "create database test$i;"
         done
     done