feat: add workflow to build riscv64 binaries and cache libpq

mliezun · dvarrazzo · commit 0f8287993604 · 2025-10-29T22:48:23.000Z
This commit introduces a new GitHub Actions workflow for building and caching the libpq package, ensuring that the libpq artifacts are available for subsequent builds.
Additionally, it updates the build scripts to release riscv64 binaries for linux.
diff --git a/.github/workflows/build-and-cache-libpq.yml b/.github/workflows/build-and-cache-libpq.yml
@@ -0,0 +1,138 @@
+name: Build and cache libpq
+
+# Build the libpq package and cache the artifacts.
+#
+# Every Python version on the same architecture will use the same libpq.
+# Therefore building and caching the libpq together with the binary packages
+# result in multiple concurrent builds, and the github artifacts manageer very
+# confused.
+#
+# This job builds the libpq and then caches the artifacts so that the
+# packages.yml workflow will find the library in the cache.
+#
+# You can see the caches at https://github.com/psycopg/psycopg2/actions/caches
+#
+# Or from the API:
+#
+#   curl -fsSL -X GET \
+#     -H "Accept: application/vnd.github+json" \
+#     -H "Authorization: Bearer $GITHUB_TOKEN" \
+#     -H "X-GitHub-Api-Version: 2022-11-28" \
+#     "https://api.github.com/repos/psycopg/psycopg/actions/caches" \
+#   | jq -r '.actions_caches[].key'
+#
+# You can delete a cache using:
+#
+#   curl -fsSL -X DELETE \
+#     -H "Accept: application/vnd.github+json" \
+#     -H "Authorization: Bearer $GITHUB_TOKEN" \
+#     -H "X-GitHub-Api-Version: 2022-11-28" \
+#     "https://api.github.com/repos/psycopg/psycopg/actions/caches?key=libpq-manylinux-ppc64le-17.2-3.4.0"
+#
+# ref: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-github-actions-caches-for-a-repository-using-a-cache-key
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/build-and-cache-libpq.yml
+      - scripts/build/build_libpq.sh
+
+# TODO: move these env vars in an external env file in order to share them
+# across workflows.
+
+env:
+  LIBPQ_VERSION: "17.6"
+  OPENSSL_VERSION: "3.5.4"
+  PQ_FLAGS: ""
+
+concurrency:
+  # Cancel older requests of the same workflow in the same branch.
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+
+  linux:  # {{{
+    runs-on: ubuntu-latest
+    if: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [x86_64, ppc64le, aarch64, riscv64]
+        platform: [manylinux, musllinux]
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up QEMU for multi-arch build
+        # Check https://github.com/docker/setup-qemu-action for newer versions.
+        uses: docker/setup-qemu-action@v3
+        with:
+          # https://github.com/pypa/cibuildwheel/discussions/2256
+          image: tonistiigi/binfmt:qemu-v8.1.5
+
+      - name: Cache libpq build
+        uses: actions/cache@v4
+        with:
+          path: /tmp/libpq.build
+          key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v3.2.0
+        env:
+          CIBW_SKIP: "cp31?t-*"
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
+          CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014
+          CIBW_MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
+          CIBW_BUILD: cp313-${{matrix.platform}}_${{matrix.arch}}
+          CIBW_ARCHS_LINUX: auto aarch64 ppc64le riscv64
+          CIBW_BEFORE_ALL_LINUX: ./scripts/build/build_libpq.sh
+          CIBW_REPAIR_WHEEL_COMMAND: >-
+            ./scripts/build/strip_wheel.sh {wheel}
+            && auditwheel repair -w {dest_dir} {wheel}
+          CIBW_ENVIRONMENT_PASS_LINUX: LIBPQ_VERSION OPENSSL_VERSION
+          CIBW_ENVIRONMENT: >-
+            LIBPQ_BUILD_PREFIX=/host/tmp/libpq.build
+            PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH"
+            LD_LIBRARY_PATH="$LIBPQ_BUILD_PREFIX/lib:$LIBPQ_BUILD_PREFIX/lib64"
+
+
+  # }}}
+
+  macos:  # {{{
+    runs-on: macos-latest
+    if: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [x86_64, arm64]
+
+    steps:
+      - name: Checkout repos
+        uses: actions/checkout@v5
+
+      - name: Cache libpq build
+        uses: actions/cache@v4
+        with:
+          path: /tmp/libpq.build
+          key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v3.2.0
+        env:
+          CIBW_SKIP: "cp31?t-*"
+          CIBW_BUILD: cp313-macosx_${{matrix.arch}}
+          CIBW_ARCHS_MACOS: ${{matrix.arch}}
+          MACOSX_ARCHITECTURE: ${{matrix.arch}}
+          CIBW_BEFORE_ALL_MACOS: ./scripts/build/build_libpq.sh
+          CIBW_ENVIRONMENT: >-
+            PSYCOPG_IMPL=binary
+            LIBPQ_BUILD_PREFIX=/tmp/libpq.build
+            PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH"
+
+  # }}}
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
@@ -1,12 +1,21 @@
 ---
 name: Build packages
+
+# Note: these jobs also build and cache the libpq, but, because every Python
+# version will try to build and cache the same libpq instance, there is a race
+# condition and most likely the artifacts manager will refuse to cache.
+#
+# Please run the `build-and-cache-libpq.yml` workflow when the libpq/openssl
+# version change in order to update the cache.
+
 on:
   - workflow_dispatch
 
 env:
   PIP_BREAK_SYSTEM_PACKAGES: "1"
   LIBPQ_VERSION: "17.6"
   OPENSSL_VERSION: "3.5.4"
+  PQ_FLAGS: ""
 
 jobs:
   sdist:  # {{{
@@ -65,7 +74,7 @@ jobs:
       fail-fast: false
       matrix:
         platform: [manylinux, musllinux]
-        arch: [x86_64, aarch64, ppc64le]
+        arch: [x86_64, aarch64, ppc64le, riscv64]
         pyver: [cp39, cp310, cp311, cp312, cp313, cp314]
 
     runs-on: ubuntu-latest
@@ -80,7 +89,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: /tmp/libpq.build
-          key: libpq-${{ env.LIBPQ_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }}
+          key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
 
       - name: Build wheels
         uses: pypa/cibuildwheel@v3.2.0
@@ -90,8 +99,9 @@ jobs:
           CIBW_MANYLINUX_I686_IMAGE: manylinux2014
           CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
           CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014
+          CIBW_MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
           CIBW_BUILD: ${{matrix.pyver}}-${{matrix.platform}}_${{matrix.arch}}
-          CIBW_ARCHS_LINUX: auto aarch64 ppc64le
+          CIBW_ARCHS_LINUX: auto aarch64 ppc64le riscv64
           CIBW_BEFORE_ALL_LINUX: ./scripts/build/wheel_linux_before_all.sh
           CIBW_REPAIR_WHEEL_COMMAND: >-
             ./scripts/build/strip_wheel.sh {wheel}
@@ -151,7 +161,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: /tmp/libpq.build
-          key: libpq-${{ env.LIBPQ_VERSION }}-macos-${{ matrix.arch }}
+          key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
 
       - name: Build wheels
         uses: pypa/cibuildwheel@v3.2.0
diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Build a modern version of libpq and depending libs from source on Centos 5, Alpine or macOS
+# Build a modern version of libpq and depending libs from source on Centos 5, Rocky, Alpine or macOS
 
 set -euo pipefail
 
@@ -54,9 +54,9 @@ fi
 
 # Install packages required to build the libpq.
 case "$ID" in
-    centos)
+    centos|rocky)
         yum update -y
-        yum install -y flex krb5-devel pam-devel perl-IPC-Cmd perl-Time-Piece zlib-devel
+        yum install -y flex krb5-devel pam-devel perl perl-IPC-Cmd perl-Time-Piece zlib-devel
         ;;
 
     alpine)
@@ -104,7 +104,7 @@ else
     )
 fi
 
-if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
+if [ "$ID" == "centos" ] || [ "$ID" == "rocky" ] || [ "$ID" == "macos" ]; then
   if [[ ! -f "${LIBPQ_BUILD_PREFIX}/openssl.cnf" ]]; then
 
     # Build openssl if needed
@@ -160,7 +160,7 @@ if [ "$ID" == "macos" ]; then
 fi
 
 
-if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
+if [ "$ID" == "centos" ] || [ "$ID" == "rocky" ] || [ "$ID" == "macos" ]; then
   if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libsasl2.${library_suffix}" ]]; then
 
     # Build libsasl2 if needed
@@ -176,6 +176,14 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
 
         pushd "${sasl_dir}"
 
+        if [ "$ID" == "rocky" ]; then
+            # Fix missing time.h include in multiple files for newer GCC versions
+            sed -i.bak '/#include "saslint.h"/a\
+#include <time.h>' lib/saslutil.c
+            sed -i.bak '/#include "plugin_common.h"/a\
+#include <time.h>' plugins/cram.c
+        fi
+
         autoreconf -i
         ./configure "${make_configure_standard_flags[@]}" --disable-macos-framework
         make -s
@@ -193,7 +201,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
 fi
 
 
-if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
+if [ "$ID" == "centos" ] || [ "$ID" == "rocky" ] || [ "$ID" == "macos" ]; then
   if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libldap.${library_suffix}" ]]; then
 
     # Build openldap if needed
diff --git a/scripts/build/print_so_versions.sh b/scripts/build/print_so_versions.sh
@@ -26,7 +26,7 @@ case "$ID" in
         done) | sort | uniq
         ;;
 
-    centos)
+    centos|rocky)
         echo "TODO!"
         ;;
 
diff --git a/scripts/build/wheel_linux_before_all.sh b/scripts/build/wheel_linux_before_all.sh
@@ -35,7 +35,7 @@ case "$ID" in
         apt-get -y install libpq-dev
         ;;
 
-    centos)
+    centos|rocky)
         "${dir}/build_libpq.sh" > /dev/null
         ;;
 
