Merge branch 'main' into nbclassic

danielhollas · web-flow · commit c2f2802cb531 · 2025-02-11T12:52:47.000Z
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -55,7 +55,9 @@ jobs:
 
             - name: Build and upload to ghcr.io 📤
               id: build-upload
-              uses: docker/bake-action@v5
+              # We need to pin exact version here, since updates can break
+              # the extract-image-names.sh script
+              uses: docker/bake-action@v5.5.0
               with:
                   push: true
                   # Using provenance to disable default attestation so it will build only desired images:
@@ -71,9 +73,15 @@ jobs:
                       build.json
                       .github/workflows/env.hcl
 
-            - name: Set output variables
+            - name: Set output image names
               id: bake_metadata
+              # bake-action metadata output has gotten too big, so we first write it
+              # to a file. See https://github.com/aiidalab/aiidalab-docker-stack/issues/491
               run: |
-                  .github/workflows/extract-image-names.sh | tee -a "${GITHUB_OUTPUT}" | awk -F'=' '{print $2}' | jq
-              env:
-                  BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }}
+                  cat << EOF > bake_metadata.json
+                  ${{ steps.build-upload.outputs.metadata }}
+                  EOF
+                  images=$(.github/workflows/extract-image-names.sh bake_metadata.json)
+                  echo "images=${images}" >> "${GITHUB_OUTPUT}"
+                  # Pretty-print for GHA logs
+                  echo "$images" | jq
diff --git a/.github/workflows/extract-image-names.sh b/.github/workflows/extract-image-names.sh
@@ -2,13 +2,14 @@
 
 set -euo pipefail
 
+metadata_file=$1
 # Extract image names together with their sha256 digests
 # from the docker/bake-action metadata output.
 # These together uniquely identify newly built images.
 
-# The input to this script is a JSON string passed via BAKE_METADATA env variable
+# The input to this script is a json file (filename passed as first parameter to the script)
 # Here's example input (trimmed to relevant bits):
-# BAKE_METADATA: {
+# {
 #    "base": {
 #      "containerimage.descriptor": {
 #        "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
@@ -35,23 +36,17 @@ set -euo pipefail
 #    }
 #  }
 #
-# Example output (real output is on one line):
+# Example output with trimmed SHAs (real output is on one line):
 #
-# images={
-#   "BASE_IMAGE": "ghcr.io/aiidalab/base@sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
-#   "BASE_WITH_SERVICES_IMAGE": "ghcr.io/aiidalab/base-with-services@sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26",
-#   "FULL_STACK_IMAGE": "ghcr.io/aiidalab/full-stack@sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48",
-#   "LAB_IMAGE": "ghcr.io/aiidalab/lab@sha256:4d9be090da287fcdf2d4658bb82f78bad791ccd15dac9af594fb8306abe47e97"
+# {
+#   "BASE_IMAGE": "ghcr.io/aiidalab/base@sha256:8e57a52b92",
+#   "BASE_WITH_SERVICES_IMAGE": "ghcr.io/aiidalab/base-with-services@sha256:6753a809",
+#   "FULL_STACK_IMAGE": "ghcr.io/aiidalab/full-stack@sha256:85ee91f61be",
+#   "LAB_IMAGE": "ghcr.io/aiidalab/lab@sha256:4d9be090da2"
 # }
 #
 # This json output is later turned to environment variables using fromJson() GHA builtin
 # (e.g. BASE_IMAGE=ghcr.io/aiidalab/base@sha256:8e57a52b...)
 # and these are in turn read in the docker-compose.<target>.yml files for tests.
 
-if [[ -z ${BAKE_METADATA-} ]];then
-    echo "ERROR: Environment variable BAKE_METADATA is not set!"
-    exit 1
-fi
-
-images=$(echo "${BAKE_METADATA}" | jq -c '. as $base |[to_entries[] |{"key": (.key|ascii_upcase|sub("-"; "_"; "g") + "_IMAGE"), "value": [(.value."image.name"|split(",")[0]),.value."containerimage.digest"]|join("@")}] |from_entries')
-echo "images=$images"
+jq -c '. as $base |[to_entries[] |{"key": (.key|ascii_upcase|sub("-"; "_"; "g") + "_IMAGE"), "value": [(.value."image.name"|split(",")[0]),.value."containerimage.digest"]|join("@")}] |from_entries' $metadata_file
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -55,16 +55,11 @@ jobs:
             runsOn: ubuntu-22.04
             platforms: linux/amd64,linux/arm64
 
-    # To save arm64 runner resources, we run the tests only on main
-    # and only for full-stack image (same for integration tests below).
     test-arm64:
-        if: >-
-            github.repository == 'aiidalab/aiidalab-docker-stack'
-            && (github.ref_type == 'tag' || github.ref_name == 'main')
         needs: build
         uses: ./.github/workflows/test.yml
         with:
-            runsOn: buildjet-4vcpu-ubuntu-2204-arm
+            runsOn: ubuntu-22.04-arm
             images: ${{ needs.build.outputs.images }}
             target: full-stack
             integration: false
@@ -74,16 +69,8 @@ jobs:
         needs: build
         strategy:
             fail-fast: false
-            # Trick to exclude arm64 tests from PRs
-            # https://github.com/orgs/community/discussions/26253
             matrix:
-                runner: [ubuntu-22.04, buildjet-4vcpu-ubuntu-2204-arm]
-                isPR:
-                    - ${{ github.event_name == 'pull_request' }}
-                exclude:
-                    - isPR: true
-                      runner: buildjet-4vcpu-ubuntu-2204-arm
-
+                runner: [ubuntu-22.04, ubuntu-22.04-arm]
         uses: ./.github/workflows/test.yml
         with:
             runsOn: ${{ matrix.runner }}
@@ -120,7 +107,7 @@ jobs:
         needs: [publish-dockerhub]
 
         steps:
-            - uses: softprops/action-gh-release@v2.0.6
+            - uses: softprops/action-gh-release@v2
               name: Create release
               with:
                   generate_release_notes: true
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -51,9 +51,7 @@ jobs:
                   cache: pip
 
             - name: Install dependencies 📦
-              run: |
-                  pip install -r requirements.txt
-                  pip freeze
+              run: pip install -r requirements.txt
 
             - name: Run tests
               run: pytest -s -m "${{ inputs.integration && 'integration' || 'not integration' }}" --target ${{inputs.target}}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ ci:
 repos:
 
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.6.0
+      rev: v5.0.0
       hooks:
           - id: check-json
           - id: end-of-file-fixer
@@ -18,14 +18,14 @@ repos:
             args: [--preserve-quotes]
 
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.5.0
+      rev: v0.9.2
       hooks:
           - id: ruff-format
           - id: ruff
             args: [--fix, --exit-non-zero-on-fix, --show-fixes]
 
     - repo: https://github.com/python-jsonschema/check-jsonschema
-      rev: "0.28.6"
+      rev: "0.31.0"
       hooks:
           - id: check-dependabot
           - id: check-github-workflows
diff --git a/build.json b/build.json
@@ -10,13 +10,13 @@
       "default": "3.9.13"
     },
     "AIIDA_VERSION": {
-      "default": "2.5.1"
+      "default": "2.6.3"
     },
     "AIIDALAB_VERSION": {
-      "default": "24.07.0"
+      "default": "24.09.0"
     },
     "AIIDALAB_HOME_VERSION": {
-      "default": "23.03.1"
+      "default": "25.01.0"
     }
   }
 }
diff --git a/bumpver.toml b/bumpver.toml
@@ -1,5 +1,5 @@
 [bumpver]
-current_version = "v2024.1021"
+current_version = "v2025.1025"
 version_pattern = "vYYYY.BUILD[-TAG]"
 commit_message = "Bump version {old_version} -> {new_version}."
 commit = true
diff --git a/requirements.in b/requirements.in
@@ -0,0 +1,7 @@
+docker
+requests
+pre-commit
+pytest~=7.0
+pytest-docker
+doit
+dunamai
diff --git a/requirements.txt b/requirements.txt
@@ -1,7 +1,32 @@
-docker
-requests
-pre-commit
-pytest~=7.0
-pytest-docker
-doit
-dunamai
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --universal --python 3.9 requirements.in --annotation-style=line -o requirements.txt
+attrs==24.3.0             # via pytest-docker
+certifi==2024.12.14       # via requests
+cfgv==3.4.0               # via pre-commit
+charset-normalizer==3.4.1  # via requests
+cloudpickle==3.1.1        # via doit
+colorama==0.4.6 ; sys_platform == 'win32'  # via pytest
+distlib==0.3.9            # via virtualenv
+docker==7.1.0             # via -r requirements.in
+doit==0.36.0              # via -r requirements.in
+dunamai==1.23.0           # via -r requirements.in
+exceptiongroup==1.2.2 ; python_full_version < '3.11'  # via pytest
+filelock==3.17.0          # via virtualenv
+identify==2.6.6           # via pre-commit
+idna==3.10                # via requests
+importlib-metadata==8.6.1  # via doit
+iniconfig==2.0.0          # via pytest
+nodeenv==1.9.1            # via pre-commit
+packaging==24.2           # via dunamai, pytest
+platformdirs==4.3.6       # via virtualenv
+pluggy==1.5.0             # via pytest
+pre-commit==4.1.0         # via -r requirements.in
+pytest==7.4.4             # via pytest-docker, -r requirements.in
+pytest-docker==3.1.1      # via -r requirements.in
+pywin32==308 ; sys_platform == 'win32'  # via docker
+pyyaml==6.0.2             # via pre-commit
+requests==2.32.3          # via docker, -r requirements.in
+tomli==2.2.1 ; python_full_version < '3.11'  # via pytest
+urllib3==2.3.0            # via docker, requests
+virtualenv==20.29.1       # via pre-commit
+zipp==3.21.0              # via importlib-metadata
diff --git a/stack/base/Dockerfile b/stack/base/Dockerfile
@@ -10,7 +10,8 @@ USER root
 # https://pymatgen.org/installation.html#installation-tips-for-optional-libraries
 # rsync: needed to support the new AiiDA backup command
 # povray: rendering engine used in aiidalab-widgets-base
-ENV EXTRA_APT_PACKAGES "curl povray rsync build-essential"
+# bc: needed to compute the resources for computer setup
+ENV EXTRA_APT_PACKAGES "curl povray rsync build-essential bc"
 
 # For ARM64 we need to install erlang as it is not available on conda-forge
 # (this is needed later as rabbitmq dependency in base-with-services image,
@@ -41,18 +42,31 @@ RUN cat /opt/requirements.txt | xargs -I{} conda config --system --add pinned_pa
 
 # Configure pip to use the same requirements file as constraints file.
 ENV PIP_CONSTRAINT /opt/requirements.txt
-# Ensure that pip installs packages to ~/.local by default
-ENV PIP_USER 1
+# Ensure that pip installs packages to '~/.local/lib/python3.X/site-packages/' by default
+# by implicitly passing the '--user' option to 'pip install'
+# Otherwise, pip would install into /opt/conda and such packages would be lost
+# when the container exits.
+# NOTE: We specifically chose the location '/opt/conda/pip.conf'
+# which represents the 'site' config file when VIRTUAL_ENV is not set, per:
+# https://pip.pypa.io/en/stable/topics/configuration/#configuration-files
+# Other locations such as '~/.config/pip/pip.conf' or '/etc/pip.conf' would interfere with virtual environments,
+# for example those used by pre-commit.
+# We can't use the PIP_USER env variable for the same reason.
+# To better understand this, try running `pip config debug` and see
+# https://github.com/aiidalab/aiidalab-docker-stack/issues/501
+COPY pip.conf "${CONDA_DIR}/pip.conf"
 
 # Upgrade pip and mamba to latest
+# Update async_generator, certipy to satisfy `pip check`
+# https://github.com/aiidalab/aiidalab-docker-stack/issues/490
 # Install aiida-core and other shared requirements.
-RUN mamba update -y pip zstandard && \
+RUN mamba update -y pip zstandard async_generator certipy && \
      mamba install --yes \
      aiida-core==${AIIDA_VERSION} \
      mamba-bash-completion \
+     traitlets=5.9.0 \
      && mamba clean --all -f -y && \
-     fix-permissions "${CONDA_DIR}" && \
-     fix-permissions "/home/${NB_USER}"
+     fix-permissions "${CONDA_DIR}"
 
 # Enable verdi autocompletion.
 RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \
diff --git a/stack/base/before-notebook.d/40_prepare-aiida.sh b/stack/base/before-notebook.d/40_prepare-aiida.sh
@@ -80,6 +80,3 @@ else
   verdi storage migrate --force
 
 fi
-
-# Daemon will start only if the database exists and is migrated to the latest version.
-verdi daemon start || echo "ERROR: AiiDA daemon is not running!"
diff --git a/stack/base/before-notebook.d/90_start_aiida_daemon.sh b/stack/base/before-notebook.d/90_start_aiida_daemon.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# This script is executed whenever the docker container is (re)started.
+set -x
+
+export SHELL=/bin/bash
+
+# Daemon will start only if the database exists and is migrated to the latest version.
+verdi daemon start || echo "ERROR: AiiDA daemon is not running!"
diff --git a/stack/base/pip.conf b/stack/base/pip.conf
@@ -0,0 +1,2 @@
+[install]
+user = true
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -3,7 +3,7 @@
 
 import pytest
 import requests
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError  # noqa: A004
 
 TARGETS = ("base", "lab", "base-with-services", "full-stack")
 
@@ -112,11 +112,11 @@ def pip_install(aiidalab_exec, nb_user):
     def _pip_install(pkg, **args):
         nonlocal package
         package = pkg
-        return aiidalab_exec(f"pip install {pkg}", **args)
+        return aiidalab_exec(f"pip install {pkg}", user=nb_user, **args)
 
     yield _pip_install
     if package:
-        aiidalab_exec(f"pip uninstall --yes {package}")
+        aiidalab_exec(f"pip uninstall --yes {package}", user=nb_user)
 
 
 @pytest.fixture(scope="session")
diff --git a/tests/test_base.py b/tests/test_base.py
@@ -1,11 +1,19 @@
 """This module contains tests for the base image, which are AiiDA and package management related tests."""
 
+import email
 import json
 
 import pytest
 from packaging.version import parse
 
 
+@pytest.fixture
+def venv(tmp_path, aiidalab_exec):
+    venv_path = tmp_path / ".venv"
+    aiidalab_exec(f"python -m venv {venv_path}")
+    return venv_path
+
+
 @pytest.mark.parametrize("pkg_manager", ["pip", "mamba"])
 def test_prevent_installation_of_aiida(
     aiidalab_exec, nb_user, aiida_version, pkg_manager
@@ -76,8 +84,6 @@ def test_path_local_pip(aiidalab_exec, nb_user):
 
 def test_pip_user_install(aiidalab_exec, pip_install, nb_user):
     """Test that pip installs packages to ~/.local/ by default"""
-    import email
-
     # We use 'tuna' as an example of python-only package without dependencies
     pkg = "tuna"
     pip_install(pkg)
@@ -86,3 +92,16 @@ def test_pip_user_install(aiidalab_exec, pip_install, nb_user):
     # `pip show` output is in the RFC-compliant email header format
     msg = email.message_from_string(output)
     assert msg.get("Location").startswith(f"/home/{nb_user}/.local/")
+
+
+def test_pip_install_in_venv(aiidalab_exec, venv, nb_user):
+    """Test that pip installs packages to an activated venv"""
+
+    pkg = "tuna"
+    pip = venv / "bin/pip"
+
+    aiidalab_exec(f"{pip} install {pkg}")
+
+    output = aiidalab_exec(f"{pip} show {pkg}")
+    msg = email.message_from_string(output)
+    assert msg.get("Location").startswith(f"{venv}/lib")
diff --git a/tests/test_common.py b/tests/test_common.py
@@ -10,8 +10,11 @@ def test_notebook_service_available(notebook_service):
 
 def test_verdi_status(aiidalab_exec, nb_user):
     output = aiidalab_exec("verdi status", user=nb_user).strip()
-    assert "Connected to RabbitMQ" in output
+    for status in ("version", "config", "profile", "storage", "broker", "daemon"):
+        assert f"✔ {status}" in output
+    assert "/home/jovyan/.aiida" in output
     assert "Daemon is running" in output
+    assert "Unable to connect to broker" not in output
 
 
 def test_ssh_agent_is_running(aiidalab_exec, nb_user):

Original file line number	Diff line number	Diff line change
`@@ -10,13 +10,13 @@`
`10`	`10`	`"default": "3.9.13"`
`11`	`11`	`},`
`12`	`12`	`"AIIDA_VERSION": {`
`13`		`- "default": "2.5.1"`
	`13`	`+ "default": "2.6.3"`
`14`	`14`	`},`
`15`	`15`	`"AIIDALAB_VERSION": {`
`16`		`- "default": "24.07.0"`
	`16`	`+ "default": "24.09.0"`
`17`	`17`	`},`
`18`	`18`	`"AIIDALAB_HOME_VERSION": {`
`19`		`- "default": "23.03.1"`
	`19`	`+ "default": "25.01.0"`
`20`	`20`	`}`
`21`	`21`	`}`
`22`	`22`	`}`