build(docker): upgrade to Ubuntu 24.04 and Python 3.12 (#692)

Closes reanahub/reana#808

Author: mdonadoni

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on: [push, pull_request]
 
 jobs:
   lint-commitlint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -36,7 +36,7 @@ jobs:
           ./run-tests.sh --check-commitlint ${{ github.event.pull_request.head.sha }}~${{ github.event.pull_request.commits }} ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.number }}
 
   lint-shellcheck:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -47,79 +47,79 @@ jobs:
           ./run-tests.sh --check-shellcheck
 
   lint-black:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.12"
 
       - name: Check Python code formatting
         run: |
           pip install black
           ./run-tests.sh --check-black
 
   lint-flake8:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.12"
 
       - name: Check compliance with pep8, pyflakes and circular complexity
         run: |
           pip install flake8
           ./run-tests.sh --check-flake8
 
   lint-pydocstyle:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.12"
 
       - name: Check compliance with Python docstring conventions
         run: |
           pip install pydocstyle
           ./run-tests.sh --check-pydocstyle
 
   lint-check-manifest:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.12"
 
       - name: Check Python manifest completeness
         run: |
           pip install check-manifest
           ./run-tests.sh --check-manifest
 
   docs-sphinx:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.12"
 
       - name: Install system dependencies
         run: |
@@ -135,16 +135,15 @@ jobs:
         run: ./run-tests.sh --check-sphinx
 
   python-tests:
-    runs-on: ubuntu-20.04
-
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.12"
 
       - name: Install Python dependencies
         run: |
@@ -164,7 +163,7 @@ jobs:
           files: coverage.xml
 
   lint-dockerfile:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -173,7 +172,7 @@ jobs:
         run: ./run-tests.sh --check-dockerfile
 
   docker-build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -182,7 +181,7 @@ jobs:
         run: ./run-tests.sh --check-docker-build
 
   release-docker:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     if: >
       vars.RELEASE_DOCKER == 'true' &&
       github.event_name == 'push' &&

.readthedocs.yaml

@@ -7,9 +7,9 @@
 version: 2
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "3.8"
+    python: "3.12"
 
 sphinx:
   configuration: docs/conf.py

Dockerfile

@@ -5,15 +5,13 @@
 # under the terms of the MIT License; see LICENSE file for more details.
 
 # Use Ubuntu LTS base image
-FROM docker.io/library/ubuntu:20.04
+FROM docker.io/library/ubuntu:24.04
 
 # Use default answers in installation commands
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Use distutils provided by the standard Python library instead of the vendored one in
-# setuptools, so that editable installations are stored in the right directory.
-# See https://github.com/pypa/setuptools/issues/3301
-ENV SETUPTOOLS_USE_DISTUTILS=stdlib
+# Allow pip to install packages in the system site-packages dir
+ENV PIP_BREAK_SYSTEM_PACKAGES=true
 
 # Prepare list of Python dependencies
 COPY requirements.txt /code/
@@ -27,17 +25,17 @@ RUN apt-get update -y && \
       libffi-dev \
       libpcre3 \
       libpcre3-dev \
-      libpython3.8 \
+      libpython3.12 \
       procps \
       python3-pip \
-      python3.8 \
-      python3.8-dev \
+      python3.12 \
+      python3.12-dev \
       vim-tiny && \
-    pip install --no-cache-dir --upgrade pip 'setuptools<71' && \
+    pip install --no-cache-dir --upgrade setuptools && \
     pip install --no-cache-dir -r /code/requirements.txt && \
     apt-get remove -y \
       gcc \
-      python3.8-dev && \
+      python3.12-dev && \
     apt-get autoremove -y && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
@@ -68,7 +66,7 @@ RUN if test -e modules/reana-commons; then \
     fi
 
 # A quick fix to allow eduGAIN and social login users that wouldn't otherwise match Invenio username rules
-RUN sed -i 's|^username_regex = re.compile\(.*\)$|username_regex = re.compile("^\\S+$")|g' /usr/local/lib/python3.8/dist-packages/invenio_userprofiles/validators.py
+RUN sed -i 's|^username_regex = re.compile\(.*\)$|username_regex = re.compile("^\\S+$")|g' /usr/local/lib/python3.12/dist-packages/invenio_userprofiles/validators.py
 
 # Check for any broken Python dependencies
 # hadolint ignore=DL3059

docs/openapi.json

@@ -7,10 +7,6 @@
   "paths": {
     "/account/settings/linkedaccounts/": {},
     "/account/settings/linkedaccounts/static/{filename}": {},
-    "/account/settings/login": {},
-    "/account/settings/security/": {},
-    "/account/settings/sessions/revoke/": {},
-    "/account/settings/static/{filename}": {},
     "/api/config": {
       "get": {
         "description": "This resource provides configuration needed by Reana-UI.",
@@ -4543,6 +4539,7 @@
     "/oauth/disconnect/{remote_app}/": {},
     "/oauth/login": {},
     "/oauth/login/{remote_app}/": {},
+    "/oauth/logout": {},
     "/oauth/signup/{remote_app}/": {},
     "/oauth/static/{filename}": {},
     "/signin": {},

reana_server/app.py

@@ -1,16 +1,19 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2017, 2018, 2019, 2020, 2021 CERN.
+# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2024 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
 
 """Main entrypoint for REANA-Server."""
 
-from reana_server.factory import create_app
+from invenio_app.factory import create_app
 
 # Needed for flask.with_appcontext decorator to work.
+#
+# Note that this is the full Flask app including all the necessary Invenio modules.
+# See `factory.py` for more details.
 app = create_app()
 
 if __name__ == "__main__":

reana_server/ext.py

@@ -53,7 +53,13 @@ def handle_args_validation_error(error: UnprocessableEntity):
 
 
 class REANA(object):
-    """REANA Invenio app."""
+    """REANA Invenio app.
+
+    This is used to initialise REANA as a Flask/Invenio extension,
+    and this is used in production.
+
+    See the docsting of `reana_server/factory.py` for more details.
+    """
 
     def __init__(self, app=None):
         """Extension initialization."""

reana_server/factory.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022 CERN.
+# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022, 2024 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -11,11 +11,10 @@
 import logging
 
 from flask import Flask, current_app
-from flask_babelex import Babel
+from invenio_i18n import Babel, InvenioI18N
 from flask_menu import Menu as FlaskMenu
 from flask_oauthlib.client import OAuth as FlaskOAuth
 from invenio_accounts import InvenioAccounts
-from invenio_accounts.views import blueprint as blueprint_user
 from invenio_db import InvenioDB
 from invenio_oauthclient import InvenioOAuthClient
 from invenio_oauthclient.views.client import blueprint as blueprint_client
@@ -24,8 +23,23 @@
 from reana_db.database import Session
 
 
-def create_app(config_mapping=None):
-    """REANA Server application factory."""
+def create_minimal_app(config_mapping=None):
+    """REANA Server application factory.
+
+    Create a minimal Flask app containing all of REANA's endpoints and the needed
+    Invenio modules. Use `invenio_app.factory.create_app` the create the full Invenio
+    app that is also used in production or when invoking `invenio run`.
+
+    This method is used to create the Flask app in the tests and in the
+    `generate_openapi_spec.py` script.
+
+    In general, this is how Flask apps are created:
+    - When running in debug mode, `invenio run ...` is invoked. This calls `invenio_app.factory.create_app`.
+    - When running in production mode, `uwsgi` is used, and the module configured is `invenio_app.wsgi:application`. This calls `invenio_app.factory.create_app`.
+    - When running `flask reana-admin` commands, flask auto-detects the app present in `app.py`, which is created with `invenio_app.factory.create_app`.
+    - When running the tests, `reana_server.factory.create_minimal_app` is called.
+    - When running `generate_openapi_spec.py`, the app is created with `reana_server.factory.create_minimal_app`.
+    """
     logging.basicConfig(level=REANA_LOG_LEVEL, format=REANA_LOG_FORMAT, force=True)
     app = Flask(__name__)
     app.config.from_object("reana_server.config")
@@ -35,15 +49,16 @@ def create_app(config_mapping=None):
 
     app.session = Session
 
+    # Inspired from https://github.com/inveniosoftware/invenio-accounts/blob/345abfc2d3bf4af0be898a1b4ee1fe45edd16053/tests/conftest.py#L66
     Babel(app)
     FlaskMenu(app)
     InvenioDB(app)
+    InvenioI18N(app)
     InvenioAccounts(app)
     FlaskOAuth(app)
     InvenioOAuthClient(app)
 
     # Register Invenio OAuth endpoints
-    app.register_blueprint(blueprint_user)
     app.register_blueprint(blueprint_client)
     app.register_blueprint(blueprint_settings)