Create lint pipeline and fix errors

.github/workflows/lint.yml

@@ -0,0 +1,24 @@
+name: lint
+on:
+  push
+jobs:
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '>=1.22'
+      - name: Install dependencies
+        run: make setup-lint
+      - name: Run lint
+        run: . venv/bin/activate && make lint

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
@@ -31,7 +31,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 

Makefile

@@ -74,19 +74,31 @@ setup-test-e2e:
 	( cd racetrack_commons && make setup )
 	@echo Activate your venv: . venv/bin/activate
 
+setup-lint:
+	uv venv venv &&\
+	. venv/bin/activate &&\
+	uv pip install -r requirements-test.txt -r requirements-dev.txt \
+		-r racetrack_client/requirements.txt -e racetrack_client@racetrack_client \
+		-r racetrack_commons/requirements.txt -e racetrack_commons@racetrack_commons \
+		-r lifecycle/requirements.txt -e lifecycle@lifecycle \
+		-r image_builder/requirements.txt -e image_builder@image_builder \
+		-r dashboard/requirements.txt -e dashboard@dashboard
+	@echo Activate your venv: . venv/bin/activate
+
 install-racetrack-client:
 	( cd racetrack_client && pip install -e . )
 
 lint:
-	-python -m mypy --ignore-missing-imports --exclude 'racetrack_client/build' racetrack_client
-	-python -m mypy --ignore-missing-imports racetrack_commons
-	-python -m mypy --ignore-missing-imports --exclude 'lifecycle/lifecycle/django/registry/migrations' lifecycle
-	-python -m mypy --ignore-missing-imports image_builder
-	-python -m mypy --ignore-missing-imports dashboard
-	-python -m flake8 --ignore E501 --per-file-ignores="__init__.py:F401" \
-		lifecycle image_builder dashboard
-	-python -m pylint --disable=R,C,W \
-		lifecycle/lifecycle image_builder/image_builder dashboard/dashboard
+	python -m mypy --ignore-missing-imports --exclude 'racetrack_client/build' racetrack_client; e1=$$?;\
+	python -m mypy --ignore-missing-imports racetrack_commons; e2=$$?;\
+	python -m mypy --ignore-missing-imports --exclude 'lifecycle/lifecycle/django/registry/migrations' lifecycle; e3=$$?;\
+	python -m mypy --ignore-missing-imports image_builder; e4=$$?;\
+	python -m mypy --ignore-missing-imports dashboard; e5=$$?;\
+	python -m flake8 --ignore E501 --per-file-ignores="__init__.py:F401 lifecycle/lifecycle/event_stream/server.py:E402" \
+    lifecycle image_builder dashboard; e6=$$?;\
+	python -m pylint --disable=R,C,W \
+		lifecycle/lifecycle image_builder/image_builder dashboard/dashboard; e7=$$?;\
+	exit "$$(( e1 || e2 || e3 || e4 || e5 || e6 || e7 ))"
 
 format:
 	python -m black -S --diff --color -l 120 \

dashboard/dashboard/api/api.py

@@ -1,7 +1,7 @@
 import os
 
 import httpx
-from fastapi import FastAPI, Request, Response, Body
+from fastapi import FastAPI, Request, Body
 from starlette.background import BackgroundTask
 from starlette.responses import StreamingResponse
 from starlette.datastructures import MutableHeaders
@@ -30,7 +30,7 @@ def _status():
             'grafana_url': get_external_grafana_url(),
             'site_name': os.environ.get('SITE_NAME', ''),
         }
-    
+
     setup_docs_endpoints(app)
     setup_proxy_endpoints(app)
 
@@ -40,7 +40,7 @@ def setup_proxy_endpoints(app: FastAPI):
     lifecycle_api_url = trim_url(os.environ.get('LIFECYCLE_URL', 'http://127.0.0.1:7202'))
     logger.info(f'Forwarding API requests to "{lifecycle_api_url}"')
     client = httpx.AsyncClient(base_url=f"{lifecycle_api_url}/")
-    
+
     async def _proxy_api_call(request: Request, path: str, payload=Body(default={})):
         """Forward API call to Lifecycle service"""
         subpath = f'/api/v1/{request.path_params["path"]}'

dashboard/dashboard/api/docs.py

@@ -35,7 +35,7 @@ def _get_docs_index() -> dict:
             'doc_pages': sorted(doc_pages, key=lambda x: x['title'].lower()),
             'plugin_pages': sorted(plugin_pages, key=lambda x: x['title'].lower()),
         }
-    
+
     @app.get('/api/docs/page/{doc_path:path}')
     def _get_docs_page(doc_path: str) -> dict:
         docs_path = _get_docs_root_dir()
@@ -51,7 +51,7 @@ def _get_docs_page(doc_path: str) -> dict:
             'doc_name': doc_path,
             'html_content': html,
         }
-    
+
     @app.get('/api/docs/plugin/{plugin_name}')
     def _get_docs_plugin_page(plugin_name: str) -> dict:
         plugin_client = LifecyclePluginClient()

dashboard/dashboard/api/webview.py

@@ -13,11 +13,11 @@ def setup_web_views(app: FastAPI):
     @app.get("/", tags=['ui'])
     def _ui_root_view():
         return RedirectResponse('/dashboard/ui/')
-    
+
     @app.get("/ui", tags=['ui'])
     def _ui_home_view():
         return FileResponse('static/index.html')
-    
+
     @app.get("/ui/", tags=['ui'])
     def _ui_home_slash_view():
         return FileResponse('static/index.html')
@@ -26,7 +26,7 @@ def _ui_home_slash_view():
         app.mount("/ui/assets", StaticFiles(directory="static/assets/"), name="static_front")
     else:
         logger.warning('No static/assets directory found')
-    
+
     @app.get("/ui/{subpath:path}", tags=['ui'])
     def _ui_home_any_view(subpath: str):
         return FileResponse('static/index.html')

image_builder/image_builder/api.py

@@ -9,7 +9,6 @@
 from image_builder.health import health_response
 from image_builder.scheduler import schedule_tasks_async
 from racetrack_client.client_config.io import load_credentials_from_dict
-from racetrack_client.log.exception import log_exception
 from racetrack_client.log.logs import configure_logs
 from racetrack_client.manifest.load import load_manifest_from_dict
 from racetrack_client.utils.config import load_config

image_builder/image_builder/build.py

@@ -18,7 +18,6 @@
 from image_builder.progress import update_deployment_phase
 from image_builder.verify import verify_manifest_consistency
 from image_builder.warnings import update_deployment_warnings
-from racetrack_client.client.env import merge_env_vars
 from racetrack_client.client_config.client_config import Credentials
 from racetrack_client.log.context_error import wrap_context
 from racetrack_client.log.logs import get_logger

image_builder/image_builder/docker/builder.py

@@ -57,7 +57,7 @@ def build(
         built_images: list[str] = []
         images_num = job_type.images_num
         for image_index in range(images_num):
-            progress = f'({image_index+1}/{images_num})'
+            progress = f'({image_index + 1}/{images_num})'
 
             try:
                 _build_job_image(
@@ -97,7 +97,7 @@ def _build_job_image(
     built_images: list[str],
     build_flags: list[str],
 ):
-    progress = f'({image_index+1}/{images_num})'
+    progress = f'({image_index + 1}/{images_num})'
     image_def: JobTypeImageDef = job_type.images[image_index]
     with phase_context(f'building image {progress}', metric_labels, deployment_id, config, metric_phase='building image'):
 
@@ -161,7 +161,7 @@ def _build_container_image(
         'DOCKER_BUILDKIT=1 docker buildx build',
         f'-t {image_name}',
         f'-f {dockerfile_path}',
-        f'--network=host',
+        '--network=host',
         f'--build-context {JOBTYPE_BUILD_CONTEXT}="{jobtype_dir}"',
     ]
     cmd_parts.extend(build_flags)

image_builder/image_builder/verify.py

@@ -42,11 +42,14 @@ def verify_manifest_consistency(submitted_yaml: str, workspace: Path, repo_dir:
         logger.info(f'Manifest file in Job repository:\n{repo_dict}')
         difference = _differentiate_dicts(repo_dict, submitted_dict)
         warning = ('Submitted job manifest is not consistent with the file found in a repository. '
-                    'Did you forget to do "git push"? '
-                    f'Difference: {difference}')
+                   'Did you forget to do "git push"? '
+                   f'Difference: {difference}')
         logger.warning(warning)
         return warning
 
+    return None
+
+
 def _find_workspace_manifest_file(workspace: Path, repo_dir: Path) -> Optional[Path]:
     paths_to_check = [
         workspace / JOB_MANIFEST_FILENAME,

lifecycle/lifecycle/auth/authenticate_password.py

@@ -7,6 +7,7 @@
 UNUSABLE_PASSWORD_SUFFIX_LENGTH = 40
 UNUSABLE_PASSWORD_PREFIX = "!"
 
+
 def authenticate(username: str, password: str) -> Optional[User]:
     """
     If the given credentials are valid, return a User object.
@@ -18,17 +19,22 @@ def authenticate(username: str, password: str) -> Optional[User]:
 
     return user
 
+
 def authenticate_user(username: str, password: str) -> Optional[User]:
-        try:
-            user: User = LifecycleCache.record_mapper().find_one(User, username=username)
-        except EntityNotFound:
-            # Run the default password hasher once to reduce the timing
-            # difference between an existing and a nonexistent user
-
-            make_password(password)
-        else:
-            if check_password(password, user.password) and user.is_active:
-                return user
+    try:
+        user: User = LifecycleCache.record_mapper().find_one(User, username=username)
+    except EntityNotFound:
+        # Run the default password hasher once to reduce the timing
+        # difference between an existing and a nonexistent user
+
+        make_password(password)
+
+        return None
+
+    if check_password(password, user.password) and user.is_active:
+        return user
+
+    return None
 
 
 class PermissionDenied(Exception):

lifecycle/lifecycle/auth/authorize.py

@@ -243,7 +243,7 @@ def list_permitted_jobs(
         family_to_job_ids[job.name].append(job.id)
 
     job_ids = set()
-    for permission in permissions: 
+    for permission in permissions:
         if permission.job_family_id is None and permission.job_id is None:
             return all_jobs
 

lifecycle/lifecycle/auth/hasher.py

@@ -1,10 +1,17 @@
+from __future__ import annotations
+
 import base64
 import functools
 import hashlib
 import math
-from typing import Optional
+from typing import Optional, Protocol
 import secrets
 
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from _typeshed import ReadableBuffer
+
+
 RANDOM_STRING_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
 
 
@@ -14,6 +21,11 @@ def make_password(password: str) -> str:
     return hasher.encode(password, salt)
 
 
+class Hash(Protocol):
+    def __call__(self, string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> hashlib._Hash:
+        ...
+
+
 class PBKDF2PasswordHasher:
     """
     Secure password hashing using the PBKDF2 algorithm (recommended)
@@ -25,10 +37,10 @@ class PBKDF2PasswordHasher:
 
     algorithm: str = "pbkdf2_sha256"
     iterations: int = 600000
-    digest: 'hashlib._Hash' = hashlib.sha256
+    digest: Hash = hashlib.sha256
     salt_entropy: int = 128
 
-    def encode(self, password: str, salt, iterations: Optional[int]=None):
+    def encode(self, password: str, salt, iterations: Optional[int] = None):
         iterations = iterations or self.iterations
         hash = pbkdf2(password, salt, iterations, digest=self.digest)
         hash = base64.b64encode(hash).decode("ascii").strip()
@@ -60,7 +72,7 @@ def salt(self) -> str:
         return get_random_string(char_count, allowed_chars=RANDOM_STRING_CHARS)
 
 
-def get_random_string(length: int, allowed_chars: str=RANDOM_STRING_CHARS) -> str:
+def get_random_string(length: int, allowed_chars: str = RANDOM_STRING_CHARS) -> str:
     """
     Return a securely generated random string.
 
@@ -74,14 +86,14 @@ def get_random_string(length: int, allowed_chars: str=RANDOM_STRING_CHARS) -> st
     return "".join(secrets.choice(allowed_chars) for i in range(length))
 
 
-def pbkdf2(password: str, salt: str, iterations: int, dklen: int=0, digest: Optional['hashlib._Hash']=None):
+def pbkdf2(password: str, salt: str, iterations: int, dklen: int = 0, digest: Optional[Hash] = None):
     """Return the hash of password using pbkdf2."""
     if digest is None:
         digest = hashlib.sha256
-    dklen = dklen or None
-    password = password.encode("utf-8", "strict")
-    salt = salt.encode("utf-8", "strict")
-    return hashlib.pbkdf2_hmac(digest().name, password, salt, iterations, dklen)
+
+    password_bytes = password.encode("utf-8", "strict")
+    salt_bytes = salt.encode("utf-8", "strict")
+    return hashlib.pbkdf2_hmac(digest().name, password_bytes, salt_bytes, iterations, dklen or None)
 
 
 def constant_time_compare(val1, val2):

lifecycle/lifecycle/auth/users.py

@@ -41,7 +41,7 @@ def register_user_account(username: str, password: str) -> tables.User:
         pass
 
     user = LifecycleCache.record_mapper().create_from_dict(
-        tables.User, 
+        tables.User,
         {
             "username": username,
             "password": make_password(password),
@@ -66,7 +66,7 @@ def register_user_account(username: str, password: str) -> tables.User:
     grant_permission(auth_subject, AuthScope.DEPLOY_JOB.value)
 
     logger.info(f'User account created: {username}')
-    return user
+    return user_record
 
 
 def change_user_password(username: str, old_password: str, new_password: str):

lifecycle/lifecycle/auth/validate.py

@@ -1,28 +1,32 @@
 from pathlib import Path
- 
- 
+
+
 def validate_password(password: str):
     for validator in [validate_password_length, validate_numeric_password, validate_common_password]:
         validator(password)
 
+
 def validate_password_length(password: str):
     if len(password) < 8:
         raise ValidationError("This password is too short. It must contain at least 8 characters.")
-
+
+
 def validate_numeric_password(password: str):
     if password.isdigit():
         raise ValidationError("This password is entirely numeric.")
-
+
+
 def validate_common_password(password: str):
     # password list based on https://gist.github.com/roycewilliams/226886fd01572964e1431ac8afc999ce
     # by Royce Williams
-    passwords_path: str = Path(__file__).resolve().parent / "common-passwords.txt"
-    
+    passwords_path: Path = Path(__file__).resolve().parent / "common-passwords.txt"
+
     with open(passwords_path, "rt", encoding="utf-8") as f:
         passwords = {x.strip() for x in f}
 
     if password.lower().strip() in passwords:
         raise ValidationError("This password is too common.")
 
+
 class ValidationError(Exception):
     pass