Merge pull request #1920 from aboutcode-org/pipeline-dashboard

Pipeline Dashboard improvements

Author: keshav-space

requirements.txt

@@ -28,10 +28,10 @@ decorator==5.1.1
 defusedxml==0.7.1
 distro==1.7.0
 Django==4.2.22
+django-altcha==0.2.0
 django-crispy-forms==2.3
 django-environ==0.11.2
 django-filter==24.3
-django-recaptcha==4.0.0
 django-widget-tweaks==1.5.0
 djangorestframework==3.15.2
 doc8==0.11.1

setup.cfg

@@ -65,6 +65,7 @@ install_requires =
     crispy-bootstrap4>=2024.1
     django-environ>=0.11.0
     gunicorn>=23.0.0
+    django-altcha==0.2.0
 
     # for the API doc
     drf-spectacular[sidecar]>=0.24.2
@@ -101,8 +102,6 @@ install_requires =
     python-dotenv
     texttable
 
-    django-recaptcha>=4.0.0
-
 
 [options.extras_require]
 dev =

vulnerabilities/admin.py

@@ -21,6 +21,10 @@
 from vulnerabilities.models import VulnerabilityReference
 from vulnerabilities.models import VulnerabilitySeverity
 
+admin.site.site_header = "VulnerableCode Administration"
+admin.site.site_title = "VulnerableCode Admin Portal"
+admin.site.index_title = "Welcome to VulnerableCode Management"
+
 
 @admin.register(Vulnerability)
 class VulnerabilityAdmin(admin.ModelAdmin):
@@ -125,6 +129,7 @@ def __init__(self, *args, **kwargs):
 
     def save(self, commit=True):
         group = super().save(commit=commit)
+        group.save()
         self.save_m2m()
         group.user_set.set(self.cleaned_data["users"])
         return group

vulnerabilities/api_v2.py

@@ -769,15 +769,15 @@ def has_permission(self, request, view):
 
 class PipelineRunAPISerializer(serializers.HyperlinkedModelSerializer):
     status = serializers.SerializerMethodField()
-    execution_time = serializers.SerializerMethodField()
+    runtime = serializers.SerializerMethodField()
     log = serializers.SerializerMethodField()
 
     class Meta:
         model = PipelineRun
         fields = [
             "run_id",
             "status",
-            "execution_time",
+            "runtime",
             "run_start_date",
             "run_end_date",
             "run_exitcode",
@@ -791,9 +791,9 @@ class Meta:
     def get_status(self, run):
         return run.status
 
-    def get_execution_time(self, run):
-        if run.execution_time:
-            return round(run.execution_time, 2)
+    def get_runtime(self, run):
+        if run.runtime:
+            return f"{round(run.runtime, 2)}s"
 
     def get_log(self, run):
         """Return only last 5000 character of log."""
@@ -802,7 +802,8 @@ def get_log(self, run):
 
 class PipelineScheduleAPISerializer(serializers.HyperlinkedModelSerializer):
     url = serializers.HyperlinkedIdentityField(
-        view_name="schedule-detail", lookup_field="pipeline_id"
+        view_name="pipelines-detail",
+        lookup_field="pipeline_id",
     )
     latest_run = serializers.SerializerMethodField()
     next_run_date = serializers.SerializerMethodField()
@@ -830,6 +831,12 @@ def get_latest_run(self, schedule):
             return PipelineRunAPISerializer(latest).data
         return None
 
+    def to_representation(self, schedule):
+        representation = super().to_representation(schedule)
+        representation["run_interval"] = f"{schedule.run_interval}hr"
+        representation["execution_timeout"] = f"{schedule.execution_timeout}hr"
+        return representation
+
 
 class PipelineScheduleCreateSerializer(serializers.ModelSerializer):
     class Meta:
@@ -883,6 +890,11 @@ def get_permissions(self):
             return [IsAdminWithSessionAuth()]
         return super().get_permissions()
 
+    def get_view_name(self):
+        if self.detail:
+            return "Pipeline Instance"
+        return "Pipeline Jobs"
+
 
 class AdvisoriesPackageV2ViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = PackageV2.objects.all().prefetch_related(

vulnerabilities/forms.py

@@ -10,8 +10,7 @@
 from django import forms
 from django.contrib.admin.forms import AdminAuthenticationForm
 from django.core.validators import validate_email
-from django_recaptcha.fields import ReCaptchaField
-from django_recaptcha.widgets import ReCaptchaV2Checkbox
+from django_altcha import AltchaField
 
 from vulnerabilities.models import ApiUser
 
@@ -45,12 +44,12 @@ class AdvisorySearchForm(forms.Form):
 
 
 class ApiUserCreationForm(forms.ModelForm):
-    """
-    Support a simplified creation for API-only users directly from the UI.
-    """
+    """Support a simplified creation for API-only users directly from the UI."""
 
-    captcha = ReCaptchaField(
-        error_messages={"required": ("Captcha is required")}, widget=ReCaptchaV2Checkbox
+    captcha = AltchaField(
+        floating=True,
+        hidefooter=True,
+        hidelogo=True,
     )
 
     class Meta:
@@ -110,9 +109,8 @@ class PipelineSchedulePackageForm(forms.Form):
 
 
 class AdminLoginForm(AdminAuthenticationForm):
-    captcha = ReCaptchaField(
-        error_messages={
-            "required": ("Captcha is required"),
-        },
-        widget=ReCaptchaV2Checkbox(),
+    captcha = AltchaField(
+        floating=True,
+        hidefooter=True,
+        hidelogo=True,
     )

vulnerabilities/migrations/0096_alter_pipelineschedule_run_interval.py

@@ -0,0 +1,71 @@
+# Generated by Django 4.2.22 on 2025-06-24 16:33
+
+import django.core.validators
+from django.db import migrations, models
+from datetime import datetime
+from datetime import timezone
+
+from aboutcode.pipeline import LoopProgress
+
+class Migration(migrations.Migration):
+    def update_old_interval(apps, schema_editor):
+        Pipeline = apps.get_model("vulnerabilities", "PipelineSchedule")
+        pipeline_schedule = Pipeline.objects.all()
+
+        log(f"\nUpdating interval for {pipeline_schedule.count():,d} pipeline")
+        progress = LoopProgress(
+            total_iterations=pipeline_schedule.count(),
+            progress_step=10,
+            logger=log,
+        )
+        for pipeline in progress.iter(pipeline_schedule.iterator()):
+            pipeline.run_interval = pipeline.run_interval * 24
+            pipeline.save()
+    
+    def reverse_update_old_interval(apps, schema_editor):
+        Pipeline = apps.get_model("vulnerabilities", "PipelineSchedule")
+        pipeline_schedule = Pipeline.objects.all()
+
+        log(f"\nReverse interval for {pipeline_schedule.count():,d} pipeline")
+        progress = LoopProgress(
+            total_iterations=pipeline_schedule.count(),
+            progress_step=10,
+            logger=log,
+        )
+        for pipeline in progress.iter(pipeline_schedule.iterator()):
+            pipeline.run_interval = pipeline.run_interval / 24
+            pipeline.save()
+
+    dependencies = [
+        ("vulnerabilities", "0095_alter_apiuser_options"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="pipelineschedule",
+            name="run_interval",
+            field=models.PositiveSmallIntegerField(
+                default=24,
+                help_text="Number of hours to wait between run of this pipeline.",
+                validators=[
+                    django.core.validators.MinValueValidator(
+                        1, message="Interval must be at least 1 hour."
+                    ),
+                    django.core.validators.MaxValueValidator(
+                        8760, message="Interval must be at most 8760 hours."
+                    ),
+                ],
+            ),
+        ),
+        migrations.RunPython(
+            code=update_old_interval,
+            reverse_code=reverse_update_old_interval,
+        ),
+    ]
+
+
+def log(message):
+    now_local = datetime.now(timezone.utc).astimezone()
+    timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+    message = f"{timestamp} {message}"
+    print(message)

vulnerabilities/models.py

@@ -2023,6 +2023,7 @@ class Meta:
 
     class Status(models.TextChoices):
         UNKNOWN = "unknown"
+        DISABLED = "disabled"
         RUNNING = "running"
         SUCCESS = "success"
         FAILURE = "failure"
@@ -2115,7 +2116,7 @@ def run_running(self):
         return self.status == self.Status.RUNNING
 
     @property
-    def execution_time(self):
+    def runtime(self):
         """Return the pipeline execution time."""
         if not self.run_start_date or (not self.run_end_date and not self.run_running):
             return
@@ -2278,11 +2279,11 @@ class PipelineSchedule(models.Model):
 
     run_interval = models.PositiveSmallIntegerField(
         validators=[
-            MinValueValidator(1, message="Interval must be at least 1 day."),
-            MaxValueValidator(365, message="Interval must be at most 365 days."),
+            MinValueValidator(1, message="Interval must be at least 1 hour."),
+            MaxValueValidator(8760, message="Interval must be at most 8760 hours."),
         ],
-        default=1,
-        help_text=("Number of days to wait between run of this pipeline."),
+        default=24,
+        help_text=("Number of hours to wait between run of this pipeline."),
     )
 
     schedule_work_id = models.CharField(
@@ -2360,14 +2361,21 @@ def latest_run_date(self):
         latest_run = self.pipelineruns.values("run_start_date").first()
         return latest_run["run_start_date"]
 
+    @property
+    def latest_run_end_date(self):
+        if not self.pipelineruns.exists():
+            return
+        latest_run = self.pipelineruns.values("run_end_date").first()
+        return latest_run["run_end_date"]
+
     @property
     def next_run_date(self):
         if not self.is_active:
             return
 
         current_date_time = datetime.datetime.now(tz=datetime.timezone.utc)
         if self.latest_run_date:
-            next_execution = self.latest_run_date + datetime.timedelta(days=self.run_interval)
+            next_execution = self.latest_run_date + datetime.timedelta(hours=self.run_interval)
             if next_execution > current_date_time:
                 return next_execution
 
@@ -2376,7 +2384,7 @@ def next_run_date(self):
     @property
     def status(self):
         if not self.is_active:
-            return
+            return PipelineRun.Status.DISABLED
 
         if self.pipelineruns.exists():
             latest = self.pipelineruns.only("pk").first()

vulnerabilities/schedules.py

@@ -14,7 +14,6 @@
 from redis.exceptions import ConnectionError
 
 from vulnerabilities.tasks import enqueue_pipeline
-from vulnerablecode.settings import VULNERABLECODE_PIPELINE_TIMEOUT
 
 log = logging.getLogger(__name__)
 scheduler = django_rq.get_scheduler()
@@ -29,14 +28,13 @@ def schedule_execution(pipeline_schedule, execute_now=False):
     if not execute_now:
         first_execution = pipeline_schedule.next_run_date
 
-    interval_in_seconds = pipeline_schedule.run_interval * 24 * 60 * 60
+    interval_in_seconds = pipeline_schedule.run_interval * 60 * 60
 
     job = scheduler.schedule(
         scheduled_time=first_execution,
         func=enqueue_pipeline,
         args=[pipeline_schedule.pipeline_id],
         interval=interval_in_seconds,
-        result_ttl=f"{VULNERABLECODE_PIPELINE_TIMEOUT}h",
         repeat=None,
     )
     return job._id

vulnerabilities/tasks.py

@@ -106,6 +106,18 @@ def set_run_failure(job, connection, type, value, traceback):
 
 def enqueue_pipeline(pipeline_id):
     pipeline_schedule = models.PipelineSchedule.objects.get(pipeline_id=pipeline_id)
+    if pipeline_schedule.status in [
+        models.PipelineRun.Status.RUNNING,
+        models.PipelineRun.Status.QUEUED,
+    ]:
+        logger.warning(
+            (
+                f"Cannot enqueue a new execution for {pipeline_id} "
+                "until the previous one has finished."
+            )
+        )
+        return
+
     run = models.PipelineRun.objects.create(
         pipeline=pipeline_schedule,
     )

vulnerabilities/templates/admin_login.html

@@ -3,6 +3,38 @@
 
 {% block extrastyle %}{{ block.super }}<link rel="stylesheet" href="{% static "admin/css/login.css" %}">
 {{ form.media }}
+
+<style>
+    /* Altcha theme correction for dark mode */
+
+    html[data-theme="dark"] altcha-widget {
+        --altcha-color-base: #1e1e1e;
+        --altcha-color-text: #ffffff;
+        --altcha-color-border: #444;
+        --altcha-border-radius: 8px;
+        --altcha-border-width: 2px;
+        --altcha-color-footer-bg: #2a2a2a;
+    }
+
+    html[data-theme="light"] altcha-widget {
+        --altcha-color-base: #ffffff;
+        --altcha-color-text: #000000;
+        --altcha-color-border: #ccc;
+        --altcha-border-radius: 8px;
+        --altcha-color-footer-bg: #f9f9f9;
+    }
+
+    @media (prefers-color-scheme: dark) {
+    altcha-widget {
+      --altcha-color-base: #1e1e1e;
+      --altcha-color-text: #ffffff;
+      --altcha-color-border: #444;
+      --altcha-border-radius: 8px;
+      --altcha-border-width: 2px;
+      --altcha-color-footer-bg: #2a2a2a;
+    }
+  }
+</style>
 {% endblock %}
 
 {% block bodyclass %}{{ block.super }} login{% endblock %}
@@ -57,10 +89,8 @@
             <a href="{{ password_reset_url }}">{% translate 'Forgotten your password or username?' %}</a>
         </div>
         {% endif %}
-        <div class="field" style="padding-top: 1rem; text-align: center;">
-            <div class="control" style="display: inline-block;">
-                {{ form.captcha }}
-            </div>
+        <div class="field" style="padding-top: 0.5rem;">
+            {{ form.captcha }}
         </div>
         <div class="submit-row">
             <input type="submit" value="{% translate 'Log in' %}">