ref: add processor label to queue size metric (#232)

Author: quitrk

skynet/modules/monitoring.py

@@ -54,6 +54,14 @@
     subsystem=PROMETHEUS_SUMMARIES_SUBSYSTEM,
 )
 
+SUMMARY_QUEUE_SIZE_BY_PROCESSOR_METRIC = Gauge(
+    'summary_queue_size_by_processor',
+    documentation='Number of jobs in the queue per processor',
+    namespace=PROMETHEUS_NAMESPACE,
+    subsystem=PROMETHEUS_SUMMARIES_SUBSYSTEM,
+    labelnames=['processor'],
+)
+
 SUMMARY_ERROR_COUNTER = Counter(
     'summary_errors',
     documentation='Number of jobs that have failed',

skynet/modules/ttt/summaries/jobs.py

@@ -33,6 +33,7 @@
     SUMMARY_ERROR_COUNTER,
     SUMMARY_FULL_DURATION_METRIC,
     SUMMARY_INPUT_LENGTH_METRIC,
+    SUMMARY_QUEUE_SIZE_BY_PROCESSOR_METRIC,
     SUMMARY_QUEUE_SIZE_METRIC,
     SUMMARY_TIME_IN_QUEUE_METRIC,
 )
@@ -117,6 +118,9 @@ async def update_summary_queue_metric() -> None:
         processor_queue_size = await db.llen(pending_key)
         total_queue_size += processor_queue_size
 
+        # Set individual processor queue size metric
+        SUMMARY_QUEUE_SIZE_BY_PROCESSOR_METRIC.labels(processor=processor.value).set(processor_queue_size)
+
     SUMMARY_QUEUE_SIZE_METRIC.set(total_queue_size)