Convert a bunch of gauges (#34513)

GitOrigin-RevId: 0889bfc972867ec934018ab0f117ea2b8a754c92

Author: gautamg795

crates/application/src/application_function_runner/metrics.rs

@@ -6,9 +6,8 @@ use metrics::{
     log_counter,
     log_counter_with_labels,
     log_distribution,
-    log_gauge_with_labels,
+    log_distribution_with_labels,
     register_convex_counter,
-    register_convex_gauge,
     register_convex_histogram,
     StaticMetricLabel,
     StatusTimer,
@@ -69,7 +68,7 @@ pub enum OutstandingFunctionState {
     Waiting,
 }
 
-register_convex_gauge!(
+register_convex_histogram!(
     APPLICATION_FUNCTION_RUNNER_OUTSTANDING_TOTAL,
     "The number of currently outstanding functions of a given type. Includes both running and \
      waiting functions",
@@ -88,7 +87,7 @@ pub fn log_outstanding_functions(
             OutstandingFunctionState::Waiting => "waiting",
         },
     );
-    log_gauge_with_labels(
+    log_distribution_with_labels(
         &APPLICATION_FUNCTION_RUNNER_OUTSTANDING_TOTAL,
         total as f64,
         vec![udf_type.metric_label(), state_label, env.metric_label()],

crates/application/src/cache/metrics.rs

@@ -165,6 +165,8 @@ pub fn log_validate_system_time_in_the_future() {
     log_counter(&CACHE_VALIDATE_SYSTEM_TIME_IN_THE_FUTURE_TOTAL, 1);
 }
 
+// n.b. this gauge is safe in a multi-instance context because it is shared
+// across all instances.
 register_convex_gauge!(CACHE_SIZE_BYTES, "Size of the cache in bytes");
 pub fn log_cache_size(size: usize) {
     log_gauge(&CACHE_SIZE_BYTES, size as f64)
@@ -194,6 +196,8 @@ register_convex_counter!(
     QUERY_CACHE_EVICTED_TOTAL,
     "The total number of records evicted",
 );
+// n.b. this gauge is safe in a multi-instance context because it is shared
+// across all instances.
 register_convex_gauge!(
     QUERY_CACHE_EVICTED_AGE_SECONDS,
     "The age of the last evicted entry",

crates/application/src/cron_jobs/metrics.rs

@@ -4,9 +4,7 @@ use errors::ErrorMetadataAnyhowExt;
 use metrics::{
     log_counter_with_labels,
     log_distribution,
-    log_gauge,
     register_convex_counter,
-    register_convex_gauge,
     register_convex_histogram,
     StaticMetricLabel,
     STATUS_LABEL,
@@ -38,7 +36,7 @@ pub fn log_cron_job_failure(e: &anyhow::Error) {
     )
 }
 
-register_convex_gauge!(CRON_JOB_EXECUTION_LAG_SECONDS, "Cron job execution lag");
+register_convex_histogram!(CRON_JOB_EXECUTION_LAG_SECONDS, "Cron job execution lag");
 pub fn log_cron_job_execution_lag(lag: Duration) {
-    log_gauge(&CRON_JOB_EXECUTION_LAG_SECONDS, lag.as_secs_f64());
+    log_distribution(&CRON_JOB_EXECUTION_LAG_SECONDS, lag.as_secs_f64());
 }

crates/application/src/metrics.rs

@@ -1,9 +1,7 @@
 use metrics::{
     log_counter_with_labels,
     log_distribution_with_labels,
-    log_gauge_with_labels,
     register_convex_counter,
-    register_convex_gauge,
     register_convex_histogram,
     StaticMetricLabel,
     StatusTimer,
@@ -53,28 +51,15 @@ pub struct AppWorkerStatus {
 
 impl Drop for AppWorkerStatus {
     fn drop(&mut self) {
-        log_worker_status(false, self.name);
+        tracing::debug!("Worker {} finished", self.name);
     }
 }
 
-register_convex_gauge!(
-    APP_WORKER_IN_PROGRESS_TOTAL,
-    "1 if a worker is working, 0 otherwise",
-    &["worker"],
-);
 pub fn log_worker_starting(name: &'static str) -> AppWorkerStatus {
-    log_worker_status(true, name);
+    tracing::debug!("Worker {} started", name);
     AppWorkerStatus { name }
 }
 
-fn log_worker_status(is_working: bool, name: &'static str) {
-    log_gauge_with_labels(
-        &APP_WORKER_IN_PROGRESS_TOTAL,
-        if is_working { 1f64 } else { 0f64 },
-        vec![StaticMetricLabel::new("worker", name)],
-    )
-}
-
 register_convex_counter!(
     TABLE_SUMMARY_CHECKPOINT_TOTAL,
     "Number of table summary checkpoint writes",

crates/application/src/scheduled_jobs/metrics.rs

@@ -3,11 +3,10 @@ use std::time::Duration;
 use errors::ErrorMetadataAnyhowExt;
 use metrics::{
     log_counter_with_labels,
+    log_distribution,
     log_distribution_with_labels,
-    log_gauge,
     prometheus::VMHistogram,
     register_convex_counter,
-    register_convex_gauge,
     register_convex_histogram,
     StaticMetricLabel,
     Timer,
@@ -50,20 +49,20 @@ pub fn log_scheduled_job_failure(e: &anyhow::Error, prev_failures: u32) {
     );
 }
 
-register_convex_gauge!(
+register_convex_histogram!(
     SCHEDULED_JOB_EXECUTION_LAG_SECONDS,
     "Schedule job execution lag"
 );
 pub fn log_scheduled_job_execution_lag(lag: Duration) {
-    log_gauge(&SCHEDULED_JOB_EXECUTION_LAG_SECONDS, lag.as_secs_f64());
+    log_distribution(&SCHEDULED_JOB_EXECUTION_LAG_SECONDS, lag.as_secs_f64());
 }
 
-register_convex_gauge!(
+register_convex_histogram!(
     SCHEDULED_JOB_NUM_RUNNING_TOTAL,
     "Number of currently executing scheduled jobs"
 );
 pub fn log_num_running_jobs(num_running: usize) {
-    log_gauge(&SCHEDULED_JOB_NUM_RUNNING_TOTAL, num_running as f64);
+    log_distribution(&SCHEDULED_JOB_NUM_RUNNING_TOTAL, num_running as f64);
 }
 
 register_convex_histogram!(

crates/common/src/metrics.rs

@@ -43,6 +43,8 @@ pub fn log_undefined_filter() {
     log_counter(&COMMON_UNDEFINED_FILTER_TOTAL, 1);
 }
 
+// Note: These Codel queue gauges are incorrect if the process contains multiple
+// queues.
 register_convex_gauge!(COMMON_CODEL_QUEUE_LENGTH_TOTAL, "Length of the CoDel queue");
 pub fn log_codel_queue_size(size: usize) {
     log_gauge(&COMMON_CODEL_QUEUE_LENGTH_TOTAL, size as f64)

crates/database/src/index_worker.rs

@@ -99,7 +99,6 @@ use crate::{
     metrics::{
         log_index_backfilled,
         log_num_indexes_to_backfill,
-        log_worker_starting,
     },
     retention::LeaderRetentionManager,
     Database,
@@ -295,7 +294,6 @@ impl<RT: Runtime> IndexWorker<RT> {
     async fn run(&mut self) -> anyhow::Result<()> {
         tracing::info!("Starting IndexWorker");
         loop {
-            let status = log_worker_starting("IndexWorker");
             // Get all the documents from the `_index` table.
             let mut tx = self.database.begin(Identity::system()).await?;
             // Index doesn't have `by_creation_time` index, and thus can't be queried via
@@ -357,7 +355,6 @@ impl<RT: Runtime> IndexWorker<RT> {
             if self.should_terminate {
                 return Ok(());
             }
-            drop(status);
 
             let token = tx.into_token()?;
             let subscription = self.database.subscribe(token).await?;

crates/database/src/index_workers/fast_forward.rs

@@ -57,7 +57,6 @@ use crate::{
         retriable_worker::RetriableWorker,
         timeout_with_jitter,
     },
-    metrics::log_worker_starting,
     text_index_worker::fast_forward::TextFastForward,
     vector_index_worker::fast_forward::VectorFastForward,
     Database,
@@ -120,7 +119,6 @@ impl FastForwardIndexWorker {
         let mut vector_search_last_fast_forward_info: Option<LastFastForwardInfo> = None;
 
         loop {
-            let status = log_worker_starting("TextSearchFastForward");
             tracing::debug!("FastForwardWorker checking if we can fast forward");
             Self::fast_forward::<RT, TextSnapshotVersion, TextFastForward>(
                 "TextSearch",
@@ -129,16 +127,13 @@ impl FastForwardIndexWorker {
                 &mut text_search_last_fast_forward_info,
             )
             .await?;
-            drop(status);
-            let status = log_worker_starting("VectorSearchFastForward");
             Self::fast_forward::<RT, (), VectorFastForward>(
                 "VectorSearch",
                 rt,
                 db,
                 &mut vector_search_last_fast_forward_info,
             )
             .await?;
-            drop(status);
 
             backoff.reset();
             timeout_with_jitter(rt, *DATABASE_WORKERS_POLL_INTERVAL).await

crates/database/src/index_workers/search_worker.rs

@@ -40,7 +40,6 @@ use crate::{
         timeout_with_jitter,
         writer::SearchIndexMetadataWriter,
     },
-    metrics::log_worker_starting,
     text_index_worker::{
         compactor::{
             new_text_compactor,
@@ -215,9 +214,7 @@ impl<RT: Runtime> SearchIndexWorker<RT> {
         backoff: &mut Backoff,
     ) -> anyhow::Result<()> {
         loop {
-            let status = log_worker_starting(name);
             let (metrics, token) = self.step().await?;
-            drop(status);
 
             if !metrics.is_empty() {
                 // We did some useful work this loop iteration that we expect is committed.

crates/database/src/metrics.rs

@@ -12,10 +12,7 @@ use metrics::{
     log_counter_with_labels,
     log_distribution,
     log_distribution_with_labels,
-    log_gauge,
-    log_gauge_with_labels,
     register_convex_counter,
-    register_convex_gauge,
     register_convex_histogram,
     IntoLabel,
     StaticMetricLabel,
@@ -33,25 +30,25 @@ use crate::{
     Transaction,
 };
 
-register_convex_gauge!(
+register_convex_histogram!(
     DOCUMENTS_SIZE_BYTES,
     "Total size of document store in bytes"
 );
 pub fn log_document_store_size(total_size: u64) {
-    log_gauge(&DOCUMENTS_SIZE_BYTES, total_size as f64);
+    log_distribution(&DOCUMENTS_SIZE_BYTES, total_size as f64);
 }
 
-register_convex_gauge!(DOCUMENTS_KEYS_TOTAL, "Total number of document keys");
+register_convex_histogram!(DOCUMENTS_KEYS_TOTAL, "Total number of document keys");
 pub fn log_num_keys(num_keys: u64) {
-    log_gauge(&DOCUMENTS_KEYS_TOTAL, num_keys as f64);
+    log_distribution(&DOCUMENTS_KEYS_TOTAL, num_keys as f64);
 }
 
-register_convex_gauge!(
+register_convex_histogram!(
     INDEXES_TO_BACKFILL_TOTAL,
     "Number of indexes needing backfill"
 );
 pub fn log_num_indexes_to_backfill(num_indexes: usize) {
-    log_gauge(&INDEXES_TO_BACKFILL_TOTAL, num_indexes as f64);
+    log_distribution(&INDEXES_TO_BACKFILL_TOTAL, num_indexes as f64);
 }
 
 register_convex_counter!(INDEXES_BACKFILLED_TOTAL, "Number of indexes backfilled");
@@ -406,49 +403,49 @@ pub fn retention_delete_document_chunk_timer() -> Timer<VMHistogram> {
     Timer::new(&RETENTION_DELETE_DOCUMENT_CHUNK_SECONDS)
 }
 
-register_convex_gauge!(RETENTION_CURSOR_AGE_SECONDS, "Age of the retention cursor");
+register_convex_histogram!(RETENTION_CURSOR_AGE_SECONDS, "Age of the retention cursor");
 pub fn log_retention_cursor_age(age_secs: f64) {
-    log_gauge(&RETENTION_CURSOR_AGE_SECONDS, age_secs)
+    log_distribution(&RETENTION_CURSOR_AGE_SECONDS, age_secs)
 }
 
-register_convex_gauge!(
+register_convex_histogram!(
     RETENTION_CURSOR_LAG_SECONDS,
     "Lag between the retention cursor and the min index snapshot"
 );
 pub fn log_retention_cursor_lag(age_secs: f64) {
-    log_gauge(&RETENTION_CURSOR_LAG_SECONDS, age_secs)
+    log_distribution(&RETENTION_CURSOR_LAG_SECONDS, age_secs)
 }
 
-register_convex_gauge!(
+register_convex_histogram!(
     DOCUMENT_RETENTION_CURSOR_AGE_SECONDS,
     "Age of the document retention cursor"
 );
 pub fn log_document_retention_cursor_age(age_secs: f64) {
-    log_gauge(&DOCUMENT_RETENTION_CURSOR_AGE_SECONDS, age_secs)
+    log_distribution(&DOCUMENT_RETENTION_CURSOR_AGE_SECONDS, age_secs)
 }
 
-register_convex_gauge!(
+register_convex_histogram!(
     DOCUMENT_RETENTION_CURSOR_LAG_SECONDS,
     "Lag between the retention cursor and the min document snapshot"
 );
 pub fn log_document_retention_cursor_lag(age_secs: f64) {
-    log_gauge(&DOCUMENT_RETENTION_CURSOR_LAG_SECONDS, age_secs)
+    log_distribution(&DOCUMENT_RETENTION_CURSOR_LAG_SECONDS, age_secs)
 }
 
-register_convex_gauge!(
+register_convex_counter!(
     RETENTION_MISSING_CURSOR_INFO,
     "Index retention has no cursor"
 );
 pub fn log_retention_no_cursor() {
-    log_gauge(&RETENTION_MISSING_CURSOR_INFO, 1.0)
+    log_counter(&RETENTION_MISSING_CURSOR_INFO, 1)
 }
 
-register_convex_gauge!(
+register_convex_counter!(
     DOCUMENT_RETENTION_MISSING_CURSOR_INFO,
     "Document retention has no cursor"
 );
 pub fn log_document_retention_no_cursor() {
-    log_gauge(&DOCUMENT_RETENTION_MISSING_CURSOR_INFO, 1.0)
+    log_counter(&DOCUMENT_RETENTION_MISSING_CURSOR_INFO, 1)
 }
 
 register_convex_counter!(
@@ -645,34 +642,6 @@ pub fn log_virtual_table_query() {
     log_counter(&VIRTUAL_TABLE_QUERY_REQUESTS_TOTAL, 1);
 }
 
-pub struct DatabaseWorkerStatus {
-    name: &'static str,
-}
-
-impl Drop for DatabaseWorkerStatus {
-    fn drop(&mut self) {
-        log_worker_status(false, self.name);
-    }
-}
-
-register_convex_gauge!(
-    DATABASE_WORKER_IN_PROGRESS_TOTAL,
-    "1 if a worker is working, 0 otherwise",
-    &["worker"],
-);
-pub fn log_worker_starting(name: &'static str) -> DatabaseWorkerStatus {
-    log_worker_status(true, name);
-    DatabaseWorkerStatus { name }
-}
-
-fn log_worker_status(is_working: bool, name: &'static str) {
-    log_gauge_with_labels(
-        &DATABASE_WORKER_IN_PROGRESS_TOTAL,
-        if is_working { 1f64 } else { 0f64 },
-        vec![StaticMetricLabel::new("worker", name)],
-    )
-}
-
 register_convex_histogram!(
     SEARCH_AND_VECTOR_BOOTSTRAP_SECONDS,
     "Time taken to bootstrap text and vector indexes",