Merge pull request #8 from msalinas92/feature/refresh-cache

Probabilistic Cache Refreshing

Author: msalinas92

Cargo.lock

@@ -56,7 +56,8 @@ metrics = "0.24.2"
 metrics-macros = "0.7.1"
 metrics-exporter-prometheus = "0.17.0"
 futures = "0.3"
-
+rand = "0.8"
+tower = "0.5.2"
 
 
 [dev-dependencies]

Cargo.toml

@@ -91,36 +91,86 @@ Check if URI is marked as degraded (should_failover)
                                    └── Downstream failed --> try_cache fallback
 ```
 
+---
+## 🔁 Probabilistic Cache Refreshing
+
+To ensure cached responses stay fresh over time, CacheBolt supports **probabilistic refreshes**.  
+You can configure a percentage of requests that will intentionally bypass the cache and fetch a fresh version from the backend.
+
+```yaml
+cache:
+  refresh_percentage: 10
+```
+
+In the example above, approximately 1 in every 10 requests to the same cache key will bypass the memory and persistent cache and trigger a revalidation from the upstream server.
+The refreshed response is then stored again in both memory and persistent storage backends.
+
+This strategy helps:
+
+Keep long-lived cache entries updated
+
+Avoid cache staleness without needing manual invalidation
+
+Distribute backend load gradually and intelligently
+
+If set to 0, no automatic refresh will occur unless the cache is manually purged.
 
 ---
 ## 🔧 Configuration
 
 The config is written in YAML. Example:
 
 ```yaml
+# 🔧 Unique identifier for this CacheBolt instance
 app_id: my-service
 
+# 🚦 Maximum number of concurrent outbound requests to the downstream service
 max_concurrent_requests: 200
+
+# 🌐 Base URL of the upstream API/backend to which requests are proxied
 downstream_base_url: http://localhost:4000
+
+# ⏱️ Timeout (in seconds) for downstream requests before failing
 downstream_timeout_secs: 5
 
-storage_backend: s3  # options: gcs, s3, azure, local
+# 💾 Backend used for persistent cache storage
+# Available options: gcs, s3, azure, local
+storage_backend: s3
+
+# 🪣 Name of the Google Cloud Storage bucket (used if storage_backend is 'gcs')
 gcs_bucket: cachebolt
+
+# 🪣 Name of the Amazon S3 bucket (used if storage_backend is 's3')
 s3_bucket: my-cachebolt-bucket
+
+# 📦 Name of the Azure Blob Storage container (used if storage_backend is 'azure')
 azure_container: cachebolt-container
 
-memory_eviction:
-  threshold_percent: 90
+# 🧠 Memory cache configuration
+cache:
+  # 🚨 System memory usage threshold (%) above which in-memory cache will start evicting entries
+  memory_threshold: 80
+
+  # 🔁 Percentage of requests (per key) that should trigger a refresh from backend instead of using cache
+  # Example: 10% means 1 in every 10 requests will bypass cache
+  refresh_percentage: 10
 
+# ⚠️ Latency-based failover configuration
 latency_failover:
-  default_max_latency_ms: 300
+  # ⌛ Default maximum allowed latency in milliseconds for any request
+  default_max_latency_ms: 3000
+
+  # 🛣️ Path-specific latency thresholds
   path_rules:
     - pattern: "^/api/v1/products/.*"
-      max_latency_ms: 150
+      max_latency_ms: 1500
     - pattern: "^/auth/.*"
-      max_latency_ms: 100
+      max_latency_ms: 1000
+
+# 🚫 List of request headers to ignore when computing cache keys (case-insensitive)
 ignored_headers:
   - postman-token
+  - if-none-match
 ```
 
 ---

README.md

@@ -1,26 +1,50 @@
+# 🔧 Unique identifier for this CacheBolt instance
 app_id: my-service
 
+# 🚦 Maximum number of concurrent outbound requests to the downstream service
 max_concurrent_requests: 200
+
+# 🌐 Base URL of the upstream API/backend to which requests are proxied
 downstream_base_url: http://localhost:4000
+
+# ⏱️ Timeout (in seconds) for downstream requests before failing
 downstream_timeout_secs: 5
 
-storage_backend: s3 # opciones: gcs, s3, azure, local
+# 💾 Backend used for persistent cache storage
+# Available options: gcs, s3, azure, local
+storage_backend: s3
+
+# 🪣 Name of the Google Cloud Storage bucket (used if storage_backend is 'gcs')
 gcs_bucket: cachebolt
+
+# 🪣 Name of the Amazon S3 bucket (used if storage_backend is 's3')
 s3_bucket: my-cachebolt-bucket
+
+# 📦 Name of the Azure Blob Storage container (used if storage_backend is 'azure')
 azure_container: cachebolt-container
 
+# 🧠 Memory cache configuration
+cache:
+  # 🚨 System memory usage threshold (%) above which in-memory cache will start evicting entries
+  memory_threshold: 80
 
-memory_eviction:
-  threshold_percent: 90
+  # 🔁 Percentage of requests (per key) that should trigger a refresh from backend instead of using cache
+  # Example: 10% means 1 in every 10 requests will bypass cache
+  refresh_percentage: 10
 
+# ⚠️ Latency-based failover configuration
 latency_failover:
-  default_max_latency_ms: 300
+  # ⌛ Default maximum allowed latency in milliseconds for any request
+  default_max_latency_ms: 3000
+
+  # 🛣️ Path-specific latency thresholds
   path_rules:
     - pattern: "^/api/v1/products/.*"
-      max_latency_ms: 150
+      max_latency_ms: 1500
     - pattern: "^/auth/.*"
-      max_latency_ms: 100
+      max_latency_ms: 1000
 
+# 🚫 List of request headers to ignore when computing cache keys (case-insensitive)
 ignored_headers:
   - postman-token
-  - if-none-match
+  - if-none-match

config.yaml

@@ -1,3 +1,17 @@
+// Copyright (C) 2025 Matías Salinas ([email protected])
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use crate::memory::memory::MEMORY_CACHE;
 use axum::{extract::Query, http::StatusCode, response::IntoResponse, Json};
 use serde::{Deserialize, Serialize};

src/admin/clean.rs

@@ -17,7 +17,6 @@ use serde::Deserialize;
 use std::{collections::HashSet, error::Error, fs};
 
 /// Supported persistent storage backends for the cache.
-/// This enum is deserialized from lowercase strings in the YAML config.
 #[derive(Debug, Deserialize, PartialEq, Clone)]
 #[serde(rename_all = "lowercase")]
 pub enum StorageBackend {
@@ -27,20 +26,23 @@ pub enum StorageBackend {
     Local,
 }
 
-/// Configuration for memory-based eviction strategy.
-/// Eviction triggers when system memory usage exceeds a certain percentage.
+/// Cache-related settings for memory usage and re-cache policies.
 #[derive(Debug, Deserialize, Clone)]
-pub struct MemoryEviction {
+pub struct CacheSettings {
     /// Memory usage threshold as a percentage (e.g., 80 = 80%).
-    pub threshold_percent: usize,
+    pub memory_threshold: usize,
+
+     /// Percentage of fallback requests that should attempt revalidation.
+    #[serde(default)]
+    pub refresh_percentage: u8, 
 }
 
 /// Describes latency thresholds per path to decide when to fallback to the cache.
-/// Useful for protecting the system when downstream responses become too slow.
 #[derive(Debug, Deserialize, Clone)]
 pub struct MaxLatencyRule {
     /// Regex pattern to match request paths (e.g., ^/api/products).
     pub pattern: String,
+
     /// Maximum allowable response time in milliseconds for this pattern.
     pub max_latency_ms: u64,
 }
@@ -50,6 +52,7 @@ pub struct MaxLatencyRule {
 pub struct LatencyFailover {
     /// Default latency limit in milliseconds if no rule matches.
     pub default_max_latency_ms: u64,
+
     /// Specific path-based rules, applied in order.
     pub path_rules: Vec<MaxLatencyRule>,
 }
@@ -79,8 +82,8 @@ pub struct Config {
     /// Timeout for downstream requests in seconds.
     pub downstream_timeout_secs: u64,
 
-    /// Memory eviction policy settings.
-    pub memory_eviction: MemoryEviction,
+    /// Cache settings including memory limits and re-cache rules.
+    pub cache: CacheSettings,
 
     /// Latency-based failover rules.
     pub latency_failover: LatencyFailover,
@@ -115,18 +118,28 @@ impl Config {
             StorageBackend::Gcs if parsed.gcs_bucket.trim().is_empty() => {
                 return Err("GCS backend selected but gcs_bucket is empty.".into());
             }
+            StorageBackend::S3 if parsed.s3_bucket.trim().is_empty() => {
+                return Err("S3 backend selected but s3_bucket is empty.".into());
+            }
+            StorageBackend::Azure if parsed.azure_container.trim().is_empty() => {
+                return Err("Azure backend selected but azure_container is empty.".into());
+            }
             _ => {}
         }
 
-        // Provide info logs about latency fallback rules
+        // Validate memory threshold
+        if parsed.cache.memory_threshold == 0 || parsed.cache.memory_threshold > 100 {
+            return Err("cache.memory_threshold must be between 1 and 100.".into());
+        }
+
+        // Log latency failover rules
         if parsed.latency_failover.path_rules.is_empty() {
             tracing::info!(
                 "No per-path latency rules defined. Using default max latency: {}ms",
                 parsed.latency_failover.default_max_latency_ms
             );
         } else {
             for rule in &parsed.latency_failover.path_rules {
-                
                 tracing::info!(
                     "Latency rule: pattern = '{}', max_latency = {}ms",
                     rule.pattern,
@@ -138,6 +151,7 @@ impl Config {
         Ok(parsed)
     }
 
+    /// Returns the list of headers to ignore (lowercased).
     pub fn ignored_headers_set(&self) -> HashSet<String> {
         self.ignored_headers
             .clone()
@@ -146,4 +160,4 @@ impl Config {
             .map(|h| h.to_ascii_lowercase())
             .collect()
     }
-}
+}

src/config.rs

@@ -192,6 +192,7 @@ async fn main() {
     // ------------------------------------------------------
     let app = Router::new()
         .route("/metrics", get(move || async move { handle.render() }))
+        .route("/", get(proxy::proxy_handler)) 
         .route("/*path", get(proxy::proxy_handler))
         .route("/cache", delete(invalidate_handler));
 

src/main.rs

@@ -73,14 +73,14 @@ pub async fn load_into_memory(data: Vec<(String, CachedResponse)>) {
 /// Monitors system memory usage and evicts LRU entries if usage exceeds the configured threshold.
 /// This function is designed to prevent the application from consuming too much system memory.
 ///
-/// The threshold is defined in `config.yaml` under `memory_eviction.threshold_percent`.
+/// The threshold is defined in `config.yaml` under `cache.memory_threshold`.
 ///
 /// # Arguments
 /// * `cache` - A mutable reference to the global LRU cache to perform eviction on.
 pub async fn maybe_evict_if_needed(cache: &mut LruCache<String, CachedResponse, RandomState>) {
     let config = CONFIG.get();
     let threshold_percent = config
-        .map(|c| c.memory_eviction.threshold_percent)
+        .map(|c| c.cache.memory_threshold)
         .unwrap_or(80);
 
     let (used_kib, total_kib) = get_memory_usage_kib();

src/memory/memory.rs

@@ -17,14 +17,16 @@ use hyper::client::HttpConnector;
 use hyper::{Body, Client, Request, Response};
 use once_cell::sync::Lazy;
 use sha2::{Digest, Sha256};
-use std::sync::Arc;
+use std::sync::{Arc};
 use tokio::sync::{Semaphore, mpsc};
 use tokio::time::{Duration, Instant, timeout};
 
 use crate::config::{CONFIG, StorageBackend};
 use crate::memory::memory;
 use crate::rules::latency::{get_max_latency_for_path, mark_latency_fail, should_failover};
 use crate::storage::{azure, gcs, local, s3};
+use crate::rules::refresh::should_refresh;
+
 use metrics::{counter, histogram}; // ✅
 
 // ------------------------------------------
@@ -117,17 +119,23 @@ pub async fn proxy_handler(req: Request<Body>) -> impl IntoResponse {
     let key_source = format!("{}|{}", uri, relevant_headers);
     let key = hash_uri(&key_source);
 
+    let force_refresh = should_refresh(&key);
+
+
+
     // If the URI is in failover mode, serve from cache
-    if should_failover(&uri) {
+    if should_failover(&uri) && !force_refresh{
         tracing::info!("⚠️ Using fallback for '{}'", uri);
         counter!("cachebolt_failover_total", "uri" => uri.clone()).increment(1);
         return try_cache(&key).await;
     }
 
     // Try memory cache first
     if let Some(cached) = memory::get_from_memory(&key).await {
-        counter!("cachebolt_memory_hits_total", "uri" => uri.clone()).increment(1);
-        return build_response(cached.body.clone(), cached.headers.clone());
+        if !force_refresh {
+            counter!("cachebolt_memory_hits_total", "uri" => uri.clone()).increment(1);
+            return build_response(cached.body.clone(), cached.headers.clone());
+        }
     }
 
     // Try to acquire concurrency slot

src/proxy.rs

@@ -32,7 +32,7 @@ pub fn should_failover(uri: &str) -> bool {
     let now = Instant::now();
     let map = LATENCY_FAILS.read().unwrap();
     if let Some(&last_fail) = map.get(&key) {
-        now.duration_since(last_fail) < Duration::from_secs(300)
+        now.duration_since(last_fail) < Duration::from_secs(10)
     } else {
         false
     }