Skip to content

Commit 4579278

Browse files
zhravanzhravan
authored
fix: add retry mechanism for service registration to caddy (#656)
1 parent 1483f6f commit 4579278

File tree

3 files changed

+176
-35
lines changed

3 files changed

+176
-35
lines changed

‎cli/app/commands/proxy/proxy.py‎

Lines changed: 80 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
from app.utils.config import CADDY_BASE_URL, LOAD_ENDPOINT, PROXY_PORT, get_active_config, get_yaml_value
88
from app.utils.protocols import LoggerProtocol
9+
from app.utils.retry import retry_with_backoff, wait_for_condition
910

1011
from .messages import (
1112
caddy_connection_failed,
@@ -31,10 +32,45 @@ def _get_caddy_url(port: int, endpoint: str) -> str:
3132
return f"{caddy_base_url.format(port=port)}{endpoint}"
3233

3334

35+
def _check_caddy_ready(proxy_port: int) -> bool:
36+
"""Check if Caddy is ready to accept connections."""
37+
url = _get_caddy_url(proxy_port, "/config/")
38+
try:
39+
response = requests.get(url, timeout=5)
40+
# 200 = config exists, 404 = no config yet but Caddy is responding
41+
return response.status_code in (200, 404)
42+
except (requests.exceptions.ConnectionError, requests.exceptions.RequestException):
43+
return False
44+
45+
46+
def _wait_for_caddy(
47+
proxy_port: int,
48+
logger: Optional[LoggerProtocol] = None,
49+
) -> tuple[bool, Optional[str]]:
50+
"""Wait for Caddy to be ready to accept connections with exponential backoff."""
51+
52+
def on_retry(attempt: int, delay: float) -> None:
53+
if logger:
54+
logger.debug(f"Waiting for Caddy to be ready (attempt {attempt}, retrying in {delay:.1f}s)")
55+
56+
success, error = wait_for_condition(
57+
check_func=lambda: _check_caddy_ready(proxy_port),
58+
on_retry=on_retry,
59+
timeout_message="Caddy not ready",
60+
)
61+
62+
if success and logger:
63+
logger.debug("Caddy is ready")
64+
65+
return success, error
66+
67+
3468
def load_config(
35-
config_file: str, proxy_port: int = default_proxy_port, logger: Optional[LoggerProtocol] = None
69+
config_file: str,
70+
proxy_port: int = default_proxy_port,
71+
logger: Optional[LoggerProtocol] = None,
3672
) -> tuple[bool, Optional[str]]:
37-
"""Load Caddy proxy configuration from a JSON file."""
73+
"""Load Caddy proxy configuration from a JSON file with retry logic."""
3874
if not config_file:
3975
return False, "Configuration file is required"
4076

@@ -47,29 +83,13 @@ def load_config(
4783
try:
4884
if logger:
4985
logger.debug(debug_loading_config_file.format(file=config_file))
50-
86+
5187
with open(config_file, "r") as f:
5288
config_data = json.load(f)
53-
54-
if logger:
55-
logger.debug(debug_config_parsed)
5689

57-
url = _get_caddy_url(proxy_port, caddy_load_endpoint)
5890
if logger:
59-
logger.debug(debug_posting_config.format(url=url))
91+
logger.debug(debug_config_parsed)
6092

61-
response = requests.post(url, json=config_data, headers={"Content-Type": "application/json"}, timeout=10)
62-
63-
if response.status_code == 200:
64-
if logger:
65-
logger.debug(debug_config_loaded_success)
66-
return True, None
67-
else:
68-
error_msg = response.text.strip() if response.text else http_error.format(code=response.status_code)
69-
if logger:
70-
logger.debug(f"Failed to load config: {error_msg}")
71-
return False, error_msg
72-
7393
except FileNotFoundError:
7494
error_msg = config_file_not_found.format(file=config_file)
7595
if logger:
@@ -80,19 +100,45 @@ def load_config(
80100
if logger:
81101
logger.debug(error_msg)
82102
return False, error_msg
83-
except requests.exceptions.ConnectionError as e:
84-
error_msg = caddy_connection_failed.format(error=str(e))
85-
if logger:
86-
logger.debug(error_msg)
87-
return False, error_msg
88-
except requests.exceptions.RequestException as e:
89-
error_msg = request_failed_error.format(error=str(e))
90-
if logger:
91-
logger.debug(error_msg)
92-
return False, error_msg
93-
except Exception as e:
94-
error_msg = unexpected_error.format(error=str(e))
103+
104+
# Wait for Caddy to be ready before attempting to load config
105+
if logger:
106+
logger.debug("Waiting for Caddy to be ready...")
107+
ready, ready_error = _wait_for_caddy(proxy_port, logger)
108+
if not ready:
109+
return False, ready_error
110+
111+
url = _get_caddy_url(proxy_port, caddy_load_endpoint)
112+
113+
def post_config() -> tuple[bool, Optional[str]]:
114+
"""Attempt to post config to Caddy."""
115+
try:
116+
if logger:
117+
logger.debug(debug_posting_config.format(url=url))
118+
119+
response = requests.post(url, json=config_data, headers={"Content-Type": "application/json"}, timeout=10)
120+
121+
if response.status_code == 200:
122+
return True, None
123+
else:
124+
error_msg = response.text.strip() if response.text else http_error.format(code=response.status_code)
125+
return False, error_msg
126+
127+
except requests.exceptions.ConnectionError as e:
128+
return False, caddy_connection_failed.format(error=str(e))
129+
except requests.exceptions.RequestException as e:
130+
return False, request_failed_error.format(error=str(e))
131+
except Exception as e:
132+
return False, unexpected_error.format(error=str(e))
133+
134+
def on_retry(attempt: int, delay: float, last_error: Optional[str]) -> None:
95135
if logger:
96-
logger.debug(error_msg)
97-
return False, error_msg
136+
logger.debug(f"Failed to load config (attempt {attempt}): {last_error}")
137+
logger.debug(f"Retrying in {delay:.1f}s...")
138+
139+
success, error = retry_with_backoff(func=post_config, on_retry=on_retry)
140+
141+
if success and logger:
142+
logger.debug(debug_config_loaded_success)
98143

144+
return success, error

‎cli/app/utils/retry.py‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
import time
2+
from typing import Callable, Optional, Tuple, TypeVar
3+
4+
T = TypeVar("T")
5+
6+
DEFAULT_MAX_RETRIES = 10
7+
DEFAULT_INITIAL_DELAY = 2.0
8+
DEFAULT_MAX_DELAY = 30.0
9+
DEFAULT_BACKOFF_FACTOR = 1.5
10+
11+
12+
def retry_with_backoff(
13+
func: Callable[[], Tuple[bool, Optional[str]]],
14+
max_retries: int = DEFAULT_MAX_RETRIES,
15+
initial_delay: float = DEFAULT_INITIAL_DELAY,
16+
max_delay: float = DEFAULT_MAX_DELAY,
17+
backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
18+
on_retry: Optional[Callable[[int, float, Optional[str]], None]] = None,
19+
) -> Tuple[bool, Optional[str]]:
20+
"""
21+
Execute a function with exponential backoff retry logic.
22+
23+
Args:
24+
func: A callable that returns (success: bool, error: Optional[str])
25+
max_retries: Maximum number of retry attempts
26+
initial_delay: Initial delay between retries in seconds
27+
max_delay: Maximum delay between retries in seconds
28+
backoff_factor: Multiplier for delay after each retry
29+
on_retry: Optional callback called before each retry with (attempt, delay, last_error)
30+
31+
Returns:
32+
Tuple of (success: bool, error: Optional[str])
33+
"""
34+
delay = initial_delay
35+
last_error: Optional[str] = None
36+
37+
for attempt in range(1, max_retries + 1):
38+
try:
39+
success, error = func()
40+
if success:
41+
return True, None
42+
last_error = error
43+
except Exception as e:
44+
last_error = str(e)
45+
46+
if attempt < max_retries:
47+
if on_retry:
48+
on_retry(attempt, delay, last_error)
49+
time.sleep(delay)
50+
delay = min(delay * backoff_factor, max_delay)
51+
52+
return False, last_error
53+
54+
55+
def wait_for_condition(
56+
check_func: Callable[[], bool],
57+
max_retries: int = DEFAULT_MAX_RETRIES,
58+
initial_delay: float = DEFAULT_INITIAL_DELAY,
59+
max_delay: float = DEFAULT_MAX_DELAY,
60+
backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
61+
on_retry: Optional[Callable[[int, float], None]] = None,
62+
timeout_message: str = "Condition not met after max retries",
63+
) -> Tuple[bool, Optional[str]]:
64+
"""
65+
Wait for a condition to become true with exponential backoff.
66+
67+
Args:
68+
check_func: A callable that returns True when condition is met
69+
max_retries: Maximum number of retry attempts
70+
initial_delay: Initial delay between retries in seconds
71+
max_delay: Maximum delay between retries in seconds
72+
backoff_factor: Multiplier for delay after each retry
73+
on_retry: Optional callback called before each retry with (attempt, delay)
74+
timeout_message: Error message if condition is never met
75+
76+
Returns:
77+
Tuple of (success: bool, error: Optional[str])
78+
"""
79+
delay = initial_delay
80+
81+
for attempt in range(1, max_retries + 1):
82+
try:
83+
if check_func():
84+
return True, None
85+
except Exception:
86+
pass
87+
88+
if attempt < max_retries:
89+
if on_retry:
90+
on_retry(attempt, delay)
91+
time.sleep(delay)
92+
delay = min(delay * backoff_factor, max_delay)
93+
94+
return False, f"{timeout_message} ({max_retries} attempts)"
95+

‎cli/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "nixopus"
3-
version = "0.1.29"
3+
version = "0.1.30"
44
description = "A CLI for Nixopus"
55
authors = ["Nixopus <[email protected]>"]
66
readme = "README.md"

0 commit comments

Comments
 (0)