Skip to content

Commit 24dbbbe

Browse files
committed
check for any running processes
1 parent 0affd4a commit 24dbbbe

File tree

1 file changed

+81
-0
lines changed

1 file changed

+81
-0
lines changed

packages/sandbox/src/sandbox.ts

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ export class Sandbox<Env = unknown> extends Container<Env> implements ISandbox {
6767
private readonly ACTIVITY_RENEWAL_THROTTLE_MS = 5000; // Throttle renewals to once per 5 seconds
6868
private readonly STREAM_HEALTH_CHECK_INTERVAL_MS = 30000; // Check sandbox health every 30 seconds during streaming
6969
private readonly STREAM_READ_TIMEOUT_MS = 300000; // 5 minutes timeout for stream reads (detects hung streams)
70+
private readonly PROCESS_MONITOR_INTERVAL_MS = 5000; // Check for running processes every 5 seconds
71+
private processMonitorInterval: ReturnType<typeof setInterval> | null = null;
72+
private lastProcessMonitorRenewal: number = 0;
7073

7174
constructor(ctx: DurableObject['ctx'], env: Env) {
7275
super(ctx, env);
@@ -190,6 +193,57 @@ export class Sandbox<Env = unknown> extends Container<Env> implements ISandbox {
190193
}
191194
}
192195

196+
/**
197+
* Start the global process monitor that keeps the container alive
198+
* while any processes are running, even without active streams.
199+
*/
200+
private startProcessMonitor(): void {
201+
// Don't start if already running
202+
if (this.processMonitorInterval) {
203+
return;
204+
}
205+
206+
this.logger.debug('Starting global process monitor');
207+
208+
this.processMonitorInterval = setInterval(async () => {
209+
try {
210+
const processList = await this.client.processes.listProcesses();
211+
const runningProcesses = processList.processes.filter(
212+
p => p.status === 'running' || p.status === 'starting'
213+
);
214+
215+
if (runningProcesses.length > 0) {
216+
const now = Date.now();
217+
if (now - this.lastProcessMonitorRenewal >= this.ACTIVITY_RENEWAL_THROTTLE_MS) {
218+
this.renewActivityTimeout();
219+
this.lastProcessMonitorRenewal = now;
220+
this.logger.debug(
221+
`Global process monitor renewed activity timeout due to ${runningProcesses.length} running process(es): ${runningProcesses.map(p => p.id).join(', ')}`
222+
);
223+
}
224+
} else {
225+
// No running processes, stop the monitor
226+
this.logger.debug('No running processes found, stopping global process monitor');
227+
this.stopProcessMonitor();
228+
}
229+
} catch (error) {
230+
// Non-fatal: if we can't check processes, just log and continue
231+
this.logger.debug(`Global process monitor failed to check running processes: ${error instanceof Error ? error.message : String(error)}`);
232+
}
233+
}, this.PROCESS_MONITOR_INTERVAL_MS);
234+
}
235+
236+
/**
237+
* Stop the global process monitor
238+
*/
239+
private stopProcessMonitor(): void {
240+
if (this.processMonitorInterval) {
241+
clearInterval(this.processMonitorInterval);
242+
this.processMonitorInterval = null;
243+
this.logger.debug('Stopped global process monitor');
244+
}
245+
}
246+
193247
// Override fetch to route internal container requests to appropriate ports
194248
override async fetch(request: Request): Promise<Response> {
195249
// Extract or generate trace ID from request
@@ -475,6 +529,10 @@ export class Sandbox<Env = unknown> extends Container<Env> implements ISandbox {
475529
exitCode: undefined
476530
}, session);
477531

532+
// Start the global process monitor to keep container alive
533+
// even if no one is streaming
534+
this.startProcessMonitor();
535+
478536
// Call onStart callback if provided
479537
if (options?.onStart) {
480538
options.onStart(processObj);
@@ -611,6 +669,7 @@ export class Sandbox<Env = unknown> extends Container<Env> implements ISandbox {
611669
let currentTimeoutHandle: ReturnType<typeof setTimeout> | undefined;
612670

613671
// Set up periodic health monitoring to detect container crashes
672+
// AND to keep container alive while processes are running even without output
614673
healthCheckInterval = setInterval(async () => {
615674
if (!streamActive) {
616675
if (healthCheckInterval) {
@@ -633,6 +692,28 @@ export class Sandbox<Env = unknown> extends Container<Env> implements ISandbox {
633692
await reader.cancel(error.message);
634693
}
635694
}
695+
696+
// Check for running processes and renew activity if any exist
697+
try {
698+
const processList = await self.client.processes.listProcesses();
699+
const runningProcesses = processList.processes.filter(
700+
p => p.status === 'running' || p.status === 'starting'
701+
);
702+
703+
if (runningProcesses.length > 0) {
704+
const now = Date.now();
705+
if (now - lastActivityRenewal >= self.ACTIVITY_RENEWAL_THROTTLE_MS) {
706+
self.renewActivityTimeout();
707+
lastActivityRenewal = now;
708+
self.logger.debug(
709+
`Renewed activity timeout due to ${runningProcesses.length} running process(es): ${runningProcesses.map(p => p.id).join(', ')}`
710+
);
711+
}
712+
}
713+
} catch (error) {
714+
// Non-fatal: if we can't check processes, just log and continue
715+
self.logger.debug(`Failed to check running processes: ${error instanceof Error ? error.message : String(error)}`);
716+
}
636717
} catch (error) {
637718
// If getState() fails, container is likely dead
638719
isHealthy = false;

0 commit comments

Comments
 (0)