resolve helper container exit code from a terminal created or dead state

src/lib/server/docker.ts

@@ -17,7 +17,7 @@ import { pumpWebStreamToWritable } from './stream-pump';
 import { toWebReadableStream } from './node-readable-stream';
 import { buildImagePruneFilters } from './image-prune-core';
 import { computeRequestTimeoutMs } from './backups/request-timeout';
-import { helperWaitDeadline } from './helper-wait-core';
+import { helperWaitDeadline, helperExitFromState } from './helper-wait-core';
 import type { Environment } from './db';
 import { getSetting } from './db';
 import { getAdditionalVolumeBinds, dedupeVolumesForRecreate } from './mount-dedupe';
@@ -5129,14 +5129,22 @@ export async function runContainerWithStreaming(options: {
 					try {
 						const insp = await dockerFetch(`/containers/${containerId}/json`, {}, options.envId);
 						if (insp.ok) {
-							const data = await insp.json() as { State?: { Status?: string; Running?: boolean; Dead?: boolean; ExitCode?: number } };
+							const data = await insp.json() as { State?: { Status?: string; Running?: boolean; Dead?: boolean; ExitCode?: number; Error?: string } };
 							const st = data.State;
-							if (st && st.Running === false && st.Status === 'exited' && typeof st.ExitCode === 'number') {
-								exitCode = st.ExitCode;
-								console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode} (poll mode)`);
+							// Resolves for a normal `exited` AND for a container the daemon left
+							// terminally `created`/`dead` after a start failure (e.g. Docker 29.x
+							// exit 128 "failed to mount overlay: device or resource busy", #1487) -
+							// otherwise the unbounded poll waits forever for an `exited` that never comes.
+							const resolved = helperExitFromState(st);
+							if (resolved !== undefined) {
+								exitCode = resolved;
+								console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode} (poll mode, status=${st?.Status})`);
 								break;
 							}
-							if (st?.Dead || st?.Status === 'removing') break; // gone -> fail-closed diagnostics below
+							// A `dead`/`removing` container that helperExitFromState didn't resolve
+							// (e.g. dead with a 0 exit and no Error) is still terminal - break to the
+							// fail-closed diagnostics rather than polling it to the deadline.
+							if (st?.Dead || st?.Status === 'removing') break;
 						} else if (insp.status === 404) {
 							console.warn(`[runContainerWithStreaming] Poll: ${options.name ?? containerId.slice(0, 12)} not found (404) — container removed`);
 							break;
@@ -5188,9 +5196,10 @@ export async function runContainerWithStreaming(options: {
 								StartedAt: st?.StartedAt, FinishedAt: st?.FinishedAt, Pid: st?.Pid,
 							})}`
 						);
-						if (st && st.Running === false && st.Status === 'exited' && typeof st.ExitCode === 'number') {
-							exitCode = st.ExitCode;
-							console.log(`[runContainerWithStreaming] Resolved exit code via inspect: ${exitCode}`);
+						const resolved = helperExitFromState(st);
+						if (resolved !== undefined) {
+							exitCode = resolved;
+							console.log(`[runContainerWithStreaming] Resolved exit code via inspect: ${exitCode} (status=${st?.Status})`);
 						}
 					} else {
 						console.warn(`[runContainerWithStreaming] EXIT-CODE INDETERMINATE for ${options.name ?? containerId.slice(0, 12)}: inspect returned HTTP ${insp.status} (container may already be gone/removed)`);

src/lib/server/helper-wait-core.ts

@@ -26,3 +26,35 @@ export function helperWaitDeadline(timeout: number | undefined, now: number): nu
 	const cap = helperWaitCapMs(timeout);
 	return cap > 0 ? now + cap : Infinity;
 }
+
+/** The subset of Docker's container State the exit classifier reads. */
+export interface HelperContainerState {
+	Status?: string;
+	Running?: boolean;
+	ExitCode?: number;
+	Error?: string;
+}
+
+/**
+ * The definitive exit code for a helper container's State, or undefined when the
+ * container is not yet in a terminal state (still creating/running -> keep waiting).
+ *
+ * A normal helper goes created -> running -> `exited`. But the daemon can also leave a
+ * container that FAILED TO START in a terminal `created` (or `dead`) state with a
+ * non-zero ExitCode and a State.Error - e.g. Docker 29.x / containerd leaving exit 128
+ * "failed to mount overlay: device or resource busy" (#1487). That is terminal too: the
+ * container will never run, so its exit code must be resolved (fail-closed) instead of
+ * the poll loop waiting on an `exited` status that never arrives.
+ *
+ * A brand-new `created` container that is about to start (ExitCode 0, no Error) is NOT
+ * terminal - runContainerWithStreaming calls /start before polling, so a `created` seen
+ * with a non-zero ExitCode or an Error is a genuine start failure, not a pre-start race.
+ */
+export function helperExitFromState(state: HelperContainerState | undefined | null): number | undefined {
+	if (!state || state.Running !== false || typeof state.ExitCode !== 'number') return undefined;
+	if (state.Status === 'exited') return state.ExitCode;
+	if ((state.Status === 'created' || state.Status === 'dead') && (state.ExitCode !== 0 || !!state.Error)) {
+		return state.ExitCode;
+	}
+	return undefined;
+}

tests/helper-wait-core.test.ts

@@ -1,6 +1,6 @@
 // @ts-expect-error -- bun:test is a runtime built-in with no types installed
 import { describe, expect, test } from 'bun:test';
-import { helperWaitCapMs, helperWaitDeadline } from '../src/lib/server/helper-wait-core';
+import { helperWaitCapMs, helperWaitDeadline, helperExitFromState } from '../src/lib/server/helper-wait-core';
 
 describe('helperWaitCapMs', () => {
 	test('a positive timeout is the cap', () => {
@@ -36,3 +36,46 @@ describe('helperWaitDeadline', () => {
 		expect(Number.MAX_SAFE_INTEGER < helperWaitDeadline(undefined, 0)).toBe(true);
 	});
 });
+
+describe('helperExitFromState', () => {
+	test('a normal exited container returns its exit code', () => {
+		expect(helperExitFromState({ Status: 'exited', Running: false, ExitCode: 0 })).toBe(0);
+		expect(helperExitFromState({ Status: 'exited', Running: false, ExitCode: 1 })).toBe(1);
+	});
+
+	test('a still-running container is not terminal (keep waiting)', () => {
+		expect(helperExitFromState({ Status: 'running', Running: true, ExitCode: 0 })).toBeUndefined();
+	});
+
+	test('a brand-new created container about to start is NOT terminal (#1487 race guard)', () => {
+		// created + ExitCode 0 + no Error = pre-start, must keep waiting, not resolve to 0.
+		expect(helperExitFromState({ Status: 'created', Running: false, ExitCode: 0 })).toBeUndefined();
+	});
+
+	test('the #1487 case: created + non-zero ExitCode + mount Error IS terminal', () => {
+		// Docker 29.x / containerd leaving the helper unstarted: exit 128, State.Error set.
+		expect(helperExitFromState({
+			Status: 'created', Running: false, ExitCode: 128,
+			Error: 'failed to mount /var/lib/docker/rootfs/overlayfs/...: device or resource busy'
+		})).toBe(128);
+	});
+
+	test('created with a non-zero ExitCode (no Error) is terminal', () => {
+		expect(helperExitFromState({ Status: 'created', Running: false, ExitCode: 125 })).toBe(125);
+	});
+
+	test('created with ExitCode 0 but an Error present is terminal', () => {
+		expect(helperExitFromState({ Status: 'created', Running: false, ExitCode: 0, Error: 'oci runtime error' })).toBe(0);
+	});
+
+	test('a dead container with a non-zero exit code is terminal', () => {
+		expect(helperExitFromState({ Status: 'dead', Running: false, ExitCode: 137 })).toBe(137);
+	});
+
+	test('missing / partial state is not terminal', () => {
+		expect(helperExitFromState(undefined)).toBeUndefined();
+		expect(helperExitFromState(null)).toBeUndefined();
+		expect(helperExitFromState({ Status: 'exited', Running: false })).toBeUndefined(); // no ExitCode
+		expect(helperExitFromState({ Running: true })).toBeUndefined();
+	});
+});