1.0.18

src/lib/server/db.ts

@@ -779,6 +779,7 @@ export const NOTIFICATION_EVENT_TYPES = [
 	{ id: 'container_restarted', label: 'Container restarted', description: 'When a container restarts (manual or automatic)', group: 'container', scope: 'environment' },
 	{ id: 'container_exited', label: 'Container exited', description: 'When a container exits unexpectedly', group: 'container', scope: 'environment' },
 	{ id: 'container_unhealthy', label: 'Container unhealthy', description: 'When a container health check fails', group: 'container', scope: 'environment' },
+	{ id: 'container_healthy', label: 'Container healthy', description: 'When a container health check recovers', group: 'container', scope: 'environment' },
 	{ id: 'container_oom', label: 'Out of memory', description: 'When a container is killed due to out of memory', group: 'container', scope: 'environment' },
 	{ id: 'container_updated', label: 'Container updated', description: 'When a container image is updated', group: 'container', scope: 'environment' },
 	{ id: 'image_pulled', label: 'Image pulled', description: 'When a new image is pulled', group: 'container', scope: 'environment' },
@@ -2982,7 +2983,7 @@ export async function deleteOldScans(keepDays = 30): Promise<number> {
 export type AuditAction =
 	| 'create' | 'update' | 'delete' | 'start' | 'stop' | 'restart' | 'down'
 	| 'pause' | 'unpause' | 'pull' | 'push' | 'prune' | 'login'
-	| 'logout' | 'view' | 'exec' | 'connect' | 'disconnect' | 'deploy' | 'sync' | 'rename';
+	| 'logout' | 'view' | 'exec' | 'connect' | 'disconnect' | 'deploy' | 'sync' | 'rename' | 'webhook';
 
 export type AuditEntityType =
 	| 'container' | 'image' | 'stack' | 'volume' | 'network'

src/lib/server/docker.ts

@@ -807,7 +807,7 @@ export async function unpauseContainer(id: string, envId?: number | null) {
 
 export async function removeContainer(id: string, force = false, envId?: number | null) {
 	const response = await dockerFetch(`/containers/${id}?force=${force}`, { method: 'DELETE' }, envId);
-	if (!response.ok) {
+	if (!response.ok && response.status !== 404) {
 		const errorBody = await response.text();
 		let errorMessage = `Failed to remove container ${id}`;
 		try {
@@ -1338,6 +1338,43 @@ export async function createContainer(options: CreateContainerOptions, envId?: n
 	return { id: result.Id, start: () => startContainer(result.Id, envId) };
 }
 
+/**
+ * Deep-diff two objects recursively, returning all paths that differ.
+ */
+export function deepDiff(a: any, b: any, path = ''): string[] {
+	const diffs: string[] = [];
+
+	if (a === b) return diffs;
+	if (a === null || b === null || typeof a !== typeof b) {
+		diffs.push(`${path}: ${JSON.stringify(a)} → ${JSON.stringify(b)}`);
+		return diffs;
+	}
+	if (typeof a !== 'object') {
+		if (a !== b) diffs.push(`${path}: ${JSON.stringify(a)} → ${JSON.stringify(b)}`);
+		return diffs;
+	}
+	if (Array.isArray(a) || Array.isArray(b)) {
+		const aStr = JSON.stringify(a);
+		const bStr = JSON.stringify(b);
+		if (aStr !== bStr) diffs.push(`${path}: ${aStr} → ${bStr}`);
+		return diffs;
+	}
+
+	const allKeys = Array.from(new Set([...Object.keys(a), ...Object.keys(b)]));
+	for (const key of allKeys) {
+		const childPath = path ? `${path}.${key}` : key;
+		if (!(key in a)) {
+			diffs.push(`${childPath}: <missing> → ${JSON.stringify(b[key])}`);
+		} else if (!(key in b)) {
+			diffs.push(`${childPath}: ${JSON.stringify(a[key])} → <missing>`);
+		} else {
+			diffs.push(...deepDiff(a[key], b[key], childPath));
+		}
+	}
+
+	return diffs;
+}
+
 /**
  * Recreate a container using full Config/HostConfig passthrough from inspect data.
  * Passes Config and HostConfig directly from inspect to create, only changing
@@ -1519,7 +1556,23 @@ export async function recreateContainerFromInspect(
 		}
 	}
 
-	// 8. Remove old container (best effort)
+	// 8. Log config diff between old and new container
+	try {
+		const newInspect = await inspectContainer(newContainerId, envId);
+		const diffs = deepDiff(inspectData, newInspect);
+		if (diffs.length === 0) {
+			log?.(`[${name}] Config diff: no differences (all settings preserved)`);
+		} else {
+			log?.(`[${name}] Config diff: ${diffs.length} difference(s):`);
+			for (const d of diffs) {
+				log?.(`  [${name}] ${d}`);
+			}
+		}
+	} catch {
+		// Non-critical, don't fail the update
+	}
+
+	// 9. Remove old container (best effort)
 	log?.('Removing old container...');
 	await removeContainer(oldContainerId, true, envId).catch(() => {});
 
@@ -2649,13 +2702,17 @@ export async function getHawserInfo(envId: number): Promise<{
 	mode: string;
 	uptime: number;
 } | null> {
-	try {
-		const response = await dockerFetch('/_hawser/info', {}, envId);
-		if (response.ok) {
-			return await response.json();
+	for (let attempt = 0; attempt < 2; attempt++) {
+		try {
+			const response = await dockerFetch('/_hawser/info', {}, envId);
+			if (response.ok) {
+				return await response.json();
+			}
+			console.warn(`[Hawser] Info endpoint returned ${response.status} for env ${envId}`);
+		} catch (error) {
+			const msg = error instanceof Error ? error.message : String(error);
+			console.warn(`[Hawser] Failed to fetch info for env ${envId} (attempt ${attempt + 1}): ${msg}`);
 		}
-	} catch {
-		// Hawser info not available
 	}
 	return null;
 }
@@ -3275,7 +3332,7 @@ export async function runContainer(options: {
 
 		// Wait for container to finish
 		console.log(`[runContainer] Waiting for container ${containerId} to finish...`);
-		const waitResponse = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST' }, options.envId);
+		const waitResponse = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST', streaming: true }, options.envId);
 		const waitResult = await waitResponse.json().catch(() => ({}));
 		console.log(`[runContainer] Container ${containerId} finished with exit code:`, waitResult?.StatusCode);
 
@@ -3367,7 +3424,7 @@ export async function runContainerWithStreaming(options: {
 			// Wait for container to exit - this is the reliable signal
 			let exitCode: number | undefined;
 			try {
-				const waitResult = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST' }, options.envId);
+				const waitResult = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST', streaming: true }, options.envId);
 				const waitData = await waitResult.json() as { StatusCode?: number };
 				exitCode = waitData.StatusCode;
 				console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode}`);

src/lib/server/git.ts

@@ -49,6 +49,80 @@ interface GitEnv {
 	[key: string]: string;
 }
 
+const NSS_WRAPPER_LIB = '/usr/lib/libnss_wrapper.so';
+const TMP_PASSWD = '/tmp/dockhand-passwd';
+const TMP_GROUP = '/tmp/dockhand-group';
+
+// Cache the check so we only do it once per process
+let _nssWrapperChecked = false;
+let _nssWrapperNeeded = false;
+
+/**
+ * Ensures the current UID exists in /etc/passwd for git/SSH operations.
+ * SSH calls getpwuid() which fails with "No user exists for uid XXXX" if the
+ * UID isn't in /etc/passwd (common with Docker --user or read-only containers).
+ * Creates a temp passwd file and configures LD_PRELOAD with libnss_wrapper.
+ */
+async function ensurePasswdEntry(env: GitEnv): Promise<void> {
+	if (_nssWrapperChecked) {
+		if (_nssWrapperNeeded) {
+			env.LD_PRELOAD = NSS_WRAPPER_LIB;
+			env.NSS_WRAPPER_PASSWD = TMP_PASSWD;
+			env.NSS_WRAPPER_GROUP = TMP_GROUP;
+		}
+		return;
+	}
+	_nssWrapperChecked = true;
+
+	// Check if current UID is in /etc/passwd
+	const uid = process.getuid?.();
+	if (uid === undefined || uid === 0) return; // root or not available
+
+	try {
+		const passwd = await Bun.file('/etc/passwd').text();
+		const uidStr = `:${uid}:`;
+		if (passwd.split('\n').some(line => {
+			const parts = line.split(':');
+			return parts[2] === String(uid);
+		})) {
+			return; // UID exists, nothing to do
+		}
+	} catch {
+		return; // can't read passwd, bail
+	}
+
+	// UID not found — check if libnss_wrapper is available
+	if (!existsSync(NSS_WRAPPER_LIB)) {
+		console.warn(`[git] UID ${uid} not in /etc/passwd and libnss_wrapper not found — SSH may fail`);
+		return;
+	}
+
+	// Create temp passwd/group with the missing entry
+	try {
+		const gid = process.getgid?.() ?? uid;
+		const passwd = await Bun.file('/etc/passwd').text();
+		const group = await Bun.file('/etc/group').text();
+
+		const passwdEntry = `dockhand:x:${uid}:${gid}:Dockhand:/home/dockhand:/bin/sh`;
+		await Bun.write(TMP_PASSWD, passwd.trimEnd() + '\n' + passwdEntry + '\n');
+
+		const gidExists = group.split('\n').some(line => line.split(':')[2] === String(gid));
+		if (gidExists) {
+			await Bun.write(TMP_GROUP, group);
+		} else {
+			await Bun.write(TMP_GROUP, group.trimEnd() + '\n' + `dockhand:x:${gid}:\n`);
+		}
+
+		_nssWrapperNeeded = true;
+		env.LD_PRELOAD = NSS_WRAPPER_LIB;
+		env.NSS_WRAPPER_PASSWD = TMP_PASSWD;
+		env.NSS_WRAPPER_GROUP = TMP_GROUP;
+		console.log(`[git] Created temp passwd for UID ${uid} with libnss_wrapper`);
+	} catch (err) {
+		console.warn(`[git] Failed to create temp passwd:`, err);
+	}
+}
+
 async function buildGitEnv(credential: GitCredential | null): Promise<GitEnv> {
 	const env: GitEnv = {
 		...process.env as GitEnv,
@@ -57,6 +131,9 @@ async function buildGitEnv(credential: GitCredential | null): Promise<GitEnv> {
 		SSH_AUTH_SOCK: ''
 	};
 
+	// Ensure current UID is resolvable for SSH/git operations
+	await ensurePasswdEntry(env);
+
 	if (credential?.authType === 'ssh' && credential.sshPrivateKey) {
 		// Create a temporary SSH key file (use absolute path so SSH can find it)
 		const sshKeyPath = resolve(join(GIT_REPOS_DIR, `.ssh-key-${credential.id}`));
@@ -72,9 +149,20 @@ async function buildGitEnv(credential: GitCredential | null): Promise<GitEnv> {
 		// Bun.write's mode option doesn't always work reliably, so use chmodSync
 		chmodSync(sshKeyPath, 0o600);
 
+		// If key has a passphrase, decrypt it in-place so SSH can use it non-interactively
+		if (credential.sshPassphrase) {
+			const result = Bun.spawnSync([
+				'ssh-keygen', '-p', '-f', sshKeyPath,
+				'-P', credential.sshPassphrase, '-N', ''
+			], { env, stderr: 'pipe' });
+			if (result.exitCode !== 0) {
+				const stderr = result.stderr.toString().trim();
+				console.warn(`[git] Failed to decrypt SSH key: ${stderr}`);
+			}
+		}
+
 		// Configure SSH to use ONLY this key (no agent, no default keys)
-		const sshCommand = `ssh -i "${sshKeyPath}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes`;
-		env.GIT_SSH_COMMAND = sshCommand;
+		env.GIT_SSH_COMMAND = `ssh -i "${sshKeyPath}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes`;
 	} else {
 		// No SSH credential - prevent using any keys (IdentitiesOnly=yes with no -i means no keys)
 		env.GIT_SSH_COMMAND = 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no -o PubkeyAuthentication=no';

src/lib/server/host-path.ts

@@ -32,7 +32,7 @@ let cachedMounts: Array<{ source: string; destination: string }> | null = null;
 /**
  * Get our own container ID
  */
-function getOwnContainerId(): string | null {
+export function getOwnContainerId(): string | null {
 	// Method 1: From cgroup (works in most cases)
 	try {
 		const cgroup = readFileSync('/proc/self/cgroup', 'utf-8');

src/lib/server/notifications.ts

@@ -299,14 +299,19 @@ async function sendTelegram(appriseUrl: string, payload: NotificationPayload): P
 // Gotify
 async function sendGotify(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
 	// gotify://hostname/token or gotifys://hostname/token
+	// gotify://hostname/subpath/token (subpath support)
 	const match = appriseUrl.match(/^gotifys?:\/\/([^/]+)\/(.+)/);
 	if (!match) {
 		return { success: false, error: 'Invalid Gotify URL format. Expected: gotify://hostname/token' };
 	}
 
-	const [, hostname, token] = match;
+	const [, hostname, pathPart] = match;
 	const protocol = appriseUrl.startsWith('gotifys') ? 'https' : 'http';
-	const url = `${protocol}://${hostname}/message?token=${token}`;
+	// Token is always the last path segment; anything before it is a subpath
+	const lastSlash = pathPart.lastIndexOf('/');
+	const subpath = lastSlash >= 0 ? pathPart.substring(0, lastSlash) : '';
+	const token = lastSlash >= 0 ? pathPart.substring(lastSlash + 1) : pathPart;
+	const url = `${protocol}://${hostname}${subpath ? '/' + subpath : ''}/message?token=${token}`;
 
 	try {
 		const response = await fetch(url, {
@@ -506,6 +511,7 @@ function mapActionToEventType(action: string): NotificationEventType | null {
 		'kill': 'container_exited',
 		'oom': 'container_oom',
 		'health_status: unhealthy': 'container_unhealthy',
+		'health_status: healthy': 'container_healthy',
 		'pull': 'image_pulled'
 	};
 	return mapping[action] || null;

src/lib/server/scheduler/tasks/env-update-check.ts

@@ -124,6 +124,11 @@ export async function runEnvUpdateCheckJob(
 					continue;
 				}
 
+				if (isSystemContainer(imageName)) {
+					await log(`  [${container.name}] Skipping - system container`);
+					continue;
+				}
+
 				checkedCount++;
 				await log(`  Checking: ${container.name} (${imageName})`);
 
@@ -222,16 +227,6 @@ export async function runEnvUpdateCheckJob(
 			const blockedContainers: { name: string; reason: string; scannerResults?: { scanner: string; critical: number; high: number; medium: number; low: number }[] }[] = [];
 
 			for (const update of updatesAvailable) {
-				// Skip system containers (Dockhand/Hawser) - cannot update themselves
-				const systemContainerType = isSystemContainer(update.imageName);
-				if (systemContainerType) {
-					const reason = systemContainerType === 'dockhand'
-						? 'cannot auto-update Dockhand itself'
-						: 'cannot auto-update Hawser agent';
-					await log(`\n[${update.containerName}] Skipping - ${reason}`);
-					continue;
-				}
-
 				try {
 					await log(`\nUpdating: ${update.containerName}`);
 

src/lib/server/scheduler/tasks/image-prune.ts

@@ -116,12 +116,14 @@ export async function runImagePrune(
 			}
 		});
 
-		// Send success notification
-		await sendEventNotification('image_prune_success', {
-			title: 'Image prune completed',
-			message: `${imagesRemoved} unused images removed, ${formatBytes(spaceReclaimed)} disk space reclaimed`,
-			type: 'success'
-		}, envId);
+		// Send success notification only when something was actually cleaned up
+		if (imagesRemoved > 0) {
+			await sendEventNotification('image_prune_success', {
+				title: 'Image prune completed',
+				message: `${imagesRemoved} unused images removed, ${formatBytes(spaceReclaimed)} disk space reclaimed`,
+				type: 'success'
+			}, envId);
+		}
 
 	} catch (error: any) {
 		await log(`Error: ${error.message}`);

src/lib/server/stack-scanner.ts

@@ -44,7 +44,7 @@ export interface ScanResult {
 /**
  * Normalize a stack name to be valid (lowercase alphanumeric with hyphens/underscores)
  */
-function normalizeStackName(name: string): string {
+export function normalizeStackName(name: string): string {
 	return name
 		.toLowerCase()
 		.replace(/[^a-z0-9_-]/g, '-')
@@ -223,7 +223,7 @@ export async function adoptStack(
 	}
 
 	// Check for name conflict within the same environment
-	let finalName = stack.name;
+	let finalName = normalizeStackName(stack.name);
 	const existingNames = new Set(
 		existingSources
 			.filter((s) => s.environmentId === environmentId)

src/lib/server/stacks.ts

@@ -1562,6 +1562,43 @@ export async function getStackPathHints(
 	return { workingDir, configFiles };
 }
 
+/**
+ * Stop or remove orphan containers that belong to a stack but aren't defined in the compose file.
+ * These are dynamically-spawned child containers (e.g., nextcloud-aio master creates worker containers).
+ * Best-effort: errors are logged but don't fail the overall operation.
+ */
+async function cleanupOrphanStackContainers(
+	stackName: string,
+	envId: number | null | undefined,
+	operation: 'stop' | 'remove' | 'restart'
+): Promise<void> {
+	try {
+		const containers = await getStackContainers(stackName, envId);
+		const targets = containers.filter(
+			(c) => c.state === 'running' || c.state === 'restarting'
+		);
+		if (targets.length === 0) return;
+
+		const { stopContainer, removeContainer, restartContainer } = await import('./docker.js');
+		const results = await Promise.allSettled(
+			targets.map((c) => {
+				if (operation === 'remove') return removeContainer(c.id, true, envId);
+				if (operation === 'restart') return restartContainer(c.id, envId);
+				return stopContainer(c.id, envId);
+			})
+		);
+
+		const failures = results.filter((r) => r.status === 'rejected');
+		if (failures.length > 0) {
+			console.warn(
+				`[stacks] ${failures.length} orphan container(s) failed to ${operation} for stack "${stackName}"`
+			);
+		}
+	} catch (err) {
+		console.warn(`[stacks] Failed to cleanup orphan containers for stack "${stackName}":`, err);
+	}
+}
+
 /**
  * Helper to perform container-based operations for external stacks
  * Used as fallback when no compose file exists.
@@ -1767,13 +1804,18 @@ export async function stopStack(
 		return withContainerFallback(stackName, envId, 'stop');
 	}
 
-	return executeComposeCommand(
+	const composeResult = await executeComposeCommand(
 		'stop',
 		{ stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath },
 		result.content!,
 		result.nonSecretVars,
 		result.secretVars
 	);
+
+	// Stop any dynamically-spawned child containers not in the compose file
+	await cleanupOrphanStackContainers(stackName, envId, 'stop');
+
+	return composeResult;
 }
 
 /**
@@ -1791,13 +1833,18 @@ export async function restartStack(
 		return withContainerFallback(stackName, envId, 'restart');
 	}
 
-	return executeComposeCommand(
+	const composeResult = await executeComposeCommand(
 		'restart',
 		{ stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath },
 		result.content!,
 		result.nonSecretVars,
 		result.secretVars
 	);
+
+	// Restart any dynamically-spawned child containers not in the compose file
+	await cleanupOrphanStackContainers(stackName, envId, 'restart');
+
+	return composeResult;
 }
 
 /**
@@ -1816,13 +1863,18 @@ export async function downStack(
 		return withContainerFallback(stackName, envId, 'stop');
 	}
 
-	return executeComposeCommand(
+	const composeResult = await executeComposeCommand(
 		'down',
 		{ stackName, envId, removeVolumes, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath },
 		result.content!,
 		result.nonSecretVars,
 		result.secretVars
 	);
+
+	// Remove any dynamically-spawned child containers not in the compose file
+	await cleanupOrphanStackContainers(stackName, envId, 'remove');
+
+	return composeResult;
 }
 
 /**
@@ -1861,6 +1913,9 @@ export async function removeStack(
 			if (!downResult.success && !force) {
 				return downResult;
 			}
+
+			// Remove any dynamically-spawned child containers not handled by compose
+			await cleanupOrphanStackContainers(stackName, envId, 'remove');
 		} else {
 			// External stack - remove containers directly in parallel
 			const { removeContainer } = await import('./docker.js');

src/lib/server/subprocesses/event-subprocess.ts

@@ -20,7 +20,7 @@ const MAX_RECONNECT_DELAY = 60000; // 1 minute max
 const environmentOnlineStatus: Map<number, boolean> = new Map();
 
 // Active collectors per environment (for streaming mode)
-const collectors: Map<number, AbortController> = new Map();
+const collectors: Map<number, { controller: AbortController; reconnectTimeout: ReturnType<typeof setTimeout> | null }> = new Map();
 
 // Poll intervals per environment (for polling mode)
 const pollIntervals: Map<number, ReturnType<typeof setInterval>> = new Map();
@@ -313,7 +313,8 @@ async function startEnvironmentCollector(envId: number, envName: string) {
 	stopEnvironmentCollector(envId);
 
 	const controller = new AbortController();
-	collectors.set(envId, controller);
+	const entry = { controller, reconnectTimeout: null as ReturnType<typeof setTimeout> | null };
+	collectors.set(envId, entry);
 
 	let reconnectDelay = RECONNECT_DELAY;
 
@@ -415,7 +416,8 @@ async function startEnvironmentCollector(envId: number, envName: string) {
 		if (controller.signal.aborted || isShuttingDown) return;
 
 		console.log(`[EventSubprocess] Reconnecting to ${envName} in ${reconnectDelay / 1000}s...`);
-		setTimeout(() => {
+		entry.reconnectTimeout = setTimeout(() => {
+			entry.reconnectTimeout = null;
 			if (!controller.signal.aborted && !isShuttingDown) {
 				connect();
 			}
@@ -468,9 +470,12 @@ function stopEnvironmentPoller(envId: number) {
  * Stop collecting events for a specific environment (streaming mode)
  */
 function stopEnvironmentCollector(envId: number) {
-	const controller = collectors.get(envId);
-	if (controller) {
-		controller.abort();
+	const entry = collectors.get(envId);
+	if (entry) {
+		if (entry.reconnectTimeout !== null) {
+			clearTimeout(entry.reconnectTimeout);
+		}
+		entry.controller.abort();
 		collectors.delete(envId);
 		environmentOnlineStatus.delete(envId);
 	}
@@ -528,6 +533,15 @@ async function refreshEventCollectors() {
 			}
 		}
 
+		// Clean up stale map entries for deleted environments
+		const allEnvIds = new Set(environments.map((e) => e.id));
+		for (const envId of environmentOnlineStatus.keys()) {
+			if (!allEnvIds.has(envId)) environmentOnlineStatus.delete(envId);
+		}
+		for (const envId of lastPollTime.keys()) {
+			if (!allEnvIds.has(envId)) lastPollTime.delete(envId);
+		}
+
 		// Start collectors based on mode
 		for (const env of environments) {
 			// Skip Hawser Edge (handled by main process)

src/lib/server/subprocesses/metrics-subprocess.ts

@@ -368,6 +368,12 @@ async function checkDiskSpace() {
 				console.warn(`[MetricsSubprocess] Environment "${enabledEnvs[index].name}" disk check failed: ${reason}`);
 			}
 		});
+
+		// Clean up stale lastDiskWarning entries for deleted environments
+		const activeEnvIds = new Set(environments.map((e) => e.id));
+		for (const envId of lastDiskWarning.keys()) {
+			if (!activeEnvIds.has(envId)) lastDiskWarning.delete(envId);
+		}
 	} catch (error) {
 		const message = error instanceof Error ? error.message : String(error);
 		console.error(`[MetricsSubprocess] Disk space check error: ${message}`);