1.0.13

src/lib/server/stacks.ts

@@ -6,10 +6,9 @@
  */
 
 import { existsSync, mkdirSync, rmSync, readdirSync, cpSync, statSync, unlinkSync, renameSync, readFileSync, writeFileSync } from 'node:fs';
-import { join, resolve, dirname } from 'node:path';
+import { join, resolve, dirname, basename } from 'node:path';
 import {
 	getEnvironment,
-	getStackEnvVarsAsRecord,
 	getSecretEnvVarsAsRecord,
 	getNonSecretEnvVarsAsRecord,
 	getStackEnvVars,
@@ -95,7 +94,6 @@ export interface DeployStackOptions {
 	name: string;
 	compose: string;
 	envId?: number | null;
-	envFileVars?: Record<string, string>;
 	sourceDir?: string; // Directory to copy all files from (for git stacks)
 	forceRecreate?: boolean;
 	composePath?: string; // Custom compose file path (for adopted/imported stacks)
@@ -313,25 +311,6 @@ export async function findStackDir(stackName: string, envId?: number | null): Pr
 	return null;
 }
 
-/**
- * List stacks that have compose files stored locally
- */
-export function listManagedStacks(): string[] {
-	const stacksDir = getStacksDir();
-	if (!existsSync(stacksDir)) {
-		return [];
-	}
-
-	return readdirSync(stacksDir, { withFileTypes: true })
-		.filter((dirent) => dirent.isDirectory())
-		.filter((dirent) => {
-			// Check all valid compose filenames
-			const composeNames = ['compose.yaml', 'compose.yml', 'docker-compose.yml', 'docker-compose.yaml'];
-			return composeNames.some(name => existsSync(join(stacksDir, dirent.name, name)));
-		})
-		.map((dirent) => dirent.name);
-}
-
 // =============================================================================
 // COMPOSE FILE MANAGEMENT
 // =============================================================================
@@ -761,6 +740,8 @@ interface ComposeCommandOptions {
 	composePath?: string;
 	/** Full path to the env file (for --env-file flag, supports custom names) */
 	envPath?: string;
+	/** When true, write non-secret envVars to .env.dockhand override file (git stacks only) */
+	useOverrideFile?: boolean;
 }
 
 /**
@@ -785,7 +766,8 @@ async function executeLocalCompose(
 	envId?: number | null,
 	workingDir?: string,
 	customComposePath?: string,
-	customEnvPath?: string
+	customEnvPath?: string,
+	useOverrideFile?: boolean
 ): Promise<StackOperationResult> {
 	const logPrefix = `[Stack:${stackName}]`;
 
@@ -894,12 +876,29 @@ async function executeLocalCompose(
 	const useStdin = finalComposeContent !== composeContent;
 	const args = ['docker', 'compose', '-p', stackName, '-f', useStdin ? '-' : composeFile];
 
-	// Add --env-file flag if env file exists
-	// This makes Docker Compose load the .env file automatically (like Portainer)
-	// Uses custom path if provided, otherwise defaults to .env in stack directory
-	const envFilePath = customEnvPath || join(stackDir, '.env');
-	if (existsSync(envFilePath)) {
-		args.push('--env-file', envFilePath);
+	// Always auto-detect .env in compose directory
+	const defaultEnvPath = join(stackDir, '.env');
+	if (existsSync(defaultEnvPath)) {
+		args.push('--env-file', defaultEnvPath);
+	}
+
+	// Add custom env file if configured and different from auto-detected .env
+	if (customEnvPath && resolve(customEnvPath) !== resolve(defaultEnvPath) && existsSync(customEnvPath)) {
+		args.push('--env-file', customEnvPath);
+	}
+
+	// For git stacks: write non-secret overrides to .env.dockhand and add as second --env-file
+	// Docker Compose applies env files in order, so later files override earlier ones.
+	// This lets the repo's .env provide defaults while our overrides take precedence.
+	// Secrets are still injected via shell env only (never written to disk).
+	// Only written when useOverrideFile is true (git stacks). Internal/adopted stacks
+	// already have their non-secrets in the .env file written by the UI.
+	if (useOverrideFile && envVars && Object.keys(envVars).length > 0) {
+		const overrideEnvPath = join(stackDir, '.env.dockhand');
+		const header = '# Auto-generated by Dockhand. Do not edit - changes will be overwritten on next deploy.\n';
+		const lines = Object.entries(envVars).map(([k, v]) => `${k}=${v}`);
+		await Bun.write(overrideEnvPath, header + lines.join('\n') + '\n');
+		args.push('--env-file', overrideEnvPath);
 	}
 
 	if (useStdin) {
@@ -1199,7 +1198,7 @@ async function executeComposeCommand(
 	envVars?: Record<string, string>,
 	secretVars?: Record<string, string>
 ): Promise<StackOperationResult> {
-	const { stackName, envId, forceRecreate, removeVolumes, stackFiles, workingDir, composePath, envPath } = options;
+	const { stackName, envId, forceRecreate, removeVolumes, stackFiles, workingDir, composePath, envPath, useOverrideFile } = options;
 
 	// Get environment configuration
 	const env = envId ? await getEnvironment(envId) : null;
@@ -1219,7 +1218,8 @@ async function executeComposeCommand(
 			envId,
 			workingDir,
 			composePath,
-			envPath
+			envPath,
+			useOverrideFile
 		);
 	}
 
@@ -1263,7 +1263,8 @@ async function executeComposeCommand(
 				envId,
 				workingDir,
 				composePath,
-				envPath
+				envPath,
+				useOverrideFile
 			);
 		}
 
@@ -1282,7 +1283,8 @@ async function executeComposeCommand(
 				envId,
 				workingDir,
 				composePath,
-				envPath
+				envPath,
+				useOverrideFile
 			);
 	}
 }
@@ -1487,7 +1489,6 @@ async function withContainerFallback(
 export interface RequireComposeResult {
 	success: boolean;
 	content?: string;
-	envVars?: Record<string, string>;
 	secretVars?: Record<string, string>;
 	needsFileLocation?: boolean;
 	error?: string;
@@ -1495,19 +1496,18 @@ export interface RequireComposeResult {
 	stackDir?: string;
 	/** Full path to the compose file (for imported stacks) */
 	composePath?: string;
+	/** Full path to the env file (for --env-file flag) */
+	envPath?: string;
 }
 
 /**
- * Get compose file and env vars for stack operations.
+ * Get compose file and secret vars for stack operations.
  *
  * Returns:
  * - content: The compose file content
- * - envVars: Non-secret variables (from .env file, with DB fallback)
  * - secretVars: Secret variables (from DB only, for shell injection)
+ * - envPath: Path to the .env file (Docker Compose reads non-secrets from it)
  * - needsFileLocation: true if stack needs user to specify file paths
- *
- * SECURITY: Secrets are NEVER written to .env files. They are stored in the database
- * and injected via shell environment variables at runtime.
  */
 async function requireComposeFile(
 	stackName: string,
@@ -1534,10 +1534,7 @@ async function requireComposeFile(
 	// These are NEVER written to disk
 	const secretVars = await getSecretEnvVarsAsRecord(stackName, envId);
 
-	// Get non-secret variables from database (for backward compatibility)
-	const dbNonSecretVars = await getNonSecretEnvVarsAsRecord(stackName, envId);
-
-	// Read non-secret vars from .env file
+	// Determine env file path for --env-file flag
 	// For stacks with custom composePath (adopted/external), derive envPath from same directory
 	// For internal stacks, use the default data directory
 	let envFilePath: string | null = null;
@@ -1560,38 +1557,11 @@ async function requireComposeFile(
 		envFilePath = join(stackDir, '.env');
 	}
 
-	let fileEnvVars: Record<string, string> = {};
-
-	if (envFilePath && existsSync(envFilePath)) {
-		try {
-			const content = await Bun.file(envFilePath).text();
-			for (const line of content.split('\n')) {
-				const trimmed = line.trim();
-				if (!trimmed || trimmed.startsWith('#')) continue;
-				const eqIndex = trimmed.indexOf('=');
-				if (eqIndex > 0) {
-					const key = trimmed.substring(0, eqIndex).trim();
-					let value = trimmed.substring(eqIndex + 1);
-					if ((value.startsWith('"') && value.endsWith('"')) ||
-					    (value.startsWith("'") && value.endsWith("'"))) {
-						value = value.slice(1, -1);
-					}
-					fileEnvVars[key] = value;
-				}
-			}
-		} catch {
-			// Ignore file read errors
-		}
-	}
-
-	// Merge non-secret vars: DB as fallback, file values override
-	// This ensures external edits to .env are respected during deployment
-	const envVars = { ...dbNonSecretVars, ...fileEnvVars };
-
+	// Docker Compose reads non-secrets from the .env file via --env-file.
+	// Only secrets need to be injected via shell environment.
 	return {
 		success: true,
 		content: composeResult.content!,
-		envVars,
 		secretVars,
 		stackDir: composeResult.stackDir,
 		composePath: composeResult.composePath ?? undefined,
@@ -1618,7 +1588,7 @@ export async function startStack(
 		'up',
 		{ stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath },
 		result.content!,
-		result.envVars,
+		undefined,
 		result.secretVars
 	);
 }
@@ -1642,7 +1612,7 @@ export async function stopStack(
 		'stop',
 		{ stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath },
 		result.content!,
-		result.envVars,
+		undefined,
 		result.secretVars
 	);
 }
@@ -1666,7 +1636,7 @@ export async function restartStack(
 		'restart',
 		{ stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath },
 		result.content!,
-		result.envVars,
+		undefined,
 		result.secretVars
 	);
 }
@@ -1691,7 +1661,7 @@ export async function downStack(
 		'down',
 		{ stackName, envId, removeVolumes, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath },
 		result.content!,
-		result.envVars,
+		undefined,
 		result.secretVars
 	);
 }
@@ -1707,7 +1677,7 @@ export async function removeStack(
 ): Promise<StackOperationResult> {
 	return withStackLock(stackName, async () => {
 		// Get compose file (may not exist for external stacks)
-		const composeResult = await getStackComposeFile(stackName);
+		const composeResult = await getStackComposeFile(stackName, envId);
 
 		// Get stack containers BEFORE removing them (for cleanup later)
 		const stackContainers = await getStackContainers(stackName, envId);
@@ -1804,16 +1774,17 @@ export async function removeStack(
 			const resolvedStacksDir = resolve(stacksDir);
 
 			// Only delete if the directory is within DATA_DIR/stacks/ (files we created)
-			// AND it looks like a stack directory (contains stackName for safety)
+			// AND the directory basename matches the stack name exactly (for safety)
 			if (resolvedCustomDir.startsWith(resolvedStacksDir) &&
-				customDir.includes(stackName) &&
+				basename(resolvedCustomDir) === stackName &&
 				existsSync(customDir)) {
 				stackDir = customDir;
 			}
 		}
 
-		// Fall back to default paths (always within DATA_DIR/stacks/)
-		if (!stackDir) {
+		// Fall back to default paths ONLY if no custom path was set in DB
+		// (Don't delete default-path files when an adopted stack has custom path outside DATA_DIR)
+		if (!stackDir && !stackSource?.composePath) {
 			const defaultDir = await findStackDir(stackName, envId) || await getStackDir(stackName, envId);
 			if (existsSync(defaultDir)) {
 				stackDir = defaultDir;
@@ -1853,14 +1824,14 @@ export async function removeStack(
 			const gitStack = await getGitStackByName(stackName, envId);
 			if (gitStack) {
 				await deleteGitStack(gitStack.id);
-				deleteGitStackFiles(gitStack.id);
+				await deleteGitStackFiles(gitStack.id, gitStack.stackName, gitStack.environmentId);
 			}
 			// Also cleanup any orphaned git stacks with NULL environment_id for this stack name
 			if (envId !== undefined && envId !== null) {
 				const orphanedGitStack = await getGitStackByName(stackName, null);
 				if (orphanedGitStack) {
 					await deleteGitStack(orphanedGitStack.id);
-					deleteGitStackFiles(orphanedGitStack.id);
+					await deleteGitStackFiles(orphanedGitStack.id, orphanedGitStack.stackName, orphanedGitStack.environmentId);
 				}
 			}
 		} catch (err: any) {
@@ -1890,7 +1861,7 @@ export async function removeStack(
  * Uses stack locking to prevent concurrent deployments.
  */
 export async function deployStack(options: DeployStackOptions): Promise<StackOperationResult> {
-	const { name, compose, envId, envFileVars, sourceDir, forceRecreate, composePath, envPath, composeFileName, envFileName } = options;
+	const { name, compose, envId, sourceDir, forceRecreate, composePath, envPath, composeFileName, envFileName } = options;
 	const logPrefix = `[Stack:${name}]`;
 
 	console.log(`${logPrefix} ========================================`);
@@ -1903,11 +1874,6 @@ export async function deployStack(options: DeployStackOptions): Promise<StackOpe
 	console.log(`${logPrefix} Custom env path:`, envPath ?? '(none)');
 	console.log(`${logPrefix} Compose filename:`, composeFileName ?? '(none)');
 	console.log(`${logPrefix} Env filename:`, envFileName ?? '(none)');
-	console.log(`${logPrefix} Env file vars provided:`, envFileVars ? Object.keys(envFileVars).length : 0);
-	if (envFileVars && Object.keys(envFileVars).length > 0) {
-		console.log(`${logPrefix} Env file var keys:`, Object.keys(envFileVars).join(', '));
-		console.log(`${logPrefix} Env file vars (masked):`, JSON.stringify(maskSecrets(envFileVars), null, 2));
-	}
 
 	// Validate stack name - Docker Compose requires lowercase alphanumeric, hyphens, underscores
 	// Must also start with a letter or number
@@ -1943,7 +1909,16 @@ export async function deployStack(options: DeployStackOptions): Promise<StackOpe
 			if (composeFileName) {
 				actualComposePath = join(workingDir, composeFileName);
 				console.log(`${logPrefix} Using compose filename from git config:`, composeFileName);
-				console.log(`${logPrefix} Actual compose path will be:`, actualComposePath);
+			} else {
+				// Detect compose file in source directory
+				const composeNames = ['docker-compose.yaml', 'docker-compose.yml', 'compose.yaml', 'compose.yml'];
+				for (const cn of composeNames) {
+					if (existsSync(join(sourceDir, cn))) {
+						actualComposePath = join(workingDir, cn);
+						console.log(`${logPrefix} Detected compose file:`, cn);
+						break;
+					}
+				}
 			}
 
 			// Set actualEnvPath using the provided env filename from git stack config
@@ -1967,31 +1942,38 @@ export async function deployStack(options: DeployStackOptions): Promise<StackOpe
 			cpSync(sourceDir, workingDir, { recursive: true });
 			console.log(`${logPrefix} Copied ${sourceDir} -> ${workingDir}`);
 		} else {
-			// Internal stack: compose file should already exist (written by saveStackComposeFile)
-			// Just determine the working directory
-			workingDir = await getStackDir(name, envId);
-			console.log(`${logPrefix} Using internal stack directory:`, workingDir);
+			// Internal stack: check if a custom path exists in DB (adopted/imported stacks)
+			const source = await getStackSource(name, envId);
+			if (source?.composePath) {
+				workingDir = dirname(source.composePath);
+				actualComposePath = source.composePath;
+				if (source.envPath) {
+					actualEnvPath = source.envPath;
+				}
+				console.log(`${logPrefix} Using custom path from DB:`, workingDir);
+			} else {
+				// Default: compose file should already exist (written by saveStackComposeFile)
+				workingDir = await getStackDir(name, envId);
+				console.log(`${logPrefix} Using internal stack directory:`, workingDir);
+			}
 		}
 
 		console.log(`${logPrefix} Compose content length:`, compose.length, 'chars');
 		console.log(`${logPrefix} Compose content (full):`);
 		console.log(compose);
 
-		// Fetch stack environment variables from database (these are user overrides)
-		const dbEnvVars = await getStackEnvVarsAsRecord(name, envId);
-		console.log(`${logPrefix} DB env vars count:`, Object.keys(dbEnvVars).length);
-		if (Object.keys(dbEnvVars).length > 0) {
-			console.log(`${logPrefix} DB env var keys:`, Object.keys(dbEnvVars).join(', '));
-			console.log(`${logPrefix} DB env vars (masked):`, JSON.stringify(maskSecrets(dbEnvVars), null, 2));
-		}
+		// Fetch overrides and secrets from DB
+		const dbNonSecretVars = await getNonSecretEnvVarsAsRecord(name, envId);
+		const secretVars = await getSecretEnvVarsAsRecord(name, envId);
+		console.log(`${logPrefix} DB non-secret override vars:`, Object.keys(dbNonSecretVars).length);
+		console.log(`${logPrefix} DB secret vars:`, Object.keys(secretVars).length);
 
-		// Merge: env file vars as base, database overrides take precedence
-		const envVars = { ...envFileVars, ...dbEnvVars };
-		console.log(`${logPrefix} Merged env vars count:`, Object.keys(envVars).length);
-		if (Object.keys(envVars).length > 0) {
-			console.log(`${logPrefix} Merged env var keys:`, Object.keys(envVars).join(', '));
-			console.log(`${logPrefix} Merged env vars (masked):`, JSON.stringify(maskSecrets(envVars), null, 2));
-		}
+		// For git stacks (sourceDir provided), use the override file (.env.dockhand)
+		// to layer editor overrides on top of the repo's .env file.
+		// Only DB overrides go into .env.dockhand - repo values are already in the repo's env file.
+		// For internal/adopted stacks, the .env file is already the editor's output,
+		// so no override file is needed - only pass secrets for shell injection.
+		const isGitStack = !!sourceDir;
 
 		console.log(`${logPrefix} Calling executeComposeCommand...`);
 		const result = await executeComposeCommand(
@@ -2003,10 +1985,12 @@ export async function deployStack(options: DeployStackOptions): Promise<StackOpe
 				stackFiles,
 				workingDir,
 				composePath: actualComposePath,
-				envPath: actualEnvPath
+				envPath: actualEnvPath,
+				useOverrideFile: isGitStack
 			},
 			compose,
-			envVars
+			isGitStack ? dbNonSecretVars : undefined,
+			secretVars
 		);
 		console.log(`${logPrefix} ========================================`);
 		console.log(`${logPrefix} DEPLOY STACK RESULT`);
@@ -2042,7 +2026,7 @@ export async function pullStackImages(
 		'pull',
 		{ stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath },
 		result.content!,
-		result.envVars,
+		undefined,
 		result.secretVars
 	);
 }
@@ -2077,9 +2061,22 @@ export async function writeStackEnvFile(
 	envId?: number | null,
 	customEnvPath?: string
 ): Promise<void> {
-	const envFilePath = customEnvPath
-		? customEnvPath
-		: join(await findStackDir(stackName, envId) || await getStackDir(stackName, envId), '.env');
+	let envFilePath: string;
+	if (customEnvPath) {
+		envFilePath = customEnvPath;
+	} else {
+		// Check if stack has a custom path in DB
+		const source = await getStackSource(stackName, envId);
+		if (source?.envPath) {
+			envFilePath = source.envPath;
+		} else if (source?.composePath) {
+			// Derive env path from custom compose path location
+			envFilePath = join(dirname(source.composePath), '.env');
+		} else {
+			// Fall back to default location
+			envFilePath = join(await findStackDir(stackName, envId) || await getStackDir(stackName, envId), '.env');
+		}
+	}
 
 	// Ensure parent directory exists
 	const dir = dirname(envFilePath);
@@ -2109,9 +2106,22 @@ export async function writeRawStackEnvFile(
 	envId?: number | null,
 	customEnvPath?: string
 ): Promise<void> {
-	const envFilePath = customEnvPath
-		? customEnvPath
-		: join(await findStackDir(stackName, envId) || await getStackDir(stackName, envId), '.env');
+	let envFilePath: string;
+	if (customEnvPath) {
+		envFilePath = customEnvPath;
+	} else {
+		// Check if stack has a custom path in DB
+		const source = await getStackSource(stackName, envId);
+		if (source?.envPath) {
+			envFilePath = source.envPath;
+		} else if (source?.composePath) {
+			// Derive env path from custom compose path location
+			envFilePath = join(dirname(source.composePath), '.env');
+		} else {
+			// Fall back to default location
+			envFilePath = join(await findStackDir(stackName, envId) || await getStackDir(stackName, envId), '.env');
+		}
+	}
 
 	// Ensure parent directory exists
 	const dir = dirname(envFilePath);