diff --git a/src/lib/server/backups/backup-script.ts b/src/lib/server/backups/backup-script.ts
index 3b0d0d8..e587be3 100644
--- a/src/lib/server/backups/backup-script.ts
+++ b/src/lib/server/backups/backup-script.ts
@@ -41,7 +41,9 @@ export interface MetadataFile {
* @param resticArgs the full restic argv (starting with 'backup') — already
* tag/exclude/flag-assembled by the caller.
*/
-export function buildBackupScript(resticArgs: string[], stackDirProbe?: { volumeKey: string; composeFileName: string; label?: string; hostPath?: string }): string {
+import type { StackDirProbeHint } from './stackdir-plan';
+
+export function buildBackupScript(resticArgs: string[], stackDirProbe?: { volumeKey: string; composeFileName: string; label?: string; hostPath?: string; hint?: StackDirProbeHint }): string {
// restic runs backgrounded so the SIGINT-forwarding trap can reach it on cancel
// (SIGINT lets restic release its repo lock; a SIGKILL orphans it and hangs the
// next backup on --retry-lock). The kill-0 re-wait loop only fires when a signal
@@ -61,14 +63,28 @@ export function buildBackupScript(resticArgs: string[], stackDirProbe?: { volume
if (stackDirProbe) {
const composePath = shellQuote(`/volumes/${stackDirProbe.volumeKey}/${stackDirProbe.composeFileName}`);
const label = stackDirProbe.label ? ` ${stackDirProbe.label}` : '';
- // Name the REAL host path we mounted (/) so the operator knows
- // exactly where to look, and what to do: redeploy the stack to stage its files there.
- const hostHint = stackDirProbe.hostPath
- ? ` The stack folder is empty or missing at ${stackDirProbe.hostPath} on the Docker host - redeploy the stack to stage its files there.`
- : '';
+ // The compose is missing under the host dir we bind-mounted. For a remote env (hawser or a
+ // user-set path) tell the operator to point "Remote stack path (for backup)" at the real host
+ // path; for a local stack a redeploy stages the files.
+ const hint = stackDirProbe.hint;
+ const remote = hint?.kind === 'hawser-defaulted' || hint?.kind === 'user-set';
+ const where = remote ? hint.hostPath : stackDirProbe.hostPath;
+ // "on " when we know the environment name, else a plain "on the host".
+ const on = remote && hint.envName ? ` on ${hint.envName}` : ` on the host`;
+ // Name the EXACT settings location so the operator can go straight there.
+ const settingsLoc = remote && hint.envName ? `Settings > Environments > ${hint.envName}` : `Settings > Environments`;
+ const fix = remote
+ ? ` Set "Remote stack path (for backup)" in ${settingsLoc} to the real host path where this stack's files live.`
+ : ` Redeploy the stack to stage its files there.`;
+ const at = where ? ` in ${where}` : '';
+ // Build the whole message as ONE string and shell-quote it so the operator-facing text
+ // (which contains quotes, parens, and `>` - e.g. `"Remote stack path (for backup)"` and
+ // `Settings > Environments > `) can never break the `sh` syntax. The timestamp stays
+ // a live $(date ...) OUTSIDE the quoted literal, concatenated in.
+ const msg = `${label} STACKDIR PROBE FAILED: compose ${stackDirProbe.composeFileName} not found${at}${on}.${fix}`;
probe =
`if [ ! -f ${composePath} ]; then ` +
- `echo "[backup] $(date -u +%Y-%m-%dT%H:%M:%SZ)${label} STACKDIR PROBE FAILED: the stack's compose (${stackDirProbe.composeFileName}) is not present under the mounted stack dir.${hostHint} Refusing to write a silent-empty snapshot." 1>&2; ` +
+ `echo "[backup] $(date -u +%Y-%m-%dT%H:%M:%SZ)"${shellQuote(msg)} 1>&2; ` +
`exit 1; fi; `;
}
diff --git a/src/lib/server/backups/backup-service.ts b/src/lib/server/backups/backup-service.ts
index ca51526..6865ff2 100644
--- a/src/lib/server/backups/backup-service.ts
+++ b/src/lib/server/backups/backup-service.ts
@@ -43,7 +43,7 @@ import { EXIT_MARKER } from './restic-script';
import { parseRetention, buildForgetArgs, checkWouldWipe, retentionActive } from './retention';
import { liveTargetKey } from './locks';
import type { DiscoveredVolume } from './discovery-core';
-import { STACKDIR_VOLUME_KEY } from './stackdir-plan';
+import { STACKDIR_VOLUME_KEY, type StackDirProbeHint } from './stackdir-plan';
/** What the service needs from the outside world. All injected for testability. */
export interface BackupPorts {
@@ -53,18 +53,16 @@ export interface BackupPorts {
/** Discover volumes; MUST throw (fail closed) on any inspect failure. */
discoverVolumes(containers: Array<{ id: string; name: string }>, envId: number | null | undefined, selected: string[] | null):
Promise<{ volumes: DiscoveredVolume[]; skipped: Array<{ type: string; destination: string; reason: string }> }>;
- /** Decide how a STACK's directory is captured. Normally: locate the HOST folder and build the
- * synthetic __dockhand_stackdir__ volume the helper bind-mounts (`kind: 'candidate'`).
- * `kind: 'tar'` is the direct-remote-with-no-remote_stacks_dir case: nothing is staged on the
- * host, so the compose/config is tarred from Dockhand's local copy instead (no host bind).
- * `composeFileName` feeds the in-helper probe (assert the compose is
- * visible under the mount). `kind: 'unknown'` means the stack is not compose-managed (no
- * working_dir label) -> the caller HARD-FAILS the backup. Logged inside this port. */
+ /** Decide how a STACK's directory is captured: locate the HOST folder and build the synthetic
+ * __dockhand_stackdir__ volume the helper bind-mounts (`kind: 'candidate'`). The stack files
+ * are ALWAYS read from the host where the stack runs - never from Dockhand's own copy.
+ * `composeFileName` feeds the in-helper probe (assert the compose is visible under the mount).
+ * `kind: 'unknown'` means the host folder can't be located (not compose-managed, or a remote
+ * env with no Remote stacks directory) -> the caller HARD-FAILS the backup. Logged inside this port. */
planStackDirVolume(targetName: string, envId: number | null | undefined, excludedStackFiles?: string[]):
Promise<
| { kind: 'unknown'; reason: string }
- | { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[] }
- | { kind: 'tar'; localStackDir: string; composeFileName: string }
+ | { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[]; probeHint?: StackDirProbeHint }
>;
/** Stop the target for a consistent backup; returns a restart closure. */
stopForBackup(type: BackupTargetType, targetName: string, containers: Array<{ id: string; name: string; state: string }>, envId: number | null | undefined):
@@ -72,17 +70,14 @@ export interface BackupPorts {
/** Run the backup session container; returns the restic run. metadataFiles (metadata.json
* + the light stack-dir listing) are put-archived into /metadata (not the Cmd) so they
* can't blow ARG_MAX. The stack dir's bytes ride a host bind mount, not these files. */
- runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }>; onStderr?: (line: string) => void; onStdout?: (line: string) => void }):
+ runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; onStderr?: (line: string) => void; onStdout?: (line: string) => void }):
Promise;
/** Run a repo-only restic command on the host (verify / retention). */
runLocal(args: string[], tier?: 'interactive' | 'data'): Promise;
/** Collect the restore metadata files: metadata.json plus, for a stack, a light file
- * listing. The stack dir's bytes normally ride the helper's host bind mount, not these
- * files. EXCEPTION - `stackDirTar`: for a direct-remote stack with no remote_stacks_dir
- * there is no host bind, so the collector reads localStackDir and tars its files into the
- * snapshot at /volumes/__dockhand_stackdir__ (the same path restore reads). */
- collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[], stackDirTar?: { localStackDir: string; composeFileName: string } | null):
- Promise<{ files: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }> }>;
+ * listing. The stack dir's bytes ride the helper's host bind mount, not these files. */
+ collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[]):
+ Promise<{ files: MetadataFile[] }>;
/** The Dockhand host name for restic --host. */
host(): string;
/** This installation's stable instance id. */
@@ -151,13 +146,10 @@ export class BackupService {
// __dockhand_stackdir__ volume). `stackDirProbe` carries what the helper needs to
// assert that mount is real (test -f the compose) before restic runs. Null for
// containers (no stack dir).
- let stackDirProbe: { volumeKey: string; composeFileName: string; hostPath?: string } | null = null;
+ let stackDirProbe: { volumeKey: string; composeFileName: string; hostPath?: string; hint?: StackDirProbeHint } | null = null;
// restic --exclude paths for compose bind dirs inside the stackdir volume (they are
// captured separately as their own /volumes/, honoring volume selection).
let stackDirExcludes: string[] = [];
- // Set (instead of a host bind) for a direct-remote stack with no remote_stacks_dir: the
- // compose/config is tarred into /metadata from Dockhand's local copy, not the host.
- let stackDirTar: { localStackDir: string; composeFileName: string } | null = null;
try {
const t = await this.ports.resolveTargets(job.type, job.targetName, envId);
containers = t.containers;
@@ -184,16 +176,11 @@ export class BackupService {
if (p.kind === 'unknown') {
return this.earlySkipOrError(job, triggeredBy,
`Cannot locate the stack folder on the host for "${job.targetName}" (${p.reason}). ` +
- `For remote environments, set a Remote stacks directory or use matching paths so the stack files exist on the Docker host. ` +
+ `For remote environments, set a Remote stack path (for backup) in Settings > Environments so the stack files exist on the Docker host. ` +
`Refusing to write a backup missing the stack's compose and config.`);
- } else if (p.kind === 'tar') {
- // direct-remote, no remote_stacks_dir: no host bind (nothing staged), so the
- // compose/config is captured from Dockhand's local copy into /metadata via a tar.
- // No synthetic volume, no probe - the metadata collector reads localStackDir.
- stackDirTar = { localStackDir: p.localStackDir, composeFileName: p.composeFileName };
} else {
volumes = [...volumes, p.syntheticVolume];
- stackDirProbe = { volumeKey: p.volumeKey, composeFileName: p.composeFileName, hostPath: p.syntheticVolume.source ?? undefined };
+ stackDirProbe = { volumeKey: p.volumeKey, composeFileName: p.composeFileName, hostPath: p.syntheticVolume.source ?? undefined, hint: p.probeHint };
stackDirExcludes = p.excludePaths;
}
}
@@ -221,7 +208,7 @@ export class BackupService {
try {
// --- serialize on the destination (share the repo, one at a time) ---
return await this.ports.serializeDestination(job.destinationId, () =>
- this.runLocked(job, triggeredBy, containers, volumes, stackDirProbe, stackDirExcludes, stackDirTar, startTime, releaseOnce));
+ this.runLocked(job, triggeredBy, containers, volumes, stackDirProbe, stackDirExcludes, startTime, releaseOnce));
} finally {
releaseOnce();
}
@@ -237,9 +224,8 @@ export class BackupService {
triggeredBy: 'cron' | 'manual' | 'webhook',
containers: Array<{ id: string; name: string; state: string }>,
volumes: DiscoveredVolume[],
- stackDirProbe: { volumeKey: string; composeFileName: string } | null,
+ stackDirProbe: { volumeKey: string; composeFileName: string; hostPath?: string; hint?: StackDirProbeHint } | null,
stackDirExcludes: string[],
- stackDirTar: { localStackDir: string; composeFileName: string } | null,
startTime: number,
releaseLiveTarget: () => void,
): Promise {
@@ -287,7 +273,7 @@ export class BackupService {
// --- metadata + backup args + session script ---
op.progress('metadata', 'Collecting metadata...');
- const { files: metadata, stackDirStreamSources } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes, stackDirTar);
+ const { files: metadata } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes);
const instanceId = await this.ports.instanceId();
const tags = buildSnapshotTags({ instanceId, configId: job.configId, environmentId: envId, targetName: job.targetName, type: job.type });
// Cross-shard identity. If two shards log the SAME instanceId + configId
@@ -295,14 +281,12 @@ export class BackupService {
// collide. host() + destinationId show which repo this run targets, so a
// CI log across shards reveals any overlap.
diag('IDENTITY', `instanceId=${instanceId} host=${this.ports.host()} destId=${job.destinationId} tags=[${tags.join(',')}]`);
- // restic must back up /volumes/ when there's anything to capture there: real bind/named
- // volumes (volumes.length), OR a tar-mode stack whose compose/config was put-archived
- // into /volumes/__dockhand_stackdir__ (no synthetic volume in `volumes`, so it wouldn't
- // count otherwise - and /metadata-only would silently drop the tarred stack files).
+ // restic backs up /volumes/ when there's anything to capture there: real bind/named
+ // volumes OR the synthetic __dockhand_stackdir__ volume (both are in `volumes`).
const args = buildBackupArgs({
host: this.ports.host(),
tags,
- hasVolumes: volumes.length > 0 || stackDirTar !== null,
+ hasVolumes: volumes.length > 0,
// User excludes PLUS the compose bind dirs inside the stackdir volume (those
// ride their own /volumes/, so excluding them here avoids a duplicate and
// keeps volume DESELECTION meaningful for in-folder binds).
@@ -381,7 +365,7 @@ export class BackupService {
// This flag drives that fallback.
let liveStdoutSeen = false;
const run = await this.ports.runInHelper({
- script, binds, envId, name: job.helperName, metadataFiles: metadata, stackDirStreamSources,
+ script, binds, envId, name: job.helperName, metadataFiles: metadata,
// Stream restic's own output to the UI LIVE: restic writes its --json
// progress (per-file %) to STDOUT and its status messages to STDERR. Wire
// BOTH so the log fills in as the backup runs, not all at once at the end.
diff --git a/src/lib/server/backups/index.ts b/src/lib/server/backups/index.ts
index bdf0bc5..11bfe85 100644
--- a/src/lib/server/backups/index.ts
+++ b/src/lib/server/backups/index.ts
@@ -29,6 +29,8 @@ import { sendEventNotification } from '../notifications';
import type { BackupResult, RestoreResult, ResticRun, BackupTargetType } from './models';
import type { MetadataFile } from './backup-script';
import type { DiscoveredVolume } from './discovery-core';
+import type { StackDirProbeHint } from './stackdir-plan';
+import { normalizeBaseDir, stackDirIn } from '../stack-paths';
import { volumeBind } from './discovery-core';
import { resolveBindFromMetadata } from './restore-core';
import { parseSnapshotLsEntries } from './browse-core';
@@ -122,8 +124,7 @@ async function planStackDirVolume(
attempts = 7,
): Promise<
| { kind: 'unknown'; reason: string }
- | { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[] }
- | { kind: 'tar'; localStackDir: string; composeFileName: string }
+ | { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[]; probeHint?: StackDirProbeHint }
> {
const { getStackComposeFile } = await import('../stacks');
const { dirname, join, basename } = await import('path');
@@ -174,16 +175,23 @@ async function planStackDirVolume(
let envTcpHost: string | null = null;
let remoteStacksDir: string | null = null;
let isHawser = false;
+ // True when a hawser env had NO remote_stacks_dir set and we fell back to the agent's default
+ // STACKS_DIR (/data/stacks). Steers the probe-fail message: a defaulted path that comes up empty
+ // means the agent stores stacks under a DIFFERENT host dir (e.g. a containerized agent whose
+ // /data/stacks is mounted from another host path), so the user must set the HOST-side path.
+ let remoteStacksDirDefaulted = false;
+ let envName: string | null = null;
if (envId != null) {
try {
const { getEnvironment, getEnvSetting } = await import('../db');
const env = await getEnvironment(envId);
envConnType = env?.connectionType ?? null;
+ envName = env?.name ?? null;
envTcpHost = env?.host && env?.port ? `tcp://${env.host}:${env.port}` : null;
isHawser = envConnType === 'hawser-standard' || envConnType === 'hawser-edge';
if (envConnType === 'direct') {
const rsd = await getEnvSetting('remote_stacks_dir', envId);
- remoteStacksDir = typeof rsd === 'string' && rsd.trim() ? rsd.trim().replace(/\/+$/, '') : null;
+ remoteStacksDir = typeof rsd === 'string' && rsd.trim() ? normalizeBaseDir(rsd) : null;
} else if (isHawser) {
// Same `remote_stacks_dir` setting as direct, but for hawser it is a BACKUP-ONLY
// declaration of where the AGENT keeps stack files on its host - it does NOT steer
@@ -191,9 +199,9 @@ async function planStackDirVolume(
// to bind-mount. Falls back to the agent's default STACKS_DIR (/data/stacks) so a
// standard agent needs no configuration.
const rsd = await getEnvSetting('remote_stacks_dir', envId);
- remoteStacksDir = typeof rsd === 'string' && rsd.trim()
- ? rsd.trim().replace(/\/+$/, '')
- : HAWSER_DEFAULT_STACKS_DIR;
+ const userSet = typeof rsd === 'string' && rsd.trim();
+ remoteStacksDir = userSet ? normalizeBaseDir(rsd) : HAWSER_DEFAULT_STACKS_DIR;
+ remoteStacksDirDefaulted = !userSet;
}
} catch { /* treat as unknown -> non-local (safe: skips the wrong-host translation) */ }
}
@@ -206,9 +214,8 @@ async function planStackDirVolume(
// direct env the decision is explicit, not guessed:
// - remote_stacks_dir SET -> the deploy staged the files at / on the host, so
// that IS the host path (bind-derived can still win below if a bind pins it more exactly).
- // - remote_stacks_dir UNSET -> nothing was staged; capture the compose/config from
- // Dockhand's OWN copy via a tar into the snapshot (no host bind, so no :ro mask). The
- // caller signals this with kind: 'tar'.
+ // - remote_stacks_dir UNSET -> the host folder can't be located; resolution returns UNKNOWN
+ // and the backup HARD-FAILS with an actionable message (set a Remote stacks directory).
const directRemote = envConnType === 'direct' && !localDaemon;
// The working_dir label is only trustworthy for hawser (agent ran compose on its host) or a
// LOCAL daemon. For direct-remote it's Dockhand's path, so drop it from the candidates.
@@ -217,10 +224,11 @@ async function planStackDirVolume(
// AND hawser. For hawser this is the agent's STACKS_DIR (default /data/stacks) - the DETERMINISTIC
// host location, so it wins over the flaky working_dir label (which is "/" for a start-from-stdin
// hawser stack). The in-helper probe confirms the compose is actually there.
- const remoteStacksDirHostPath = (directRemote || isHawser) && remoteStacksDir ? `${remoteStacksDir}/${targetName}` : null;
+ // SAME formula the deploy plan STAGES to (stackDirIn) - deploy WRITES here, backup READS here.
+ const remoteStacksDirHostPath = (directRemote || isHawser) && remoteStacksDir ? stackDirIn(remoteStacksDir, targetName) : null;
// bind-derived is a PHANTOM empty dir for a direct-remote stdin deploy with no remote_stacks_dir
- // (see trustBindDerivedForEnv) - distrust it there so resolution falls through to kind:'tar'.
+ // (see trustBindDerivedForEnv) - distrust it there so resolution falls through to UNKNOWN (hard-fail).
const trustedBindDerived = trustBindDerivedForEnv(bindDerivedHostPath, { directRemote, hasRemoteStacksDir: !!remoteStacksDir });
// Fallback host-path candidates (LOCAL daemon only). dataDirHostPath: DATA_DIR -> HOST_DATA_DIR
@@ -243,14 +251,10 @@ async function planStackDirVolume(
workingDirLabel: trustedWorkingDirLabel,
});
if (resolution.kind === 'unknown') {
- // A direct-REMOTE env with NO remote_stacks_dir has nothing staged on the host, but
- // Dockhand HAS the stack's compose/config locally - capture it via a tar into the
- // snapshot instead of hard-failing. (Only compose/config; volume data rides its own
- // /volumes/. Secrets are never on disk - carried as ciphertext in metadata.)
- if (directRemote && !remoteStacksDir && dockhandStackDir) {
- console.log(`[Backup] stackdir plan for "${targetName}": TAR (direct-remote, no remote_stacks_dir) - capturing compose/config from Dockhand's local copy in folder ${dockhandStackDir}`);
- return { kind: 'tar', localStackDir: dockhandStackDir, composeFileName: composeFileName0 ?? 'docker-compose.yml' };
- }
+ // Backup ALWAYS reads the stack files from the host where the stack runs - never from
+ // Dockhand's own copy. When the host folder can't be located (a remote env with no
+ // Remote stacks directory / no matching paths), we HARD-FAIL with an actionable message
+ // rather than silently capturing a possibly-stale local copy.
// Report every candidate we tried so an UNKNOWN is diagnosable without SSH: which
// inputs were null tells us WHY (e.g. workingDir null = 0 containers listed).
console.log(`[Backup] stackdir plan for "${targetName}": UNKNOWN reason="${resolution.reason}" dockhandStackDir=${dockhandStackDir ?? 'none'} | connType=${envConnType} localDaemon=${localDaemon} remoteStacksDir=${remoteStacksDir ?? 'null'} containers=${stackContainers.length} bindDerived=${bindDerivedHostPath ?? 'null'} dataDir=${dataDirHostPath ?? 'null'} mount=${mountHostPath ?? 'null'} workingDirLabel=${workingDirLabel ?? 'null'}`);
@@ -293,6 +297,15 @@ async function planStackDirVolume(
}
}
+ // The probe (in the helper) hard-fails when the compose is missing under the mounted host dir.
+ // Give it the context to tell the operator what to DO: a defaulted hawser path that comes up
+ // empty means the agent keeps stacks under a different HOST dir, so the fix is to set the
+ // HOST-side "Remote stack path (for backup)" - NOT to redeploy.
+ const probeHint: StackDirProbeHint =
+ isHawser && remoteStacksDirDefaulted ? { kind: 'hawser-defaulted', hostPath, envName }
+ : (isHawser || directRemote) && remoteStacksDir ? { kind: 'user-set', hostPath, envName }
+ : { kind: 'local' };
+
return {
kind: 'candidate',
syntheticVolume: {
@@ -306,6 +319,7 @@ async function planStackDirVolume(
composeFileName,
excludePaths,
bindSources,
+ probeHint,
};
}
@@ -318,13 +332,9 @@ async function planStackDirVolume(
export async function previewStackBackupPath(
targetName: string,
envId: number | null | undefined,
-): Promise<{ kind: 'candidate'; hostPath: string; composeFile: string } | { kind: 'tar'; composeFile: string } | { kind: 'unknown'; reason: string }> {
+): Promise<{ kind: 'candidate'; hostPath: string; composeFile: string } | { kind: 'unknown'; reason: string }> {
const plan = await planStackDirVolume(targetName, envId);
if (plan.kind === 'unknown') return { kind: 'unknown', reason: plan.reason };
- // TAR mode (direct-remote, no remote_stacks_dir): the stack folder is captured from
- // Dockhand's own copy into the snapshot, not from a host path. Surface that to the UI so the
- // preview says "captured from Dockhand" rather than showing a misleading host path.
- if (plan.kind === 'tar') return { kind: 'tar', composeFile: plan.composeFileName };
return { kind: 'candidate', hostPath: plan.syntheticVolume.source ?? '', composeFile: plan.composeFileName };
}
@@ -342,7 +352,6 @@ export async function probeStackDir(
envId: number | null | undefined,
): Promise<
| { kind: 'listed'; hostPath: string; entries: import('./stackdir-plan').StackDirEntry[] }
- | { kind: 'tar'; localStackDir: string; entries: import('./stackdir-plan').StackDirEntry[] }
| { kind: 'helper-failed'; reason: string }
| { kind: 'unknown'; reason: string }
> {
@@ -351,30 +360,14 @@ export async function probeStackDir(
const { ensureHelperImage } = await import('./restic');
const { INSTANCE_LABEL } = await import('./reap-core');
- // Ask the real capture planner how this stack resolves. TAR mode (direct-remote, no
- // remote_stacks_dir) has no host folder to mount - the backup streams Dockhand's OWN local
- // copy (compose/config only; bind data lives on the remote host, out of reach). Surface THAT
- // to the picker so the user knows exactly what a backup would contain, instead of a bare
- // "unknown". A `candidate`/`unknown` plan falls through to the host-mount probe below.
+ // Ask the real capture planner how this stack resolves. A `candidate` plan falls through to
+ // the host-mount probe below; an `unknown` plan means the host folder can't be located, so the
+ // picker surfaces that (the user must set the env's stack path).
// attempts=1: config-time probe. The stack already exists (the user is editing its backup),
// so a not-listable container is legitimately stopped/absent - the 15s just-deployed retry
// would only hang the picker. (planStackDirVolume calls collectStackContainerHostInfo ONCE and
// returns its bindSources, so the probe does NOT inspect the containers a second time.)
const plan = await planStackDirVolume(targetName, envId, undefined, 1);
- if (plan.kind === 'tar') {
- const { readdirSync, statSync } = await import('fs');
- const { join } = await import('path');
- let entries: import('./stackdir-plan').StackDirEntry[] = [];
- try {
- entries = readdirSync(plan.localStackDir).map((name) => {
- let type: 'dir' | 'file' = 'file'; let size = 0;
- try { const st = statSync(join(plan.localStackDir, name)); type = st.isDirectory() ? 'dir' : 'file'; size = st.isFile() ? st.size : 0; } catch { /* keep defaults */ }
- return { name, type, size };
- }).sort((a, b) => (a.type === b.type ? a.name.localeCompare(b.name) : a.type === 'dir' ? -1 : 1));
- } catch { /* empty local copy -> empty list, still tar kind */ }
- return { kind: 'tar', localStackDir: plan.localStackDir, entries };
- }
-
if (plan.kind === 'unknown') {
return { kind: 'unknown', reason: plan.reason };
}
@@ -532,12 +525,8 @@ async function collectMetadata(
targetName: string,
envId: number | null | undefined,
volumes: DiscoveredVolume[],
- stackDirTar?: { localStackDir: string; composeFileName: string } | null,
-): Promise<{ files: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }> }> {
+): Promise<{ files: MetadataFile[] }> {
const files: MetadataFile[] = [];
- // tar-mode ONLY: disk paths of the stack dir files to stream into the helper (no base64, no
- // cap). Filled in the stack block below when stackDirTar is set; empty otherwise.
- const stackDirStreamSources: Array<{ diskPath: string; archivePath: string }> = [];
// Writes metadata.json plus, for a stack, a light file listing (path + size, no content)
// of the stack dir. The stack dir's bytes are captured by the helper's HOST bind mount
// (the synthetic __dockhand_stackdir__ volume from planStackDirVolume), not here.
@@ -582,7 +571,7 @@ async function collectMetadata(
if (type === 'stack') {
const { getStackComposeFile } = await import('../stacks');
const { readFileSync, readdirSync, lstatSync, statSync } = await import('fs');
- const { dirname, join, relative, basename, sep } = await import('path');
+ const { dirname, join, relative, basename } = await import('path');
const compose = await getStackComposeFile(targetName, envId ?? undefined);
if (compose.success && compose.composePath) {
// Presence of `stackInfo` IS hasStackFiles (derived, not a stored flag that could
@@ -627,22 +616,7 @@ async function collectMetadata(
listed.sort((a, b) => a.path.localeCompare(b.path));
if (stackInfo) { stackInfo.fileList = listed; stackInfo.excludedBindDirs = excludeRelDirs; }
- if (stackDirTar) {
- // TAR mode (direct-remote, no remote_stacks_dir): no host bind exists, so the
- // listed files are STREAMED from disk into the helper at
- // /volumes/__dockhand_stackdir__/ (O(1) RAM, no cap; restore reads the same
- // path via stackDirSource). Bind data dirs are excluded (same as listed) - their
- // runtime bytes live on the target host, out of reach; compose/config only.
- for (const { path: rel } of listed) {
- stackDirStreamSources.push({
- diskPath: join(stackDir, rel),
- archivePath: `volumes/${STACKDIR_VOLUME_KEY}/${rel.split(sep).join('/')}`,
- });
- }
- console.log(`[Backup] stackfiles "${targetName}": TAR mode - streaming ${stackDirStreamSources.length} file(s) into /volumes/${STACKDIR_VOLUME_KEY} from folder ${stackDir}`);
- } else {
- console.log(`[Backup] stackfiles "${targetName}": captured via HOST bind mount at /volumes/${STACKDIR_VOLUME_KEY} - listed ${listed.length} file(s), ${excludeRelDirs.length} bind dir(s) excluded`);
- }
+ console.log(`[Backup] stackfiles "${targetName}": captured via HOST bind mount at /volumes/${STACKDIR_VOLUME_KEY} - listed ${listed.length} file(s), ${excludeRelDirs.length} bind dir(s) excluded`);
} catch (e) {
console.warn(`[Backup] stackfiles "${targetName}": listing failed (files still captured via bind):`, e instanceof Error ? e.message : e);
}
@@ -663,7 +637,7 @@ async function collectMetadata(
}
}
- return { files, stackDirStreamSources: stackDirStreamSources.length > 0 ? stackDirStreamSources : undefined };
+ return { files };
}
// A restic helper runner bound to one destination (so no shared mutable state —
diff --git a/src/lib/server/backups/models.ts b/src/lib/server/backups/models.ts
index 6102850..6cdbc8a 100644
--- a/src/lib/server/backups/models.ts
+++ b/src/lib/server/backups/models.ts
@@ -296,14 +296,20 @@ export function readConfigIdFromTags(tags: string[]): number | null {
}
// =============================================================================
-// Timeout tiers — two named tiers, not a per-operation multiplier ladder
+// Timeout tiers
// =============================================================================
-/** Two timeout tiers, in ms. `interactive` for quick repo ops (list/ls/init);
- * `data` for streaming ops that move real bytes (backup/restore/check/prune). */
+/** Restic operations are UNBOUNDED by default (0 = no timeout). A large backup or a
+ * slow listing on a big/remote repo legitimately runs for a long time; a fixed wall-clock
+ * cap killed healthy work mid-run (#1382). A stuck op is handled by manual cancel (the
+ * helper is a named container killed by cancelBackup, and every run shows as `running` in
+ * the execution log) and the helper reaper — the same model backrest/zerobyte use (neither
+ * times out a backup). The two tiers remain so callers can still opt into a cap via env:
+ * `RESTIC_TIMEOUT` (interactive: list/ls/init) and `RESTIC_MAINTENANCE_TIMEOUT`
+ * (data: backup/restore/prune). Both default to 0 (off). */
export const TIMEOUTS = {
- interactive: Number(process.env.RESTIC_TIMEOUT ?? 300_000), // 5 min
- data: Number(process.env.RESTIC_MAINTENANCE_TIMEOUT ?? 6 * 60 * 60 * 1000), // 6 h
+ interactive: Number(process.env.RESTIC_TIMEOUT ?? 0),
+ data: Number(process.env.RESTIC_MAINTENANCE_TIMEOUT ?? 0),
} as const;
export type TimeoutTier = keyof typeof TIMEOUTS;
diff --git a/src/lib/server/backups/restic.ts b/src/lib/server/backups/restic.ts
index 8b5ae43..07d77a3 100644
--- a/src/lib/server/backups/restic.ts
+++ b/src/lib/server/backups/restic.ts
@@ -19,8 +19,7 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { runContainerWithStreaming, inspectImage, pullImage, putContainerArchive } from '../docker';
-import { buildTar, buildTarStream, type TarFileSource } from './tar';
-import { putContainerArchiveStreaming } from './put-archive-stream';
+import { buildTar } from './tar';
import type { MetadataFile } from './backup-script';
import { getInstanceId } from './identity';
import { decryptBackupDestination, getSetting } from '../db';
@@ -111,11 +110,6 @@ export interface HelperRunSpec {
/** Small in-RAM files (e.g. metadata.json) put-archived into the container BEFORE it
* starts. `path` is the full path from the container root. */
metadataFiles?: MetadataFile[];
- /** tar-mode ONLY (direct-remote, no remote_stacks_dir): the stack dir's files, streamed
- * from disk into the helper (O(1) RAM, no cap) alongside metadataFiles in the same
- * put-archive. Each source's archivePath is the full container path
- * (volumes/__dockhand_stackdir__/). Empty/absent = no streaming. */
- stackDirStreamSources?: TarFileSource[];
}
export class Restic {
@@ -247,21 +241,12 @@ export class Restic {
// the container via put-archive (docker cp), NOT the Cmd, so they can't blow
// ARG_MAX. The stack dir's bytes ride a read-only host bind mount, not these files.
const metadataFiles = spec.metadataFiles ?? [];
- const streamSources = spec.stackDirStreamSources ?? [];
const inlineEntries = metadataFiles.map((f) => ({ path: f.path.replace(/^\/+/, ''), content: Buffer.from(f.contentBase64, 'base64') }));
- const beforeStart = (metadataFiles.length > 0 || streamSources.length > 0)
+ const beforeStart = metadataFiles.length > 0
? async (containerId: string) => {
// Each file's `path` is the full path from the container root (e.g.
// `metadata/metadata.json`), placing it under the right root.
- if (streamSources.length > 0) {
- // tar-mode: STREAM the stack dir files from disk (O(1) RAM, no cap) into the
- // helper, with metadata.json as inline entries in the same tar. All under `/`.
- const tar = buildTarStream(streamSources, inlineEntries);
- const { streamed } = await putContainerArchiveStreaming(containerId, '/', tar, spec.envId);
- if (!streamed) throw new Error(`tar-mode: transport for env ${spec.envId} cannot stream the stack dir`);
- } else {
- await putContainerArchive(containerId, '/', await buildTar(inlineEntries), spec.envId);
- }
+ await putContainerArchive(containerId, '/', await buildTar(inlineEntries), spec.envId);
}
: undefined;
diff --git a/src/lib/server/backups/stackdir-plan.ts b/src/lib/server/backups/stackdir-plan.ts
index ad0cc28..f591b69 100644
--- a/src/lib/server/backups/stackdir-plan.ts
+++ b/src/lib/server/backups/stackdir-plan.ts
@@ -203,6 +203,17 @@ export function isLocalDaemon(connectionType: string | null, envTcpHost: string
return false;
}
+// Context the in-helper probe-fail message needs to give the operator an ACTIONABLE next step
+// (the fix differs by wiring). `hawser-defaulted`: a hawser env with no remote_stacks_dir fell
+// back to /data/stacks and it came up empty -> the agent stores stacks under a DIFFERENT host dir
+// (e.g. a containerized agent whose /data/stacks is mounted from another host path), so set the
+// HOST-side path. `user-set`: an explicitly-configured host path came up empty -> the path is
+// wrong. `local`: a socket/local-daemon stack -> redeploy staged its files.
+export type StackDirProbeHint =
+ | { kind: 'hawser-defaulted'; hostPath: string; envName: string | null }
+ | { kind: 'user-set'; hostPath: string; envName: string | null }
+ | { kind: 'local' };
+
/** Result of resolving the candidate host stack folder (before the runtime probe). */
export type HostStackDirResolution =
| { kind: 'candidate'; hostPath: string; composeFile: string; source: string }
diff --git a/src/routes/backups/CreateBackupModal.svelte b/src/routes/backups/CreateBackupModal.svelte
index 66fc55f..d43e2eb 100644
--- a/src/routes/backups/CreateBackupModal.svelte
+++ b/src/routes/backups/CreateBackupModal.svelte
@@ -51,7 +51,6 @@
// Stack files on the host (probe listing) — stack targets only.
type StackListing =
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
- | { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
| { kind: 'helper-failed'; reason: string }
| { kind: 'unknown'; reason: string }
| null;
diff --git a/src/routes/containers/BackupPanel.svelte b/src/routes/containers/BackupPanel.svelte
index c794c27..18df5cc 100644
--- a/src/routes/containers/BackupPanel.svelte
+++ b/src/routes/containers/BackupPanel.svelte
@@ -498,7 +498,6 @@
// picker, but stackPath is still fetched for the compose-file name shown elsewhere.
type StackListing =
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
- | { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
| { kind: 'unknown'; reason: string }
| null;
let stackListing = $state(null);