/** * backups/restore-service.ts — orchestrates ONE restore, with stronger safety * guards than backup because a restore can destroy live data. * * Two modes: * new-location (default, non-destructive): restic restores the requested * volumes into a fresh target directory. Nothing live is touched. * in-place (destructive, explicit intent required): restic restores each * volume into a staging dir on the live volume's own filesystem, * then a phase-marked swap commits it — never deleting the * original until the replacement is in place. * * Safety properties enforced here: * - the caller must OWN the snapshot (instance + environment) before any read; * - an in-place restore requires explicit confirmation (validator + service, defence in depth); * - the requested volumes must exist in the snapshot before anything live is * touched (so a typo can't wipe a volume for a restore that then fails); * - a failed/interrupted swap leaves live data intact; * - two operations on the same live target are rejected; ops on one repo serialize; * - the restore never records success on an unknown outcome (undefined exit). */ import { BackupError, resticOk, type RestoreResult, type PostRestoreOutcome, type ResticRun, } from './models'; import { buildInPlaceRestore, buildNewLocationRestore, buildCloneRestore } from './restore-script'; import { assertSafeVolumeInclude, assertSafeRestoreTarget } from './security'; import { liveTargetKey } from './locks'; import { cleanErrorMsg } from './helpers'; import type { SnapshotLayout } from './snapshot-layout'; import type { OperationHandle } from './backup-service'; import { formatResticLines } from './backup-service'; import { EXIT_MARKER } from './restic-script'; /** What the restore service needs from the outside. All injected for testability. */ export interface RestorePorts { /** Verify instance + env ownership of the snapshot; throws if not allowed. */ guardSnapshotAccess(destinationId: number, snapshotId: string): Promise; /** List the volume names present in a snapshot (for existence checks). */ listSnapshotVolumes(destinationId: number, snapshotId: string): Promise; /** Resolve the target container(s) for an in-place restore. `type` selects a * single container (name match) or a whole compose stack (project-label match); * defaults to 'container'. */ resolveTargets(targetName: string, envId: number | null | undefined, type?: 'container' | 'stack'): Promise<{ containers: Array<{ id: string; name: string; state: string }> }>; /** Stop the target container(s) before an in-place restore. Throws on failure * so the restore aborts before touching live volumes. Returns a restart fn. */ stopBeforeRestore(containers: Array<{ id: string; name: string; state: string }>, envId: number | null | undefined): Promise<{ restart: () => Promise }>; /** Stop a whole stack (native `docker compose stop`) before an in-place restore. * Throws on failure so the restore aborts before touching live volumes. Returns * a restart fn (native `docker compose start`) for the error path. */ stopStackBeforeRestore(stackName: string, envId: number | null | undefined): Promise<{ restart: () => Promise }>; /** Run the restore session container. */ runInHelper(spec: { script?: string; args?: string[]; binds: string[]; envId: number | null | undefined; name: string; onStderr?: (line: string) => void }): Promise; /** Resolve the live bind for a volume from ALREADY-READ metadata (name → `:/volumes/:rw`). * Takes the metadata the caller already loaded so the swap resolves from one read - no second * dump that could transiently fail and slug-bind a bind into an orphan volume. */ resolveVolumeBind(metadata: SnapshotLayout | null, envId: number | null | undefined, volumeName: string): Promise<{ bind: string; include: string }>; /** Resolve clone destinations through the shared resolver (the SAME one the dialog preview * uses), so the previewed path and the populated path are one computation. Returns per-volume * bind material; the caller still does the side effects (volumeExists / createTargetVolume). */ resolveCloneTargets(job: RestoreJob, volumes: string[]): Promise>; acquireLiveTarget(key: string): (() => void) | null; serializeDestination(destinationId: number, fn: () => Promise): Promise; openOperation(entityName: string, environmentId: number | null, triggeredBy: 'manual' | 'cron', onProgress?: (s: string, m: string) => void): Promise; notify(event: 'restore_success' | 'restore_failed', payload: Record, envId: number | null | undefined): Promise; helperName(snapshotId: string): string; // --- post-restore actions (in-place only) ------------------------------------ /** Does the target container still exist (running or stopped)? */ containerExists(name: string, envId: number | null | undefined): Promise; /** Does a compose stack with this name already exist on the target env (any of * its containers present)? Used by the clone path to fail-closed on a name * collision before redeploying over someone else's stack. */ stackExists(name: string, envId: number | null | undefined): Promise; /** Start an existing container; throws on failure. */ startContainer(name: string, envId: number | null | undefined): Promise; /** Recreate a container AS-IS from the snapshot's stored inspect config; throws * with a clear message if that metadata is missing/insufficient. */ recreateContainerFromMetadata(name: string, envId: number | null | undefined, snapshotMetaContainer: unknown): Promise; /** Redeploy a stack from the snapshot's stored compose files; throws if absent. * restoreSecrets (opt-in, default on): write the stack's secrets carried in the * snapshot into the target env's DB before redeploy so the stack comes up complete. */ redeployStack(name: string, envId: number | null | undefined, destinationId: number, snapshotId: string, restoreSecrets?: boolean): Promise; /** Read the snapshot's stored restore metadata (typed SnapshotLayout), or null. */ readSnapshotMetadata(destinationId: number, snapshotId: string): Promise; /** Write the snapshot's captured stack files (/volumes/__dockhand_stackdir__) into * Dockhand's LOCAL data dir at stacks/// so Dockhand can * manage/redeploy the restored stack, even when the volume data restored to a * remote env. No-op (returns false) if the snapshot has no stack files. */ writeLocalStackFiles(destinationId: number, snapshotId: string, stackName: string, targetPath: string, overwrite: boolean, envId?: number | null): Promise; /** Resolve the canonical LOCAL managed stack dir (stacks///) for a * target - where a clone's compose/.env is materialised so the restored stack is a normal * managed (internal) stack the UI can edit + redeploy. */ stackDirFor(stackName: string, envId: number | null | undefined): Promise; // --- clone (cross-env restore) ----------------------------------------------- /** Does a NAMED VOLUME with this name already exist on the target env? Used by * the clone path to fail-closed on a collision (never overwrite an existing * volume the user didn't create for this restore). */ volumeExists(name: string, envId: number | null | undefined): Promise; /** Create a fresh named volume on the target env (the caller has already checked * it does not exist). Throws on failure. */ createTargetVolume(name: string, envId: number | null | undefined): Promise; } export interface RestoreJob { destinationId: number; snapshotId: string; mode: 'in-place' | 'new-location'; /** Whether the in-place target is a single container or a compose stack. * Selects which in-place path runs (stop+restart one container, vs `docker * compose stop` + redeploy the whole stack). Defaults to 'container'. Ignored * for new-location restores. */ targetType?: 'container' | 'stack'; /** Volume names to restore; empty = all in the snapshot. */ volumes: string[]; environmentId: number | null; confirmOverwrite?: boolean; targetPath?: string | null; /** Stack restores replace the managed compose/.env dir (clearing stale files) * by default; set true to MERGE onto the existing dir instead. Independent of * confirmOverwrite (which guards live VOLUME overwrites). */ mergeStackFiles?: boolean; /** For in-place: the container/stack whose volumes are being restored. */ targetName?: string | null; restoreFlags?: string[]; /** What to do AFTER the restore. For in-place, defaults to `'start'`. For a * new-location CLONE (see volumeDestinations), defaults to `'none'` and only * `recreate`/`redeploy`/`none` are meaningful (`start` needs a live container). * - start: start the target container if it still exists (in-place). * - recreate: rebuild the container from the snapshot's stored config if it's gone. * - redeploy: `docker compose up -d` from the snapshot's stored compose files. * - none: do nothing (leave a stopped target stopped). */ postRestore?: 'start' | 'recreate' | 'redeploy' | 'none'; /** New-location only: per-volume destination on the TARGET env (environmentId). * A volume WITH an entry is populated at that destination (named volume created * on the target env, or an absolute host path bound as-is) so a subsequent * recreate/redeploy mounts it with data — a cross-env CLONE. A volume WITHOUT an * entry falls back to loose-files extraction under targetPath (today's default). * Presence of ANY entry (or a recreate/redeploy postRestore) selects the clone * path. */ volumeDestinations?: Array<{ volume: string; kind: 'volume' | 'path'; target: string }>; /** Whether to restore the stack's secrets carried in the snapshot (default true). * false = bring the stack up without secrets (the operator re-enters them). Only * meaningful for a stack redeploy. */ restoreSecrets?: boolean; /** New-location STACK restore only. When true, the snapshot's captured stack dir * (/volumes/__dockhand_stackdir__: compose/.env/config) is NOT restored - only the * volume data lands at targetPath. A data-only restore for when the operator already * has the compose and just wants the data back. Default false (stack files included and * materialised into Dockhand's managed stack dir). */ skipStackFiles?: boolean; } export class RestoreService { constructor(private ports: RestorePorts) {} // volume key → kind, read once from the snapshot metadata at the top of run(). // A fresh RestoreService is built per restore (index.ts), so this never races. private volumeTypes = new Map(); /** Emit the per-volume list ` • [BIND|VOL] ` — same format the backup log * uses, so LogConsole renders the same typed chips in both. No-op for an empty * set (a config-only snapshot). */ private emitVolumeList(op: { progress: (s: string, m: string) => void }, volumes: string[]): void { for (const v of volumes) { const kind = this.volumeTypes.get(v) === 'bind' ? 'BIND' : 'VOL'; op.progress('restoring', ` • [${kind}] ${v}`); } } async run(job: RestoreJob, triggeredBy: 'manual' | 'cron' = 'manual'): Promise { const envId = job.environmentId; // Two tiers, both stamped with wall-clock + {env,target,snapshot} so a line lines up // with the shard test log by time and target. log() is always on (started/failed); // diag() carries the internal mode/type micro-trace behind BACKUP_DIAG=1 (CI-only). const key = `${job.targetName ?? job.targetPath ?? '?'} snap=${job.snapshotId.slice(0, 8)} env=${envId}`; const t0 = Date.now(); const log = (msg: string) => console.log(`[restore] ${new Date().toISOString()} ${key} | ${msg}`); const diagOn = process.env.BACKUP_DIAG === '1'; const diag = (phase: string, extra?: string) => { if (!diagOn) return; console.log(`[restore] ${new Date().toISOString()} ${key} | ${phase} @${Date.now() - t0}ms${extra ? ' | ' + extra : ''}`); }; log(`started (${job.mode}, ${job.targetType})`); diag('START', `post=${job.postRestore ?? 'none'}`); try { // --- ownership: caller must own the snapshot (instance + env) --- await this.ports.guardSnapshotAccess(job.destinationId, job.snapshotId); // The snapshot is authoritative about what it is: if it was taken of a // stack, restore it as a stack (and recover the stack name from metadata // if the caller didn't supply one) rather than trusting the client to // remember targetType — otherwise a stack restore silently degrades to a // container restore and its stack files never land. try { const meta = await this.ports.readSnapshotMetadata(job.destinationId, job.snapshotId); if (meta?.type === 'stack') { job.targetType = 'stack'; if (!job.targetName && typeof meta.targetName === 'string') job.targetName = meta.targetName; } // Per-volume kind for the restore log's typed chips (keyed by the same // `key` that listSnapshotVolumes / the includes use). for (const v of meta?.volumes ?? []) { const k = v?.key ?? v?.name; if (k && (v.type === 'volume' || v.type === 'bind')) this.volumeTypes.set(k, v.type); } } catch { /* metadata best-effort; fall back to the caller-supplied targetType */ } // --- the requested volumes must EXIST in the snapshot --- // A metadata-only snapshot (config-only backup) has no volumes; that's valid. // An in-place restore of it just recreates/redeploys from the stored config; // a new-location restore extracts the stored config/compose files. const available = await this.ports.listSnapshotVolumes(job.destinationId, job.snapshotId); const requested = job.volumes.length > 0 ? job.volumes : available; const missing = requested.filter((v) => !available.includes(v)); if (missing.length > 0) throw new BackupError('VALIDATION', `${missing.join(', ')} not found in snapshot. Available: ${available.join(', ') || 'none'}`); // Four paths on (mode, targetType). A stack carries its compose files in // the snapshot, so its paths also restore/redeploy that definition. if (job.mode === 'new-location') { // CLONE: new-location that populates real destinations on the target env // and (optionally) brings the target up there. Selected when the caller // supplied per-volume destinations OR a recreate/redeploy action. const isClone = (job.volumeDestinations?.length ?? 0) > 0 || job.postRestore === 'recreate' || job.postRestore === 'redeploy'; if (isClone) { // Fail-closed on a TARGET-NAME collision when the clone would bring the // target up (recreate/redeploy). Never start someone else's existing // container or `compose up` over an existing stack — the user must // remove it or choose another name first. (Skip when postRestore is // 'none': no container/stack is created, so a name clash is harmless.) const willBringUp = job.postRestore === 'recreate' || job.postRestore === 'redeploy'; const name = job.targetName ?? ''; if (willBringUp && name) { if (job.targetType === 'stack') { if (await this.ports.stackExists(name, job.environmentId)) { throw new BackupError('VALIDATION', `a stack "${name}" already exists on the target environment; remove it or choose another name before cloning`); } } else if (await this.ports.containerExists(name, job.environmentId)) { throw new BackupError('VALIDATION', `a container "${name}" already exists on the target environment; remove it or choose another name before cloning`); } } return job.targetType === 'stack' ? await this.runCloneStack(job, triggeredBy, requested) : await this.runCloneContainer(job, triggeredBy, requested); } return job.targetType === 'stack' ? await this.runNewLocationStack(job, triggeredBy, requested) : await this.runNewLocationContainer(job, triggeredBy, requested); } return job.targetType === 'stack' ? await this.runInPlaceStack(job, triggeredBy, requested) : await this.runInPlaceContainer(job, triggeredBy, requested); } catch (err) { const { code, message } = errorInfo(err); log(`failed after ${((Date.now() - t0) / 1000).toFixed(1)}s: ${code} ${message}`); return { status: 'error', code, error: message }; } } /** Non-destructive CONTAINER restore into a fresh directory. No locks on live * data — restores just the volume data to targetPath. */ private runNewLocationContainer(job: RestoreJob, triggeredBy: 'manual' | 'cron', volumes: string[]): Promise { // Metadata-only snapshot: extract the stored config (/metadata) so the operator // gets the container definition at the target path. Otherwise extract the volumes. const includes = volumes.length > 0 ? volumes.map((v) => `/volumes/${v}`) : ['/metadata']; return this.runNewLocationTo(job, triggeredBy, volumes, includes); } /** Non-destructive STACK restore into a fresh directory. Restores the volume data and, * unless skipStackFiles is set, the snapshot's stored compose/.env (under * /volumes/__dockhand_stackdir__). Dockhand ALWAYS materialises the stack files into its own * managed stack dir and registers the stack, so it can manage/redeploy it - this is * load-bearing: for a REMOTE env the compose exists only on the remote host otherwise, so * Dockhand would have nothing to manage. `skipStackFiles` opts out entirely (data-only). */ private async runNewLocationStack(job: RestoreJob, triggeredBy: 'manual' | 'cron', volumes: string[]): Promise { const includes = volumes.map((v) => `/volumes/${v}`); const includeStackFiles = job.targetName && !job.skipStackFiles; if (includeStackFiles) { // The captured stack dir is always at /volumes/__dockhand_stackdir__ // (listSnapshotVolumes filters it out of `volumes`, so add it explicitly here so // the new-location restore also lands the compose/.env for a redeployable dir). const { STACKDIR_VOLUME_KEY } = await import('./stackdir-plan'); includes.push(`/volumes/${STACKDIR_VOLUME_KEY}`); } const result = await this.runNewLocationTo(job, triggeredBy, volumes, includes); // Materialise the stack files into the CANONICAL managed stack dir on Dockhand // (stackDirFor -> $DATA_DIR/stacks//, NOT job.targetPath which is the host path // where the volume DATA landed), and register the stack as managed. Always (unless // skipStackFiles): Dockhand needs the compose locally to edit/redeploy, especially for a // remote env. Best-effort - the volume restore already succeeded. if (result.status !== 'error' && includeStackFiles && job.targetName) { try { const overwrite = job.mergeStackFiles !== true; const stackTargetPath = await this.ports.stackDirFor(job.targetName, job.environmentId); await this.ports.writeLocalStackFiles(job.destinationId, job.snapshotId, job.targetName, stackTargetPath, overwrite, job.environmentId); } catch (e) { console.log(`[restore] ${new Date().toISOString()} materialise stack files threw for "${job.targetName}": ${e instanceof Error ? e.message : String(e)}`); } } return result; } /** Shared non-destructive restore: extract the given includes to targetPath. */ private async runNewLocationTo(job: RestoreJob, triggeredBy: 'manual' | 'cron', volumes: string[], includes: string[]): Promise { const op = await this.ports.openOperation(`Restore ${job.snapshotId.slice(0, 8)} → ${job.targetPath}`, job.environmentId, triggeredBy); try { // Defense-in-depth: the helper bind-mounts targetPath rw as root. Refuse a protected // system dir here even if the request bypassed API validation. (validate.ts blocks it // at the edge; this guarantees the contract at the last point before the mount.) assertSafeRestoreTarget(job.targetPath!); includes.forEach(assertSafeVolumeInclude); const args = buildNewLocationRestore(job.snapshotId, job.targetPath!, includes, job.restoreFlags ?? []); op.progress('restoring', `Restoring ${volumes.length} volume(s) to ${job.targetPath}...`); this.emitVolumeList(op, volumes); // Bind the target path OUT to the host at the same path, so restic's // `--target ${targetPath}` writes to a durable host location instead of // into the ephemeral helper container (where the extracted data would // vanish when the helper exits — a silent no-op restore). const run = await this.ports.runInHelper({ args, binds: [`${job.targetPath}:${job.targetPath}`], envId: job.environmentId, name: this.ports.helperName(job.snapshotId), onStderr: (line) => { for (const l of formatResticLines(line)) op.progress('progress', l.startsWith('[dockhand]') ? l : `[restic] ${l}`); } }); this.emitResticStdout(op, run.stdout); if (!resticOk(run)) throw new BackupError('RESTIC', run.stderr.trim() || 'restore failed', { exitCode: run.exitCode }); await op.close({ kind: 'ok' }, { snapshotId: job.snapshotId, volumes, targetPath: job.targetPath }); await this.notifyOk(job, volumes); return { status: 'success', executionId: op.id, restoredVolumes: volumes, targetPath: job.targetPath! }; } catch (err) { return await this.failOp(op, job, err); } } /** CLONE a CONTAINER to the target env: populate the mapped destinations, then * recreate the container there from the snapshot's stored config (default). */ private runCloneContainer(job: RestoreJob, triggeredBy: 'manual' | 'cron', volumes: string[]): Promise { return this.runClonePopulate(job, triggeredBy, volumes, (op) => this.applyPostRestore(job.postRestore ?? 'recreate', job, op)); } /** CLONE a STACK to the target env: populate the mapped destinations, then either * redeploy from the snapshot's stored compose files, or (postRestore 'none') just * materialise the compose/.env dir locally so the user has a stack to edit + redeploy * by hand. Redeploy materialises the dir itself (redeployStack); 'none' would otherwise * leave NO compose dir at all - the UI says "update the compose file... then redeploy", * so the dir must exist. */ private runCloneStack(job: RestoreJob, triggeredBy: 'manual' | 'cron', volumes: string[]): Promise { return this.runClonePopulate(job, triggeredBy, volumes, async (op) => { if (job.postRestore === 'none') { // Bring nothing up, but materialise the managed stack dir so it's editable + // redeployable. Best-effort (matches runNewLocationStack): the volume clone // already succeeded, so a stackfile write failure must not fail the restore. if (job.targetName) { try { const overwrite = job.mergeStackFiles !== true; const stackTargetPath = await this.ports.stackDirFor(job.targetName, job.environmentId); await this.ports.writeLocalStackFiles(job.destinationId, job.snapshotId, job.targetName, stackTargetPath, overwrite, job.environmentId); } catch (e) { console.log(`[restore] ${new Date().toISOString()} clone 'none' writeLocalStackFiles threw for "${job.targetName}": ${e instanceof Error ? e.message : String(e)}`); } } return this.applyPostRestore('none', job, op); } return this.applyPostRestore('redeploy', job, op); }); } /** * Shared CLONE machinery: restore each mapped volume into a caller-supplied * destination on the TARGET env (a fresh named volume it creates, or an absolute * host path bound as-is), then run a post-restore action to bring the target up. * * Unlike in-place there is nothing live to stop, no staging→swap (the * destination is fresh), no live-target lock, and no swap/stop intents. Fresh * volumes are populated directly. Named-volume destinations are FAIL-CLOSED: if * one already exists on the target env we abort before touching anything (never * overwrite a volume the user didn't create for this restore). Absolute-path * destinations are bound as-is — the target daemon resolves them on its own host * (the user is responsible for the path existing there). * * Volumes WITHOUT a destination fall back to loose-files extraction under * targetPath, exactly like a plain new-location restore. */ private async runClonePopulate( job: RestoreJob, triggeredBy: 'manual' | 'cron', volumes: string[], postAction: (op: OperationHandle) => Promise, ): Promise { const env = job.environmentId; const destMap = new Map((job.volumeDestinations ?? []).map((d) => [d.volume, d])); const mapped = volumes.filter((v) => destMap.has(v)); const loose = volumes.filter((v) => !destMap.has(v)); return this.ports.serializeDestination(job.destinationId, async () => { const op = await this.ports.openOperation( `Clone ${job.snapshotId.slice(0, 8)} → ${job.targetName ?? 'target'}`, env, triggeredBy); try { // --- populate the mapped destinations (fail-closed on volume collision) --- if (mapped.length > 0) { const binds: string[] = []; const mountRoots: string[] = []; // The target PATHS + bind strings come from the shared resolver (same as the dialog // preview), so preview and restore can't diverge. The loop only does the SIDE // EFFECTS (collision check / volume creation), not bind construction. const resolved = await this.ports.resolveCloneTargets(job, mapped); const byKey = new Map(resolved.map((r) => [r.key, r])); for (const v of mapped) { const r = byKey.get(v)!; assertSafeVolumeInclude(r.include); // Last line of defense (independent of validate.ts): a host-path clone target // is bind-mounted rw as root, so refuse a protected system dir even if a // crafted request reached the service directly. if (r.type === 'bind') assertSafeRestoreTarget(r.target); if (r.type === 'volume') { if (await this.ports.volumeExists(r.target, env)) { throw new BackupError('VALIDATION', `volume "${r.target}" already exists on the target environment; remove it or choose another destination before cloning`); } await this.ports.createTargetVolume(r.target, env); } // named volume (now created) or absolute host path — bind rw into the helper. binds.push(r.bind); mountRoots.push(r.include); } const script = buildCloneRestore(job.snapshotId, mountRoots, job.restoreFlags ?? []); op.progress('restoring', `Populating ${mapped.length} volume(s) on the target environment...`); this.emitVolumeList(op, mapped); const run = await this.ports.runInHelper({ script, binds, envId: env, name: this.ports.helperName(job.snapshotId), onStderr: (line) => { for (const l of formatResticLines(line)) op.progress('progress', l.startsWith('[dockhand]') ? l : `[restic] ${l}`); } }); this.emitResticStdout(op, run.stdout); if (!resticOk(run)) throw new BackupError('RESTIC', run.stderr.trim() || 'clone restore failed', { exitCode: run.exitCode }); } // --- loose-files fallback for any un-mapped volumes (extract to targetPath) --- // FAIL-CLOSED: un-mapped volumes need a targetPath to land in. Without one they // would be silently dropped while the restore still reports success and the // redeploy brings the target up with EMPTY volumes (data loss). Refuse instead. if (loose.length > 0 && !job.targetPath) { throw new BackupError('VALIDATION', `Clone restore has ${loose.length} volume(s) with no destination mapping and no targetPath ` + `to extract them to (${loose.join(', ')}). Map every volume or provide a targetPath.`); } if (loose.length > 0 && job.targetPath) { const includes = loose.map((v) => `/volumes/${v}`); includes.forEach(assertSafeVolumeInclude); const args = buildNewLocationRestore(job.snapshotId, job.targetPath, includes, job.restoreFlags ?? []); op.progress('restoring', `Extracting ${loose.length} volume(s) to ${job.targetPath}...`); this.emitVolumeList(op, loose); const run = await this.ports.runInHelper({ args, binds: [`${job.targetPath}:${job.targetPath}`], envId: env, name: this.ports.helperName(job.snapshotId), onStderr: (line) => { for (const l of formatResticLines(line)) op.progress('progress', l.startsWith('[dockhand]') ? l : `[restic] ${l}`); } }); this.emitResticStdout(op, run.stdout); if (!resticOk(run)) throw new BackupError('RESTIC', run.stderr.trim() || 'clone restore failed', { exitCode: run.exitCode }); } // --- bring the target up on the chosen env --- const outcome = await postAction(op); if (outcome.ok) { await op.close({ kind: 'ok' }, { snapshotId: job.snapshotId, volumes, postRestore: outcome }); await this.notifyOk(job, volumes); return { status: 'success', executionId: op.id, restoredVolumes: volumes, postRestore: outcome }; } // For a CLONE the whole point is to bring the target up on the new env, // so a post-restore failure is a hard ERROR (not a quiet warning). Surface // the RAW Docker / compose error verbatim so the user sees exactly what // the daemon said (e.g. a source network/image missing on the target) and // fixes it themselves — we don't parse or interpret it. The volume data is // already restored; only the container/stack wasn't brought up. const rawErr = outcome.error ?? `${outcome.action} failed`; op.progress('error', rawErr); await op.close({ kind: 'error', code: 'DOCKER', message: rawErr }, { snapshotId: job.snapshotId, volumes, postRestore: outcome }); return { status: 'error', executionId: op.id, code: 'DOCKER', error: rawErr }; } catch (err) { return await this.failOp(op, job, err); } }); } /** Destructive in-place restore of a single CONTAINER: stop it, swap its * volumes, then run the chosen post-restore action (default: start). */ private runInPlaceContainer(job: RestoreJob, triggeredBy: 'manual' | 'cron', volumes: string[]): Promise { return this.runInPlaceSwap(job, triggeredBy, volumes, { // Stop the one container (a stop FAILURE aborts before any volume is touched). stop: async (op) => { if (!job.targetName) return { restart: null }; const { containers } = await this.ports.resolveTargets(job.targetName, job.environmentId); const running = containers.filter((c) => c.state === 'running'); if (running.length === 0) return { restart: null }; op.progress('stopping', `Stopping ${job.targetName}...`); const { restart } = await this.ports.stopBeforeRestore(running, job.environmentId); return { restart }; }, // After the swap, run the caller's post-restore action (start/recreate/none). postAction: (op) => this.applyPostRestore(job.postRestore ?? 'start', job, op), }); } /** Destructive in-place restore of a STACK: `docker compose stop` the whole * stack, swap its volumes, then redeploy it from the snapshot's stored compose * files (so both the data AND the stack definition are restored as-of backup). */ private runInPlaceStack(job: RestoreJob, triggeredBy: 'manual' | 'cron', volumes: string[]): Promise { return this.runInPlaceSwap(job, triggeredBy, volumes, { // Stop the whole stack via native compose stop; a stop FAILURE aborts // before any volume is touched. stop: async (op) => { if (!job.targetName) return { restart: null }; const { containers } = await this.ports.resolveTargets(job.targetName, job.environmentId, 'stack'); const running = containers.filter((c) => c.state === 'running'); if (running.length === 0) return { restart: null }; op.progress('stopping', `Stopping stack ${job.targetName}...`); const { restart } = await this.ports.stopStackBeforeRestore(job.targetName, job.environmentId); return { restart }; }, // A stack is brought back by redeploying from its stored compose files. postAction: (op) => this.applyPostRestore('redeploy', job, op), }); } /** * The shared in-place restore machinery: resolve live binds, lock + serialize, * record durable swap/stop intents, run the phase-marked staging→commit swap in * a helper, then apply a post-restore action. `opts.stop` and `opts.postAction` * are the only container-vs-stack differences. The swap itself is per-volume and * target-agnostic. A failed/interrupted swap leaves live data intact (the * durable swap intent lets startup roll it back). */ private async runInPlaceSwap( job: RestoreJob, triggeredBy: 'manual' | 'cron', volumes: string[], opts: { stop: (op: OperationHandle) => Promise<{ restart: (() => Promise) | null }>; postAction: (op: OperationHandle) => Promise; }, ): Promise { if (job.confirmOverwrite !== true) { return { status: 'error', code: 'VALIDATION', error: 'an in-place restore overwrites live data and requires explicit confirmation' }; } const env = job.environmentId; // DATA-SAFETY: a BIND volume's live bind is the SOURCE env's absolute host path // (resolveBindFromMetadata returns entry.source, e.g. /srv/appA/data). An in-place // swap writes to that exact path on `env`'s daemon. If `env` is NOT the snapshot's // source env, we'd destructively swap into an unrelated /srv/appA/data on the WRONG // host. Named volumes are safe (they resolve to a name on the local daemon), so only // bind volumes are gated. Cross-env restores must go through the clone path. // Detect binds from the metadata directly (fail-closed: if metadata is unreadable we // can't prove the swap is safe, so we refuse rather than trust an empty volumeTypes). // Read the metadata ONCE here and reuse it to resolve the binds below. A second per-volume // read opens a TOCTOU window: a transient dump failure returns null and slug-binds a bind // into an orphan named volume (data loss). const meta = await this.ports.readSnapshotMetadata(job.destinationId, job.snapshotId).catch(() => null); { const requested = new Set(volumes); const bindVols = (meta?.volumes ?? []) .filter((v) => v.type === 'bind' && requested.has(v.key ?? v.name)) .map((v) => v.key ?? v.name); // If we requested volumes but couldn't read metadata to classify them, we cannot // prove none are binds -> fail-closed. const metaUnreadable = volumes.length > 0 && meta === null; if (bindVols.length > 0 || metaUnreadable) { const sourceEnv = meta?.environmentId; const targetIsSource = meta !== null && (sourceEnv ?? null) === (env ?? null); if (!targetIsSource) { const which = bindVols.length > 0 ? ` (${bindVols.join(', ')})` : ''; return { status: 'error', code: 'VALIDATION', error: `in-place restore of a bind-mounted volume${which} is only allowed on the snapshot's source environment; use "restore to new location" to move it across environments`, }; } } } // Past the gate, `meta` is proven non-null whenever we requested volumes (metaUnreadable // would have returned above), so the resolver never sees null for a real bind -> no // slug-bind. Resolve every volume from this ONE metadata read. // Resolve each volume's live bind + include (rw — the swap writes to it). const binds: string[] = []; const liveRoots: string[] = []; for (const v of volumes) { const { bind, include } = await this.ports.resolveVolumeBind(meta, env, v); assertSafeVolumeInclude(include); binds.push(bind); liveRoots.push(include); } // Lock the live targets (reject a concurrent op on this data). The fallback identity // matches backup-service's `${type}:${targetName}` so a backup and a restore of the // SAME config-only target still collide, while disjoint config-only targets don't. const key = liveTargetKey(env, binds, `${job.targetType ?? 'container'}:${job.targetName}`); const release = this.ports.acquireLiveTarget(key); if (!release) return { status: 'skipped', reason: `Another backup or restore is already running for "${job.targetName ?? key}".` }; try { return await this.ports.serializeDestination(job.destinationId, async () => { const op = await this.ports.openOperation(`Restore ${job.snapshotId.slice(0, 8)} → ${job.targetName ?? 'volumes'}`, env, triggeredBy); let restart: (() => Promise) | null = null; try { // Stop the target first; a stop FAILURE aborts before any volume is touched. ({ restart } = await opts.stop(op)); // Metadata-only snapshot (no volumes): there's nothing to swap — go // straight to the post-restore action (recreate/redeploy from the // stored config). Otherwise run the phase-marked volume swap. The swap // is atomic per volume (staging + rename) and self-recovers on the next // restore via the on-disk phase marker, so a crash mid-swap leaves the // live data either fully-old or fully-new, never half-swapped. if (liveRoots.length > 0) { const script = buildInPlaceRestore(job.snapshotId, liveRoots, job.restoreFlags ?? []); op.progress('restoring', `Restoring ${volumes.length} volume(s) in place...`); this.emitVolumeList(op, volumes); const run = await this.ports.runInHelper({ script, binds, envId: env, name: this.ports.helperName(job.snapshotId), onStderr: (line) => { for (const l of formatResticLines(line)) op.progress('progress', l.startsWith('[dockhand]') ? l : `[restic] ${l}`); } }); this.emitResticStdout(op, run.stdout); if (!resticOk(run)) throw new BackupError('RESTIC', run.stderr.trim() || 'restore failed', { exitCode: run.exitCode }); } else { op.progress('restoring', `Config-only snapshot — no volume data to restore for ${job.targetName ?? 'target'}.`); } // The volume data is now safely restored. Run the post-restore action — // best-effort, NEVER rolls the restore back. On failure we downgrade to a // warning (data is good, operator must act). const outcome = await opts.postAction(op); if (outcome.ok) { await op.close({ kind: 'ok' }, { snapshotId: job.snapshotId, volumes, postRestore: outcome }); await this.notifyOk(job, volumes); return { status: 'success', executionId: op.id, restoredVolumes: volumes, postRestore: outcome }; } const warnMsg = `Data restored, but the post-restore step (${outcome.action}) failed: ${outcome.error ?? 'unknown error'}`; op.progress('warning', warnMsg); await op.close({ kind: 'warning', message: warnMsg }, { snapshotId: job.snapshotId, volumes, postRestore: outcome }); await this.notifyOk(job, volumes); return { status: 'warning', executionId: op.id, restoredVolumes: volumes, postRestore: outcome }; } catch (err) { // Restart whatever we stopped for the restore. If the restart itself // fails, SURFACE it: there is no boot-time recovery, so the operator // must know the target is still stopped and start it themselves. if (restart) { try { await restart(); } catch { const msg = `Restore failed and ${job.targetName ?? 'the target'} could not be restarted — it is STILL STOPPED. Start it manually.`; op.progress('warning', msg); try { await this.ports.notify('restore_failed', { snapshotId: job.snapshotId, kind: 'restore', errorCode: 'STOPPED_RESTART_FAILED', message: msg }, job.environmentId); } catch { /* non-fatal */ } } } return await this.failOp(op, job, err); } }); } finally { release(); } } /** * Apply the chosen post-restore action after a successful in-place swap. * Best-effort: a throw becomes `{ ok:false, error }` — it NEVER propagates as * a failed restore (the data is already good). The pre-swap stop already ran; * this decides what happens to the target AFTER. */ private async applyPostRestore( action: 'start' | 'recreate' | 'redeploy' | 'none', job: RestoreJob, op: OperationHandle, ): Promise { const envId = job.environmentId; const name = job.targetName ?? ''; try { switch (action) { case 'none': return { action, ok: true }; case 'start': { if (await this.ports.containerExists(name, envId)) { op.progress('starting', `Starting ${name}...`); await this.ports.startContainer(name, envId); return { action, ok: true }; } return { action, ok: false, error: 'target container no longer exists — choose Recreate to rebuild it' }; } case 'recreate': { // Recreate is a superset of start: if the container is still there, // prefer starting it (never destroy a live container); only rebuild // from the stored config when it's actually gone. if (await this.ports.containerExists(name, envId)) { op.progress('starting', `Starting ${name}...`); await this.ports.startContainer(name, envId); return { action, ok: true }; } const meta = await this.ports.readSnapshotMetadata(job.destinationId, job.snapshotId); if (!meta || meta.container == null) { return { action, ok: false, error: 'no container metadata stored in this snapshot; cannot recreate' }; } op.progress('recreating', `Recreating ${name} from the snapshot's stored config...`); await this.ports.recreateContainerFromMetadata(name, envId, meta.container); return { action, ok: true }; } case 'redeploy': { const meta = await this.ports.readSnapshotMetadata(job.destinationId, job.snapshotId); if (!meta || meta.stack === undefined) { return { action, ok: false, error: 'no stack files stored in this snapshot; cannot redeploy' }; } op.progress('redeploying', `Redeploying stack ${name}...`); await this.ports.redeployStack(name, envId, job.destinationId, job.snapshotId, job.restoreSecrets ?? true); return { action, ok: true }; } } } catch (err) { return { action, ok: false, error: err instanceof Error ? err.message : String(err) }; } } /** restic writes its `--json` progress (per-file status + summary) to STDOUT, * which the helper returns buffered (not streamed). Emit those lines so the * restore log shows restic's actual output, not just our [dockhand] echos. * Drops the internal exit-marker line. */ private emitResticStdout(op: OperationHandle, stdout: string): void { for (const l of formatResticLines(stdout)) { if (l.includes(EXIT_MARKER)) continue; op.progress('progress', l.startsWith('[dockhand]') ? l : `[restic] ${l}`); } } private async notifyOk(job: RestoreJob, volumes: string[]): Promise { try { await this.ports.notify('restore_success', { snapshotId: job.snapshotId, kind: 'restore', volumes }, job.environmentId); } catch { /* non-fatal */ } } private async failOp(op: OperationHandle, job: RestoreJob, err: unknown): Promise { const { code, message } = errorInfo(err); await op.close({ kind: 'error', code, message }, { errorCode: code }); try { await this.ports.notify('restore_failed', { snapshotId: job.snapshotId, kind: 'restore', errorCode: code, message }, job.environmentId); } catch { /* non-fatal */ } return { status: 'error', executionId: op.id, code, error: message }; } } function errorInfo(err: unknown): { code: BackupError['code']; message: string } { // See backup-service.errorInfo — unwrap embedded daemon JSON so the UI shows a // readable message consistently across every failure path. const raw = err instanceof BackupError ? err.message : (err instanceof Error ? err.message : String(err)); const code = err instanceof BackupError ? err.code : 'UNKNOWN'; return { code, message: cleanErrorMsg(raw) }; }