backup engine fixes: shared-repo unlock, non-root uid reads, lock release, local-path repo, backblaze key id
This commit is contained in:
15 files changed
+375
-313
No files matched your search
+181
-134
@@ -25,7 +25,7 @@ import { instanceTagFilter, parseSnapshots, retentionTagFilter } from './models'
|
||||
import { buildSwapRecoveryScript } from './swap';
|
||||
import { parseOptionsJson, fireWebhook, parseResticDiff, type SnapshotDiff } from './helpers';
|
||||
import { getHostname } from '../license';
|
||||
import { getBackupConfig, getBackupConfigs, getBackupDestination, updateBackupConfig, updateBackupDestination, decryptBackupDestination, getEnvironment } from '../db';
|
||||
import { getBackupConfig, getBackupConfigs, getBackupDestination, updateBackupConfig, updateBackupDestination, decryptBackupDestination } from '../db';
|
||||
import { getInstanceId } from './identity';
|
||||
import { inspectContainer } from '../docker';
|
||||
import { sendEventNotification } from '../notifications';
|
||||
@@ -175,6 +175,18 @@ async function collectMetadata(
|
||||
console.warn(`[Backup] Stack directory for "${targetName}" exceeded the backup cap (${MAX_FILES} files / ${Math.round(MAX_TOTAL_BYTES / 1024 / 1024)} MB) — only ${count} file(s) / ${bytes} byte(s) captured. The snapshot is INCOMPLETE for redeploy; reduce the stack dir or exclude large sibling files.`);
|
||||
}
|
||||
} catch { /* directory walk best-effort — compose/.env already captured */ }
|
||||
// Carry the stack's secret env vars IN the snapshot so a restore reproduces a
|
||||
// WORKING stack — secrets and all — even after the source stack (and its DB
|
||||
// rows) are gone. Stored as their at-rest ciphertext (enc:v1:...), encrypted
|
||||
// under THIS instance's key: a restore on the same instance decrypts them
|
||||
// transparently; on a fresh instance the operator must carry over the same
|
||||
// encryption key (.encryption_key / ENCRYPTION_KEY) or they stay unreadable.
|
||||
// Values are NEVER stored plaintext (getStackSecretCiphertexts guarantees it).
|
||||
try {
|
||||
const { getStackSecretCiphertexts } = await import('../db');
|
||||
const secrets = await getStackSecretCiphertexts(targetName, envId ?? null);
|
||||
if (secrets.length > 0) metadata.secrets = secrets;
|
||||
} catch { /* best-effort — a lookup failure just omits secrets from the snapshot */ }
|
||||
// Rewrite metadata.json with hasStackFiles set.
|
||||
files[0] = { path: 'metadata.json', contentBase64: Buffer.from(JSON.stringify(metadata, null, 2)).toString('base64') };
|
||||
}
|
||||
@@ -224,6 +236,89 @@ function backupPorts(destination: any, onProgress?: (status: string, message: st
|
||||
// =============================================================================
|
||||
// RestorePorts — the real implementation, bound to one destination + access ctx
|
||||
// =============================================================================
|
||||
/**
|
||||
* Extract a snapshot's captured stack files (/metadata/stacks/<name>/stackfiles/) into
|
||||
* `targetPath` under the stacks root, with a path-traversal guard, a data-loss guard
|
||||
* (an empty extraction never wipes the live dir), and an atomic staging-swap so a
|
||||
* mid-copy failure can't leave the dir half-written. Returns false (no-op) when the
|
||||
* snapshot has no stackfiles. Shared by the writeLocalStackFiles port and the restore
|
||||
* redeploy path (which materialises into the CANONICAL stack dir). `destination` is
|
||||
* already resolved by the caller.
|
||||
*/
|
||||
async function materialiseStackFiles(destination: any, snapId: string, stackName: string, targetPath: string, overwrite: boolean): Promise<boolean> {
|
||||
const { mkdtempSync, mkdirSync, rmSync, existsSync, cpSync, readdirSync, renameSync } = await import('fs');
|
||||
const { tmpdir } = await import('os');
|
||||
const { join, dirname, resolve, sep } = await import('path');
|
||||
const { randomUUID } = await import('crypto');
|
||||
const { getStacksDir } = await import('../stacks');
|
||||
// Use `restic restore --include` (not dump→tar): restore writes a
|
||||
// DETERMINISTIC path under the target — <tmp>/metadata/stacks/<name>/
|
||||
// stackfiles/ — so there's no tar-prefix guessing.
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'dh-stackfiles-'));
|
||||
console.log(`[Backup] materialiseStackFiles: "${stackName}" snapshot=${snapId} → extract to ${tmp}, then swap into ${targetPath} (overwrite=${overwrite})`);
|
||||
try {
|
||||
const include = `/metadata/stacks/${stackName}/stackfiles`;
|
||||
const run = await restic.runLocal(destination, ['restore', snapId, '--target', tmp, '--include', include], 'data');
|
||||
if (run.exitCode !== 0) {
|
||||
console.log(`[Backup] materialiseStackFiles: restic restore failed for "${stackName}": ${run.stderr.trim() || `exit ${run.exitCode}`}`);
|
||||
return false;
|
||||
}
|
||||
const extractedDir = join(tmp, 'metadata', 'stacks', stackName, 'stackfiles');
|
||||
// DATA-LOSS guard: only proceed if extraction actually produced files.
|
||||
const extracted = existsSync(extractedDir) ? readdirSync(extractedDir) : [];
|
||||
if (extracted.length === 0) {
|
||||
console.log(`[Backup] materialiseStackFiles: no stackfiles extracted for "${stackName}" (snapshot may predate full-dir capture) — leaving ${targetPath} untouched`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Path-traversal guard: the resolved target MUST stay under the stacks root.
|
||||
const resolvedTarget = resolve(targetPath);
|
||||
const stacksRoot = resolve(getStacksDir());
|
||||
if (resolvedTarget !== stacksRoot && !resolvedTarget.startsWith(stacksRoot + sep)) {
|
||||
console.log(`[Backup] materialiseStackFiles: refusing target "${resolvedTarget}" outside the stacks root "${stacksRoot}"`);
|
||||
return false;
|
||||
}
|
||||
|
||||
mkdirSync(resolvedTarget, { recursive: true });
|
||||
|
||||
// Atomic staging-swap: build the desired contents in a sibling staging dir
|
||||
// FIRST, then swap it into place with a same-parent rename. On OVERWRITE the
|
||||
// staging holds ONLY the restored files; on merge it starts as a copy of live.
|
||||
const parent = dirname(resolvedTarget);
|
||||
const staging = mkdtempSync(join(parent, '.dockhand-stack-new-'));
|
||||
const oldAside = join(parent, `.dockhand-stack-old-${randomUUID()}`);
|
||||
try {
|
||||
if (!overwrite) {
|
||||
for (const existing of readdirSync(resolvedTarget)) {
|
||||
cpSync(join(resolvedTarget, existing), join(staging, existing), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
for (const entry of extracted) {
|
||||
cpSync(join(extractedDir, entry), join(staging, entry), { recursive: true, force: true });
|
||||
}
|
||||
renameSync(resolvedTarget, oldAside);
|
||||
try {
|
||||
renameSync(staging, resolvedTarget);
|
||||
} catch (swapErr) {
|
||||
try { renameSync(oldAside, resolvedTarget); } catch { /* best effort */ }
|
||||
throw swapErr;
|
||||
}
|
||||
rmSync(oldAside, { recursive: true, force: true });
|
||||
console.log(`[Backup] materialiseStackFiles: wrote ${extracted.length} entries to ${resolvedTarget} (${overwrite ? 'replaced' : 'merged'})`);
|
||||
return true;
|
||||
} catch (copyErr) {
|
||||
try { rmSync(staging, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
if (existsSync(oldAside) && !existsSync(resolvedTarget)) {
|
||||
try { renameSync(oldAside, resolvedTarget); } catch { /* best effort */ }
|
||||
}
|
||||
try { rmSync(oldAside, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
throw copyErr;
|
||||
}
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function restorePorts(destination: any, access: { isEnterprise: boolean; canAccessEnvironment: (id: number) => Promise<boolean> }, onProgress?: (status: string, message: string) => void): RestorePorts {
|
||||
const run = boundRunner(destination);
|
||||
return {
|
||||
@@ -313,11 +408,14 @@ function restorePorts(destination: any, access: { isEnterprise: boolean; canAcce
|
||||
// sibling configs referenced by relative paths resolve, and the file keeps its
|
||||
// real name (e.g. immich.yaml). The backup always captures the full dir plus the
|
||||
// recorded compose filename, so there is no normalized-compose.yaml fallback.
|
||||
redeployStack: async (name, envId, destId, snapId) => {
|
||||
const { redeployStackFromDir } = await import('../stacks');
|
||||
const { mkdtempSync, rmSync, existsSync, readdirSync } = await import('fs');
|
||||
const { tmpdir } = await import('os');
|
||||
redeployStack: async (name, envId, destId, snapId, restoreSecrets) => {
|
||||
const { redeployStackFromDir, getStackDir } = await import('../stacks');
|
||||
const { existsSync } = await import('fs');
|
||||
const { join } = await import('path');
|
||||
const { upsertStackSource } = await import('../db');
|
||||
|
||||
const log = (msg: string) => console.log(`[Restore:redeploy ${name}] ${msg}`);
|
||||
log(`start: env=${envId ?? 'local'} destination=${destId} snapshot=${snapId}`);
|
||||
|
||||
const meta = await getSnapshotMetadata(destId, snapId);
|
||||
const composeFileName = meta && typeof meta.composeFileName === 'string' ? meta.composeFileName : '';
|
||||
@@ -325,21 +423,57 @@ function restorePorts(destination: any, access: { isEnterprise: boolean; canAcce
|
||||
throw new Error('snapshot has no recorded compose filename; cannot redeploy this stack');
|
||||
}
|
||||
|
||||
const destination = await loadDest(destId);
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'dh-redeploy-'));
|
||||
try {
|
||||
const include = `/metadata/stacks/${name}/stackfiles`;
|
||||
const run = await restic.runLocal(destination, ['restore', snapId, '--target', tmp, '--include', include], 'data');
|
||||
const extractedDir = join(tmp, 'metadata', 'stacks', name, 'stackfiles');
|
||||
const extractedNames = run.exitCode === 0 && existsSync(extractedDir) ? readdirSync(extractedDir) : [];
|
||||
if (!extractedNames.includes(composeFileName)) {
|
||||
throw new Error(`snapshot stackfiles/ is missing the compose file "${composeFileName}"; cannot redeploy`);
|
||||
// Restore the stack's secrets FROM THE SNAPSHOT (opt-in, default on). They are
|
||||
// stored as at-rest ciphertext encrypted under this instance's key; writing
|
||||
// them into the TARGET env's DB before redeploy lets redeployStackFromDir's
|
||||
// normal secret injection pick them up. setStackEnvVars re-runs encrypt(),
|
||||
// which is a pass-through for an already-encrypted value, so the ciphertext is
|
||||
// stored verbatim (no double-encryption). On a different instance without the
|
||||
// matching key these values won't decrypt — the restore UI warns about that.
|
||||
if (restoreSecrets && Array.isArray(meta?.secrets) && meta.secrets.length > 0) {
|
||||
const { setStackEnvVars } = await import('../db');
|
||||
const secrets = meta.secrets
|
||||
.filter((s: any) => s && typeof s.key === 'string' && typeof s.value === 'string')
|
||||
.map((s: any) => ({ key: s.key, value: s.value, isSecret: true }));
|
||||
if (secrets.length > 0) {
|
||||
await setStackEnvVars(name, envId ?? null, secrets);
|
||||
log(`restored ${secrets.length} secret(s) from snapshot → env ${envId ?? 'local'}`);
|
||||
}
|
||||
const r = await redeployStackFromDir(name, extractedDir, composeFileName, envId ?? undefined);
|
||||
if (!r.success) throw new Error(r.error || 'docker compose up failed');
|
||||
} finally {
|
||||
try { rmSync(tmp, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
}
|
||||
|
||||
// Materialise the snapshot's stack files into the CANONICAL stack dir
|
||||
// (stacks/<envName>/<stackName>/) — where every managed stack lives — and
|
||||
// bring the stack up FROM THERE, so after a restore (even one that deleted
|
||||
// the stack completely) it is a normal managed stack on disk, not a
|
||||
// throwaway /tmp copy (#1329). writeLocalStackFiles carries the path-
|
||||
// traversal + data-loss + atomic-swap guards; overwrite=true because an
|
||||
// in-place restore is destructive by design.
|
||||
const stackDir = await getStackDir(name, envId ?? undefined);
|
||||
const destination = await loadDest(destId);
|
||||
log(`materialising snapshot stackfiles → canonical dir: ${stackDir} (compose: ${composeFileName})`);
|
||||
const wrote = await materialiseStackFiles(destination, snapId, name, stackDir, true);
|
||||
if (!wrote || !existsSync(join(stackDir, composeFileName))) {
|
||||
throw new Error(`could not materialise stack files for "${name}" into ${stackDir}; cannot redeploy`);
|
||||
}
|
||||
log(`materialised OK → ${stackDir}`);
|
||||
|
||||
log(`redeploying (docker compose up) from: ${stackDir}`);
|
||||
const r = await redeployStackFromDir(name, stackDir, composeFileName, envId ?? undefined);
|
||||
if (!r.success) throw new Error(r.error || 'docker compose up failed');
|
||||
log(`redeploy OK`);
|
||||
|
||||
// Register the restored stack as managed (internal) so the UI can
|
||||
// edit/redeploy it — a complete-delete-then-restore must come back managed.
|
||||
const composePath = join(stackDir, composeFileName);
|
||||
const envPath = existsSync(join(stackDir, '.env')) ? join(stackDir, '.env') : null;
|
||||
await upsertStackSource({
|
||||
stackName: name,
|
||||
environmentId: envId ?? null,
|
||||
sourceType: 'internal',
|
||||
composePath,
|
||||
envPath
|
||||
});
|
||||
log(`registered as managed (internal): composePath=${composePath} envPath=${envPath ?? '(none)'}`);
|
||||
},
|
||||
readSnapshotMetadata: (destId, snapId) => getSnapshotMetadata(destId, snapId),
|
||||
// Materialise the snapshot's captured stack files into Dockhand's LOCAL data
|
||||
@@ -347,88 +481,8 @@ function restorePorts(destination: any, access: { isEnterprise: boolean; canAcce
|
||||
// files were backed up under /metadata/stacks/<name>/stackfiles/; dump that
|
||||
// subtree as a tar and extract it flat into targetPath.
|
||||
writeLocalStackFiles: async (destId, snapId, stackName, targetPath, overwrite) => {
|
||||
const { mkdtempSync, mkdirSync, rmSync, existsSync, cpSync, readdirSync, renameSync } = await import('fs');
|
||||
const { tmpdir } = await import('os');
|
||||
const { join, dirname, resolve, sep } = await import('path');
|
||||
const { randomUUID } = await import('crypto');
|
||||
const { getStacksDir } = await import('../stacks');
|
||||
// Use `restic restore --include` (not dump→tar): restore writes a
|
||||
// DETERMINISTIC path under the target — <tmp>/metadata/stacks/<name>/
|
||||
// stackfiles/ — so there's no tar-prefix guessing. Mirrors the proven
|
||||
// approach the previous engine used (commit 5edcfa07 et al.).
|
||||
const destination = await loadDest(destId);
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'dh-stackfiles-'));
|
||||
try {
|
||||
const include = `/metadata/stacks/${stackName}/stackfiles`;
|
||||
const run = await restic.runLocal(destination, ['restore', snapId, '--target', tmp, '--include', include], 'data');
|
||||
if (run.exitCode !== 0) {
|
||||
console.log(`[Backup] writeLocalStackFiles: restic restore failed for "${stackName}": ${run.stderr.trim() || `exit ${run.exitCode}`}`);
|
||||
return false;
|
||||
}
|
||||
const extractedDir = join(tmp, 'metadata', 'stacks', stackName, 'stackfiles');
|
||||
// DATA-LOSS guard: only proceed if extraction actually produced files.
|
||||
// A snapshot that predates full-dir capture, or a corrupt path, yields
|
||||
// nothing — replacing the live dir with an empty one would be data loss.
|
||||
const extracted = existsSync(extractedDir) ? readdirSync(extractedDir) : [];
|
||||
if (extracted.length === 0) {
|
||||
console.log(`[Backup] writeLocalStackFiles: no stackfiles extracted for "${stackName}" (snapshot may predate full-dir capture) — leaving ${targetPath} untouched`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Path-traversal guard: the resolved target MUST stay under the stacks
|
||||
// root before any mkdir/rename/rm — a "../.." name must never let a
|
||||
// restore wipe an arbitrary directory.
|
||||
const resolvedTarget = resolve(targetPath);
|
||||
const stacksRoot = resolve(getStacksDir());
|
||||
if (resolvedTarget !== stacksRoot && !resolvedTarget.startsWith(stacksRoot + sep)) {
|
||||
console.log(`[Backup] writeLocalStackFiles: refusing target "${resolvedTarget}" outside the stacks root "${stacksRoot}"`);
|
||||
return false;
|
||||
}
|
||||
|
||||
mkdirSync(resolvedTarget, { recursive: true });
|
||||
|
||||
// Atomic staging-swap (audit #3 / HIGH #2): build the desired contents
|
||||
// in a sibling staging dir FIRST, then swap it into place with a
|
||||
// same-parent rename, so a mid-copy failure can never leave the live
|
||||
// stack dir half-wiped. On OVERWRITE the staging holds ONLY the restored
|
||||
// files (stale files cleared); on merge it starts as a copy of the live
|
||||
// dir so existing user files are preserved.
|
||||
const parent = dirname(resolvedTarget);
|
||||
const staging = mkdtempSync(join(parent, '.dockhand-stack-new-'));
|
||||
const oldAside = join(parent, `.dockhand-stack-old-${randomUUID()}`);
|
||||
try {
|
||||
if (!overwrite) {
|
||||
for (const existing of readdirSync(resolvedTarget)) {
|
||||
cpSync(join(resolvedTarget, existing), join(staging, existing), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
for (const entry of extracted) {
|
||||
cpSync(join(extractedDir, entry), join(staging, entry), { recursive: true, force: true });
|
||||
}
|
||||
// Copy fully succeeded — same-parent swap: live aside, staging in.
|
||||
renameSync(resolvedTarget, oldAside);
|
||||
try {
|
||||
renameSync(staging, resolvedTarget);
|
||||
} catch (swapErr) {
|
||||
try { renameSync(oldAside, resolvedTarget); } catch { /* best effort */ }
|
||||
throw swapErr;
|
||||
}
|
||||
rmSync(oldAside, { recursive: true, force: true });
|
||||
console.log(`[Backup] writeLocalStackFiles: wrote ${extracted.length} entries to ${resolvedTarget} (${overwrite ? 'replaced' : 'merged'})`);
|
||||
return true;
|
||||
} catch (copyErr) {
|
||||
// Fail closed: clean up staging/aside, restore the live dir if the
|
||||
// swap left it moved, and rethrow so the restore reports FAILED.
|
||||
try { rmSync(staging, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
if (existsSync(oldAside) && !existsSync(resolvedTarget)) {
|
||||
try { renameSync(oldAside, resolvedTarget); } catch { /* best effort */ }
|
||||
}
|
||||
try { rmSync(oldAside, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
throw copyErr;
|
||||
}
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
return materialiseStackFiles(destination, snapId, stackName, targetPath, overwrite);
|
||||
},
|
||||
// --- clone (cross-env restore) -----------------------------------------------
|
||||
volumeExists: async (name, envId) => {
|
||||
@@ -499,7 +553,7 @@ export async function previewSnapshot(
|
||||
destinationId: number,
|
||||
snapshotId: string,
|
||||
access: { isEnterprise: boolean; canAccessEnvironment: (id: number) => Promise<boolean> } = { isEnterprise: false, canAccessEnvironment: async () => true },
|
||||
): Promise<{ snapshotId: string; volumes: string[]; volumeTypes: Record<string, 'volume' | 'bind'>; volumeSources: Record<string, string>; backupTime: string | null; sourceEnvironmentId: number | null; hasMetadata: boolean; hasStackFiles: boolean }> {
|
||||
): Promise<{ snapshotId: string; volumes: string[]; volumeTypes: Record<string, 'volume' | 'bind'>; volumeSources: Record<string, string>; backupTime: string | null; sourceEnvironmentId: number | null; hasMetadata: boolean; hasStackFiles: boolean; sourceSecretKeys: string[] }> {
|
||||
const destination = await getBackupDestination(destinationId);
|
||||
if (!destination) throw new Error('backup destination not found');
|
||||
const instanceId = await getInstanceId();
|
||||
@@ -518,6 +572,13 @@ export async function previewSnapshot(
|
||||
const volumeSources: Record<string, string> = {};
|
||||
let backupTime: string | null = null;
|
||||
let sourceEnvironmentId: number | null = null;
|
||||
// Names (never values) of the secret env vars carried IN the snapshot for this stack.
|
||||
// The restore UI lists these under the "restore secrets from this backup" toggle so
|
||||
// the operator sees exactly which secrets come back. The VALUES live in the snapshot
|
||||
// as ciphertext (never exposed here) and are written to the DB by redeployStack.
|
||||
let sourceSecretKeys: string[] = [];
|
||||
let mdType: string | null = null;
|
||||
let mdTargetName: string | null = null;
|
||||
if (hasMetadata) {
|
||||
const dump = await restic.runLocal(destination, ['dump', '--no-lock', snapshotId, '/metadata/metadata.json']);
|
||||
if (dump.exitCode === 0) {
|
||||
@@ -525,6 +586,13 @@ export async function previewSnapshot(
|
||||
const md = JSON.parse(dump.stdout);
|
||||
backupTime = md?.backupTime ?? null;
|
||||
sourceEnvironmentId = typeof md?.environmentId === 'number' ? md.environmentId : null;
|
||||
mdType = typeof md?.type === 'string' ? md.type : null;
|
||||
mdTargetName = typeof md?.targetName === 'string' ? md.targetName : null;
|
||||
if (Array.isArray(md?.secrets)) {
|
||||
sourceSecretKeys = md.secrets
|
||||
.map((s: any) => s?.key)
|
||||
.filter((k: unknown): k is string => typeof k === 'string');
|
||||
}
|
||||
for (const v of md?.volumes ?? []) {
|
||||
// Key by v.key — that's what listSnapshotVolumes returns (the /volumes/<key>
|
||||
// segment) and what the restore UI indexes on. For a bind, key !== name
|
||||
@@ -537,7 +605,7 @@ export async function previewSnapshot(
|
||||
} catch { /* leave types empty on malformed metadata */ }
|
||||
}
|
||||
}
|
||||
return { snapshotId, volumes, volumeTypes, volumeSources, backupTime, sourceEnvironmentId, hasMetadata, hasStackFiles };
|
||||
return { snapshotId, volumes, volumeTypes, volumeSources, backupTime, sourceEnvironmentId, hasMetadata, hasStackFiles, sourceSecretKeys };
|
||||
}
|
||||
|
||||
// --- snapshot reads (routes already apply the env guard; we add instance own.) ---
|
||||
@@ -702,25 +770,25 @@ export type RepoTask = 'unlock' | 'check' | 'prune' | 'stats' | 'repair-index' |
|
||||
/** Run a repo-maintenance task. Returns { success } for the route + activity log. */
|
||||
export interface RepoStats { totalSize: number; totalFiles: number; snapshots: number }
|
||||
|
||||
export async function runRepoTask(destinationId: number, task: RepoTask, opts?: { maxUnused?: string; dataSubset?: string }): Promise<{ success: boolean; output?: string; error?: string; stats?: RepoStats }> {
|
||||
export async function runRepoTask(destinationId: number, task: RepoTask, opts?: { maxUnused?: string; dataSubset?: string; staleOnly?: boolean }): Promise<{ success: boolean; output?: string; error?: string; stats?: RepoStats }> {
|
||||
const destination = await loadDest(destinationId);
|
||||
if (task === 'repair-index' || task === 'repair-snapshots') {
|
||||
const sub = task === 'repair-index' ? 'index' : 'snapshots';
|
||||
// repair takes the repo lock — serialize against a concurrent backup to the same repo.
|
||||
const run = await serializeByRepo(destinationId, () => restic.runLocal(destination, ['repair', sub], 'data'));
|
||||
// repair takes the repo lock — serialize against a concurrent backup to the same
|
||||
// repo, and --retry-lock so a legitimately-running cross-instance backup doesn't
|
||||
// make repair fail REPO_LOCKED (parity with prune/forget).
|
||||
const run = await serializeByRepo(destinationId, () => restic.runLocal(destination, ['repair', sub, '--retry-lock', '5m'], 'data'));
|
||||
logRepoOp(destination.name, `repair ${sub}`, run.exitCode === 0, { output: run.stdout, error: run.stderr });
|
||||
return run.exitCode === 0 ? { success: true, output: run.stdout.trim() } : { success: false, error: run.stderr.trim() || 'repair failed' };
|
||||
}
|
||||
// prune/check hold the repo lock; run them serialized on the destination so a
|
||||
// scheduled maintenance pass can't collide with a backup writing to the same repo.
|
||||
// unlock is ALSO serialized: it now uses `--remove-all` (see unlockRepository), so
|
||||
// running it unserialized could wipe the LIVE lock of a concurrent backup/restore on
|
||||
// the same repo in THIS instance. Serializing means the only locks it clears are
|
||||
// orphans (dead owners) — exactly its documented job. stats (--no-lock) is read-safe
|
||||
// and stays unserialized.
|
||||
// unlock: the EXPLICIT user action uses `--remove-all`; an AUTOMATIC caller passes
|
||||
// staleOnly (plain unlock) so it never wipes a live foreign lock on a shared repo.
|
||||
// stats (--no-lock) is read-safe and stays unserialized.
|
||||
const r = task === 'prune' ? await serializeByRepo(destinationId, () => pruneRepository(reader(), destination, opts?.maxUnused))
|
||||
: task === 'check' ? await serializeByRepo(destinationId, () => checkRepository(reader(), destination, opts?.dataSubset))
|
||||
: task === 'unlock' ? await serializeByRepo(destinationId, () => unlockRepository(reader(), destination))
|
||||
: task === 'unlock' ? await serializeByRepo(destinationId, () => unlockRepository(reader(), destination, !opts?.staleOnly))
|
||||
: await repoStats(reader(), destination);
|
||||
logRepoOp(destination.name, task, r.ok, r.ok ? { output: r.output } : { error: r.error });
|
||||
if (!r.ok) return { success: false, error: r.error };
|
||||
@@ -924,18 +992,6 @@ async function runBackupRegistered(configId: number, triggeredBy: 'cron' | 'manu
|
||||
|
||||
/** Run a restore. The route layer has already validated the request + auth and
|
||||
* supplies the caller's access context for the ownership guard. */
|
||||
/** A filesystem repo (absolute or relative path) rather than a remote backend. */
|
||||
function isLocalRepo(repository: string): boolean {
|
||||
return repository.startsWith('/') || repository.startsWith('./');
|
||||
}
|
||||
/** An environment whose Docker daemon is NOT Dockhand's own host. */
|
||||
function isRemoteEnv(connectionType?: string | null, host?: string | null): boolean {
|
||||
if (!connectionType) return false;
|
||||
if (connectionType === 'hawser-standard' || connectionType === 'hawser-edge') return true;
|
||||
if (connectionType === 'direct' && !!host) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function runRestore(
|
||||
job: RestoreJob,
|
||||
access: { isEnterprise: boolean; canAccessEnvironment: (id: number) => Promise<boolean> } = { isEnterprise: false, canAccessEnvironment: async () => true },
|
||||
@@ -944,18 +1000,9 @@ export async function runRestore(
|
||||
const destination = await getBackupDestination(job.destinationId);
|
||||
if (!destination) return { status: 'error', code: 'VALIDATION', error: 'backup destination not found' };
|
||||
|
||||
// A local filesystem repository lives on Dockhand's own host and is bind-mounted
|
||||
// into the restore helper. That only works when the helper runs on the same host
|
||||
// (socket env); on a remote environment the repo path doesn't exist, so the
|
||||
// restore would fail deep in restic. Reject it up front with a clear message —
|
||||
// mirrors the same guard on the backup-config path.
|
||||
if (job.environmentId != null && isLocalRepo(destination.repository)) {
|
||||
const env = await getEnvironment(job.environmentId);
|
||||
if (env && isRemoteEnv(env.connectionType, env.host)) {
|
||||
return { status: 'error', code: 'VALIDATION', error: `Local repository "${destination.name}" cannot restore to remote environment "${env.name}" — the repository path only exists on Dockhand's host. Use an S3 or REST destination for cross-host restores.` };
|
||||
}
|
||||
}
|
||||
|
||||
// A local-path repo is allowed on any env: the restore helper (on the target
|
||||
// daemon) fails loud via the localRepoGuard if the repo isn't visible there
|
||||
// (wrong-host mount), so no silent bad restore — see restic-script.ts.
|
||||
return new RestoreService(restorePorts(destination, access, onProgress)).run(job, 'manual');
|
||||
}
|
||||
|
||||
|
||||
@@ -84,19 +84,24 @@ export async function pruneRepository(restic: ResticLocal, destination: any, max
|
||||
return toResult(run);
|
||||
}
|
||||
|
||||
/** Remove stale repository locks left by a crashed operation. */
|
||||
export async function unlockRepository(restic: ResticLocal, destination: any): Promise<RepoResult> {
|
||||
// `--remove-all`, not a plain `unlock`. This is an EXPLICIT, user-initiated action
|
||||
// ("Unlock" in the UI / `task:unlock`), whose documented purpose is to "remove stale
|
||||
// locks left behind when a previous restic operation crashed or was killed" and is
|
||||
// "safe if no other operation is currently running against the repo". A plain
|
||||
// `restic unlock` only removes locks it can prove stale, and it CANNOT prove that for
|
||||
// a lock whose hostname belongs to a now-gone helper CONTAINER (restic:
|
||||
// `if hn != l.Hostname { return false }`, then a 30-min age-out) — i.e. exactly the
|
||||
// orphaned lock this button exists to clear. So the plain form silently failed to do
|
||||
// what the UI/manual promise. `--remove-all` makes it actually clear those locks.
|
||||
// Only reachable via a deliberate user/API call, never automatic retention.
|
||||
const run = await restic.runLocal(destination, ['unlock', '--remove-all']);
|
||||
/**
|
||||
* Remove repository locks left by a crashed operation.
|
||||
*
|
||||
* `removeAll` (default true) uses `restic unlock --remove-all` — clears EVERY lock,
|
||||
* including one it cannot prove stale (an orphan whose helper-container hostname restic
|
||||
* won't age out for 30 min). That is what the EXPLICIT user "Unlock" button needs, and
|
||||
* it is safe only when no other op runs against the repo — the operator's call.
|
||||
*
|
||||
* `removeAll = false` is a plain `restic unlock`: it reaps ONLY locks restic can prove
|
||||
* stale and LEAVES a live foreign lock intact. AUTOMATIC callers (scheduled maintenance
|
||||
* auto-unlock) MUST use this — the per-repo serializer is per-instance and cannot see a
|
||||
* separate instance's live lock on a shared repo, so a blind --remove-all there would
|
||||
* silently wipe another instance's in-flight backup lock (same hazard backup-service
|
||||
* deliberately avoids). #1313-adjacent shared-repo data-safety fix.
|
||||
*/
|
||||
export async function unlockRepository(restic: ResticLocal, destination: any, removeAll = true): Promise<RepoResult> {
|
||||
const args = removeAll ? ['unlock', '--remove-all'] : ['unlock'];
|
||||
const run = await restic.runLocal(destination, args);
|
||||
return toResult(run);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,15 +1,8 @@
|
||||
/**
|
||||
* backups/restic-script.ts — pure helpers for building the shell command a
|
||||
* restic helper container runs, and reading its exit code back. No I/O, no DB,
|
||||
* no docker — so these are unit-testable in isolation (the Restic class in
|
||||
* ./restic that uses them is the side-effecting edge).
|
||||
*
|
||||
* Why an exit marker instead of relying on the container's exit code: restic's
|
||||
* `backup` exit code 3 ("could not read all source files") is a partial-but-
|
||||
* valid outcome we must be able to see as a warning rather than a hard failure.
|
||||
* The helper script captures restic's real exit code and prints it as its final
|
||||
* stdout line; the caller reads it back. The container's own shell always exits
|
||||
* 0 so the docker layer doesn't treat a mere restic non-zero as a helper crash.
|
||||
* Pure builders for the restic helper container's shell command + reading its
|
||||
* exit code back. The exit marker preserves restic's real code (esp. exit 3 =
|
||||
* partial backup, a warning not a failure) since the helper's shell always
|
||||
* exits 0 so docker doesn't treat a restic non-zero as a crash.
|
||||
*/
|
||||
|
||||
/** The marker line the helper prints carrying restic's real exit code. */
|
||||
@@ -26,26 +19,47 @@ export function resticCommand(args: string[]): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a command so it runs, captures its exit code, and prints the exit marker
|
||||
* as the FINAL stdout line — regardless of success or failure. The command runs
|
||||
* WITHOUT `set -e` so a restic non-zero (e.g. exit 3) doesn't abort before the
|
||||
* marker is printed.
|
||||
*
|
||||
* CONSTRAINT: `command` must RETURN its status (which restic and `sh -c '...'`
|
||||
* do), not terminate the shell with a bare `exit N` — a bare exit ends the whole
|
||||
* script before the marker line runs, and the caller would then read `undefined`
|
||||
* (treated as an unknown-outcome failure). When composing a multi-step session
|
||||
* script, wrap a step whose failure must be caught in a subshell `( ... )` or
|
||||
* `sh -c '...'` rather than letting `set -e` / `exit` end the wrapper.
|
||||
* Append the exit marker after `command` so the caller reads its real code.
|
||||
* GOTCHA: `command` must RETURN its status, not `exit N` - a bare exit ends the
|
||||
* script before the marker prints, and the caller reads `undefined` (= failure).
|
||||
* Wrap a step that might `exit`/`set -e` in a subshell `( ... )`.
|
||||
*/
|
||||
export function finishScript(command: string): string {
|
||||
return `${command}; __rc=$?; echo "${EXIT_MARKER}\${__rc}"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fail loud if a bind-mounted local repo isn't visible on the target daemon's
|
||||
* host: Docker would auto-create the missing path empty, silently backing up to
|
||||
* the wrong host. '' for non-local repos (they never bind a path).
|
||||
*/
|
||||
export function localRepoGuard(repository: string): string {
|
||||
if (!(repository.startsWith('/') || repository.startsWith('./'))) return '';
|
||||
const msg = `restic repository not found at $RESTIC_REPOSITORY on this environment's Docker host. A local-path repository only works when the environment's Docker daemon runs on the same host as Dockhand (e.g. a co-located socket-proxy). For a remote host, use an S3 or REST destination.`;
|
||||
return `test -f "$RESTIC_REPOSITORY/config" || { echo ${shellQuote(msg)} >&2; exit 1; }; `;
|
||||
}
|
||||
|
||||
/**
|
||||
* The helper runs as root (it must read root-owned source volumes), so it writes
|
||||
* the repo files as root. The main Dockhand process reads the repo as its own
|
||||
* (su-exec'd) uid, so for a LOCAL-path repo it couldn't read root-owned files
|
||||
* ("snapshot not found"). Re-own the repo to the main process's uid:gid after the
|
||||
* op so subsequent local reads work. Best-effort (`|| true`) and appended AFTER the
|
||||
* exit marker, so it never changes restic's real exit code. '' for non-local repos
|
||||
* (no bind, nothing to chown) and when ownerSpec is empty.
|
||||
* Note: this only re-owns the repo FILES; the ownership of the BACKED-UP data lives
|
||||
* in snapshot metadata and is untouched (restore fidelity is preserved).
|
||||
*/
|
||||
export function localRepoChown(repository: string, ownerSpec: string): string {
|
||||
if (!ownerSpec) return '';
|
||||
if (!(repository.startsWith('/') || repository.startsWith('./'))) return '';
|
||||
return `; chown -R ${ownerSpec} "$RESTIC_REPOSITORY" 2>/dev/null || true`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the restic exit code from a helper's stdout (the EXIT_MARKER line, read
|
||||
* from the end so trailing output wins). Returns undefined if the marker is
|
||||
* absent — meaning the script did not run to completion (killed / crashed),
|
||||
* absent - meaning the script did not run to completion (killed / crashed),
|
||||
* i.e. an unknown outcome the caller must treat as failure.
|
||||
*/
|
||||
export function readExitMarker(stdout: string): number | undefined {
|
||||
@@ -60,6 +74,35 @@ export function readExitMarker(stdout: string): number | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a spawned restic process's `close` event into the fail-closed exit/stderr
|
||||
* contract, independent of how stdout was captured (string or Buffer). A timeout kill
|
||||
* gives `code === null` + a signal, which maps to `exitCode: undefined` (unknown
|
||||
* outcome = failure for the caller), and the killing signal is appended to stderr so
|
||||
* it is visible in diagnostics.
|
||||
*/
|
||||
export function classifyProcClose(
|
||||
code: number | null,
|
||||
signal: NodeJS.Signals | null,
|
||||
stderr: string
|
||||
): { exitCode: number | undefined; stderr: string } {
|
||||
const exitCode = code === null ? undefined : code;
|
||||
const stderrOut = signal ? `${stderr}\n(process killed by ${signal})` : stderr;
|
||||
return { exitCode, stderr: stderrOut };
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a spawn `error` (restic missing / not spawnable): a real failure surfaced
|
||||
* as data with `exitCode: undefined` (never thrown), the error message appended to
|
||||
* stderr so the caller sees why.
|
||||
*/
|
||||
export function classifyProcError(
|
||||
err: Error,
|
||||
stderr: string
|
||||
): { exitCode: undefined; stderr: string } {
|
||||
return { exitCode: undefined, stderr: `${stderr}\n${err.message}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the KEY=value env array for a restic helper container: repo + password
|
||||
* + a progress-rate hint, plus the cloud credentials that survive the allowlist.
|
||||
@@ -81,8 +124,8 @@ export function buildHelperEnv(
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the bind list for a restic helper: the caller's volume binds, plus — for
|
||||
* a local (filesystem) repository only — the repo path bind so restic can reach
|
||||
* Build the bind list for a restic helper: the caller's volume binds, plus - for
|
||||
* a local (filesystem) repository only - the repo path bind so restic can reach
|
||||
* it. `resolveLocalRepoHostPath` maps the in-container repo path to a host path.
|
||||
*/
|
||||
export function buildHelperBinds(
|
||||
|
||||
@@ -37,7 +37,7 @@ import {
|
||||
} from './security';
|
||||
import { TIMEOUTS, type ResticRun, type TimeoutTier } from './models';
|
||||
import { withTimeout } from './helpers';
|
||||
import { resticCommand, finishScript, readExitMarker, buildHelperEnv, buildHelperBinds } from './restic-script';
|
||||
import { resticCommand, finishScript, readExitMarker, buildHelperEnv, buildHelperBinds, localRepoGuard, localRepoChown, classifyProcClose, classifyProcError } from './restic-script';
|
||||
import { translateContainerPathViaMount } from '../host-path';
|
||||
|
||||
/** The label key stamped on every helper container so a reaper only removes
|
||||
@@ -119,15 +119,13 @@ export class Restic {
|
||||
proc.stdout.on('data', (d) => { stdout += d; });
|
||||
proc.stderr.on('data', (d) => { stderr += d; });
|
||||
proc.on('close', (code, signal) => {
|
||||
// A timeout kill gives code === null + a signal → exit is undefined
|
||||
// (unknown outcome), which the caller treats as failure.
|
||||
const exitCode = code === null ? undefined : code;
|
||||
const stderrOut = signal ? `${stderr}\n(process killed by ${signal})` : stderr;
|
||||
const { exitCode, stderr: stderrOut } = classifyProcClose(code, signal, stderr);
|
||||
done({ exitCode, stdout, stderr: stderrOut });
|
||||
});
|
||||
proc.on('error', (err) => {
|
||||
// restic missing / not spawnable — a real failure, surfaced as data.
|
||||
done({ exitCode: undefined, stdout, stderr: `${stderr}\n${err.message}` });
|
||||
const { exitCode, stderr: stderrOut } = classifyProcError(err, stderr);
|
||||
done({ exitCode, stdout, stderr: stderrOut });
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -163,12 +161,12 @@ export class Restic {
|
||||
proc.stdout.on('data', (d: Buffer) => { chunks.push(Buffer.from(d)); });
|
||||
proc.stderr.on('data', (d) => { stderr += d; });
|
||||
proc.on('close', (code, signal) => {
|
||||
const exitCode = code === null ? undefined : code;
|
||||
const stderrOut = signal ? `${stderr}\n(process killed by ${signal})` : stderr;
|
||||
const { exitCode, stderr: stderrOut } = classifyProcClose(code, signal, stderr);
|
||||
done({ exitCode, stdout: Buffer.concat(chunks), stderr: stderrOut });
|
||||
});
|
||||
proc.on('error', (err) => {
|
||||
done({ exitCode: undefined, stdout: Buffer.concat(chunks), stderr: `${stderr}\n${err.message}` });
|
||||
const { exitCode, stderr: stderrOut } = classifyProcError(err, stderr);
|
||||
done({ exitCode, stdout: Buffer.concat(chunks), stderr: stderrOut });
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -194,9 +192,18 @@ export class Restic {
|
||||
// code. A caller-supplied script (e.g. the in-place restore swap) may use
|
||||
// `set -e`, so capture its exit in a subshell — the marker line must still
|
||||
// print on success, otherwise a clean run reads back as undefined (failure).
|
||||
// The local-repo guard runs FIRST (inside the marked command, so its exit 1
|
||||
// surfaces via the marker): a local repo whose `config` isn't visible on the
|
||||
// target daemon's host is a wrong-host mount — fail loud, not silently empty.
|
||||
const guard = localRepoGuard(destination.repository);
|
||||
// The helper runs as root; re-own a local repo to the main process's uid:gid
|
||||
// after the op so the main process (su-exec'd to that uid) can read it back.
|
||||
// Appended AFTER the exit marker so it never changes restic's exit code.
|
||||
const ownerSpec = `${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`;
|
||||
const chown = localRepoChown(destination.repository, ownerSpec);
|
||||
const cmd = spec.script
|
||||
? ['sh', '-c', finishScript(`( ${spec.script} )`)]
|
||||
: ['sh', '-c', finishScript(resticCommand([...spec.args, ...sanitizeResticFlags(destination.flags)]))];
|
||||
? ['sh', '-c', finishScript(`( ${guard}${spec.script} )`) + chown]
|
||||
: ['sh', '-c', finishScript(`${guard}${resticCommand([...spec.args, ...sanitizeResticFlags(destination.flags)])}`) + chown];
|
||||
|
||||
// Large metadata/stack files go into the container via put-archive (docker
|
||||
// cp), NOT the Cmd, so they can't blow ARG_MAX. Build the tar once; stream
|
||||
|
||||
Reference in new issue
Block a user