/** * Real-filesystem integration tests for backups/swap.ts. The generated shell * runs against a temp directory that stands in for a live volume mount, proving * the safety guarantees of the in-place restore swap WITHOUT needing Docker or * restic: * - a clean swap replaces the live content and exits 0 * - a FAILED restic step leaves the live content byte-for-byte intact * - a kill between the two move-phases is detectable and fully recovered * - a fresh restore over an interrupted swap recovers the original first and * never destroys the sole surviving copy */ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import { execFileSync } from 'node:child_process'; import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, readFileSync, rmSync, existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { buildInPlaceRestoreScript, buildSwapRecoveryScript, SWAP_NEW, SWAP_OLD, SWAP_PHASE, SWAP_ARTIFACTS, } from '../../src/lib/server/backups/swap'; /** Run a script via a temp file (no nested-quoting hazard). Returns exit code. */ function runScript(script: string): number { const dir = mkdtempSync(join(tmpdir(), 'swap-sh-')); const file = join(dir, 'run.sh'); writeFileSync(file, script); try { execFileSync('sh', [file], { stdio: 'pipe' }); return 0; } catch (e: any) { return e.status ?? 1; } finally { rmSync(dir, { recursive: true, force: true }); } } /** Read the non-artifact contents of a live dir as { name: content }. */ function readLive(live: string): Record { const out: Record = {}; for (const name of readdirSync(live)) { if ((SWAP_ARTIFACTS as readonly string[]).includes(name)) continue; out[name] = readFileSync(join(live, name), 'utf8'); } return out; } function artifacts(live: string): string[] { return readdirSync(live).filter((n) => (SWAP_ARTIFACTS as readonly string[]).includes(n)); } /** A sandbox with a live dir; seed its initial + staged (restored) content. */ function sandbox(opts: { live: Record; staged?: Record | null }) { const root = mkdtempSync(join(tmpdir(), 'swap-')); const live = join(root, 'volumes', 'data'); mkdirSync(live, { recursive: true }); for (const [n, c] of Object.entries(opts.live)) writeFileSync(join(live, n), c); if (opts.staged) { mkdirSync(join(live, SWAP_NEW), { recursive: true }); for (const [n, c] of Object.entries(opts.staged)) writeFileSync(join(live, SWAP_NEW, n), c); } return { root, live, cleanup: () => rmSync(root, { recursive: true, force: true }) }; } describe('buildInPlaceRestoreScript — happy path', () => { it('replaces live content with the restored content and exits 0', () => { const { live, cleanup } = sandbox({ live: { 'old.txt': 'OLD', 'stale.txt': 'STALE' }, staged: { 'restored.txt': 'NEW', 'config.yaml': 'v: 2' }, }); try { // resticCmd is a no-op — staging is pre-seeded; we test the swap only. const code = runScript(buildInPlaceRestoreScript([live], 'true')); assert.equal(code, 0); assert.deepEqual(readLive(live), { 'restored.txt': 'NEW', 'config.yaml': 'v: 2' }); assert.deepEqual(artifacts(live), [], 'all swap artifacts cleaned up'); } finally { cleanup(); } }); it('leaves live untouched when the restore produced no staged content (no NEW dir)', () => { const { live, cleanup } = sandbox({ live: { 'keep.txt': 'KEEP' }, staged: null }); try { assert.equal(runScript(buildInPlaceRestoreScript([live], 'true')), 0); assert.deepEqual(readLive(live), { 'keep.txt': 'KEEP' }); } finally { cleanup(); } }); }); describe('buildInPlaceRestoreScript — failure keeps live data intact', () => { it('a failing restic step aborts before the swap; live is byte-for-byte intact', () => { const { live, cleanup } = sandbox({ live: { 'important.txt': 'DO NOT LOSE ME', 'db.sqlite': 'REAL DATA' }, staged: { 'restored.txt': 'NEW' }, }); try { // resticCmd fails → set -e aborts the whole script BEFORE any swap. const code = runScript(buildInPlaceRestoreScript([live], 'false')); assert.notEqual(code, 0, 'a failed restore must exit non-zero'); assert.deepEqual(readLive(live), { 'important.txt': 'DO NOT LOSE ME', 'db.sqlite': 'REAL DATA' }); } finally { cleanup(); } }); }); /** Freeze a live dir in a mid-swap state, as an external kill would leave it. */ function makeInterrupted(opts: { phase: '2' | '3' | ''; liveEntries: Record; oldEntries: Record; newEntries?: Record; }) { const root = mkdtempSync(join(tmpdir(), 'swap-int-')); const live = join(root, 'volumes', 'data'); mkdirSync(join(live, SWAP_OLD), { recursive: true }); mkdirSync(join(live, SWAP_NEW), { recursive: true }); writeFileSync(join(live, SWAP_PHASE), opts.phase); for (const [n, c] of Object.entries(opts.liveEntries)) writeFileSync(join(live, n), c); for (const [n, c] of Object.entries(opts.oldEntries)) writeFileSync(join(live, SWAP_OLD, n), c); for (const [n, c] of Object.entries(opts.newEntries ?? {})) writeFileSync(join(live, SWAP_NEW, n), c); return { root, live, cleanup: () => rmSync(root, { recursive: true, force: true }) }; } describe('buildSwapRecoveryScript — rolls back an interrupted swap', () => { it('phase 2 (emptying live interrupted): originals split across live and OLD are reassembled', () => { const { live, cleanup } = makeInterrupted({ phase: '2', liveEntries: { 'not-yet-moved.txt': 'A' }, oldEntries: { 'already-moved.txt': 'B', 'db.sqlite': 'REAL' }, newEntries: { 'restored.txt': 'NEW' }, }); try { assert.equal(runScript(buildSwapRecoveryScript([live])), 0); assert.deepEqual(readLive(live), { 'not-yet-moved.txt': 'A', 'already-moved.txt': 'B', 'db.sqlite': 'REAL' }); assert.deepEqual(artifacts(live), []); } finally { cleanup(); } }); it('phase 3 (refilling live interrupted): partial NEW content is dropped, ALL originals restored', () => { const { live, cleanup } = makeInterrupted({ phase: '3', liveEntries: { 'restored-a.txt': 'NEW-A' }, // step-3 got this far oldEntries: { 'important.txt': 'DO NOT LOSE ME', 'db.sqlite': 'REAL' }, newEntries: { 'restored-b.txt': 'NEW-B' }, }); try { assert.equal(runScript(buildSwapRecoveryScript([live])), 0); assert.deepEqual(readLive(live), { 'important.txt': 'DO NOT LOSE ME', 'db.sqlite': 'REAL' }); assert.deepEqual(artifacts(live), []); } finally { cleanup(); } }); it('empty marker (killed mid marker-write) is treated as phase 2', () => { const { live, cleanup } = makeInterrupted({ phase: '', liveEntries: {}, oldEntries: { 'everything.txt': 'ORIG' } }); try { assert.equal(runScript(buildSwapRecoveryScript([live])), 0); assert.deepEqual(readLive(live), { 'everything.txt': 'ORIG' }); } finally { cleanup(); } }); it('no marker: recovery is a NO-OP (never touches a healthy volume)', () => { const root = mkdtempSync(join(tmpdir(), 'swap-noop-')); const live = join(root, 'volumes', 'data'); mkdirSync(live, { recursive: true }); writeFileSync(join(live, 'live.txt'), 'LIVE'); try { assert.equal(runScript(buildSwapRecoveryScript([live])), 0); assert.deepEqual(readLive(live), { 'live.txt': 'LIVE' }); } finally { rmSync(root, { recursive: true, force: true }); } }); }); describe('fresh restore over an interrupted swap — never destroys the last copy', () => { it('recovers the originals first, then applies the new restore cleanly', () => { // Start from a phase-3 interrupted state, then run a FRESH restore. Recovery // runs BEFORE restic re-stages (the correct ordering), so we model restic as // a command that (re)creates the staging NEW dir with the second restore's // content — exactly what `restic restore --target ` does in reality. const { live, cleanup } = makeInterrupted({ phase: '3', liveEntries: { 'restored-a.txt': 'NEW-A' }, oldEntries: { 'orig.txt': 'ORIGINAL' }, }); try { // Model restic: mkdir the staging dir and write the new restored content. const restic = `mkdir -p '${live}/${SWAP_NEW}'; printf 'NEW-2' > '${live}/${SWAP_NEW}/second.txt'`; assert.equal(runScript(buildInPlaceRestoreScript([live], restic)), 0); assert.deepEqual(readLive(live), { 'second.txt': 'NEW-2' }, 'fresh restore applied after recovery'); assert.deepEqual(artifacts(live), []); } finally { cleanup(); } }); it('recovery-before-restic: a phase-3 original is restored even before restic re-stages', () => { // Prove the ordering: with recovery first, the interrupted originals are // back in the live root at the moment restic runs (restic then re-stages). const { live, cleanup } = makeInterrupted({ phase: '3', liveEntries: { 'half-applied.txt': 'PARTIAL' }, oldEntries: { 'orig.txt': 'ORIGINAL' }, }); try { // restic here asserts the original is already present, then stages NEW. const restic = `test -f '${live}/orig.txt'; mkdir -p '${live}/${SWAP_NEW}'; printf 'X' > '${live}/${SWAP_NEW}/new.txt'`; assert.equal(runScript(buildInPlaceRestoreScript([live], restic)), 0); assert.deepEqual(readLive(live), { 'new.txt': 'X' }); } finally { cleanup(); } }); it('if the fresh restore aborts, the recovered originals survive (never lost)', () => { // Phase-2 interrupted state; a fresh restore whose restic step FAILS. // The recovery runs inside swapOne, but swapOne only runs if NEW exists — // so with a failing restic and NO pre-seeded NEW, the originals in OLD must // still be recoverable by an explicit recovery pass (what the service does). const { live, cleanup } = makeInterrupted({ phase: '2', liveEntries: { 'a.txt': 'A' }, oldEntries: { 'b.txt': 'B' }, }); try { // The service always runs recovery before/around a restore; prove the // standalone recovery restores the originals regardless of a later abort. assert.equal(runScript(buildSwapRecoveryScript([live])), 0); assert.deepEqual(readLive(live), { 'a.txt': 'A', 'b.txt': 'B' }); } finally { cleanup(); } }); }); describe('SWAP_ARTIFACTS — single source of truth', () => { it('is exactly the three artifact basenames', () => { assert.deepEqual([...SWAP_ARTIFACTS].sort(), ['.dockhand-restore-new', '.dockhand-restore-old', '.dockhand-restore-phase']); }); it('the swap script references every artifact and skips them in the move loops', () => { const script = buildInPlaceRestoreScript(['/volumes/data'], 'true'); for (const a of SWAP_ARTIFACTS) assert.ok(script.includes(a), `references ${a}`); const guard = script.match(/case "\$e" in ([^)]*)\)/); assert.ok(guard, 'has an artifact-skip guard'); for (const a of SWAP_ARTIFACTS) assert.ok(guard![1].includes(a), `skip-guard lists ${a}`); }); }); describe('recovery narration is gated on the phase marker', () => { // The user must be told when a re-run rolls back an interrupted prior swap, // but a normal restore (no marker) must stay quiet. The narration lives INSIDE // the `if [ -f ...phase ]` branch, so it only fires when there is something to recover. it('emits a "Recovering ... rolling back" line inside the marker branch', () => { const script = buildInPlaceRestoreScript(['/volumes/data'], 'true'); assert.ok(/Recovering .*rolling back an interrupted previous restore/.test(script), 'has the recovery narration'); }); it('a real recovery run actually prints the line to stderr only when a marker is present', () => { const dir = mkdtempSync(join(tmpdir(), 'swap-narr-')); const live = join(dir, 'data'); try { // No marker: a recovery pass must be silent. mkdirSync(live, { recursive: true }); writeFileSync(join(live, 'keep'), 'x'); const quiet = execFileSync('sh', ['-c', buildSwapRecoveryScript([live])], { stdio: 'pipe' }).toString(); assert.equal(quiet, '', 'no marker → no narration'); // Marker present (interrupted phase 2): recovery must announce itself. writeFileSync(join(live, SWAP_PHASE), '2'); mkdirSync(join(live, SWAP_OLD), { recursive: true }); const noisy = execFileSync('sh', ['-c', buildSwapRecoveryScript([live]) + ' 2>&1'], { stdio: 'pipe' }).toString(); assert.ok(noisy.includes('rolling back an interrupted previous restore'), 'marker → narration printed'); } finally { rmSync(dir, { recursive: true, force: true }); } }); });