1.0.39
This commit is contained in:
16 files changed
+427
-259
No files matched your search
@@ -42,10 +42,14 @@ export interface MetadataFile {
|
||||
* tag/exclude/flag-assembled by the caller.
|
||||
*/
|
||||
export function buildBackupScript(resticArgs: string[]): string {
|
||||
// Turn off `set -e` for the restic step so exit 3 (partial) doesn't abort
|
||||
// before the marker is printed; the marker carries the real code.
|
||||
// restic runs backgrounded so the SIGINT-forwarding trap can reach it on cancel
|
||||
// (SIGINT lets restic release its repo lock; a SIGKILL orphans it and hangs the
|
||||
// next backup on --retry-lock). The kill-0 re-wait loop only fires when a signal
|
||||
// interrupted the first `wait` early; a normal exit skips it, keeping restic's
|
||||
// real code (0/3) for the marker.
|
||||
const restic = `restic ${resticArgs.map(shellQuote).join(' ')}`;
|
||||
return `mkdir -p /metadata; set +e; ${finishScript(restic)}`;
|
||||
const step = `${restic} & __rpid=$!; trap 'kill -INT $__rpid 2>/dev/null' INT TERM; wait $__rpid; __rc=$?; while kill -0 $__rpid 2>/dev/null; do wait $__rpid; __rc=$?; done; ( exit $__rc )`;
|
||||
return `mkdir -p /metadata; set +e; ${finishScript(step)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -251,6 +251,13 @@ export function cleanErrorMsg(msg: string): string {
|
||||
const message = parsed.message.replace(/\\n/g, ' ').trim();
|
||||
return prefix ? `${prefix} ${message}` : message;
|
||||
}
|
||||
if (parsed.message_type) {
|
||||
// A killed helper appends restic's last --json PROGRESS line
|
||||
// (message_type, no `message`) to the exit-code detail — no error
|
||||
// text; drop it, keep just the readable prefix.
|
||||
const p2 = clean.slice(0, jsonStart).trim().replace(/:\s*$/, '');
|
||||
if (p2) return p2;
|
||||
}
|
||||
} catch { /* not embedded JSON */ }
|
||||
}
|
||||
return clean;
|
||||
|
||||
@@ -743,33 +743,55 @@ export async function runRepoTask(destinationId: number, task: RepoTask, opts?:
|
||||
|
||||
// --- cancellation + running check ---
|
||||
|
||||
/** Kill every running helper container whose name starts with `prefix`, in the
|
||||
* given env. Returns whether anything was killed. */
|
||||
async function killHelpers(prefix: string, envId: number | null | undefined): Promise<boolean> {
|
||||
/** Signal every running helper container whose name starts with `prefix`, in the
|
||||
* given env (default SIGKILL). Returns the ids signalled. */
|
||||
async function killHelpers(prefix: string, envId: number | null | undefined, signal?: string): Promise<string[]> {
|
||||
const { listContainers, dockerFetch } = await import('../docker');
|
||||
let killed = false;
|
||||
const ids: string[] = [];
|
||||
const q = signal ? `?signal=${encodeURIComponent(signal)}` : '';
|
||||
try {
|
||||
const containers = await listContainers(true, envId ?? undefined);
|
||||
for (const c of containers) {
|
||||
if (c.state === 'running' && c.name?.startsWith(prefix)) {
|
||||
try { await dockerFetch(`/containers/${c.id}/kill`, { method: 'POST' }, envId ?? undefined); killed = true; } catch { /* best effort */ }
|
||||
try { await dockerFetch(`/containers/${c.id}/kill${q}`, { method: 'POST' }, envId ?? undefined); ids.push(c.id); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
} catch { /* daemon unreachable */ }
|
||||
return killed;
|
||||
return ids;
|
||||
}
|
||||
|
||||
/** Cancel a running backup: kill its helper so restic exits non-zero and the
|
||||
* service's failure path runs. */
|
||||
/** Cancel a running backup GRACEFULLY: send SIGINT so restic releases its repo
|
||||
* lock before exiting (a SIGKILL orphans the lock and the next backup hangs on
|
||||
* --retry-lock). Safety net: if the helper is still alive ~15s later, SIGKILL it
|
||||
* so a cancel never itself hangs. */
|
||||
export async function cancelBackup(configId: number): Promise<boolean> {
|
||||
const config = await getBackupConfig(configId);
|
||||
return killHelpers(`dockhand-backup-${configId}`, config?.environmentId ?? undefined);
|
||||
const envId = config?.environmentId ?? undefined;
|
||||
const prefix = `dockhand-backup-${configId}`;
|
||||
const signalled = await killHelpers(prefix, envId, 'SIGINT');
|
||||
if (signalled.length === 0) return false;
|
||||
// Force-kill any helper that ignored SIGINT (pathological restic hang), so the
|
||||
// user's cancel resolves regardless. Best-effort, out of band.
|
||||
void (async () => {
|
||||
const { listContainers, dockerFetch } = await import('../docker');
|
||||
for (let i = 0; i < 15; i++) {
|
||||
await new Promise((r) => setTimeout(r, 1000));
|
||||
try {
|
||||
const alive = (await listContainers(true, envId)).some((c) => c.state === 'running' && signalled.includes(c.id));
|
||||
if (!alive) return;
|
||||
} catch { return; }
|
||||
}
|
||||
for (const id of signalled) {
|
||||
try { await dockerFetch(`/containers/${id}/kill`, { method: 'POST' }, envId); } catch { /* gone */ }
|
||||
}
|
||||
})();
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Cancel a running restore. A blanket cancel (no snapshotId) never kills a
|
||||
* swap-recovery helper (that is a data-safety op, not a user restore). */
|
||||
export async function cancelRestore(snapshotId?: string, environmentId?: number): Promise<boolean> {
|
||||
if (snapshotId) return killHelpers(`dockhand-restore-${snapshotId.slice(0, 12)}`, environmentId);
|
||||
if (snapshotId) return (await killHelpers(`dockhand-restore-${snapshotId.slice(0, 12)}`, environmentId)).length > 0;
|
||||
// Blanket: kill restore helpers but NOT recovery helpers.
|
||||
const { listContainers, dockerFetch } = await import('../docker');
|
||||
let killed = false;
|
||||
|
||||
@@ -204,7 +204,7 @@ export class Restic {
|
||||
const metadataFiles = spec.metadataFiles ?? [];
|
||||
const beforeStart = metadataFiles.length > 0
|
||||
? async (containerId: string) => {
|
||||
const tar = buildTar(metadataFiles.map((f) => ({
|
||||
const tar = await buildTar(metadataFiles.map((f) => ({
|
||||
path: `metadata/${f.path.replace(/^\/+/, '')}`,
|
||||
content: Buffer.from(f.contentBase64, 'base64'),
|
||||
})));
|
||||
|
||||
@@ -1,80 +1,37 @@
|
||||
/**
|
||||
* backups/tar.ts — a tiny, pure USTAR builder for the backup helper.
|
||||
* Build the tar we put-archive (docker cp) into the backup helper, replacing the
|
||||
* base64-in-Cmd approach that blew ARG_MAX on large stack files.
|
||||
*
|
||||
* The backup metadata files (metadata.json, and for stacks the compose/.env and
|
||||
* the whole stackfiles/ tree) used to be written into the helper container by
|
||||
* embedding each file's base64 as a shell argument in the container Cmd. That
|
||||
* blows the kernel ARG_MAX for large stack files (a few MB of base64 on one arg
|
||||
* line) and the helper dies with "argument list too long" before restic runs.
|
||||
*
|
||||
* Instead we ship the files into the container with the Docker put-archive API
|
||||
* (docker cp), which streams a tar over HTTP with no arg-length limit. This
|
||||
* module builds that tar. Pure (no I/O), so it is unit-testable directly.
|
||||
* `modern-tar` is zero-dependency and emits PAX headers, so paths of ANY length
|
||||
* round-trip (the old hand-rolled USTAR writer threw on names >100 bytes, breaking
|
||||
* deep stack trees). Zero deps also matters for CI: it installs into a symlinked
|
||||
* node_modules where libs with transitive deps fail to resolve them.
|
||||
*/
|
||||
|
||||
import { packTar } from 'modern-tar';
|
||||
|
||||
export interface TarEntry {
|
||||
/** Path inside the archive, e.g. "metadata/stacks/foo/compose.yaml". No leading slash. */
|
||||
/** Path inside the archive, e.g. "metadata/stacks/foo/compose.yaml". A leading slash is stripped. */
|
||||
path: string;
|
||||
content: Uint8Array;
|
||||
}
|
||||
|
||||
const BLOCK = 512;
|
||||
|
||||
function writeAscii(block: Uint8Array, offset: number, text: string): void {
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
block.set(bytes, offset);
|
||||
}
|
||||
|
||||
/** Build a POSIX USTAR header block for one regular file. */
|
||||
function header(path: string, size: number, mtimeSecs: number): Uint8Array {
|
||||
const h = new Uint8Array(BLOCK);
|
||||
// Name (100). USTAR splits long names into prefix(345)+name(100); our paths
|
||||
// are short (metadata/stacks/<name>/stackfiles/<rel>), so the 100-byte name
|
||||
// field is enough. Truncation would corrupt the path, so guard callers.
|
||||
writeAscii(h, 0, path.slice(0, 100));
|
||||
writeAscii(h, 100, '0000644\0'); // mode 0644
|
||||
writeAscii(h, 108, '0000000\0'); // uid 0
|
||||
writeAscii(h, 116, '0000000\0'); // gid 0
|
||||
writeAscii(h, 124, size.toString(8).padStart(11, '0') + '\0'); // size (octal)
|
||||
writeAscii(h, 136, mtimeSecs.toString(8).padStart(11, '0') + '\0'); // mtime (octal)
|
||||
writeAscii(h, 148, ' '); // checksum placeholder (spaces) while summing
|
||||
h[156] = 48; // type flag '0' = regular file
|
||||
writeAscii(h, 257, 'ustar\0'); // magic
|
||||
writeAscii(h, 263, '00'); // version
|
||||
writeAscii(h, 265, 'root');
|
||||
writeAscii(h, 297, 'root');
|
||||
|
||||
let checksum = 0;
|
||||
for (let i = 0; i < BLOCK; i++) checksum += h[i];
|
||||
writeAscii(h, 148, checksum.toString(8).padStart(6, '0') + '\0 ');
|
||||
return h;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a USTAR archive from the given entries. Paths longer than 100 bytes are
|
||||
* rejected (the callers use short, bounded paths) rather than silently truncated.
|
||||
* `mtimeSecs` is injectable so the builder stays pure/deterministic for tests.
|
||||
*/
|
||||
export function buildTar(entries: TarEntry[], mtimeSecs = Math.floor(Date.now() / 1000)): Uint8Array {
|
||||
const blocks: Uint8Array[] = [];
|
||||
for (const e of entries) {
|
||||
const path = e.path.replace(/^\/+/, '');
|
||||
if (new TextEncoder().encode(path).length > 100) {
|
||||
throw new Error(`tar entry path too long for USTAR name field (>100 bytes): ${path}`);
|
||||
}
|
||||
blocks.push(header(path, e.content.length, mtimeSecs));
|
||||
const padded = Math.ceil(e.content.length / BLOCK) * BLOCK;
|
||||
const body = new Uint8Array(padded);
|
||||
body.set(e.content, 0);
|
||||
blocks.push(body);
|
||||
}
|
||||
// Two zero blocks terminate the archive.
|
||||
blocks.push(new Uint8Array(BLOCK));
|
||||
blocks.push(new Uint8Array(BLOCK));
|
||||
|
||||
const total = blocks.reduce((n, b) => n + b.length, 0);
|
||||
const out = new Uint8Array(total);
|
||||
let off = 0;
|
||||
for (const b of blocks) { out.set(b, off); off += b.length; }
|
||||
return out;
|
||||
/** Build a tar (any path length via PAX). `mtimeSecs` injectable for deterministic tests. */
|
||||
export function buildTar(entries: TarEntry[], mtimeSecs = Math.floor(Date.now() / 1000)): Promise<Uint8Array> {
|
||||
return packTar(
|
||||
entries.map((e) => ({
|
||||
header: {
|
||||
name: e.path.replace(/^\/+/, ''),
|
||||
size: e.content.length,
|
||||
mode: 0o644,
|
||||
mtime: new Date(mtimeSecs * 1000),
|
||||
uid: 0,
|
||||
gid: 0,
|
||||
uname: 'root',
|
||||
gname: 'root',
|
||||
type: 'file' as const,
|
||||
},
|
||||
body: e.content,
|
||||
})),
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user