backup engine internals - restic timeout and stack-dir capture
This commit is contained in:
9 files changed
+113
-152
No files matched your search
@@ -17,7 +17,6 @@
|
|||||||
type Entry = { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' };
|
type Entry = { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' };
|
||||||
type Listing =
|
type Listing =
|
||||||
| { kind: 'listed'; hostPath: string; entries: Entry[] }
|
| { kind: 'listed'; hostPath: string; entries: Entry[] }
|
||||||
| { kind: 'tar'; localStackDir: string; entries: Entry[] }
|
|
||||||
| { kind: 'helper-failed'; reason: string }
|
| { kind: 'helper-failed'; reason: string }
|
||||||
| { kind: 'unknown'; reason: string }
|
| { kind: 'unknown'; reason: string }
|
||||||
| null;
|
| null;
|
||||||
@@ -68,7 +67,7 @@
|
|||||||
return isLoadBearing(e.name) || e.capturedAs != null;
|
return isLoadBearing(e.name) || e.capturedAs != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const entries = $derived(listing?.kind === 'listed' || listing?.kind === 'tar' ? listing.entries : []);
|
const entries = $derived(listing?.kind === 'listed' ? listing.entries : []);
|
||||||
const selectable = $derived(entries.filter((e) => !isLocked(e)));
|
const selectable = $derived(entries.filter((e) => !isLocked(e)));
|
||||||
// "All files" is on when nothing selectable is excluded.
|
// "All files" is on when nothing selectable is excluded.
|
||||||
const allFiles = $derived(excludedStackFiles.length === 0);
|
const allFiles = $derived(excludedStackFiles.length === 0);
|
||||||
@@ -99,8 +98,7 @@
|
|||||||
<Loader2 class="h-4 w-4 shrink-0 animate-spin" />
|
<Loader2 class="h-4 w-4 shrink-0 animate-spin" />
|
||||||
Probing the host for the stack folder...
|
Probing the host for the stack folder...
|
||||||
</div>
|
</div>
|
||||||
{:else if listing?.kind === 'listed' || listing?.kind === 'tar'}
|
{:else if listing?.kind === 'listed'}
|
||||||
{#if listing.kind === 'listed'}
|
|
||||||
<!-- Resolved host path callout (copied from BackupPanel's stack-path candidate). -->
|
<!-- Resolved host path callout (copied from BackupPanel's stack-path candidate). -->
|
||||||
<div class="flex items-start gap-2 rounded-md border bg-muted/30 p-2.5 text-xs">
|
<div class="flex items-start gap-2 rounded-md border bg-muted/30 p-2.5 text-xs">
|
||||||
<FolderOpen class="mt-0.5 h-4 w-4 shrink-0 text-sky-500" />
|
<FolderOpen class="mt-0.5 h-4 w-4 shrink-0 text-sky-500" />
|
||||||
@@ -180,17 +178,6 @@
|
|||||||
<div class="mt-0.5 break-all font-mono text-muted-foreground">{listing.hostPath}</div>
|
<div class="mt-0.5 break-all font-mono text-muted-foreground">{listing.hostPath}</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{:else}
|
|
||||||
<!-- TAR mode: no host folder to read (direct-remote, no stack path set). Only Dockhand's
|
|
||||||
own copy of the compose/config is captured; bind-mount DATA on the remote host is NOT. -->
|
|
||||||
<div class="flex items-start gap-2 rounded-md border border-amber-500/40 bg-amber-500/10 p-2.5 text-xs">
|
|
||||||
<AlertTriangle class="mt-0.5 h-4 w-4 shrink-0 text-amber-500" />
|
|
||||||
<div class="min-w-0">
|
|
||||||
<div class="font-medium text-amber-600 dark:text-amber-400">Only these compose/config files will be captured</div>
|
|
||||||
<div class="mt-0.5 text-muted-foreground">This environment has no stack path set, so the folder can't be read on the remote host. Dockhand backs up its own copy of the files below. <strong>Bind-mount data on the remote host is NOT included.</strong> Set a stack path on this environment for a full backup.</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{/if}
|
|
||||||
|
|
||||||
<div class="border rounded-md overflow-hidden">
|
<div class="border rounded-md overflow-hidden">
|
||||||
<div class="flex items-center gap-3 px-3 py-2 bg-muted/30 border-b">
|
<div class="flex items-center gap-3 px-3 py-2 bg-muted/30 border-b">
|
||||||
|
|||||||
@@ -41,7 +41,9 @@ export interface MetadataFile {
|
|||||||
* @param resticArgs the full restic argv (starting with 'backup') — already
|
* @param resticArgs the full restic argv (starting with 'backup') — already
|
||||||
* tag/exclude/flag-assembled by the caller.
|
* tag/exclude/flag-assembled by the caller.
|
||||||
*/
|
*/
|
||||||
export function buildBackupScript(resticArgs: string[], stackDirProbe?: { volumeKey: string; composeFileName: string; label?: string; hostPath?: string }): string {
|
import type { StackDirProbeHint } from './stackdir-plan';
|
||||||
|
|
||||||
|
export function buildBackupScript(resticArgs: string[], stackDirProbe?: { volumeKey: string; composeFileName: string; label?: string; hostPath?: string; hint?: StackDirProbeHint }): string {
|
||||||
// restic runs backgrounded so the SIGINT-forwarding trap can reach it on cancel
|
// restic runs backgrounded so the SIGINT-forwarding trap can reach it on cancel
|
||||||
// (SIGINT lets restic release its repo lock; a SIGKILL orphans it and hangs the
|
// (SIGINT lets restic release its repo lock; a SIGKILL orphans it and hangs the
|
||||||
// next backup on --retry-lock). The kill-0 re-wait loop only fires when a signal
|
// next backup on --retry-lock). The kill-0 re-wait loop only fires when a signal
|
||||||
@@ -61,14 +63,28 @@ export function buildBackupScript(resticArgs: string[], stackDirProbe?: { volume
|
|||||||
if (stackDirProbe) {
|
if (stackDirProbe) {
|
||||||
const composePath = shellQuote(`/volumes/${stackDirProbe.volumeKey}/${stackDirProbe.composeFileName}`);
|
const composePath = shellQuote(`/volumes/${stackDirProbe.volumeKey}/${stackDirProbe.composeFileName}`);
|
||||||
const label = stackDirProbe.label ? ` ${stackDirProbe.label}` : '';
|
const label = stackDirProbe.label ? ` ${stackDirProbe.label}` : '';
|
||||||
// Name the REAL host path we mounted (<remote_stacks_dir>/<stack>) so the operator knows
|
// The compose is missing under the host dir we bind-mounted. For a remote env (hawser or a
|
||||||
// exactly where to look, and what to do: redeploy the stack to stage its files there.
|
// user-set path) tell the operator to point "Remote stack path (for backup)" at the real host
|
||||||
const hostHint = stackDirProbe.hostPath
|
// path; for a local stack a redeploy stages the files.
|
||||||
? ` The stack folder is empty or missing at ${stackDirProbe.hostPath} on the Docker host - redeploy the stack to stage its files there.`
|
const hint = stackDirProbe.hint;
|
||||||
: '';
|
const remote = hint?.kind === 'hawser-defaulted' || hint?.kind === 'user-set';
|
||||||
|
const where = remote ? hint.hostPath : stackDirProbe.hostPath;
|
||||||
|
// "on <env>" when we know the environment name, else a plain "on the host".
|
||||||
|
const on = remote && hint.envName ? ` on ${hint.envName}` : ` on the host`;
|
||||||
|
// Name the EXACT settings location so the operator can go straight there.
|
||||||
|
const settingsLoc = remote && hint.envName ? `Settings > Environments > ${hint.envName}` : `Settings > Environments`;
|
||||||
|
const fix = remote
|
||||||
|
? ` Set "Remote stack path (for backup)" in ${settingsLoc} to the real host path where this stack's files live.`
|
||||||
|
: ` Redeploy the stack to stage its files there.`;
|
||||||
|
const at = where ? ` in ${where}` : '';
|
||||||
|
// Build the whole message as ONE string and shell-quote it so the operator-facing text
|
||||||
|
// (which contains quotes, parens, and `>` - e.g. `"Remote stack path (for backup)"` and
|
||||||
|
// `Settings > Environments > <env>`) can never break the `sh` syntax. The timestamp stays
|
||||||
|
// a live $(date ...) OUTSIDE the quoted literal, concatenated in.
|
||||||
|
const msg = `${label} STACKDIR PROBE FAILED: compose ${stackDirProbe.composeFileName} not found${at}${on}.${fix}`;
|
||||||
probe =
|
probe =
|
||||||
`if [ ! -f ${composePath} ]; then ` +
|
`if [ ! -f ${composePath} ]; then ` +
|
||||||
`echo "[backup] $(date -u +%Y-%m-%dT%H:%M:%SZ)${label} STACKDIR PROBE FAILED: the stack's compose (${stackDirProbe.composeFileName}) is not present under the mounted stack dir.${hostHint} Refusing to write a silent-empty snapshot." 1>&2; ` +
|
`echo "[backup] $(date -u +%Y-%m-%dT%H:%M:%SZ)"${shellQuote(msg)} 1>&2; ` +
|
||||||
`exit 1; fi; `;
|
`exit 1; fi; `;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ import { EXIT_MARKER } from './restic-script';
|
|||||||
import { parseRetention, buildForgetArgs, checkWouldWipe, retentionActive } from './retention';
|
import { parseRetention, buildForgetArgs, checkWouldWipe, retentionActive } from './retention';
|
||||||
import { liveTargetKey } from './locks';
|
import { liveTargetKey } from './locks';
|
||||||
import type { DiscoveredVolume } from './discovery-core';
|
import type { DiscoveredVolume } from './discovery-core';
|
||||||
import { STACKDIR_VOLUME_KEY } from './stackdir-plan';
|
import { STACKDIR_VOLUME_KEY, type StackDirProbeHint } from './stackdir-plan';
|
||||||
|
|
||||||
/** What the service needs from the outside world. All injected for testability. */
|
/** What the service needs from the outside world. All injected for testability. */
|
||||||
export interface BackupPorts {
|
export interface BackupPorts {
|
||||||
@@ -53,18 +53,16 @@ export interface BackupPorts {
|
|||||||
/** Discover volumes; MUST throw (fail closed) on any inspect failure. */
|
/** Discover volumes; MUST throw (fail closed) on any inspect failure. */
|
||||||
discoverVolumes(containers: Array<{ id: string; name: string }>, envId: number | null | undefined, selected: string[] | null):
|
discoverVolumes(containers: Array<{ id: string; name: string }>, envId: number | null | undefined, selected: string[] | null):
|
||||||
Promise<{ volumes: DiscoveredVolume[]; skipped: Array<{ type: string; destination: string; reason: string }> }>;
|
Promise<{ volumes: DiscoveredVolume[]; skipped: Array<{ type: string; destination: string; reason: string }> }>;
|
||||||
/** Decide how a STACK's directory is captured. Normally: locate the HOST folder and build the
|
/** Decide how a STACK's directory is captured: locate the HOST folder and build the synthetic
|
||||||
* synthetic __dockhand_stackdir__ volume the helper bind-mounts (`kind: 'candidate'`).
|
* __dockhand_stackdir__ volume the helper bind-mounts (`kind: 'candidate'`). The stack files
|
||||||
* `kind: 'tar'` is the direct-remote-with-no-remote_stacks_dir case: nothing is staged on the
|
* are ALWAYS read from the host where the stack runs - never from Dockhand's own copy.
|
||||||
* host, so the compose/config is tarred from Dockhand's local copy instead (no host bind).
|
* `composeFileName` feeds the in-helper probe (assert the compose is visible under the mount).
|
||||||
* `composeFileName` feeds the in-helper probe (assert the compose is
|
* `kind: 'unknown'` means the host folder can't be located (not compose-managed, or a remote
|
||||||
* visible under the mount). `kind: 'unknown'` means the stack is not compose-managed (no
|
* env with no Remote stacks directory) -> the caller HARD-FAILS the backup. Logged inside this port. */
|
||||||
* working_dir label) -> the caller HARD-FAILS the backup. Logged inside this port. */
|
|
||||||
planStackDirVolume(targetName: string, envId: number | null | undefined, excludedStackFiles?: string[]):
|
planStackDirVolume(targetName: string, envId: number | null | undefined, excludedStackFiles?: string[]):
|
||||||
Promise<
|
Promise<
|
||||||
| { kind: 'unknown'; reason: string }
|
| { kind: 'unknown'; reason: string }
|
||||||
| { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[] }
|
| { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[]; probeHint?: StackDirProbeHint }
|
||||||
| { kind: 'tar'; localStackDir: string; composeFileName: string }
|
|
||||||
>;
|
>;
|
||||||
/** Stop the target for a consistent backup; returns a restart closure. */
|
/** Stop the target for a consistent backup; returns a restart closure. */
|
||||||
stopForBackup(type: BackupTargetType, targetName: string, containers: Array<{ id: string; name: string; state: string }>, envId: number | null | undefined):
|
stopForBackup(type: BackupTargetType, targetName: string, containers: Array<{ id: string; name: string; state: string }>, envId: number | null | undefined):
|
||||||
@@ -72,17 +70,14 @@ export interface BackupPorts {
|
|||||||
/** Run the backup session container; returns the restic run. metadataFiles (metadata.json
|
/** Run the backup session container; returns the restic run. metadataFiles (metadata.json
|
||||||
* + the light stack-dir listing) are put-archived into /metadata (not the Cmd) so they
|
* + the light stack-dir listing) are put-archived into /metadata (not the Cmd) so they
|
||||||
* can't blow ARG_MAX. The stack dir's bytes ride a host bind mount, not these files. */
|
* can't blow ARG_MAX. The stack dir's bytes ride a host bind mount, not these files. */
|
||||||
runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }>; onStderr?: (line: string) => void; onStdout?: (line: string) => void }):
|
runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; onStderr?: (line: string) => void; onStdout?: (line: string) => void }):
|
||||||
Promise<ResticRun>;
|
Promise<ResticRun>;
|
||||||
/** Run a repo-only restic command on the host (verify / retention). */
|
/** Run a repo-only restic command on the host (verify / retention). */
|
||||||
runLocal(args: string[], tier?: 'interactive' | 'data'): Promise<ResticRun>;
|
runLocal(args: string[], tier?: 'interactive' | 'data'): Promise<ResticRun>;
|
||||||
/** Collect the restore metadata files: metadata.json plus, for a stack, a light file
|
/** Collect the restore metadata files: metadata.json plus, for a stack, a light file
|
||||||
* listing. The stack dir's bytes normally ride the helper's host bind mount, not these
|
* listing. The stack dir's bytes ride the helper's host bind mount, not these files. */
|
||||||
* files. EXCEPTION - `stackDirTar`: for a direct-remote stack with no remote_stacks_dir
|
collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[]):
|
||||||
* there is no host bind, so the collector reads localStackDir and tars its files into the
|
Promise<{ files: MetadataFile[] }>;
|
||||||
* snapshot at /volumes/__dockhand_stackdir__ (the same path restore reads). */
|
|
||||||
collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[], stackDirTar?: { localStackDir: string; composeFileName: string } | null):
|
|
||||||
Promise<{ files: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }> }>;
|
|
||||||
/** The Dockhand host name for restic --host. */
|
/** The Dockhand host name for restic --host. */
|
||||||
host(): string;
|
host(): string;
|
||||||
/** This installation's stable instance id. */
|
/** This installation's stable instance id. */
|
||||||
@@ -151,13 +146,10 @@ export class BackupService {
|
|||||||
// __dockhand_stackdir__ volume). `stackDirProbe` carries what the helper needs to
|
// __dockhand_stackdir__ volume). `stackDirProbe` carries what the helper needs to
|
||||||
// assert that mount is real (test -f the compose) before restic runs. Null for
|
// assert that mount is real (test -f the compose) before restic runs. Null for
|
||||||
// containers (no stack dir).
|
// containers (no stack dir).
|
||||||
let stackDirProbe: { volumeKey: string; composeFileName: string; hostPath?: string } | null = null;
|
let stackDirProbe: { volumeKey: string; composeFileName: string; hostPath?: string; hint?: StackDirProbeHint } | null = null;
|
||||||
// restic --exclude paths for compose bind dirs inside the stackdir volume (they are
|
// restic --exclude paths for compose bind dirs inside the stackdir volume (they are
|
||||||
// captured separately as their own /volumes/<key>, honoring volume selection).
|
// captured separately as their own /volumes/<key>, honoring volume selection).
|
||||||
let stackDirExcludes: string[] = [];
|
let stackDirExcludes: string[] = [];
|
||||||
// Set (instead of a host bind) for a direct-remote stack with no remote_stacks_dir: the
|
|
||||||
// compose/config is tarred into /metadata from Dockhand's local copy, not the host.
|
|
||||||
let stackDirTar: { localStackDir: string; composeFileName: string } | null = null;
|
|
||||||
try {
|
try {
|
||||||
const t = await this.ports.resolveTargets(job.type, job.targetName, envId);
|
const t = await this.ports.resolveTargets(job.type, job.targetName, envId);
|
||||||
containers = t.containers;
|
containers = t.containers;
|
||||||
@@ -184,16 +176,11 @@ export class BackupService {
|
|||||||
if (p.kind === 'unknown') {
|
if (p.kind === 'unknown') {
|
||||||
return this.earlySkipOrError(job, triggeredBy,
|
return this.earlySkipOrError(job, triggeredBy,
|
||||||
`Cannot locate the stack folder on the host for "${job.targetName}" (${p.reason}). ` +
|
`Cannot locate the stack folder on the host for "${job.targetName}" (${p.reason}). ` +
|
||||||
`For remote environments, set a Remote stacks directory or use matching paths so the stack files exist on the Docker host. ` +
|
`For remote environments, set a Remote stack path (for backup) in Settings > Environments so the stack files exist on the Docker host. ` +
|
||||||
`Refusing to write a backup missing the stack's compose and config.`);
|
`Refusing to write a backup missing the stack's compose and config.`);
|
||||||
} else if (p.kind === 'tar') {
|
|
||||||
// direct-remote, no remote_stacks_dir: no host bind (nothing staged), so the
|
|
||||||
// compose/config is captured from Dockhand's local copy into /metadata via a tar.
|
|
||||||
// No synthetic volume, no probe - the metadata collector reads localStackDir.
|
|
||||||
stackDirTar = { localStackDir: p.localStackDir, composeFileName: p.composeFileName };
|
|
||||||
} else {
|
} else {
|
||||||
volumes = [...volumes, p.syntheticVolume];
|
volumes = [...volumes, p.syntheticVolume];
|
||||||
stackDirProbe = { volumeKey: p.volumeKey, composeFileName: p.composeFileName, hostPath: p.syntheticVolume.source ?? undefined };
|
stackDirProbe = { volumeKey: p.volumeKey, composeFileName: p.composeFileName, hostPath: p.syntheticVolume.source ?? undefined, hint: p.probeHint };
|
||||||
stackDirExcludes = p.excludePaths;
|
stackDirExcludes = p.excludePaths;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -221,7 +208,7 @@ export class BackupService {
|
|||||||
try {
|
try {
|
||||||
// --- serialize on the destination (share the repo, one at a time) ---
|
// --- serialize on the destination (share the repo, one at a time) ---
|
||||||
return await this.ports.serializeDestination(job.destinationId, () =>
|
return await this.ports.serializeDestination(job.destinationId, () =>
|
||||||
this.runLocked(job, triggeredBy, containers, volumes, stackDirProbe, stackDirExcludes, stackDirTar, startTime, releaseOnce));
|
this.runLocked(job, triggeredBy, containers, volumes, stackDirProbe, stackDirExcludes, startTime, releaseOnce));
|
||||||
} finally {
|
} finally {
|
||||||
releaseOnce();
|
releaseOnce();
|
||||||
}
|
}
|
||||||
@@ -237,9 +224,8 @@ export class BackupService {
|
|||||||
triggeredBy: 'cron' | 'manual' | 'webhook',
|
triggeredBy: 'cron' | 'manual' | 'webhook',
|
||||||
containers: Array<{ id: string; name: string; state: string }>,
|
containers: Array<{ id: string; name: string; state: string }>,
|
||||||
volumes: DiscoveredVolume[],
|
volumes: DiscoveredVolume[],
|
||||||
stackDirProbe: { volumeKey: string; composeFileName: string } | null,
|
stackDirProbe: { volumeKey: string; composeFileName: string; hostPath?: string; hint?: StackDirProbeHint } | null,
|
||||||
stackDirExcludes: string[],
|
stackDirExcludes: string[],
|
||||||
stackDirTar: { localStackDir: string; composeFileName: string } | null,
|
|
||||||
startTime: number,
|
startTime: number,
|
||||||
releaseLiveTarget: () => void,
|
releaseLiveTarget: () => void,
|
||||||
): Promise<BackupResult> {
|
): Promise<BackupResult> {
|
||||||
@@ -287,7 +273,7 @@ export class BackupService {
|
|||||||
|
|
||||||
// --- metadata + backup args + session script ---
|
// --- metadata + backup args + session script ---
|
||||||
op.progress('metadata', 'Collecting metadata...');
|
op.progress('metadata', 'Collecting metadata...');
|
||||||
const { files: metadata, stackDirStreamSources } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes, stackDirTar);
|
const { files: metadata } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes);
|
||||||
const instanceId = await this.ports.instanceId();
|
const instanceId = await this.ports.instanceId();
|
||||||
const tags = buildSnapshotTags({ instanceId, configId: job.configId, environmentId: envId, targetName: job.targetName, type: job.type });
|
const tags = buildSnapshotTags({ instanceId, configId: job.configId, environmentId: envId, targetName: job.targetName, type: job.type });
|
||||||
// Cross-shard identity. If two shards log the SAME instanceId + configId
|
// Cross-shard identity. If two shards log the SAME instanceId + configId
|
||||||
@@ -295,14 +281,12 @@ export class BackupService {
|
|||||||
// collide. host() + destinationId show which repo this run targets, so a
|
// collide. host() + destinationId show which repo this run targets, so a
|
||||||
// CI log across shards reveals any overlap.
|
// CI log across shards reveals any overlap.
|
||||||
diag('IDENTITY', `instanceId=${instanceId} host=${this.ports.host()} destId=${job.destinationId} tags=[${tags.join(',')}]`);
|
diag('IDENTITY', `instanceId=${instanceId} host=${this.ports.host()} destId=${job.destinationId} tags=[${tags.join(',')}]`);
|
||||||
// restic must back up /volumes/ when there's anything to capture there: real bind/named
|
// restic backs up /volumes/ when there's anything to capture there: real bind/named
|
||||||
// volumes (volumes.length), OR a tar-mode stack whose compose/config was put-archived
|
// volumes OR the synthetic __dockhand_stackdir__ volume (both are in `volumes`).
|
||||||
// into /volumes/__dockhand_stackdir__ (no synthetic volume in `volumes`, so it wouldn't
|
|
||||||
// count otherwise - and /metadata-only would silently drop the tarred stack files).
|
|
||||||
const args = buildBackupArgs({
|
const args = buildBackupArgs({
|
||||||
host: this.ports.host(),
|
host: this.ports.host(),
|
||||||
tags,
|
tags,
|
||||||
hasVolumes: volumes.length > 0 || stackDirTar !== null,
|
hasVolumes: volumes.length > 0,
|
||||||
// User excludes PLUS the compose bind dirs inside the stackdir volume (those
|
// User excludes PLUS the compose bind dirs inside the stackdir volume (those
|
||||||
// ride their own /volumes/<key>, so excluding them here avoids a duplicate and
|
// ride their own /volumes/<key>, so excluding them here avoids a duplicate and
|
||||||
// keeps volume DESELECTION meaningful for in-folder binds).
|
// keeps volume DESELECTION meaningful for in-folder binds).
|
||||||
@@ -381,7 +365,7 @@ export class BackupService {
|
|||||||
// This flag drives that fallback.
|
// This flag drives that fallback.
|
||||||
let liveStdoutSeen = false;
|
let liveStdoutSeen = false;
|
||||||
const run = await this.ports.runInHelper({
|
const run = await this.ports.runInHelper({
|
||||||
script, binds, envId, name: job.helperName, metadataFiles: metadata, stackDirStreamSources,
|
script, binds, envId, name: job.helperName, metadataFiles: metadata,
|
||||||
// Stream restic's own output to the UI LIVE: restic writes its --json
|
// Stream restic's own output to the UI LIVE: restic writes its --json
|
||||||
// progress (per-file %) to STDOUT and its status messages to STDERR. Wire
|
// progress (per-file %) to STDOUT and its status messages to STDERR. Wire
|
||||||
// BOTH so the log fills in as the backup runs, not all at once at the end.
|
// BOTH so the log fills in as the backup runs, not all at once at the end.
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ import { sendEventNotification } from '../notifications';
|
|||||||
import type { BackupResult, RestoreResult, ResticRun, BackupTargetType } from './models';
|
import type { BackupResult, RestoreResult, ResticRun, BackupTargetType } from './models';
|
||||||
import type { MetadataFile } from './backup-script';
|
import type { MetadataFile } from './backup-script';
|
||||||
import type { DiscoveredVolume } from './discovery-core';
|
import type { DiscoveredVolume } from './discovery-core';
|
||||||
|
import type { StackDirProbeHint } from './stackdir-plan';
|
||||||
|
import { normalizeBaseDir, stackDirIn } from '../stack-paths';
|
||||||
import { volumeBind } from './discovery-core';
|
import { volumeBind } from './discovery-core';
|
||||||
import { resolveBindFromMetadata } from './restore-core';
|
import { resolveBindFromMetadata } from './restore-core';
|
||||||
import { parseSnapshotLsEntries } from './browse-core';
|
import { parseSnapshotLsEntries } from './browse-core';
|
||||||
@@ -122,8 +124,7 @@ async function planStackDirVolume(
|
|||||||
attempts = 7,
|
attempts = 7,
|
||||||
): Promise<
|
): Promise<
|
||||||
| { kind: 'unknown'; reason: string }
|
| { kind: 'unknown'; reason: string }
|
||||||
| { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[] }
|
| { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[]; probeHint?: StackDirProbeHint }
|
||||||
| { kind: 'tar'; localStackDir: string; composeFileName: string }
|
|
||||||
> {
|
> {
|
||||||
const { getStackComposeFile } = await import('../stacks');
|
const { getStackComposeFile } = await import('../stacks');
|
||||||
const { dirname, join, basename } = await import('path');
|
const { dirname, join, basename } = await import('path');
|
||||||
@@ -174,16 +175,23 @@ async function planStackDirVolume(
|
|||||||
let envTcpHost: string | null = null;
|
let envTcpHost: string | null = null;
|
||||||
let remoteStacksDir: string | null = null;
|
let remoteStacksDir: string | null = null;
|
||||||
let isHawser = false;
|
let isHawser = false;
|
||||||
|
// True when a hawser env had NO remote_stacks_dir set and we fell back to the agent's default
|
||||||
|
// STACKS_DIR (/data/stacks). Steers the probe-fail message: a defaulted path that comes up empty
|
||||||
|
// means the agent stores stacks under a DIFFERENT host dir (e.g. a containerized agent whose
|
||||||
|
// /data/stacks is mounted from another host path), so the user must set the HOST-side path.
|
||||||
|
let remoteStacksDirDefaulted = false;
|
||||||
|
let envName: string | null = null;
|
||||||
if (envId != null) {
|
if (envId != null) {
|
||||||
try {
|
try {
|
||||||
const { getEnvironment, getEnvSetting } = await import('../db');
|
const { getEnvironment, getEnvSetting } = await import('../db');
|
||||||
const env = await getEnvironment(envId);
|
const env = await getEnvironment(envId);
|
||||||
envConnType = env?.connectionType ?? null;
|
envConnType = env?.connectionType ?? null;
|
||||||
|
envName = env?.name ?? null;
|
||||||
envTcpHost = env?.host && env?.port ? `tcp://${env.host}:${env.port}` : null;
|
envTcpHost = env?.host && env?.port ? `tcp://${env.host}:${env.port}` : null;
|
||||||
isHawser = envConnType === 'hawser-standard' || envConnType === 'hawser-edge';
|
isHawser = envConnType === 'hawser-standard' || envConnType === 'hawser-edge';
|
||||||
if (envConnType === 'direct') {
|
if (envConnType === 'direct') {
|
||||||
const rsd = await getEnvSetting('remote_stacks_dir', envId);
|
const rsd = await getEnvSetting('remote_stacks_dir', envId);
|
||||||
remoteStacksDir = typeof rsd === 'string' && rsd.trim() ? rsd.trim().replace(/\/+$/, '') : null;
|
remoteStacksDir = typeof rsd === 'string' && rsd.trim() ? normalizeBaseDir(rsd) : null;
|
||||||
} else if (isHawser) {
|
} else if (isHawser) {
|
||||||
// Same `remote_stacks_dir` setting as direct, but for hawser it is a BACKUP-ONLY
|
// Same `remote_stacks_dir` setting as direct, but for hawser it is a BACKUP-ONLY
|
||||||
// declaration of where the AGENT keeps stack files on its host - it does NOT steer
|
// declaration of where the AGENT keeps stack files on its host - it does NOT steer
|
||||||
@@ -191,9 +199,9 @@ async function planStackDirVolume(
|
|||||||
// to bind-mount. Falls back to the agent's default STACKS_DIR (/data/stacks) so a
|
// to bind-mount. Falls back to the agent's default STACKS_DIR (/data/stacks) so a
|
||||||
// standard agent needs no configuration.
|
// standard agent needs no configuration.
|
||||||
const rsd = await getEnvSetting('remote_stacks_dir', envId);
|
const rsd = await getEnvSetting('remote_stacks_dir', envId);
|
||||||
remoteStacksDir = typeof rsd === 'string' && rsd.trim()
|
const userSet = typeof rsd === 'string' && rsd.trim();
|
||||||
? rsd.trim().replace(/\/+$/, '')
|
remoteStacksDir = userSet ? normalizeBaseDir(rsd) : HAWSER_DEFAULT_STACKS_DIR;
|
||||||
: HAWSER_DEFAULT_STACKS_DIR;
|
remoteStacksDirDefaulted = !userSet;
|
||||||
}
|
}
|
||||||
} catch { /* treat as unknown -> non-local (safe: skips the wrong-host translation) */ }
|
} catch { /* treat as unknown -> non-local (safe: skips the wrong-host translation) */ }
|
||||||
}
|
}
|
||||||
@@ -206,9 +214,8 @@ async function planStackDirVolume(
|
|||||||
// direct env the decision is explicit, not guessed:
|
// direct env the decision is explicit, not guessed:
|
||||||
// - remote_stacks_dir SET -> the deploy staged the files at <base>/<stack> on the host, so
|
// - remote_stacks_dir SET -> the deploy staged the files at <base>/<stack> on the host, so
|
||||||
// that IS the host path (bind-derived can still win below if a bind pins it more exactly).
|
// that IS the host path (bind-derived can still win below if a bind pins it more exactly).
|
||||||
// - remote_stacks_dir UNSET -> nothing was staged; capture the compose/config from
|
// - remote_stacks_dir UNSET -> the host folder can't be located; resolution returns UNKNOWN
|
||||||
// Dockhand's OWN copy via a tar into the snapshot (no host bind, so no :ro mask). The
|
// and the backup HARD-FAILS with an actionable message (set a Remote stacks directory).
|
||||||
// caller signals this with kind: 'tar'.
|
|
||||||
const directRemote = envConnType === 'direct' && !localDaemon;
|
const directRemote = envConnType === 'direct' && !localDaemon;
|
||||||
// The working_dir label is only trustworthy for hawser (agent ran compose on its host) or a
|
// The working_dir label is only trustworthy for hawser (agent ran compose on its host) or a
|
||||||
// LOCAL daemon. For direct-remote it's Dockhand's path, so drop it from the candidates.
|
// LOCAL daemon. For direct-remote it's Dockhand's path, so drop it from the candidates.
|
||||||
@@ -217,10 +224,11 @@ async function planStackDirVolume(
|
|||||||
// AND hawser. For hawser this is the agent's STACKS_DIR (default /data/stacks) - the DETERMINISTIC
|
// AND hawser. For hawser this is the agent's STACKS_DIR (default /data/stacks) - the DETERMINISTIC
|
||||||
// host location, so it wins over the flaky working_dir label (which is "/" for a start-from-stdin
|
// host location, so it wins over the flaky working_dir label (which is "/" for a start-from-stdin
|
||||||
// hawser stack). The in-helper probe confirms the compose is actually there.
|
// hawser stack). The in-helper probe confirms the compose is actually there.
|
||||||
const remoteStacksDirHostPath = (directRemote || isHawser) && remoteStacksDir ? `${remoteStacksDir}/${targetName}` : null;
|
// SAME formula the deploy plan STAGES to (stackDirIn) - deploy WRITES here, backup READS here.
|
||||||
|
const remoteStacksDirHostPath = (directRemote || isHawser) && remoteStacksDir ? stackDirIn(remoteStacksDir, targetName) : null;
|
||||||
|
|
||||||
// bind-derived is a PHANTOM empty dir for a direct-remote stdin deploy with no remote_stacks_dir
|
// bind-derived is a PHANTOM empty dir for a direct-remote stdin deploy with no remote_stacks_dir
|
||||||
// (see trustBindDerivedForEnv) - distrust it there so resolution falls through to kind:'tar'.
|
// (see trustBindDerivedForEnv) - distrust it there so resolution falls through to UNKNOWN (hard-fail).
|
||||||
const trustedBindDerived = trustBindDerivedForEnv(bindDerivedHostPath, { directRemote, hasRemoteStacksDir: !!remoteStacksDir });
|
const trustedBindDerived = trustBindDerivedForEnv(bindDerivedHostPath, { directRemote, hasRemoteStacksDir: !!remoteStacksDir });
|
||||||
|
|
||||||
// Fallback host-path candidates (LOCAL daemon only). dataDirHostPath: DATA_DIR -> HOST_DATA_DIR
|
// Fallback host-path candidates (LOCAL daemon only). dataDirHostPath: DATA_DIR -> HOST_DATA_DIR
|
||||||
@@ -243,14 +251,10 @@ async function planStackDirVolume(
|
|||||||
workingDirLabel: trustedWorkingDirLabel,
|
workingDirLabel: trustedWorkingDirLabel,
|
||||||
});
|
});
|
||||||
if (resolution.kind === 'unknown') {
|
if (resolution.kind === 'unknown') {
|
||||||
// A direct-REMOTE env with NO remote_stacks_dir has nothing staged on the host, but
|
// Backup ALWAYS reads the stack files from the host where the stack runs - never from
|
||||||
// Dockhand HAS the stack's compose/config locally - capture it via a tar into the
|
// Dockhand's own copy. When the host folder can't be located (a remote env with no
|
||||||
// snapshot instead of hard-failing. (Only compose/config; volume data rides its own
|
// Remote stacks directory / no matching paths), we HARD-FAIL with an actionable message
|
||||||
// /volumes/<key>. Secrets are never on disk - carried as ciphertext in metadata.)
|
// rather than silently capturing a possibly-stale local copy.
|
||||||
if (directRemote && !remoteStacksDir && dockhandStackDir) {
|
|
||||||
console.log(`[Backup] stackdir plan for "${targetName}": TAR (direct-remote, no remote_stacks_dir) - capturing compose/config from Dockhand's local copy in folder ${dockhandStackDir}`);
|
|
||||||
return { kind: 'tar', localStackDir: dockhandStackDir, composeFileName: composeFileName0 ?? 'docker-compose.yml' };
|
|
||||||
}
|
|
||||||
// Report every candidate we tried so an UNKNOWN is diagnosable without SSH: which
|
// Report every candidate we tried so an UNKNOWN is diagnosable without SSH: which
|
||||||
// inputs were null tells us WHY (e.g. workingDir null = 0 containers listed).
|
// inputs were null tells us WHY (e.g. workingDir null = 0 containers listed).
|
||||||
console.log(`[Backup] stackdir plan for "${targetName}": UNKNOWN reason="${resolution.reason}" dockhandStackDir=${dockhandStackDir ?? 'none'} | connType=${envConnType} localDaemon=${localDaemon} remoteStacksDir=${remoteStacksDir ?? 'null'} containers=${stackContainers.length} bindDerived=${bindDerivedHostPath ?? 'null'} dataDir=${dataDirHostPath ?? 'null'} mount=${mountHostPath ?? 'null'} workingDirLabel=${workingDirLabel ?? 'null'}`);
|
console.log(`[Backup] stackdir plan for "${targetName}": UNKNOWN reason="${resolution.reason}" dockhandStackDir=${dockhandStackDir ?? 'none'} | connType=${envConnType} localDaemon=${localDaemon} remoteStacksDir=${remoteStacksDir ?? 'null'} containers=${stackContainers.length} bindDerived=${bindDerivedHostPath ?? 'null'} dataDir=${dataDirHostPath ?? 'null'} mount=${mountHostPath ?? 'null'} workingDirLabel=${workingDirLabel ?? 'null'}`);
|
||||||
@@ -293,6 +297,15 @@ async function planStackDirVolume(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The probe (in the helper) hard-fails when the compose is missing under the mounted host dir.
|
||||||
|
// Give it the context to tell the operator what to DO: a defaulted hawser path that comes up
|
||||||
|
// empty means the agent keeps stacks under a different HOST dir, so the fix is to set the
|
||||||
|
// HOST-side "Remote stack path (for backup)" - NOT to redeploy.
|
||||||
|
const probeHint: StackDirProbeHint =
|
||||||
|
isHawser && remoteStacksDirDefaulted ? { kind: 'hawser-defaulted', hostPath, envName }
|
||||||
|
: (isHawser || directRemote) && remoteStacksDir ? { kind: 'user-set', hostPath, envName }
|
||||||
|
: { kind: 'local' };
|
||||||
|
|
||||||
return {
|
return {
|
||||||
kind: 'candidate',
|
kind: 'candidate',
|
||||||
syntheticVolume: {
|
syntheticVolume: {
|
||||||
@@ -306,6 +319,7 @@ async function planStackDirVolume(
|
|||||||
composeFileName,
|
composeFileName,
|
||||||
excludePaths,
|
excludePaths,
|
||||||
bindSources,
|
bindSources,
|
||||||
|
probeHint,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -318,13 +332,9 @@ async function planStackDirVolume(
|
|||||||
export async function previewStackBackupPath(
|
export async function previewStackBackupPath(
|
||||||
targetName: string,
|
targetName: string,
|
||||||
envId: number | null | undefined,
|
envId: number | null | undefined,
|
||||||
): Promise<{ kind: 'candidate'; hostPath: string; composeFile: string } | { kind: 'tar'; composeFile: string } | { kind: 'unknown'; reason: string }> {
|
): Promise<{ kind: 'candidate'; hostPath: string; composeFile: string } | { kind: 'unknown'; reason: string }> {
|
||||||
const plan = await planStackDirVolume(targetName, envId);
|
const plan = await planStackDirVolume(targetName, envId);
|
||||||
if (plan.kind === 'unknown') return { kind: 'unknown', reason: plan.reason };
|
if (plan.kind === 'unknown') return { kind: 'unknown', reason: plan.reason };
|
||||||
// TAR mode (direct-remote, no remote_stacks_dir): the stack folder is captured from
|
|
||||||
// Dockhand's own copy into the snapshot, not from a host path. Surface that to the UI so the
|
|
||||||
// preview says "captured from Dockhand" rather than showing a misleading host path.
|
|
||||||
if (plan.kind === 'tar') return { kind: 'tar', composeFile: plan.composeFileName };
|
|
||||||
return { kind: 'candidate', hostPath: plan.syntheticVolume.source ?? '', composeFile: plan.composeFileName };
|
return { kind: 'candidate', hostPath: plan.syntheticVolume.source ?? '', composeFile: plan.composeFileName };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -342,7 +352,6 @@ export async function probeStackDir(
|
|||||||
envId: number | null | undefined,
|
envId: number | null | undefined,
|
||||||
): Promise<
|
): Promise<
|
||||||
| { kind: 'listed'; hostPath: string; entries: import('./stackdir-plan').StackDirEntry[] }
|
| { kind: 'listed'; hostPath: string; entries: import('./stackdir-plan').StackDirEntry[] }
|
||||||
| { kind: 'tar'; localStackDir: string; entries: import('./stackdir-plan').StackDirEntry[] }
|
|
||||||
| { kind: 'helper-failed'; reason: string }
|
| { kind: 'helper-failed'; reason: string }
|
||||||
| { kind: 'unknown'; reason: string }
|
| { kind: 'unknown'; reason: string }
|
||||||
> {
|
> {
|
||||||
@@ -351,30 +360,14 @@ export async function probeStackDir(
|
|||||||
const { ensureHelperImage } = await import('./restic');
|
const { ensureHelperImage } = await import('./restic');
|
||||||
const { INSTANCE_LABEL } = await import('./reap-core');
|
const { INSTANCE_LABEL } = await import('./reap-core');
|
||||||
|
|
||||||
// Ask the real capture planner how this stack resolves. TAR mode (direct-remote, no
|
// Ask the real capture planner how this stack resolves. A `candidate` plan falls through to
|
||||||
// remote_stacks_dir) has no host folder to mount - the backup streams Dockhand's OWN local
|
// the host-mount probe below; an `unknown` plan means the host folder can't be located, so the
|
||||||
// copy (compose/config only; bind data lives on the remote host, out of reach). Surface THAT
|
// picker surfaces that (the user must set the env's stack path).
|
||||||
// to the picker so the user knows exactly what a backup would contain, instead of a bare
|
|
||||||
// "unknown". A `candidate`/`unknown` plan falls through to the host-mount probe below.
|
|
||||||
// attempts=1: config-time probe. The stack already exists (the user is editing its backup),
|
// attempts=1: config-time probe. The stack already exists (the user is editing its backup),
|
||||||
// so a not-listable container is legitimately stopped/absent - the 15s just-deployed retry
|
// so a not-listable container is legitimately stopped/absent - the 15s just-deployed retry
|
||||||
// would only hang the picker. (planStackDirVolume calls collectStackContainerHostInfo ONCE and
|
// would only hang the picker. (planStackDirVolume calls collectStackContainerHostInfo ONCE and
|
||||||
// returns its bindSources, so the probe does NOT inspect the containers a second time.)
|
// returns its bindSources, so the probe does NOT inspect the containers a second time.)
|
||||||
const plan = await planStackDirVolume(targetName, envId, undefined, 1);
|
const plan = await planStackDirVolume(targetName, envId, undefined, 1);
|
||||||
if (plan.kind === 'tar') {
|
|
||||||
const { readdirSync, statSync } = await import('fs');
|
|
||||||
const { join } = await import('path');
|
|
||||||
let entries: import('./stackdir-plan').StackDirEntry[] = [];
|
|
||||||
try {
|
|
||||||
entries = readdirSync(plan.localStackDir).map((name) => {
|
|
||||||
let type: 'dir' | 'file' = 'file'; let size = 0;
|
|
||||||
try { const st = statSync(join(plan.localStackDir, name)); type = st.isDirectory() ? 'dir' : 'file'; size = st.isFile() ? st.size : 0; } catch { /* keep defaults */ }
|
|
||||||
return { name, type, size };
|
|
||||||
}).sort((a, b) => (a.type === b.type ? a.name.localeCompare(b.name) : a.type === 'dir' ? -1 : 1));
|
|
||||||
} catch { /* empty local copy -> empty list, still tar kind */ }
|
|
||||||
return { kind: 'tar', localStackDir: plan.localStackDir, entries };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (plan.kind === 'unknown') {
|
if (plan.kind === 'unknown') {
|
||||||
return { kind: 'unknown', reason: plan.reason };
|
return { kind: 'unknown', reason: plan.reason };
|
||||||
}
|
}
|
||||||
@@ -532,12 +525,8 @@ async function collectMetadata(
|
|||||||
targetName: string,
|
targetName: string,
|
||||||
envId: number | null | undefined,
|
envId: number | null | undefined,
|
||||||
volumes: DiscoveredVolume[],
|
volumes: DiscoveredVolume[],
|
||||||
stackDirTar?: { localStackDir: string; composeFileName: string } | null,
|
): Promise<{ files: MetadataFile[] }> {
|
||||||
): Promise<{ files: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }> }> {
|
|
||||||
const files: MetadataFile[] = [];
|
const files: MetadataFile[] = [];
|
||||||
// tar-mode ONLY: disk paths of the stack dir files to stream into the helper (no base64, no
|
|
||||||
// cap). Filled in the stack block below when stackDirTar is set; empty otherwise.
|
|
||||||
const stackDirStreamSources: Array<{ diskPath: string; archivePath: string }> = [];
|
|
||||||
// Writes metadata.json plus, for a stack, a light file listing (path + size, no content)
|
// Writes metadata.json plus, for a stack, a light file listing (path + size, no content)
|
||||||
// of the stack dir. The stack dir's bytes are captured by the helper's HOST bind mount
|
// of the stack dir. The stack dir's bytes are captured by the helper's HOST bind mount
|
||||||
// (the synthetic __dockhand_stackdir__ volume from planStackDirVolume), not here.
|
// (the synthetic __dockhand_stackdir__ volume from planStackDirVolume), not here.
|
||||||
@@ -582,7 +571,7 @@ async function collectMetadata(
|
|||||||
if (type === 'stack') {
|
if (type === 'stack') {
|
||||||
const { getStackComposeFile } = await import('../stacks');
|
const { getStackComposeFile } = await import('../stacks');
|
||||||
const { readFileSync, readdirSync, lstatSync, statSync } = await import('fs');
|
const { readFileSync, readdirSync, lstatSync, statSync } = await import('fs');
|
||||||
const { dirname, join, relative, basename, sep } = await import('path');
|
const { dirname, join, relative, basename } = await import('path');
|
||||||
const compose = await getStackComposeFile(targetName, envId ?? undefined);
|
const compose = await getStackComposeFile(targetName, envId ?? undefined);
|
||||||
if (compose.success && compose.composePath) {
|
if (compose.success && compose.composePath) {
|
||||||
// Presence of `stackInfo` IS hasStackFiles (derived, not a stored flag that could
|
// Presence of `stackInfo` IS hasStackFiles (derived, not a stored flag that could
|
||||||
@@ -627,22 +616,7 @@ async function collectMetadata(
|
|||||||
listed.sort((a, b) => a.path.localeCompare(b.path));
|
listed.sort((a, b) => a.path.localeCompare(b.path));
|
||||||
if (stackInfo) { stackInfo.fileList = listed; stackInfo.excludedBindDirs = excludeRelDirs; }
|
if (stackInfo) { stackInfo.fileList = listed; stackInfo.excludedBindDirs = excludeRelDirs; }
|
||||||
|
|
||||||
if (stackDirTar) {
|
console.log(`[Backup] stackfiles "${targetName}": captured via HOST bind mount at /volumes/${STACKDIR_VOLUME_KEY} - listed ${listed.length} file(s), ${excludeRelDirs.length} bind dir(s) excluded`);
|
||||||
// TAR mode (direct-remote, no remote_stacks_dir): no host bind exists, so the
|
|
||||||
// listed files are STREAMED from disk into the helper at
|
|
||||||
// /volumes/__dockhand_stackdir__/<rel> (O(1) RAM, no cap; restore reads the same
|
|
||||||
// path via stackDirSource). Bind data dirs are excluded (same as listed) - their
|
|
||||||
// runtime bytes live on the target host, out of reach; compose/config only.
|
|
||||||
for (const { path: rel } of listed) {
|
|
||||||
stackDirStreamSources.push({
|
|
||||||
diskPath: join(stackDir, rel),
|
|
||||||
archivePath: `volumes/${STACKDIR_VOLUME_KEY}/${rel.split(sep).join('/')}`,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
console.log(`[Backup] stackfiles "${targetName}": TAR mode - streaming ${stackDirStreamSources.length} file(s) into /volumes/${STACKDIR_VOLUME_KEY} from folder ${stackDir}`);
|
|
||||||
} else {
|
|
||||||
console.log(`[Backup] stackfiles "${targetName}": captured via HOST bind mount at /volumes/${STACKDIR_VOLUME_KEY} - listed ${listed.length} file(s), ${excludeRelDirs.length} bind dir(s) excluded`);
|
|
||||||
}
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.warn(`[Backup] stackfiles "${targetName}": listing failed (files still captured via bind):`, e instanceof Error ? e.message : e);
|
console.warn(`[Backup] stackfiles "${targetName}": listing failed (files still captured via bind):`, e instanceof Error ? e.message : e);
|
||||||
}
|
}
|
||||||
@@ -663,7 +637,7 @@ async function collectMetadata(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { files, stackDirStreamSources: stackDirStreamSources.length > 0 ? stackDirStreamSources : undefined };
|
return { files };
|
||||||
}
|
}
|
||||||
|
|
||||||
// A restic helper runner bound to one destination (so no shared mutable state —
|
// A restic helper runner bound to one destination (so no shared mutable state —
|
||||||
|
|||||||
@@ -296,14 +296,20 @@ export function readConfigIdFromTags(tags: string[]): number | null {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
// Timeout tiers — two named tiers, not a per-operation multiplier ladder
|
// Timeout tiers
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
|
|
||||||
/** Two timeout tiers, in ms. `interactive` for quick repo ops (list/ls/init);
|
/** Restic operations are UNBOUNDED by default (0 = no timeout). A large backup or a
|
||||||
* `data` for streaming ops that move real bytes (backup/restore/check/prune). */
|
* slow listing on a big/remote repo legitimately runs for a long time; a fixed wall-clock
|
||||||
|
* cap killed healthy work mid-run (#1382). A stuck op is handled by manual cancel (the
|
||||||
|
* helper is a named container killed by cancelBackup, and every run shows as `running` in
|
||||||
|
* the execution log) and the helper reaper — the same model backrest/zerobyte use (neither
|
||||||
|
* times out a backup). The two tiers remain so callers can still opt into a cap via env:
|
||||||
|
* `RESTIC_TIMEOUT` (interactive: list/ls/init) and `RESTIC_MAINTENANCE_TIMEOUT`
|
||||||
|
* (data: backup/restore/prune). Both default to 0 (off). */
|
||||||
export const TIMEOUTS = {
|
export const TIMEOUTS = {
|
||||||
interactive: Number(process.env.RESTIC_TIMEOUT ?? 300_000), // 5 min
|
interactive: Number(process.env.RESTIC_TIMEOUT ?? 0),
|
||||||
data: Number(process.env.RESTIC_MAINTENANCE_TIMEOUT ?? 6 * 60 * 60 * 1000), // 6 h
|
data: Number(process.env.RESTIC_MAINTENANCE_TIMEOUT ?? 0),
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
export type TimeoutTier = keyof typeof TIMEOUTS;
|
export type TimeoutTier = keyof typeof TIMEOUTS;
|
||||||
|
|||||||
@@ -19,8 +19,7 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
|
|||||||
import { tmpdir } from 'node:os';
|
import { tmpdir } from 'node:os';
|
||||||
import { join } from 'node:path';
|
import { join } from 'node:path';
|
||||||
import { runContainerWithStreaming, inspectImage, pullImage, putContainerArchive } from '../docker';
|
import { runContainerWithStreaming, inspectImage, pullImage, putContainerArchive } from '../docker';
|
||||||
import { buildTar, buildTarStream, type TarFileSource } from './tar';
|
import { buildTar } from './tar';
|
||||||
import { putContainerArchiveStreaming } from './put-archive-stream';
|
|
||||||
import type { MetadataFile } from './backup-script';
|
import type { MetadataFile } from './backup-script';
|
||||||
import { getInstanceId } from './identity';
|
import { getInstanceId } from './identity';
|
||||||
import { decryptBackupDestination, getSetting } from '../db';
|
import { decryptBackupDestination, getSetting } from '../db';
|
||||||
@@ -111,11 +110,6 @@ export interface HelperRunSpec {
|
|||||||
/** Small in-RAM files (e.g. metadata.json) put-archived into the container BEFORE it
|
/** Small in-RAM files (e.g. metadata.json) put-archived into the container BEFORE it
|
||||||
* starts. `path` is the full path from the container root. */
|
* starts. `path` is the full path from the container root. */
|
||||||
metadataFiles?: MetadataFile[];
|
metadataFiles?: MetadataFile[];
|
||||||
/** tar-mode ONLY (direct-remote, no remote_stacks_dir): the stack dir's files, streamed
|
|
||||||
* from disk into the helper (O(1) RAM, no cap) alongside metadataFiles in the same
|
|
||||||
* put-archive. Each source's archivePath is the full container path
|
|
||||||
* (volumes/__dockhand_stackdir__/<rel>). Empty/absent = no streaming. */
|
|
||||||
stackDirStreamSources?: TarFileSource[];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export class Restic {
|
export class Restic {
|
||||||
@@ -247,21 +241,12 @@ export class Restic {
|
|||||||
// the container via put-archive (docker cp), NOT the Cmd, so they can't blow
|
// the container via put-archive (docker cp), NOT the Cmd, so they can't blow
|
||||||
// ARG_MAX. The stack dir's bytes ride a read-only host bind mount, not these files.
|
// ARG_MAX. The stack dir's bytes ride a read-only host bind mount, not these files.
|
||||||
const metadataFiles = spec.metadataFiles ?? [];
|
const metadataFiles = spec.metadataFiles ?? [];
|
||||||
const streamSources = spec.stackDirStreamSources ?? [];
|
|
||||||
const inlineEntries = metadataFiles.map((f) => ({ path: f.path.replace(/^\/+/, ''), content: Buffer.from(f.contentBase64, 'base64') }));
|
const inlineEntries = metadataFiles.map((f) => ({ path: f.path.replace(/^\/+/, ''), content: Buffer.from(f.contentBase64, 'base64') }));
|
||||||
const beforeStart = (metadataFiles.length > 0 || streamSources.length > 0)
|
const beforeStart = metadataFiles.length > 0
|
||||||
? async (containerId: string) => {
|
? async (containerId: string) => {
|
||||||
// Each file's `path` is the full path from the container root (e.g.
|
// Each file's `path` is the full path from the container root (e.g.
|
||||||
// `metadata/metadata.json`), placing it under the right root.
|
// `metadata/metadata.json`), placing it under the right root.
|
||||||
if (streamSources.length > 0) {
|
await putContainerArchive(containerId, '/', await buildTar(inlineEntries), spec.envId);
|
||||||
// tar-mode: STREAM the stack dir files from disk (O(1) RAM, no cap) into the
|
|
||||||
// helper, with metadata.json as inline entries in the same tar. All under `/`.
|
|
||||||
const tar = buildTarStream(streamSources, inlineEntries);
|
|
||||||
const { streamed } = await putContainerArchiveStreaming(containerId, '/', tar, spec.envId);
|
|
||||||
if (!streamed) throw new Error(`tar-mode: transport for env ${spec.envId} cannot stream the stack dir`);
|
|
||||||
} else {
|
|
||||||
await putContainerArchive(containerId, '/', await buildTar(inlineEntries), spec.envId);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
|
|||||||
@@ -203,6 +203,17 @@ export function isLocalDaemon(connectionType: string | null, envTcpHost: string
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Context the in-helper probe-fail message needs to give the operator an ACTIONABLE next step
|
||||||
|
// (the fix differs by wiring). `hawser-defaulted`: a hawser env with no remote_stacks_dir fell
|
||||||
|
// back to /data/stacks and it came up empty -> the agent stores stacks under a DIFFERENT host dir
|
||||||
|
// (e.g. a containerized agent whose /data/stacks is mounted from another host path), so set the
|
||||||
|
// HOST-side path. `user-set`: an explicitly-configured host path came up empty -> the path is
|
||||||
|
// wrong. `local`: a socket/local-daemon stack -> redeploy staged its files.
|
||||||
|
export type StackDirProbeHint =
|
||||||
|
| { kind: 'hawser-defaulted'; hostPath: string; envName: string | null }
|
||||||
|
| { kind: 'user-set'; hostPath: string; envName: string | null }
|
||||||
|
| { kind: 'local' };
|
||||||
|
|
||||||
/** Result of resolving the candidate host stack folder (before the runtime probe). */
|
/** Result of resolving the candidate host stack folder (before the runtime probe). */
|
||||||
export type HostStackDirResolution =
|
export type HostStackDirResolution =
|
||||||
| { kind: 'candidate'; hostPath: string; composeFile: string; source: string }
|
| { kind: 'candidate'; hostPath: string; composeFile: string; source: string }
|
||||||
|
|||||||
@@ -51,7 +51,6 @@
|
|||||||
// Stack files on the host (probe listing) — stack targets only.
|
// Stack files on the host (probe listing) — stack targets only.
|
||||||
type StackListing =
|
type StackListing =
|
||||||
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
||||||
| { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
|
||||||
| { kind: 'helper-failed'; reason: string }
|
| { kind: 'helper-failed'; reason: string }
|
||||||
| { kind: 'unknown'; reason: string }
|
| { kind: 'unknown'; reason: string }
|
||||||
| null;
|
| null;
|
||||||
|
|||||||
@@ -498,7 +498,6 @@
|
|||||||
// picker, but stackPath is still fetched for the compose-file name shown elsewhere.
|
// picker, but stackPath is still fetched for the compose-file name shown elsewhere.
|
||||||
type StackListing =
|
type StackListing =
|
||||||
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
||||||
| { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
|
||||||
| { kind: 'unknown'; reason: string }
|
| { kind: 'unknown'; reason: string }
|
||||||
| null;
|
| null;
|
||||||
let stackListing = $state<StackListing>(null);
|
let stackListing = $state<StackListing>(null);
|
||||||
|
|||||||
Reference in new issue
Block a user