direct remote environments - remote stacks dir configurable, for putting the whole stack onto the remote
This commit is contained in:
19 files changed
+2316
-506
No files matched your search
@@ -0,0 +1,254 @@
|
||||
<script lang="ts">
|
||||
/**
|
||||
* "Stack files on the host" — probes the target host at backup-config time, shows the
|
||||
* resolved HOST path of the stack folder, and lets the user pick which entries to back up.
|
||||
* Load-bearing files (compose, .env) are always kept (non-deselectable). When the folder
|
||||
* can't be located on the host, an info callout tells the user to set the env's stack path
|
||||
* and re-configure. Mirrors VolumePicker's list + BackupPanel's host-path callout.
|
||||
*/
|
||||
import { Label } from '$lib/components/ui/label';
|
||||
import { Checkbox } from '$lib/components/ui/checkbox';
|
||||
import { TogglePill } from '$lib/components/ui/toggle-pill';
|
||||
import { FolderOpen, Folder, FileText, Info, Loader2, AlertTriangle, HelpCircle, Unplug, Route, UndoDot, Icon } from 'lucide-svelte';
|
||||
import { whale } from '@lucide/lab';
|
||||
import EnvironmentIcon from '$lib/components/EnvironmentIcon.svelte';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
|
||||
type Entry = { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' };
|
||||
type Listing =
|
||||
| { kind: 'listed'; hostPath: string; entries: Entry[] }
|
||||
| { kind: 'tar'; localStackDir: string; entries: Entry[] }
|
||||
| { kind: 'helper-failed'; reason: string }
|
||||
| { kind: 'unknown'; reason: string }
|
||||
| null;
|
||||
|
||||
interface Props {
|
||||
listing: Listing;
|
||||
loading?: boolean;
|
||||
/** Names of stack-dir entries the user has DEselected (excluded from the backup). */
|
||||
excludedStackFiles: string[];
|
||||
/** Environment connection type - the tooltip explains the host path differently per type. */
|
||||
connectionType?: string;
|
||||
/** Environment name, shown in the tooltip ("<name> is a socket environment"). */
|
||||
envName?: string;
|
||||
/** Environment icon (Lucide name or custom) + its id, for the env glyph in the tooltip. */
|
||||
envIcon?: string;
|
||||
envId?: number;
|
||||
/** The user-set "Remote stack path (for backup)" for direct/hawser envs (empty if unset). */
|
||||
configuredStackPath?: string;
|
||||
}
|
||||
|
||||
let { listing, loading = false, excludedStackFiles = $bindable(), connectionType, envName, envIcon, envId, configuredStackPath = '' }: Props = $props();
|
||||
|
||||
// Falls back to a generic subject when the env name isn't available.
|
||||
const envLabel = $derived(envName || 'This environment');
|
||||
|
||||
// The stack folder resolves to a host path by a DIFFERENT route per environment type, so the
|
||||
// "where this path comes from" help must differ too (undefined = socket, the default).
|
||||
const envKind = $derived(
|
||||
connectionType === 'direct' ? 'direct'
|
||||
: connectionType === 'hawser-standard' || connectionType === 'hawser-edge' ? 'hawser'
|
||||
: 'socket'
|
||||
);
|
||||
|
||||
// A stack always needs its compose + env files - never deselectable (matches the server
|
||||
// isLoadBearingStackFile guard, which refuses to exclude them even if the config lists them).
|
||||
function isLoadBearing(name: string): boolean {
|
||||
const n = name.toLowerCase();
|
||||
return (
|
||||
n === 'compose.yaml' || n === 'compose.yml' ||
|
||||
n === 'docker-compose.yml' || n === 'docker-compose.yaml' ||
|
||||
n === '.env' || n.startsWith('.env.')
|
||||
);
|
||||
}
|
||||
|
||||
// An entry the user cannot deselect HERE: either load-bearing (always kept) or captured via
|
||||
// its own bind/volume channel (controlled in the Volumes section, not here).
|
||||
function isLocked(e: Entry): boolean {
|
||||
return isLoadBearing(e.name) || e.capturedAs != null;
|
||||
}
|
||||
|
||||
const entries = $derived(listing?.kind === 'listed' || listing?.kind === 'tar' ? listing.entries : []);
|
||||
const selectable = $derived(entries.filter((e) => !isLocked(e)));
|
||||
// "All files" is on when nothing selectable is excluded.
|
||||
const allFiles = $derived(excludedStackFiles.length === 0);
|
||||
|
||||
function toggleAll(on: boolean) {
|
||||
excludedStackFiles = on ? [] : selectable.map((e) => e.name);
|
||||
}
|
||||
|
||||
function toggleEntry(e: Entry) {
|
||||
if (isLocked(e)) return;
|
||||
excludedStackFiles = excludedStackFiles.includes(e.name)
|
||||
? excludedStackFiles.filter((n) => n !== e.name)
|
||||
: [...excludedStackFiles, e.name];
|
||||
}
|
||||
|
||||
function fmtSize(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`;
|
||||
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="space-y-2">
|
||||
<div class="text-sm font-medium">Stack files on the host</div>
|
||||
|
||||
{#if loading}
|
||||
<div class="flex items-center gap-2 rounded-md border bg-muted/30 p-2.5 text-xs text-muted-foreground">
|
||||
<Loader2 class="h-4 w-4 shrink-0 animate-spin" />
|
||||
Probing the host for the stack folder...
|
||||
</div>
|
||||
{:else if listing?.kind === 'listed' || listing?.kind === 'tar'}
|
||||
{#if listing.kind === 'listed'}
|
||||
<!-- Resolved host path callout (copied from BackupPanel's stack-path candidate). -->
|
||||
<div class="flex items-start gap-2 rounded-md border bg-muted/30 p-2.5 text-xs">
|
||||
<FolderOpen class="mt-0.5 h-4 w-4 shrink-0 text-sky-500" />
|
||||
<div class="min-w-0 flex-1">
|
||||
<div class="flex items-center gap-1.5">
|
||||
<span class="font-medium text-foreground">Captured from the host at</span>
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger type="button" class="text-muted-foreground hover:text-foreground">
|
||||
<HelpCircle class="h-3.5 w-3.5" />
|
||||
</Tooltip.Trigger>
|
||||
<Tooltip.Content class="w-80 z-[200]" side="right">
|
||||
<div class="space-y-2">
|
||||
<p class="font-medium">Where this path comes from</p>
|
||||
{#if envKind === 'socket'}
|
||||
<p class="flex items-center gap-1.5 text-muted-foreground">
|
||||
{#if envIcon && envId != null}<EnvironmentIcon icon={envIcon} {envId} class="h-3.5 w-3.5 shrink-0" />{/if}
|
||||
<Unplug class="h-3.5 w-3.5 shrink-0 text-cyan-500" />
|
||||
<span><span class="font-medium text-foreground">{envLabel}</span> is a socket
|
||||
environment. Backup reads the stack folder from Dockhand's own stack
|
||||
directory on this host.</span>
|
||||
</p>
|
||||
<p class="text-muted-foreground">
|
||||
If this path looks wrong, set <code class="bg-muted px-1 rounded">HOST_DATA_DIR</code>
|
||||
when Dockhand runs in a container, so it points to where the data volume
|
||||
lives on the host.
|
||||
</p>
|
||||
{:else if envKind === 'direct'}
|
||||
<p class="flex items-center gap-1.5 text-muted-foreground">
|
||||
{#if envIcon && envId != null}<EnvironmentIcon icon={envIcon} {envId} class="h-3.5 w-3.5 shrink-0" />{/if}
|
||||
<Icon iconNode={whale} class="h-3.5 w-3.5 shrink-0 text-blue-500" />
|
||||
<span><span class="font-medium text-foreground">{envLabel}</span> is a direct
|
||||
environment. Backup reads the stack folder from this path on the remote
|
||||
Docker host.</span>
|
||||
</p>
|
||||
{#if configuredStackPath}
|
||||
<p class="text-muted-foreground">
|
||||
Remote stack path set on this environment:
|
||||
<code class="bg-muted px-1 rounded break-all">{configuredStackPath}</code>
|
||||
</p>
|
||||
{:else}
|
||||
<p class="text-muted-foreground">
|
||||
It is derived from the stack's bind mounts. To set it explicitly, use
|
||||
<span class="font-medium text-foreground">Environments > (edit) > Remote stack path (for backup)</span>
|
||||
and re-open this dialog.
|
||||
</p>
|
||||
{/if}
|
||||
{:else}
|
||||
<p class="flex items-center gap-1.5 text-muted-foreground">
|
||||
{#if envIcon && envId != null}<EnvironmentIcon icon={envIcon} {envId} class="h-3.5 w-3.5 shrink-0" />{/if}
|
||||
{#if connectionType === 'hawser-edge'}
|
||||
<UndoDot class="h-3.5 w-3.5 shrink-0 text-green-500" />
|
||||
{:else}
|
||||
<Route class="h-3.5 w-3.5 shrink-0 text-purple-500" />
|
||||
{/if}
|
||||
<span><span class="font-medium text-foreground">{envLabel}</span> is a Hawser
|
||||
environment. Backup reads the stack folder from the Hawser agent's stack
|
||||
directory on its host.</span>
|
||||
</p>
|
||||
{#if configuredStackPath}
|
||||
<p class="text-muted-foreground">
|
||||
Remote stack path set on this environment:
|
||||
<code class="bg-muted px-1 rounded break-all">{configuredStackPath}</code>
|
||||
</p>
|
||||
{:else}
|
||||
<p class="text-muted-foreground">
|
||||
It defaults to <code class="bg-muted px-1 rounded">/data/stacks</code>. If the
|
||||
agent uses a custom <code class="bg-muted px-1 rounded">STACKS_DIR</code>, set it
|
||||
under <span class="font-medium text-foreground">Environments > (edit) > Remote stack path (for backup)</span>
|
||||
and re-open this dialog.
|
||||
</p>
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
</div>
|
||||
<div class="mt-0.5 break-all font-mono text-muted-foreground">{listing.hostPath}</div>
|
||||
</div>
|
||||
</div>
|
||||
{:else}
|
||||
<!-- TAR mode: no host folder to read (direct-remote, no stack path set). Only Dockhand's
|
||||
own copy of the compose/config is captured; bind-mount DATA on the remote host is NOT. -->
|
||||
<div class="flex items-start gap-2 rounded-md border border-amber-500/40 bg-amber-500/10 p-2.5 text-xs">
|
||||
<AlertTriangle class="mt-0.5 h-4 w-4 shrink-0 text-amber-500" />
|
||||
<div class="min-w-0">
|
||||
<div class="font-medium text-amber-600 dark:text-amber-400">Only these compose/config files will be captured</div>
|
||||
<div class="mt-0.5 text-muted-foreground">This environment has no stack path set, so the folder can't be read on the remote host. Dockhand backs up its own copy of the files below. <strong>Bind-mount data on the remote host is NOT included.</strong> Set a stack path on this environment for a full backup.</div>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class="border rounded-md overflow-hidden">
|
||||
<div class="flex items-center gap-3 px-3 py-2 bg-muted/30 border-b">
|
||||
<Label class="text-xs">Backup all files ({selectable.length})</Label>
|
||||
<TogglePill checked={allFiles} onLabel="Yes" offLabel="No" onchange={() => toggleAll(!allFiles)} />
|
||||
</div>
|
||||
<div class="divide-y max-h-40 overflow-y-auto">
|
||||
{#each entries as entry}
|
||||
{@const locked = isLocked(entry)}
|
||||
<label class="flex items-center gap-2 px-3 py-1.5 text-xs" class:cursor-pointer={!allFiles && !locked} class:opacity-60={locked}>
|
||||
<Checkbox
|
||||
checked={!excludedStackFiles.includes(entry.name)}
|
||||
disabled={allFiles || locked}
|
||||
onCheckedChange={() => toggleEntry(entry)}
|
||||
class="h-3.5 w-3.5"
|
||||
/>
|
||||
{#if entry.type === 'dir'}
|
||||
<Folder class="h-3.5 w-3.5 shrink-0 text-sky-500" />
|
||||
{:else}
|
||||
<FileText class="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
|
||||
{/if}
|
||||
<span class="font-mono truncate">{entry.name}</span>
|
||||
{#if entry.capturedAs}
|
||||
<span class="text-muted-foreground ml-auto shrink-0 italic">captured as a {entry.capturedAs} below</span>
|
||||
{:else if isLoadBearing(entry.name)}
|
||||
<span class="text-muted-foreground ml-auto shrink-0 italic">always kept</span>
|
||||
{:else if entry.type === 'file'}
|
||||
<span class="text-muted-foreground ml-auto shrink-0">{fmtSize(entry.size)}</span>
|
||||
{/if}
|
||||
</label>
|
||||
{/each}
|
||||
</div>
|
||||
</div>
|
||||
{:else if listing?.kind === 'helper-failed'}
|
||||
<!-- The probe helper container couldn't run on the target. A backup can't run either (it
|
||||
uses the same helper), so this is a hard block, not a soft warning. -->
|
||||
<div class="flex items-start gap-2 rounded-md border border-destructive/50 bg-destructive/10 p-2.5 text-xs">
|
||||
<AlertTriangle class="mt-0.5 h-4 w-4 shrink-0 text-destructive" />
|
||||
<div class="min-w-0">
|
||||
<div class="font-medium text-destructive">The backup helper can't run on this environment</div>
|
||||
<div class="mt-0.5 break-all text-muted-foreground">{listing.reason}</div>
|
||||
<div class="mt-1 text-muted-foreground">A backup can't be created until this is fixed - the same helper container captures the stack files.</div>
|
||||
</div>
|
||||
</div>
|
||||
{:else if listing?.kind === 'unknown'}
|
||||
<div class="flex items-start gap-2 rounded-md border bg-muted/30 p-2.5 text-xs">
|
||||
<Info class="mt-0.5 h-4 w-4 shrink-0 text-sky-500" />
|
||||
<div class="min-w-0">
|
||||
<div class="font-medium text-foreground">No stack folder found on the host</div>
|
||||
<div class="mt-0.5 text-muted-foreground">{listing.reason}</div>
|
||||
{#if envKind === 'socket'}
|
||||
<div class="mt-1 text-muted-foreground">On a socket environment this usually means <code class="bg-muted px-1 rounded">HOST_DATA_DIR</code> does not match where the data volume is mounted on the host. Fix it, then cancel and re-configure this backup.</div>
|
||||
{:else if envKind === 'direct'}
|
||||
<div class="mt-1 text-muted-foreground">Set the environment's stack path (Environments > edit > Remote stack path for backup), then cancel and re-configure this backup.</div>
|
||||
{:else}
|
||||
<div class="mt-1 text-muted-foreground">On Hawser this usually means the agent uses a custom <code class="bg-muted px-1 rounded">STACKS_DIR</code> (not the default <code class="bg-muted px-1 rounded">/data/stacks</code>). Set the environment's stack path (Environments > edit > Remote stack path for backup), then cancel and re-configure this backup.</div>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
@@ -27,9 +27,9 @@ export interface MetadataFile {
|
||||
* The metadata files (metadata.json, and for stacks the compose/.env and the
|
||||
* whole stackfiles/ tree) are NOT written by this script — they are streamed into
|
||||
* the container's /metadata directory via the Docker put-archive API BEFORE the
|
||||
* container starts (see restic.ts runInHelper). Embedding their base64 inline in
|
||||
* the Cmd used to blow the kernel ARG_MAX for large stack files ("argument list
|
||||
* too long"); put-archive has no such limit.
|
||||
* container starts (see restic.ts runInHelper). Inlining their base64 in the Cmd
|
||||
* would hit the kernel ARG_MAX for large stack files ("argument list too long");
|
||||
* put-archive has no such limit.
|
||||
*
|
||||
* So the script only:
|
||||
* 1. ensures /metadata exists (put-archive already created it, but be defensive
|
||||
@@ -41,7 +41,7 @@ export interface MetadataFile {
|
||||
* @param resticArgs the full restic argv (starting with 'backup') — already
|
||||
* tag/exclude/flag-assembled by the caller.
|
||||
*/
|
||||
export function buildBackupScript(resticArgs: string[]): string {
|
||||
export function buildBackupScript(resticArgs: string[], stackDirProbe?: { volumeKey: string; composeFileName: string; label?: string; hostPath?: string }): string {
|
||||
// restic runs backgrounded so the SIGINT-forwarding trap can reach it on cancel
|
||||
// (SIGINT lets restic release its repo lock; a SIGKILL orphans it and hangs the
|
||||
// next backup on --retry-lock). The kill-0 re-wait loop only fires when a signal
|
||||
@@ -49,7 +49,30 @@ export function buildBackupScript(resticArgs: string[]): string {
|
||||
// real code (0/3) for the marker.
|
||||
const restic = `restic ${resticArgs.map(shellQuote).join(' ')}`;
|
||||
const step = `${restic} & __rpid=$!; trap 'kill -INT $__rpid 2>/dev/null' INT TERM; wait $__rpid; __rc=$?; while kill -0 $__rpid 2>/dev/null; do wait $__rpid; __rc=$?; done; ( exit $__rc )`;
|
||||
return `mkdir -p /metadata; set +e; ${finishScript(step)}`;
|
||||
|
||||
// STACK-DIR VOLUME PROBE (100% runtime guard for the 'volume' capture mode). The plan
|
||||
// decided the target daemon is local and bind-mounted the stack dir at
|
||||
// /volumes/<volumeKey>. If that mount is a phantom (a remote daemon Docker
|
||||
// auto-created an EMPTY dir for), the compose file will be MISSING under it — backing
|
||||
// that up would be a silent-empty snapshot. So before restic runs, assert the compose
|
||||
// is visible; if not, exit non-zero (the finishScript marker turns this into a hard
|
||||
// backup failure), never a silent success.
|
||||
let probe = '';
|
||||
if (stackDirProbe) {
|
||||
const composePath = shellQuote(`/volumes/${stackDirProbe.volumeKey}/${stackDirProbe.composeFileName}`);
|
||||
const label = stackDirProbe.label ? ` ${stackDirProbe.label}` : '';
|
||||
// Name the REAL host path we mounted (<remote_stacks_dir>/<stack>) so the operator knows
|
||||
// exactly where to look, and what to do: redeploy the stack to stage its files there.
|
||||
const hostHint = stackDirProbe.hostPath
|
||||
? ` The stack folder is empty or missing at ${stackDirProbe.hostPath} on the Docker host - redeploy the stack to stage its files there.`
|
||||
: '';
|
||||
probe =
|
||||
`if [ ! -f ${composePath} ]; then ` +
|
||||
`echo "[backup] $(date -u +%Y-%m-%dT%H:%M:%SZ)${label} STACKDIR PROBE FAILED: the stack's compose (${stackDirProbe.composeFileName}) is not present under the mounted stack dir.${hostHint} Refusing to write a silent-empty snapshot." 1>&2; ` +
|
||||
`exit 1; fi; `;
|
||||
}
|
||||
|
||||
return `mkdir -p /metadata; set +e; ${probe}${finishScript(step)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -42,18 +42,8 @@ import { buildBackupScript, buildBackupArgs, type MetadataFile } from './backup-
|
||||
import { EXIT_MARKER } from './restic-script';
|
||||
import { parseRetention, buildForgetArgs, checkWouldWipe, retentionActive } from './retention';
|
||||
import { liveTargetKey } from './locks';
|
||||
import { stopIntentKey as stopIntentKeyFor } from './stop-recovery-core';
|
||||
import type { DiscoveredVolume } from './discovery-core';
|
||||
|
||||
/** What a fresh process needs to restart a target stopped for a consistent
|
||||
* backup (or an in-place restore) after a crash between stop and restart. */
|
||||
export interface StopIntent {
|
||||
type: BackupTargetType;
|
||||
targetName: string;
|
||||
envId: number | null;
|
||||
/** For a container target: the ids/names that were running and got stopped. */
|
||||
containers: Array<{ id: string; name: string }>;
|
||||
}
|
||||
import { STACKDIR_VOLUME_KEY } from './stackdir-plan';
|
||||
|
||||
/** What the service needs from the outside world. All injected for testability. */
|
||||
export interface BackupPorts {
|
||||
@@ -63,34 +53,45 @@ export interface BackupPorts {
|
||||
/** Discover volumes; MUST throw (fail closed) on any inspect failure. */
|
||||
discoverVolumes(containers: Array<{ id: string; name: string }>, envId: number | null | undefined, selected: string[] | null):
|
||||
Promise<{ volumes: DiscoveredVolume[]; skipped: Array<{ type: string; destination: string; reason: string }> }>;
|
||||
/** Decide how a STACK's directory is captured. Normally: locate the HOST folder and build the
|
||||
* synthetic __dockhand_stackdir__ volume the helper bind-mounts (`kind: 'candidate'`).
|
||||
* `kind: 'tar'` is the direct-remote-with-no-remote_stacks_dir case: nothing is staged on the
|
||||
* host, so the compose/config is tarred from Dockhand's local copy instead (no host bind).
|
||||
* `composeFileName` feeds the in-helper probe (assert the compose is
|
||||
* visible under the mount). `kind: 'unknown'` means the stack is not compose-managed (no
|
||||
* working_dir label) -> the caller HARD-FAILS the backup. Logged inside this port. */
|
||||
planStackDirVolume(targetName: string, envId: number | null | undefined, excludedStackFiles?: string[]):
|
||||
Promise<
|
||||
| { kind: 'unknown'; reason: string }
|
||||
| { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[] }
|
||||
| { kind: 'tar'; localStackDir: string; composeFileName: string }
|
||||
>;
|
||||
/** Stop the target for a consistent backup; returns a restart closure. */
|
||||
stopForBackup(type: BackupTargetType, targetName: string, containers: Array<{ id: string; name: string; state: string }>, envId: number | null | undefined):
|
||||
Promise<{ restart: () => Promise<{ failed: Array<{ name: string; error: string }> }> }>;
|
||||
/** Persist a durable "this target was stopped for backup" intent BEFORE the
|
||||
* stop, so a process death between stop and restart is recoverable on startup.
|
||||
* Keyed on a stable per-target id (see stopIntentKey). */
|
||||
recordStopIntent(key: string, intent: StopIntent): Promise<void>;
|
||||
/** Clear the durable stop intent once the in-process restart has succeeded. */
|
||||
clearStopIntent(key: string): Promise<void>;
|
||||
/** Run the backup session container; returns the restic run. metadataFiles are
|
||||
* streamed into /metadata via put-archive (not the Cmd) so large stack files
|
||||
* don't blow ARG_MAX. */
|
||||
runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; onStderr?: (line: string) => void; onStdout?: (line: string) => void }):
|
||||
/** Run the backup session container; returns the restic run. metadataFiles (metadata.json
|
||||
* + the light stack-dir listing) are put-archived into /metadata (not the Cmd) so they
|
||||
* can't blow ARG_MAX. The stack dir's bytes ride a host bind mount, not these files. */
|
||||
runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }>; onStderr?: (line: string) => void; onStdout?: (line: string) => void }):
|
||||
Promise<ResticRun>;
|
||||
/** Run a repo-only restic command on the host (verify / retention). */
|
||||
runLocal(args: string[], tier?: 'interactive' | 'data'): Promise<ResticRun>;
|
||||
/** Collect the restore metadata files (metadata.json + stack files).
|
||||
* `stackFilesTruncated` is true when a stack dir exceeded the capture cap, so
|
||||
* the snapshot is an incomplete stack-file capture — the run is downgraded to
|
||||
* a warning. */
|
||||
collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[]):
|
||||
Promise<{ files: MetadataFile[]; stackFilesTruncated: boolean }>;
|
||||
/** Collect the restore metadata files: metadata.json plus, for a stack, a light file
|
||||
* listing. The stack dir's bytes normally ride the helper's host bind mount, not these
|
||||
* files. EXCEPTION - `stackDirTar`: for a direct-remote stack with no remote_stacks_dir
|
||||
* there is no host bind, so the collector reads localStackDir and tars its files into the
|
||||
* snapshot at /volumes/__dockhand_stackdir__ (the same path restore reads). */
|
||||
collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[], stackDirTar?: { localStackDir: string; composeFileName: string } | null):
|
||||
Promise<{ files: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }> }>;
|
||||
/** The Dockhand host name for restic --host. */
|
||||
host(): string;
|
||||
/** This installation's stable instance id. */
|
||||
instanceId(): Promise<string>;
|
||||
/** The live-target mutex: returns a release fn, or null if already held. */
|
||||
acquireLiveTarget(key: string): (() => void) | null;
|
||||
/** True if the user asked to cancel this config's in-flight backup — used to
|
||||
* report a killed helper's non-zero exit as "cancelled", not a restic failure. */
|
||||
isCancelling(configId: number): boolean;
|
||||
/** Serialize on a destination; runs fn behind the per-destination queue. */
|
||||
serializeDestination<T>(destinationId: number, fn: () => Promise<T>): Promise<T>;
|
||||
/** Open the operation record; returns handles to update + close it. */
|
||||
@@ -128,7 +129,7 @@ export interface BackupJob {
|
||||
selectedVolumes: string[] | null;
|
||||
stopBeforeBackup: boolean;
|
||||
retention: string | null | Record<string, unknown>;
|
||||
options: { excludePatterns?: string[]; excludeCaches?: boolean; compression?: string; limitUpload?: number; limitDownload?: number; webhookSuccess?: string; webhookFailure?: string };
|
||||
options: { excludePatterns?: string[]; excludeCaches?: boolean; compression?: string; limitUpload?: number; limitDownload?: number; webhookSuccess?: string; webhookFailure?: string; excludedStackFiles?: string[] };
|
||||
helperName: string;
|
||||
}
|
||||
|
||||
@@ -146,6 +147,17 @@ export class BackupService {
|
||||
// --- discover targets first (need the volume identity for the lock key) ---
|
||||
let containers: Array<{ id: string; name: string; state: string }>;
|
||||
let volumes: DiscoveredVolume[];
|
||||
// The stack dir is captured by the helper's HOST bind mount (a synthetic
|
||||
// __dockhand_stackdir__ volume). `stackDirProbe` carries what the helper needs to
|
||||
// assert that mount is real (test -f the compose) before restic runs. Null for
|
||||
// containers (no stack dir).
|
||||
let stackDirProbe: { volumeKey: string; composeFileName: string; hostPath?: string } | null = null;
|
||||
// restic --exclude paths for compose bind dirs inside the stackdir volume (they are
|
||||
// captured separately as their own /volumes/<key>, honoring volume selection).
|
||||
let stackDirExcludes: string[] = [];
|
||||
// Set (instead of a host bind) for a direct-remote stack with no remote_stacks_dir: the
|
||||
// compose/config is tarred into /metadata from Dockhand's local copy, not the host.
|
||||
let stackDirTar: { localStackDir: string; composeFileName: string } | null = null;
|
||||
try {
|
||||
const t = await this.ports.resolveTargets(job.type, job.targetName, envId);
|
||||
containers = t.containers;
|
||||
@@ -161,12 +173,37 @@ export class BackupService {
|
||||
// A target with no volumes/binds is fine: metadata.json (the container/stack
|
||||
// config, and for stacks the compose files) is always captured, so the
|
||||
// snapshot is a config-only backup — restorable via recreate/redeploy.
|
||||
} catch (err) {
|
||||
return this.errorResult(err);
|
||||
}
|
||||
|
||||
// For a STACK, always capture the whole stack dir by bind-mounting it from the
|
||||
// TARGET daemon's HOST (__dockhand_stackdir__). The plan locates the host folder
|
||||
// from the compose working_dir label; a stack with no such label is not
|
||||
// compose-managed, so we HARD-FAIL rather than silently skip its files (option A).
|
||||
// The synthetic volume rides the same helper-bind + lock machinery as any volume.
|
||||
if (job.type === 'stack') {
|
||||
const p = await this.ports.planStackDirVolume(job.targetName, envId, job.options.excludedStackFiles);
|
||||
if (p.kind === 'unknown') {
|
||||
return this.earlySkipOrError(job, triggeredBy,
|
||||
`Cannot locate the stack folder on the host for "${job.targetName}" (${p.reason}). ` +
|
||||
`For remote environments, set a Remote stacks directory or use matching paths so the stack files exist on the Docker host. ` +
|
||||
`Refusing to write a backup missing the stack's compose and config.`);
|
||||
} else if (p.kind === 'tar') {
|
||||
// direct-remote, no remote_stacks_dir: no host bind (nothing staged), so the
|
||||
// compose/config is captured from Dockhand's local copy into /metadata via a tar.
|
||||
// No synthetic volume, no probe - the metadata collector reads localStackDir.
|
||||
stackDirTar = { localStackDir: p.localStackDir, composeFileName: p.composeFileName };
|
||||
} else {
|
||||
volumes = [...volumes, p.syntheticVolume];
|
||||
stackDirProbe = { volumeKey: p.volumeKey, composeFileName: p.composeFileName, hostPath: p.syntheticVolume.source ?? undefined };
|
||||
stackDirExcludes = p.excludePaths;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
return this.errorResult(err);
|
||||
}
|
||||
|
||||
// --- claim the live-target mutex (reject a concurrent op on this data) ---
|
||||
const key = liveTargetKey(envId, volumes.map((v) => v.bind));
|
||||
// fallback identity keeps config-only targets (no volumes) from colliding on ${env}::
|
||||
const key = liveTargetKey(envId, volumes.map((v) => v.bind), `${job.type}:${job.targetName}`);
|
||||
const release = this.ports.acquireLiveTarget(key);
|
||||
if (!release) {
|
||||
return { status: 'skipped', reason: `Another backup or restore is already running for "${job.targetName}".` };
|
||||
@@ -184,7 +221,7 @@ export class BackupService {
|
||||
try {
|
||||
// --- serialize on the destination (share the repo, one at a time) ---
|
||||
return await this.ports.serializeDestination(job.destinationId, () =>
|
||||
this.runLocked(job, triggeredBy, containers, volumes, startTime, releaseOnce));
|
||||
this.runLocked(job, triggeredBy, containers, volumes, stackDirProbe, stackDirExcludes, stackDirTar, startTime, releaseOnce));
|
||||
} finally {
|
||||
releaseOnce();
|
||||
}
|
||||
@@ -200,6 +237,9 @@ export class BackupService {
|
||||
triggeredBy: 'cron' | 'manual' | 'webhook',
|
||||
containers: Array<{ id: string; name: string; state: string }>,
|
||||
volumes: DiscoveredVolume[],
|
||||
stackDirProbe: { volumeKey: string; composeFileName: string } | null,
|
||||
stackDirExcludes: string[],
|
||||
stackDirTar: { localStackDir: string; composeFileName: string } | null,
|
||||
startTime: number,
|
||||
releaseLiveTarget: () => void,
|
||||
): Promise<BackupResult> {
|
||||
@@ -207,26 +247,25 @@ export class BackupService {
|
||||
const op = await this.ports.openOperation(job.targetName, job.configId, envId, triggeredBy);
|
||||
let restart: (() => Promise<{ failed: Array<{ name: string; error: string }> }>) | null = null;
|
||||
let restarted = false;
|
||||
let stopIntentKey: string | null = null;
|
||||
|
||||
// ========================= @BACKUP-DIAG-TEMP =========================
|
||||
// TEMPORARY CI backup-flake diagnostics. Remove this whole block (and the other
|
||||
// `@BACKUP-DIAG-TEMP` markers below) once the flake is fixed:
|
||||
// grep -rn '@BACKUP-DIAG-TEMP' src/lib/server/backups/backup-service.ts
|
||||
// Every log line is prefixed `[BACKUP-DIAG-TEMP]` for a one-shot log filter.
|
||||
// Gated behind BACKUP_DIAG=1 so it's silent unless CI turns it on.
|
||||
// Emits per-phase timing (backup/verify/restart/retention/close) + a repo
|
||||
// pre-probe (does the repo exist? is it locked?) to settle the "destination
|
||||
// points at a wiped/uninit repo" and "stale lock" hypotheses.
|
||||
const diagOn = process.env.BACKUP_DIAG === '1';
|
||||
// Two log tiers, both stamped with a wall-clock ISO time and the {env,target}
|
||||
// key so any line can be lined up with the shard test log by time and target:
|
||||
// log() - always on. The lines an operator wants: started / snapshot created /
|
||||
// completed / failed. Plain and self-explanatory, no scary internals.
|
||||
// diag() - behind BACKUP_DIAG=1 (CI turns it on). Deep internals: repo/lock
|
||||
// pre-probes, instance identity, micro-phase timings. Noise for a user,
|
||||
// gold for a CI flake.
|
||||
const key = `${job.targetName} env=${envId}`;
|
||||
const t0 = Date.now();
|
||||
const log = (msg: string) => console.log(`[backup] ${new Date().toISOString()} ${key} | ${msg}`);
|
||||
const diagOn = process.env.BACKUP_DIAG === '1';
|
||||
const diag = (phase: string, extra?: string) => {
|
||||
if (!diagOn) return;
|
||||
const ms = Date.now() - t0;
|
||||
console.log(`[BACKUP-DIAG-TEMP] ${job.targetName} cfg=${job.configId} op=${op.id} env=${envId} | ${phase} @${ms}ms${extra ? ' | ' + extra : ''}`);
|
||||
console.log(`[backup] ${new Date().toISOString()} ${key} cfg=${job.configId} op=${op.id} | ${phase} @${Date.now() - t0}ms${extra ? ' | ' + extra : ''}`);
|
||||
};
|
||||
diag('START', `triggeredBy=${triggeredBy} stopBeforeBackup=${job.stopBeforeBackup} retention=${JSON.stringify(job.retention)}`);
|
||||
log(`started (${triggeredBy})`);
|
||||
if (diagOn) {
|
||||
diag('START', `stopBeforeBackup=${job.stopBeforeBackup} retention=${JSON.stringify(job.retention)}`);
|
||||
try {
|
||||
const cfg = await this.ports.runLocal(['cat', 'config', '--no-lock'], 'interactive');
|
||||
diag('REPO_PROBE', `cat-config exit=${cfg.exitCode} (${cfg.exitCode === 10 ? 'NOT-INITIALISED' : cfg.exitCode === 0 ? 'exists' : 'error'})`);
|
||||
@@ -237,49 +276,47 @@ export class BackupService {
|
||||
diag('LOCK_PROBE', `list-locks exit=${locks.exitCode} heldLocks=${lockLines.length}${lockLines.length ? ' [' + lockLines.join(',').slice(0, 120) + ']' : ''}`);
|
||||
} catch (e) { diag('LOCK_PROBE', `list-locks THREW ${e instanceof Error ? e.message : String(e)}`); }
|
||||
}
|
||||
// ======================= end @BACKUP-DIAG-TEMP =======================
|
||||
|
||||
try {
|
||||
// --- optionally stop the target for a consistent snapshot ---
|
||||
if (job.stopBeforeBackup) {
|
||||
op.progress('stopping', `Stopping ${job.targetName} for a consistent backup...`);
|
||||
// Record a durable stop intent BEFORE the stop, so a process death
|
||||
// between the stop and the restart leaves a row a fresh startup can
|
||||
// replay to bring the target back up (see reconcileOnStartup).
|
||||
const running = containers.filter((c) => c.state === 'running');
|
||||
if (running.length > 0) {
|
||||
stopIntentKey = stopIntentKeyFor(envId, job.type, job.targetName);
|
||||
await this.ports.recordStopIntent(stopIntentKey, {
|
||||
type: job.type, targetName: job.targetName, envId,
|
||||
containers: running.map((c) => ({ id: c.id, name: c.name })),
|
||||
});
|
||||
}
|
||||
const stopped = await this.ports.stopForBackup(job.type, job.targetName, containers, envId);
|
||||
restart = stopped.restart;
|
||||
}
|
||||
|
||||
// --- metadata + backup args + session script ---
|
||||
op.progress('metadata', 'Collecting metadata...');
|
||||
const { files: metadata, stackFilesTruncated } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes);
|
||||
const { files: metadata, stackDirStreamSources } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes, stackDirTar);
|
||||
const instanceId = await this.ports.instanceId();
|
||||
const tags = buildSnapshotTags({ instanceId, configId: job.configId, environmentId: envId, targetName: job.targetName, type: job.type });
|
||||
// @BACKUP-DIAG-TEMP: cross-shard identity. If two shards log the SAME
|
||||
// instanceId + configId (they share a cloned DB) AND the same repo, their
|
||||
// snapshots/retention collide. host() + destinationId show which repo this
|
||||
// run targets, so a CI log across shards reveals any overlap.
|
||||
// Cross-shard identity. If two shards log the SAME instanceId + configId
|
||||
// (they share a cloned DB) AND the same repo, their snapshots/retention
|
||||
// collide. host() + destinationId show which repo this run targets, so a
|
||||
// CI log across shards reveals any overlap.
|
||||
diag('IDENTITY', `instanceId=${instanceId} host=${this.ports.host()} destId=${job.destinationId} tags=[${tags.join(',')}]`);
|
||||
// restic must back up /volumes/ when there's anything to capture there: real bind/named
|
||||
// volumes (volumes.length), OR a tar-mode stack whose compose/config was put-archived
|
||||
// into /volumes/__dockhand_stackdir__ (no synthetic volume in `volumes`, so it wouldn't
|
||||
// count otherwise - and /metadata-only would silently drop the tarred stack files).
|
||||
const args = buildBackupArgs({
|
||||
host: this.ports.host(),
|
||||
tags,
|
||||
hasVolumes: volumes.length > 0,
|
||||
excludePatterns: job.options.excludePatterns,
|
||||
hasVolumes: volumes.length > 0 || stackDirTar !== null,
|
||||
// User excludes PLUS the compose bind dirs inside the stackdir volume (those
|
||||
// ride their own /volumes/<key>, so excluding them here avoids a duplicate and
|
||||
// keeps volume DESELECTION meaningful for in-folder binds).
|
||||
excludePatterns: [...(job.options.excludePatterns ?? []), ...stackDirExcludes],
|
||||
excludeCaches: job.options.excludeCaches,
|
||||
compression: job.options.compression,
|
||||
limitUpload: job.options.limitUpload,
|
||||
limitDownload: job.options.limitDownload,
|
||||
swapArtifacts: SWAP_ARTIFACTS,
|
||||
});
|
||||
const script = buildBackupScript(args);
|
||||
const script = buildBackupScript(
|
||||
args,
|
||||
stackDirProbe ? { ...stackDirProbe, label: `env=${envId} target=${job.targetName}` } : undefined
|
||||
);
|
||||
const binds = volumes.map((v) => v.bind);
|
||||
|
||||
// Echo the options actually applied to the restic run, so the execution
|
||||
@@ -291,8 +328,7 @@ export class BackupService {
|
||||
if (o.excludeCaches !== false) op.log('Exclude caches: yes');
|
||||
if (o.excludePatterns && o.excludePatterns.length) op.log(`Exclude patterns: ${o.excludePatterns.join(', ')}`);
|
||||
// Record WHICH volumes were actually selected (not just the count), so a
|
||||
// regression in the selectedVolumes filter is visible in the execution
|
||||
// log rather than silent (matches the legacy engine's log line).
|
||||
// regression in the selectedVolumes filter is visible in the execution log.
|
||||
op.log(`Volumes to backup: ${volumes.map((v) => v.name).join(', ') || 'none'}`);
|
||||
|
||||
// --- clear any STALE repo lock before backing up ---
|
||||
@@ -326,18 +362,26 @@ export class BackupService {
|
||||
} else {
|
||||
op.progress('backing-up', `Backing up ${volumes.length} ${volumes.length === 1 ? 'volume' : 'volumes'}:`);
|
||||
for (const v of volumes) {
|
||||
op.progress('backing-up', ` • [${v.type === 'bind' ? 'BIND' : 'VOL'}] ${v.name}`);
|
||||
// A bind's `name` is only its container destination, so two binds to the same
|
||||
// path (e.g. /data) read identically. Show `host-source -> destination` so each
|
||||
// is distinguishable in the log (#1373).
|
||||
const label = v.name === STACKDIR_VOLUME_KEY
|
||||
? 'Stack data'
|
||||
: v.type === 'bind'
|
||||
? `${v.source} → ${v.name}`
|
||||
: v.name;
|
||||
op.progress('backing-up', ` • [${v.type === 'bind' ? 'BIND' : 'VOL'}] ${label}`);
|
||||
}
|
||||
}
|
||||
// Did the LIVE stdout stream deliver anything? restic writes its --json
|
||||
// progress to STDOUT, which we forward via onStdout. But live stdout only
|
||||
// progress to STDOUT (forwarded via onStdout); but live stdout only
|
||||
// works on transports that stream it (local socket/http); if nothing arrives
|
||||
// live, we MUST still emit restic's output post-exit from run.stdout — else
|
||||
// the log shows only our own stage markers and no restic lines at all (a
|
||||
// regression). This flag drives that fallback.
|
||||
// This flag drives that fallback.
|
||||
let liveStdoutSeen = false;
|
||||
const run = await this.ports.runInHelper({
|
||||
script, binds, envId, name: job.helperName, metadataFiles: metadata,
|
||||
script, binds, envId, name: job.helperName, metadataFiles: metadata, stackDirStreamSources,
|
||||
// Stream restic's own output to the UI LIVE: restic writes its --json
|
||||
// progress (per-file %) to STDOUT and its status messages to STDERR. Wire
|
||||
// BOTH so the log fills in as the backup runs, not all at once at the end.
|
||||
@@ -349,7 +393,7 @@ export class BackupService {
|
||||
|
||||
// Fallback: if live stdout streaming produced nothing (transport didn't
|
||||
// stream it), emit restic's buffered stdout now so the log always shows what
|
||||
// restic actually did — the pre-live-streaming behaviour, kept as a safety net.
|
||||
// restic actually did.
|
||||
if (!liveStdoutSeen) {
|
||||
for (const l of formatResticLines(run.stdout)) {
|
||||
if (!l.includes(EXIT_MARKER)) op.progress('progress', `[restic] ${l}`);
|
||||
@@ -365,13 +409,14 @@ export class BackupService {
|
||||
// No summary ⇒ unknown outcome ⇒ NOT a success (never record a phantom).
|
||||
throw new BackupError('INTEGRITY', 'backup produced no snapshot summary — treating as failed');
|
||||
}
|
||||
// A partial is either restic's own "couldn't read all files" (exit 3) OR
|
||||
// a stack dir that exceeded the capture cap (snapshot is redeploy-incomplete).
|
||||
const partial = resticPartial(run) || stackFilesTruncated;
|
||||
// A partial is restic's own "couldn't read all files" (exit 3). A stack dir
|
||||
// over the capture cap is no longer a partial — it hard-fails in collectMetadata.
|
||||
const partial = resticPartial(run);
|
||||
|
||||
// --- VERIFY the snapshot is actually readable before declaring success ---
|
||||
log(`snapshot ${summary.snapshotId.slice(0, 8)} created, verifying`);
|
||||
op.progress('verifying', 'Verifying the snapshot is readable...');
|
||||
diag("BEFORE_VERIFY", "snapshot=" + summary.snapshotId); // @BACKUP-DIAG-TEMP
|
||||
diag("BEFORE_VERIFY", "snapshot=" + summary.snapshotId);
|
||||
await this.verifySnapshot(summary.snapshotId, instanceId);
|
||||
|
||||
// --- commit: record success + config status ---
|
||||
@@ -382,14 +427,14 @@ export class BackupService {
|
||||
// retention, then release the live-target lock: retention is a repo-only
|
||||
// forget/prune (already serialized per-destination) and a concurrent
|
||||
// restore neither reads nor writes the live volume during it.
|
||||
diag("BEFORE_RESTART"); // @BACKUP-DIAG-TEMP
|
||||
await this.restartAndSurface(restart, job, op, stopIntentKey);
|
||||
diag("BEFORE_RESTART");
|
||||
await this.restartAndSurface(restart, job, op);
|
||||
restarted = true;
|
||||
releaseLiveTarget();
|
||||
|
||||
// --- retention: ONLY now, after a confirmed + verified snapshot AND
|
||||
// after the live target is back up + its lock freed ---
|
||||
diag("BEFORE_RETENTION"); // @BACKUP-DIAG-TEMP
|
||||
diag("BEFORE_RETENTION");
|
||||
const retentionOutcome = await this.applyRetention(job, instanceId, op);
|
||||
|
||||
const details = {
|
||||
@@ -401,21 +446,36 @@ export class BackupService {
|
||||
retention: retentionOutcome,
|
||||
};
|
||||
if (partial) {
|
||||
const warnMsg = stackFilesTruncated
|
||||
? 'Backup completed with warnings: the stack directory exceeded the capture cap, so the snapshot is an incomplete stack-file capture (redeploy may be incomplete).'
|
||||
: 'Backup completed with warnings (some files could not be read).';
|
||||
await op.close({ kind: 'warning', message: warnMsg }, details);
|
||||
log(`completed with warnings (partial read) in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
|
||||
await op.close({ kind: 'warning', message: 'Backup completed with warnings (some files could not be read).' }, details);
|
||||
await this.notifySuccess(job, summary, envId, true, op.id, startTime);
|
||||
return { status: 'warning', executionId: op.id, snapshotId: summary.snapshotId, summary, warning: stackFilesTruncated ? 'stack files truncated' : 'partial read', retention: retentionOutcome };
|
||||
return { status: 'warning', executionId: op.id, snapshotId: summary.snapshotId, summary, warning: 'partial read', retention: retentionOutcome };
|
||||
}
|
||||
diag("BEFORE_CLOSE"); // @BACKUP-DIAG-TEMP
|
||||
log(`completed in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
|
||||
diag("BEFORE_CLOSE");
|
||||
await op.close({ kind: 'ok' }, details);
|
||||
await this.notifySuccess(job, summary, envId, false, op.id, startTime);
|
||||
return { status: 'success', executionId: op.id, snapshotId: summary.snapshotId, summary, retention: retentionOutcome };
|
||||
} catch (err) {
|
||||
// If the user cancelled THIS backup, its helper was SIGINT/SIGKILL'd — the
|
||||
// resulting non-zero exit (137) is not a real failure. Report it as cancelled
|
||||
// with a clean message instead of the raw restic exit spew.
|
||||
if (this.ports.isCancelling(job.configId)) {
|
||||
log('cancelled by user');
|
||||
if (!restarted) await this.restartAndSurface(restart, job, op);
|
||||
try { await this.ports.runLocal(['unlock']); } catch { /* best-effort */ }
|
||||
try { await this.ports.setConfigStatus(job.configId, 'failed'); } catch { /* non-fatal */ }
|
||||
await op.close({ kind: 'cancelled', message: 'Backup cancelled' });
|
||||
return { status: 'skipped', executionId: op.id, reason: 'Backup cancelled' };
|
||||
}
|
||||
// Log the FULL raw error to the server log (docker logs dockhand) before it's
|
||||
// cleaned for the UI — the raw text (restic JSON spew, exit code, stderr) is
|
||||
// what we actually need to diagnose a failure, and cleanErrorMsg strips it.
|
||||
log(`failed after ${((Date.now() - t0) / 1000).toFixed(1)}s`);
|
||||
console.error(`[backup] ${new Date().toISOString()} ${key} | raw error:`, err instanceof Error ? (err.stack ?? err.message) : String(err));
|
||||
// Always try to restart what we stopped, even on failure — unless the
|
||||
// success path already restarted it (avoid a double start).
|
||||
if (!restarted) await this.restartAndSurface(restart, job, op, stopIntentKey);
|
||||
if (!restarted) await this.restartAndSurface(restart, job, op);
|
||||
// Clean up the repo lock this failed/killed backup may have orphaned, so
|
||||
// the NEXT op to this repo doesn't wait out `--retry-lock` on a dead
|
||||
// owner. Still behind the per-destination serializer here. Best-effort.
|
||||
@@ -424,7 +484,7 @@ export class BackupService {
|
||||
try { await this.ports.setConfigStatus(job.configId, 'failed'); } catch { /* non-fatal */ }
|
||||
await op.close({ kind: 'error', code, message }, { errorCode: code });
|
||||
try { await this.ports.notify('backup_failed', { target: job.targetName, kind: 'backup', errorCode: code, message }, envId); } catch { /* non-fatal */ }
|
||||
// Per-config failure webhook — same payload shape as the legacy engine.
|
||||
// Per-config failure webhook.
|
||||
if (job.options.webhookFailure) {
|
||||
try {
|
||||
this.ports.fireWebhook(job.options.webhookFailure, {
|
||||
@@ -472,24 +532,26 @@ export class BackupService {
|
||||
// can see the wait, not just the eventual result.
|
||||
if (dryMs > 5000) op.log(`Retention: dry-run took ${(dryMs / 1000).toFixed(1)}s (waited on the repo lock).`);
|
||||
if (dryRun.exitCode !== 0) {
|
||||
op.progress('warning', `Retention dry-run failed (exit ${dryRun.exitCode}): ${dryRun.stderr.trim() || 'restic failed'} — skipping prune, backup is still safe.`);
|
||||
op.progress('warning', `Retention dry-run failed (exit ${dryRun.exitCode}): ${dryRun.stderr.trim() || 'restic failed'} — skipping retention, backup is still safe.`);
|
||||
return 'failed';
|
||||
}
|
||||
if (checkWouldWipe(dryRun.stdout).wouldWipe) {
|
||||
op.progress('warning', 'Retention would delete every snapshot for this config — skipping prune.');
|
||||
op.progress('warning', 'Retention would delete every snapshot for this config — skipping retention.');
|
||||
return 'skipped-would-wipe';
|
||||
}
|
||||
// forget only removes snapshot references (fast, metadata) — NO --prune here, so a
|
||||
// backup never hangs reclaiming space. The destination's prune schedule reclaims it.
|
||||
op.progress('pruning', 'Applying retention...');
|
||||
const forget = buildForgetArgs(policy, filter)!;
|
||||
const t1 = Date.now();
|
||||
const run = await this.ports.runLocal(forget, 'data');
|
||||
const forgetMs = Date.now() - t1;
|
||||
if (run.exitCode !== 0) {
|
||||
// A prune failure is non-fatal (the backup itself succeeded) but surfaced.
|
||||
// A forget failure is non-fatal (the backup itself succeeded) but surfaced.
|
||||
// The common non-fatal case: --retry-lock elapsed because another op (an
|
||||
// orphaned lock, or a second Dockhand instance sharing this repo) held the
|
||||
// lock — retention just runs next schedule; no snapshot is lost.
|
||||
op.progress('warning', `Retention prune failed after ${(forgetMs / 1000).toFixed(1)}s (exit ${run.exitCode}): ${run.stderr.trim() || 'restic failed'} — backup is safe, retention will retry next run.`);
|
||||
op.progress('warning', `Retention failed after ${(forgetMs / 1000).toFixed(1)}s (exit ${run.exitCode}): ${run.stderr.trim() || 'restic failed'} — backup is safe, retention will retry next run.`);
|
||||
return 'failed';
|
||||
}
|
||||
op.log(`Retention: applied in ${(forgetMs / 1000).toFixed(1)}s.`);
|
||||
@@ -503,7 +565,6 @@ export class BackupService {
|
||||
restart: (() => Promise<{ failed: Array<{ name: string; error: string }> }>) | null,
|
||||
job: BackupJob,
|
||||
op: OperationHandle,
|
||||
stopIntentKey: string | null,
|
||||
): Promise<void> {
|
||||
if (!restart) return;
|
||||
let failed: Array<{ name: string; error: string }> = [];
|
||||
@@ -513,14 +574,11 @@ export class BackupService {
|
||||
failed = [{ name: job.targetName, error: err instanceof Error ? err.message : String(err) }];
|
||||
}
|
||||
if (failed.length > 0) {
|
||||
// Restart failed — leave the durable intent for startup retry.
|
||||
const names = failed.map((f) => f.name).join(', ');
|
||||
const msg = `Backup finished but ${names} could not be restarted and is STILL STOPPED — start it manually.`;
|
||||
op.progress('warning', msg);
|
||||
try { await this.ports.notify('backup_failed', { target: job.targetName, kind: 'backup', errorCode: 'STOPPED_RESTART_FAILED', message: msg }, job.environmentId); } catch { /* non-fatal */ }
|
||||
} else {
|
||||
// Restart succeeded — drop the durable intent so no stale row remains.
|
||||
if (stopIntentKey) { try { await this.ports.clearStopIntent(stopIntentKey); } catch { /* non-fatal */ } }
|
||||
op.progress('restarted', `${job.targetName} restarted.`);
|
||||
}
|
||||
}
|
||||
@@ -539,7 +597,7 @@ export class BackupService {
|
||||
dataAdded: summary.dataAdded, partial,
|
||||
}, envId);
|
||||
} catch { /* notification failure never changes the outcome */ }
|
||||
// Per-config success webhook — same payload shape as the legacy engine.
|
||||
// Per-config success webhook.
|
||||
if (job.options.webhookSuccess) {
|
||||
try {
|
||||
this.ports.fireWebhook(job.options.webhookSuccess, {
|
||||
@@ -558,11 +616,15 @@ export class BackupService {
|
||||
}
|
||||
|
||||
/** The backup target (container/stack) doesn't exist — record the op and fail
|
||||
* with a validation error (there is genuinely nothing to back up). */
|
||||
* with a validation error (there is genuinely nothing to back up). Close the durable
|
||||
* execution row as a FAILED/VALIDATION error, NOT op.skip() — skip() hardcodes
|
||||
* CONCURRENCY + skipped:true, which would log a benign-looking "skipped" row while the
|
||||
* API returns VALIDATION. An operator filtering for failed backups would then never see
|
||||
* that this config has been backing up nothing (its target is gone). */
|
||||
private async earlySkipOrError(job: BackupJob, triggeredBy: 'cron' | 'manual' | 'webhook', reason: string): Promise<BackupResult> {
|
||||
try {
|
||||
const op = await this.ports.openOperation(job.targetName, job.configId, job.environmentId, triggeredBy);
|
||||
await op.skip(reason);
|
||||
await op.close({ kind: 'error', code: 'VALIDATION', message: reason });
|
||||
return { status: 'error', executionId: op.id, code: 'VALIDATION', error: reason };
|
||||
} catch {
|
||||
return { status: 'error', code: 'VALIDATION', error: reason };
|
||||
|
||||
+873
-365
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* put-archive-stream.ts — a BACKUP-ONLY streaming upload to a container's PUT /archive.
|
||||
*
|
||||
* WHY a separate function (not the shared putContainerArchive): the shared one has three
|
||||
* callers (file-browser upload, the files-upload API, and backup). It routes through
|
||||
* `dockerFetch`, whose plain-HTTP path uses global `fetch` (undici) which BUFFERS the whole
|
||||
* request body in RAM (verified: 200 MB file -> 239 MB RSS). For the backup helper we want
|
||||
* to stream a large stack-dir tar with O(1) RAM, so we go straight to node http/https core
|
||||
* (verified: 200 MB -> 43 MB RSS). Keeping this backup-only means the file-upload paths are
|
||||
* completely untouched.
|
||||
*
|
||||
* TRANSPORTS:
|
||||
* - socket / plain-http (direct, hawser-standard) -> node http.request, pipe the tar in.
|
||||
* - https / mTLS (direct, hawser-standard TLS) -> httpsAgentRequest (its ReadableStream
|
||||
* body branch streams via req.write).
|
||||
* - hawser-edge -> CANNOT stream (its WS protocol frames
|
||||
* the whole base64 tar). canStream()
|
||||
* returns false; the caller falls back
|
||||
* to the buffered putContainerArchive.
|
||||
*/
|
||||
import http from 'node:http';
|
||||
import { Readable } from 'node:stream';
|
||||
import type { ReadableStream as NodeReadableStream } from 'node:stream/web';
|
||||
import { getDockerConfig, httpsAgentRequest, putContainerArchive, type DockerClientConfig } from '../docker';
|
||||
|
||||
// Pure transport predicates + the idle watchdog live in an import-light module so they can be
|
||||
// unit-tested without pulling docker.ts (and thus better-sqlite3).
|
||||
import { transportCanStream, sanitizeArchivePath, withIdleWatchdog } from './put-archive-transport';
|
||||
export { transportCanStream, sanitizeArchivePath, withIdleWatchdog };
|
||||
|
||||
/** PUT a streaming tar into a container over a unix socket or plain HTTP, via node
|
||||
* http.request (streams the piped body — unlike undici fetch, which buffers). */
|
||||
function putViaNodeHttp(config: DockerClientConfig, containerId: string, archivePath: string, tar: Readable, extraHeaders: Record<string, string>): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const path = `/containers/${containerId}/archive?path=${encodeURIComponent(sanitizeArchivePath(archivePath))}`;
|
||||
const reqOptions: http.RequestOptions = config.type === 'socket'
|
||||
? { socketPath: config.socketPath, path, method: 'PUT', headers: { 'Content-Type': 'application/x-tar', ...extraHeaders } }
|
||||
: { host: config.host, port: config.port, path, method: 'PUT', headers: { 'Content-Type': 'application/x-tar', ...extraHeaders } };
|
||||
|
||||
const req = http.request(reqOptions, (res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
res.on('data', (c: Buffer) => chunks.push(c));
|
||||
res.on('end', () => {
|
||||
const status = res.statusCode ?? 0;
|
||||
if (status >= 200 && status < 300) resolve();
|
||||
else reject(new Error(`put-archive-stream: PUT /archive -> ${status} ${Buffer.concat(chunks).toString().slice(0, 200)}`));
|
||||
});
|
||||
res.on('error', reject);
|
||||
});
|
||||
req.on('error', reject);
|
||||
// Idle watchdog: abort if the target stops reading (backpressure stalls the tar) so a
|
||||
// half-dead agent can't hang the backup up to the 6h helper ceiling.
|
||||
const guarded = withIdleWatchdog(tar, 'put-archive-stream');
|
||||
guarded.on('error', (e) => req.destroy(e));
|
||||
guarded.pipe(req); // streams: node writes chunks as the tar produces them, O(1) RAM
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a tar (a web ReadableStream, e.g. from modern-tar's createTarPacker) into a
|
||||
* container's PUT /archive with O(1) RAM. Returns false WITHOUT uploading if the env's
|
||||
* transport can't stream (hawser-edge) — the caller must then use the buffered path.
|
||||
*/
|
||||
export async function putContainerArchiveStreaming(
|
||||
containerId: string,
|
||||
archivePath: string,
|
||||
tar: ReadableStream<Uint8Array>,
|
||||
envId: number | null | undefined,
|
||||
): Promise<{ streamed: boolean }> {
|
||||
const config = await getDockerConfig(envId ?? undefined);
|
||||
if (!transportCanStream(config)) {
|
||||
console.log(`[Backup] put-archive-stream: env ${envId} is ${config.connectionType} (cannot stream) — caller should use the buffered path`);
|
||||
return { streamed: false };
|
||||
}
|
||||
|
||||
const extraHeaders: Record<string, string> = {};
|
||||
if (config.connectionType === 'hawser-standard' && config.hawserToken) {
|
||||
extraHeaders['X-Hawser-Token'] = config.hawserToken;
|
||||
}
|
||||
|
||||
console.log(`[Backup] put-archive-stream: streaming tar to container ${containerId.slice(0, 12)} via ${config.type} (${config.connectionType ?? 'local'})`);
|
||||
|
||||
if (config.type === 'https') {
|
||||
// httpsAgentRequest streams a ReadableStream body (req.write per chunk) and, seeing a
|
||||
// stream body, skips the 30s idle timeout meant for small requests - a multi-GB tar PUT
|
||||
// can legitimately stall >30s (slow disk, mTLS backpressure, GC) without being dead. Run it
|
||||
// through the idle watchdog (node Readable) so a truly-stalled target still aborts.
|
||||
const guardedWeb = Readable.toWeb(withIdleWatchdog(Readable.fromWeb(tar as unknown as NodeReadableStream), 'put-archive-stream')) as unknown as ReadableStream<Uint8Array>;
|
||||
const res = await httpsAgentRequest(config, `/containers/${containerId}/archive?path=${encodeURIComponent(sanitizeArchivePath(archivePath))}`,
|
||||
{ method: 'PUT', headers: { 'Content-Type': 'application/x-tar', ...extraHeaders }, body: guardedWeb }, false, extraHeaders);
|
||||
if (!res.ok) throw new Error(`put-archive-stream: PUT /archive -> ${res.status} ${(await res.text()).slice(0, 200)}`);
|
||||
await res.body?.cancel().catch(() => {});
|
||||
return { streamed: true };
|
||||
}
|
||||
|
||||
// socket or plain http: node http.request, pipe the tar (converted to a node Readable).
|
||||
await putViaNodeHttp(config, containerId, archivePath, Readable.fromWeb(tar as unknown as NodeReadableStream), extraHeaders);
|
||||
return { streamed: true };
|
||||
}
|
||||
|
||||
/** Re-exported for the caller's buffered fallback (edge). */
|
||||
export { putContainerArchive };
|
||||
@@ -0,0 +1,35 @@
|
||||
// Restore-redeploy orchestration, extracted so the STEP ORDER is unit-testable without the
|
||||
// real port's IO. Invariant: register(internal) runs BEFORE deploy and is NOT gated on the
|
||||
// deploy succeeding - a failed `docker compose up` must still leave an editable managed stack.
|
||||
|
||||
export interface RedeployStackSteps {
|
||||
/** Restore snapshot secrets into the target env's DB (no-op when none / opted out). */
|
||||
restoreSecrets(): Promise<void>;
|
||||
/** Materialise compose/.env into the canonical stack dir. false -> hard error (nothing to deploy). */
|
||||
materialise(): Promise<boolean>;
|
||||
/** Register the stack as managed (internal) so the UI can edit/redeploy it. */
|
||||
register(): Promise<void>;
|
||||
/** `docker compose up` from the canonical dir. Throwing means the deploy failed. */
|
||||
deploy(): Promise<void>;
|
||||
log?(msg: string): void;
|
||||
}
|
||||
|
||||
/** Order: restoreSecrets -> materialise -> register(internal) -> deploy. materialise false is a
|
||||
* hard error; a deploy throw propagates but only AFTER register(), so it never costs the stack. */
|
||||
export async function runRedeployStack(steps: RedeployStackSteps): Promise<void> {
|
||||
const log = steps.log ?? (() => {});
|
||||
|
||||
await steps.restoreSecrets();
|
||||
|
||||
const wrote = await steps.materialise();
|
||||
if (!wrote) throw new Error('could not materialise stack files; cannot redeploy');
|
||||
log('materialised OK');
|
||||
|
||||
// Register BEFORE deploy - see the module header. This line order is the whole point.
|
||||
await steps.register();
|
||||
log('registered as managed (internal)');
|
||||
|
||||
log('redeploying (docker compose up)');
|
||||
await steps.deploy();
|
||||
log('redeploy OK');
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* stackfile-filter.ts — decide which entries under a stack directory belong in the
|
||||
* metadata "stackfiles/" capture, and which are bind-mounted DATA that restic already
|
||||
* backs up separately as a volume.
|
||||
*
|
||||
* A common stack layout keeps the bind data inside the stack dir:
|
||||
*
|
||||
* stack/
|
||||
* ├── compose.yaml <- config, MUST stay in stackfiles/ (redeploy reads it)
|
||||
* ├── .env <- config, MUST stay
|
||||
* ├── config/ <- bind-mounted -> restic volume, exclude
|
||||
* └── data/ <- bind-mounted -> restic volume, exclude
|
||||
*
|
||||
* We skip any DIRECTORY the compose file bind-mounts from inside the stack dir: it is already
|
||||
* backed up as its own restic volume, so capturing it here too would double-back-up and blow the
|
||||
* capture cap, marking the snapshot INCOMPLETE.
|
||||
*
|
||||
* WHY WE PARSE THE COMPOSE, NOT the docker Mount.Source:
|
||||
* The walk reads the stack dir in DOCKHAND's namespace (`/app/data/stacks/...`), but a
|
||||
* bind's `Mount.Source` is the DAEMON HOST path (`/docker/data/...` or `/tmp/...`) which
|
||||
* a containerized Dockhand cannot see or stat — so keying the exclusion on Mount.Source
|
||||
* silently no-ops for every Dockerized install. The compose file, which Dockhand always
|
||||
* has locally, declares binds RELATIVE to the stack dir (`./data`) — the same namespace
|
||||
* the walk already computes `relative(stackDir, abs)` in. So we derive the excluded dirs
|
||||
* from the compose text and compare on relative paths: namespace-agnostic, works
|
||||
* containerized / on-host / remote alike.
|
||||
*
|
||||
* Hard rules (adversarial review):
|
||||
* - NEVER exclude compose/`.yaml`/`.yml`/`.env` (`isLoadBearingStackFile`) — a whole-dir
|
||||
* bind `./:/app` must not strand them or restore can't redeploy.
|
||||
* - Exclude only DIRECTORY binds — a single-FILE bind (`./cfg.yaml:/x`) is config kept.
|
||||
* - Named volumes (`type: volume`, or a non-relative source) are NEVER excluded.
|
||||
* - A `${VAR}` in a bind path is left CAPTURED (conservative over-capture, never drops
|
||||
* a needed file).
|
||||
*
|
||||
* Pure and dependency-light (js-yaml is already a repo dep); the directory check is
|
||||
* injected so it stays unit-testable.
|
||||
*/
|
||||
import yaml from 'js-yaml';
|
||||
|
||||
/** Strip a trailing slash and a leading `./`; keep the rest. */
|
||||
function normRel(p: string): string {
|
||||
let s = p.trim().replace(/\/+$/, '');
|
||||
if (s.startsWith('./')) s = s.slice(2);
|
||||
return s;
|
||||
}
|
||||
|
||||
/** A load-bearing stack file that must NEVER be excluded, regardless of a bind that
|
||||
* happens to cover it: compose/`.yaml`/`.yml` (the compose graph) and `.env*` at any
|
||||
* level. Data dirs are excluded by the bind rule, not by extension. */
|
||||
export function isLoadBearingStackFile(relPath: string): boolean {
|
||||
const base = relPath.split('/').pop() ?? relPath;
|
||||
if (base === '.env' || base.startsWith('.env.')) return true;
|
||||
return /\.ya?ml$/i.test(base);
|
||||
}
|
||||
|
||||
/** The bind `source` from one compose `volumes:` entry, short or long syntax, or null
|
||||
* if it isn't a bind we care about (named volume, no source). */
|
||||
function bindSourceOf(entry: unknown): string | null {
|
||||
if (typeof entry === 'string') {
|
||||
// "src:target[:mode]" — a bind if src is a path (starts with . or /). A bare
|
||||
// name ("db_data:/x") is a NAMED VOLUME — not a bind, skip.
|
||||
const src = entry.split(':')[0];
|
||||
if (src.startsWith('.') || src.startsWith('/')) return src;
|
||||
return null;
|
||||
}
|
||||
if (entry && typeof entry === 'object') {
|
||||
const e = entry as { type?: string; source?: string };
|
||||
if (e.type === 'volume') return null; // named volume
|
||||
if (typeof e.source === 'string') {
|
||||
if (e.type === 'bind' || e.source.startsWith('.') || e.source.startsWith('/')) return e.source;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Directory bind sources declared in the compose file, expressed as paths RELATIVE to
|
||||
* the stack dir — e.g. `['data', 'config']`. `stackDir` is used only to relativize an
|
||||
* absolute bind that points into the stack dir. `isDirRel(rel)` decides whether a given
|
||||
* stack-dir-relative path is a directory (in the walk's own namespace) — a single-file
|
||||
* bind is NOT returned. `${VAR}` sources are dropped (left to be captured).
|
||||
*/
|
||||
export function relativeBindDirsFromCompose(
|
||||
composeContent: string,
|
||||
stackDir: string,
|
||||
isDirRel: (rel: string) => boolean
|
||||
): string[] {
|
||||
let doc: unknown;
|
||||
try { doc = yaml.load(composeContent); } catch { return []; }
|
||||
const services = (doc as { services?: Record<string, unknown> } | null)?.services;
|
||||
if (!services || typeof services !== 'object') return [];
|
||||
|
||||
const rels = new Set<string>();
|
||||
for (const svc of Object.values(services)) {
|
||||
const vols = (svc as { volumes?: unknown[] } | null)?.volumes;
|
||||
if (!Array.isArray(vols)) continue;
|
||||
for (const entry of vols) {
|
||||
const src = bindSourceOf(entry);
|
||||
if (!src) continue;
|
||||
if (src.includes('${')) continue; // unresolved var -> capture, don't risk dropping
|
||||
let rel: string;
|
||||
if (src.startsWith('/')) {
|
||||
// Absolute bind: keep only if it resolves INSIDE the stack dir.
|
||||
const r = relativePosix(stackDir, src);
|
||||
if (r === '' || r.startsWith('..')) continue;
|
||||
rel = r;
|
||||
} else {
|
||||
rel = normRel(src);
|
||||
if (rel === '' || rel.startsWith('..')) continue; // escapes stack dir
|
||||
}
|
||||
// A whole-dir bind (`.`, `./`, or an absolute source == the stack dir) would
|
||||
// mean "exclude everything" — refuse it: it would strip config sidecars that
|
||||
// aren't load-bearing (nginx.conf, certs/). Over-capture the whole dir instead
|
||||
// (safe: nothing lost, compose kept). isLoadBearingStackFile can't rescue a
|
||||
// non-yaml sidecar, so the only safe move here is to not exclude at all.
|
||||
if (rel === '.' || rel === '') continue;
|
||||
if (isDirRel(rel)) rels.add(rel);
|
||||
}
|
||||
}
|
||||
return [...rels];
|
||||
}
|
||||
|
||||
/**
|
||||
* Every RELATIVE bind's path below the stack dir - FILES and DIRS alike (`./config.yaml`,
|
||||
* `./data`). Unlike relativeBindDirsFromCompose (dirs only, for EXCLUDE), this feeds host-path
|
||||
* DERIVATION: a single-file bind's daemon-reported mount.Source still pins the stack dir (strip
|
||||
* the tail), so a bind-less-but-single-file stack resolves without depending on the working_dir
|
||||
* label (which needs the container to be listable - a timing race under load). Whole-dir binds
|
||||
* (`.`/`./`) are skipped (the source IS the stack dir; the bind-derived path adds nothing).
|
||||
*/
|
||||
export function relativeBindsFromCompose(composeContent: string, stackDir: string): string[] {
|
||||
let doc: unknown;
|
||||
try { doc = yaml.load(composeContent); } catch { return []; }
|
||||
const services = (doc as { services?: Record<string, unknown> } | null)?.services;
|
||||
if (!services || typeof services !== 'object') return [];
|
||||
|
||||
const rels = new Set<string>();
|
||||
for (const svc of Object.values(services)) {
|
||||
const vols = (svc as { volumes?: unknown[] } | null)?.volumes;
|
||||
if (!Array.isArray(vols)) continue;
|
||||
for (const entry of vols) {
|
||||
const src = bindSourceOf(entry);
|
||||
if (!src || src.includes('${')) continue;
|
||||
let rel: string;
|
||||
if (src.startsWith('/')) {
|
||||
const r = relativePosix(stackDir, src);
|
||||
if (r === '' || r.startsWith('..')) continue;
|
||||
rel = r;
|
||||
} else {
|
||||
rel = normRel(src);
|
||||
if (rel === '' || rel.startsWith('..')) continue;
|
||||
}
|
||||
if (rel === '.' || rel === '') continue;
|
||||
rels.add(rel);
|
||||
}
|
||||
}
|
||||
return [...rels];
|
||||
}
|
||||
|
||||
/** posix `relative(from, to)` on already-clean absolute paths (no realpath). */
|
||||
function relativePosix(from: string, to: string): string {
|
||||
const a = from.replace(/\/+$/, '').split('/');
|
||||
const b = to.replace(/\/+$/, '').split('/');
|
||||
let i = 0;
|
||||
while (i < a.length && i < b.length && a[i] === b[i]) i++;
|
||||
const up = a.slice(i).map(() => '..');
|
||||
return [...up, ...b.slice(i)].join('/');
|
||||
}
|
||||
|
||||
/** True when `relPath` IS one of the excluded relative dirs, or lives inside one.
|
||||
* Segment-aware so `data-backup` doesn't match `data`. */
|
||||
export function isUnderRelDir(relPath: string, relDirs: string[]): boolean {
|
||||
const t = normRel(relPath);
|
||||
for (const raw of relDirs) {
|
||||
const d = normRel(raw);
|
||||
if (t === d) return true;
|
||||
if (t.startsWith(d + '/')) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -1,14 +1,15 @@
|
||||
/**
|
||||
* Build the tar we put-archive (docker cp) into the backup helper, replacing the
|
||||
* base64-in-Cmd approach that blew ARG_MAX on large stack files.
|
||||
* Build the tar we put-archive (docker cp) into the backup helper.
|
||||
*
|
||||
* `modern-tar` is zero-dependency and emits PAX headers, so paths of ANY length
|
||||
* round-trip (the old hand-rolled USTAR writer threw on names >100 bytes, breaking
|
||||
* deep stack trees). Zero deps also matters for CI: it installs into a symlinked
|
||||
* node_modules where libs with transitive deps fail to resolve them.
|
||||
* round-trip (USTAR caps names at 100 bytes, breaking deep stack trees). Zero deps
|
||||
* also matters for CI: it installs into a symlinked node_modules where libs with
|
||||
* transitive deps fail to resolve them.
|
||||
*/
|
||||
|
||||
import { packTar } from 'modern-tar';
|
||||
import { packTar, createTarPacker } from 'modern-tar';
|
||||
import { createReadStream, statSync } from 'node:fs';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
export interface TarEntry {
|
||||
/** Path inside the archive, e.g. "metadata/stacks/foo/compose.yaml". A leading slash is stripped. */
|
||||
@@ -16,6 +17,53 @@ export interface TarEntry {
|
||||
content: Uint8Array;
|
||||
}
|
||||
|
||||
/** A file to stream into the tar lazily from disk (no in-RAM contents). */
|
||||
export interface TarFileSource {
|
||||
/** Absolute path on disk to read from. */
|
||||
diskPath: string;
|
||||
/** Path inside the archive (leading slash stripped). */
|
||||
archivePath: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a tar as a streaming web ReadableStream, reading each file from disk LAZILY (O(1)
|
||||
* RAM regardless of total size). Used by the backup helper's streaming put-archive so a
|
||||
* huge stack dir never buffers in memory. Entries are produced in the given order; the
|
||||
* consumer (the HTTP request) drives the pace via backpressure.
|
||||
*
|
||||
* `inlineEntries` (e.g. the small metadata.json) are written first from memory; then each
|
||||
* file source is streamed from disk. NOTE: the header size is fixed from statSync but the
|
||||
* body is read lazily via createReadStream, so a file that GROWS or SHRINKS between stat and
|
||||
* read makes modern-tar's packer throw ("exceeds given size" / "Size mismatch") and errors
|
||||
* the whole stream - the run FAILS loudly (it is NOT silently tolerated). This is acceptable
|
||||
* because the backup helper's stack dir is quiescent (the stack is stopped or the files are
|
||||
* config); a mid-capture size change is a real problem the caller should see, not hide.
|
||||
*/
|
||||
export function buildTarStream(sources: TarFileSource[], inlineEntries: TarEntry[] = [], mtimeSecs = Math.floor(Date.now() / 1000)): ReadableStream<Uint8Array> {
|
||||
const packer = createTarPacker();
|
||||
const mtime = new Date(mtimeSecs * 1000);
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
for (const e of inlineEntries) {
|
||||
const w = packer.controller.add({ name: e.path.replace(/^\/+/, ''), size: e.content.length, mode: 0o644, mtime, uid: 0, gid: 0, uname: 'root', gname: 'root', type: 'file' });
|
||||
await Readable.toWeb(Readable.from(Buffer.from(e.content))).pipeTo(w);
|
||||
}
|
||||
for (const s of sources) {
|
||||
const size = statSync(s.diskPath).size;
|
||||
const w = packer.controller.add({ name: s.archivePath.replace(/^\/+/, ''), size, mode: 0o644, mtime, uid: 0, gid: 0, uname: 'root', gname: 'root', type: 'file' });
|
||||
await Readable.toWeb(createReadStream(s.diskPath)).pipeTo(w);
|
||||
}
|
||||
packer.controller.finalize();
|
||||
} catch (err) {
|
||||
// Surface the failure to the consumer by erroring the stream.
|
||||
packer.controller.error(err);
|
||||
}
|
||||
})();
|
||||
|
||||
return packer.readable;
|
||||
}
|
||||
|
||||
/** Build a tar (any path length via PAX). `mtimeSecs` injectable for deterministic tests. */
|
||||
export function buildTar(entries: TarEntry[], mtimeSecs = Math.floor(Date.now() / 1000)): Promise<Uint8Array> {
|
||||
return packTar(
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* Pure decision logic for staging a stack's files onto a `direct` remote host (no server
|
||||
* deps - safe to unit test). The IO (helper container + put-archive) lives in
|
||||
* stage-remote-stackfiles.ts; this module only decides IF and WHERE.
|
||||
*/
|
||||
|
||||
export interface RemoteStagingPlan {
|
||||
stage: boolean;
|
||||
/** Absolute project dir on the remote host (<remoteBase>/<stack>), when stage is true. */
|
||||
projectDir?: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a `direct` deploy should stage files onto the remote host, and where.
|
||||
*
|
||||
* Staging happens when ALL hold:
|
||||
* - operation is 'up' (down/stop/start/restart don't re-stage),
|
||||
* - the env has a non-empty remote_stacks_dir configured,
|
||||
* - there are stack files to stage.
|
||||
* Setting remote_stacks_dir is the user's explicit "keep my stacks here on the host", so we
|
||||
* stage the WHOLE stack dir (compose + .env + sibling config) - not only when the compose has
|
||||
* relative binds. Relative binds still need it to resolve, but a bind-less stack needs its
|
||||
* compose on the host just as much: to be manageable and BACKUPABLE (the helper bind-mounts the
|
||||
* host dir; without the compose there, a backup can't capture it).
|
||||
* Any miss => stage:false => the deploy takes the exact current code path (no --project-directory).
|
||||
*/
|
||||
export function planRemoteStaging(input: {
|
||||
operation: string;
|
||||
remoteStacksDir: string | null | undefined;
|
||||
stackName: string;
|
||||
composeContent: string;
|
||||
hasStackFiles: boolean;
|
||||
}): RemoteStagingPlan {
|
||||
if (input.operation !== 'up') return { stage: false, reason: `operation ${input.operation} does not stage` };
|
||||
if (!input.hasStackFiles) return { stage: false, reason: 'no stack files to stage' };
|
||||
const base = typeof input.remoteStacksDir === 'string' ? input.remoteStacksDir.trim().replace(/\/+$/, '') : '';
|
||||
if (!base) return { stage: false, reason: 'no remote_stacks_dir configured on the env' };
|
||||
return { stage: true, projectDir: `${base}/${input.stackName}`, reason: 'direct env with remote_stacks_dir' };
|
||||
}
|
||||
+132
-17
@@ -41,6 +41,7 @@ import {
|
||||
import { unregisterSchedule } from './scheduler';
|
||||
import { sendEventNotification } from './notifications';
|
||||
import { deleteGitStackFiles, parseEnvFileContent } from './git';
|
||||
import { isDeletableStackDir } from './stack-delete-guard';
|
||||
import { cleanPem } from '$lib/utils/pem';
|
||||
import { rewriteComposeVolumePaths, getHostDataDir } from './host-path';
|
||||
import { getOrderValue } from './container-labels';
|
||||
@@ -988,7 +989,8 @@ async function executeLocalCompose(
|
||||
serviceName?: string,
|
||||
build?: boolean,
|
||||
noBuildCache?: boolean,
|
||||
pullPolicy?: string
|
||||
pullPolicy?: string,
|
||||
remoteProjectDir?: string
|
||||
): Promise<StackOperationResult> {
|
||||
const logPrefix = `[Stack:${stackName}]`;
|
||||
|
||||
@@ -1142,6 +1144,15 @@ async function executeLocalCompose(
|
||||
const useStdin = finalComposeContent !== composeContent;
|
||||
const args = ['docker', 'compose', '-p', stackName];
|
||||
|
||||
// For a `direct` remote daemon we can't share Dockhand's filesystem, so relative binds
|
||||
// (./config, ./data) are staged onto the remote host under `remoteProjectDir` and the
|
||||
// compose is pointed there with --project-directory: compose resolves `./x` against that
|
||||
// remote path (which need not exist locally) instead of the client cwd, so the daemon
|
||||
// binds the staged files. undefined = current behavior (no staging, no flag).
|
||||
if (remoteProjectDir) {
|
||||
args.push('--project-directory', remoteProjectDir);
|
||||
}
|
||||
|
||||
// Temp file for path-translated override content (cleaned up in finally block)
|
||||
let tempOverridePath: string | undefined;
|
||||
|
||||
@@ -1169,6 +1180,12 @@ async function executeLocalCompose(
|
||||
args.push('-f', overrideFile);
|
||||
console.log(`${logPrefix} Including override file: ${basename(overrideFile)}`);
|
||||
}
|
||||
} else if (remoteProjectDir) {
|
||||
// Internal stack + remote staging: --project-directory repoints compose's base to the
|
||||
// remote path, which breaks the usual cwd auto-discovery of compose.yaml (that file is
|
||||
// on Dockhand's host, not under the remote project dir). Point -f at the local file
|
||||
// explicitly so compose reads it locally while resolving ./binds against the remote dir.
|
||||
args.push('-f', composeFile);
|
||||
}
|
||||
// else: internal stack without path translation - no -f needed.
|
||||
// Docker Compose auto-discovers compose.yaml + compose.override.yaml from cwd.
|
||||
@@ -1683,6 +1700,34 @@ async function executeComposeCommand(
|
||||
skipVerify: env.tlsSkipVerify ?? false
|
||||
} : undefined;
|
||||
|
||||
// A direct daemon shares no filesystem with Dockhand, so relative binds
|
||||
// (./config, ./data) would resolve to a path the remote daemon can't see and
|
||||
// Docker would auto-create empty dirs. When the env has a `remote_stacks_dir`
|
||||
// set AND the compose has relative binds, stage the stack files onto the remote
|
||||
// host under <remoteDir>/<stack> and point compose there with --project-directory.
|
||||
// The plan() decides IF/WHERE (pure, unit-tested); anything it declines takes the
|
||||
// exact current path (remoteProjectDir stays undefined = no flag, no staging).
|
||||
let remoteProjectDir: string | undefined;
|
||||
{
|
||||
const { getEnvSetting } = await import('./db');
|
||||
const { planRemoteStaging } = await import('./remote-staging-plan');
|
||||
const remoteStacksDir = await getEnvSetting('remote_stacks_dir', envId ?? undefined);
|
||||
// Stage the WHOLE local stack dir (compose + relative-bind files like nginx.conf,
|
||||
// ./data). The local stack dir is config-only - runtime data written by the remote
|
||||
// container never comes back here - so this can't clobber live data. The tar is
|
||||
// STREAMED from disk (O(1) RAM), so a large ./data doesn't buffer in memory.
|
||||
const hasLocalDir = !!(operation === 'up' && workingDir && existsSync(workingDir));
|
||||
const plan = planRemoteStaging({
|
||||
operation, remoteStacksDir, stackName, composeContent, hasStackFiles: hasLocalDir,
|
||||
});
|
||||
if (plan.stage && plan.projectDir && workingDir) {
|
||||
const { stageStackDirOnRemote } = await import('./stage-remote-stackfiles');
|
||||
const { staged } = await stageStackDirOnRemote(envId!, plan.projectDir, workingDir);
|
||||
console.log(`[Stack:${stackName}] direct env: staged ${staged} file(s) to ${plan.projectDir} on the remote host (${plan.reason})`);
|
||||
remoteProjectDir = plan.projectDir;
|
||||
}
|
||||
}
|
||||
|
||||
return executeLocalCompose(
|
||||
operation,
|
||||
stackName,
|
||||
@@ -1701,7 +1746,8 @@ async function executeComposeCommand(
|
||||
serviceName,
|
||||
build,
|
||||
noBuildCache,
|
||||
pullPolicy
|
||||
pullPolicy,
|
||||
remoteProjectDir
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2331,11 +2377,66 @@ export async function downStack(
|
||||
* Remove a stack completely (compose down + delete files + cleanup database)
|
||||
* Uses stack locking to prevent concurrent operations.
|
||||
*/
|
||||
/**
|
||||
* Compute exactly which on-disk directories a `removeStack(..., deleteFiles)` would delete,
|
||||
* WITHOUT deleting anything. The delete-preview endpoint uses this so the confirm modal
|
||||
* shows the user the same paths the backend will actually remove — one source of truth, no
|
||||
* frontend/backend drift (the class of bug behind #675). Adopted stacks whose files live
|
||||
* outside DATA_DIR are reported as `null` (Dockhand never deletes those).
|
||||
*/
|
||||
export async function computeStackDeletionPaths(
|
||||
stackName: string,
|
||||
envId?: number | null
|
||||
): Promise<{ stackDir: string | null; gitDir: string | null; sourceType: string | null; namedVolumes: string[] }> {
|
||||
const stackSource = await getStackSource(stackName, envId);
|
||||
const stacksDir = getStacksDir();
|
||||
|
||||
// Named volumes compose created for this stack — exactly what `down --volumes` would
|
||||
// remove (compose-managed, labeled with the project). Best-effort: a docker/API hiccup
|
||||
// just yields an empty list (the delete still works; the modal just won't preview them).
|
||||
let namedVolumes: string[] = [];
|
||||
try {
|
||||
const { listVolumes } = await import('./docker.js');
|
||||
const vols = await listVolumes(envId);
|
||||
namedVolumes = vols
|
||||
.filter((v) => v.labels?.['com.docker.compose.project'] === stackName)
|
||||
.map((v) => v.name)
|
||||
.sort();
|
||||
} catch { /* best-effort */ }
|
||||
|
||||
let stackDir: string | null = null;
|
||||
if (stackSource?.composePath) {
|
||||
const customDir = dirname(stackSource.composePath);
|
||||
// SAME strict guard as removeStack (#675): strict subdir + basename match.
|
||||
if (isDeletableStackDir(customDir, stacksDir, stackName) && existsSync(customDir)) {
|
||||
stackDir = customDir;
|
||||
}
|
||||
}
|
||||
if (!stackDir && !stackSource?.composePath) {
|
||||
const defaultDir = await findStackDir(stackName, envId) || await getStackDir(stackName, envId);
|
||||
if (existsSync(defaultDir)) stackDir = defaultDir;
|
||||
}
|
||||
|
||||
// Git stacks additionally have a cloned repo dir that removeStack deletes.
|
||||
let gitDir: string | null = null;
|
||||
const gitStack = await getGitStackByName(stackName, envId);
|
||||
if (gitStack) {
|
||||
try {
|
||||
const { getStackRepoPath } = await import('./git');
|
||||
const repoPath = await getStackRepoPath(gitStack.id, gitStack.stackName, gitStack.environmentId);
|
||||
if (repoPath && existsSync(repoPath)) gitDir = repoPath;
|
||||
} catch { /* best-effort: no git dir shown if we can't resolve it */ }
|
||||
}
|
||||
|
||||
return { stackDir, gitDir, sourceType: stackSource?.sourceType ?? null, namedVolumes };
|
||||
}
|
||||
|
||||
export async function removeStack(
|
||||
stackName: string,
|
||||
envId?: number | null,
|
||||
force = false,
|
||||
removeVolumes = false
|
||||
removeVolumes = false,
|
||||
deleteFiles = true
|
||||
): Promise<StackOperationResult> {
|
||||
return withStackLock(stackName, async () => {
|
||||
// Get compose file (may not exist for external stacks)
|
||||
@@ -2395,6 +2496,23 @@ export async function removeStack(
|
||||
|
||||
// Remove any dynamically-spawned child containers not handled by compose
|
||||
await cleanupOrphanStackContainers(stackName, envId, 'remove');
|
||||
|
||||
// Local stack files ARE deleted below, but only under the DATA_DIR strict guard
|
||||
// (#675) - Dockhand owns that dir. A direct env's REMOTE staged dir has no such
|
||||
// guard: remote_stacks_dir is a user path that can hold co-located user data, and
|
||||
// nothing distinguishes a staged file from the user's own there. So we never delete
|
||||
// it - a stale compose is safe residue; an rm -rf could wipe user data.
|
||||
if (deleteFiles && envId != null) {
|
||||
try {
|
||||
const { getEnvironment, getEnvSetting } = await import('./db');
|
||||
const env = await getEnvironment(envId);
|
||||
if (env?.connectionType === 'direct') {
|
||||
const remoteStacksDir = await getEnvSetting('remote_stacks_dir', envId);
|
||||
const base = typeof remoteStacksDir === 'string' ? remoteStacksDir.trim().replace(/\/+$/, '') : '';
|
||||
if (base) console.log(`[Stack:${stackName}] leaving staged files at ${base}/${stackName} on the remote host (not deleting - may hold user data)`);
|
||||
}
|
||||
} catch { /* log-only, never blocks removal */ }
|
||||
}
|
||||
} else {
|
||||
// External stack - remove containers directly in parallel
|
||||
const { removeContainer } = await import('./docker.js');
|
||||
@@ -2461,16 +2579,10 @@ export async function removeStack(
|
||||
let stackDir: string | null = null;
|
||||
|
||||
if (stackSource?.composePath) {
|
||||
// Check if the compose path is within Dockhand's stacks directory
|
||||
// Only delete if the compose dir is a STRICT SUBDIR of DATA_DIR/stacks/ whose
|
||||
// basename matches the stack name (#675 guard; see stack-delete-guard.ts).
|
||||
const customDir = dirname(stackSource.composePath);
|
||||
const resolvedCustomDir = resolve(customDir);
|
||||
const resolvedStacksDir = resolve(stacksDir);
|
||||
|
||||
// Only delete if the directory is within DATA_DIR/stacks/ (files we created)
|
||||
// AND the directory basename matches the stack name exactly (for safety)
|
||||
if (resolvedCustomDir.startsWith(resolvedStacksDir) &&
|
||||
basename(resolvedCustomDir) === stackName &&
|
||||
existsSync(customDir)) {
|
||||
if (isDeletableStackDir(customDir, stacksDir, stackName) && existsSync(customDir)) {
|
||||
stackDir = customDir;
|
||||
}
|
||||
}
|
||||
@@ -2484,8 +2596,10 @@ export async function removeStack(
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the directory if found
|
||||
if (stackDir) {
|
||||
// Delete the directory if found — but ONLY when the caller asked to remove files.
|
||||
// "Remove stack" (deleteFiles=false) leaves the compose/.env/data on disk; "Remove
|
||||
// stack + files" (default) deletes them.
|
||||
if (stackDir && deleteFiles) {
|
||||
try {
|
||||
rmSync(stackDir, { recursive: true, force: true });
|
||||
} catch (err: any) {
|
||||
@@ -2512,19 +2626,20 @@ export async function removeStack(
|
||||
cleanupErrors.push(`env vars: ${err.message}`);
|
||||
}
|
||||
|
||||
// If git stack, clean up git stack record
|
||||
// If git stack, clean up git stack record. The DB record always goes (the stack is
|
||||
// gone); the cloned repo FILES on disk go only when deleteFiles is set.
|
||||
try {
|
||||
const gitStack = await getGitStackByName(stackName, envId);
|
||||
if (gitStack) {
|
||||
await deleteGitStack(gitStack.id);
|
||||
await deleteGitStackFiles(gitStack.id, gitStack.stackName, gitStack.environmentId);
|
||||
if (deleteFiles) await deleteGitStackFiles(gitStack.id, gitStack.stackName, gitStack.environmentId);
|
||||
}
|
||||
// Also cleanup any orphaned git stacks with NULL environment_id for this stack name
|
||||
if (envId !== undefined && envId !== null) {
|
||||
const orphanedGitStack = await getGitStackByName(stackName, null);
|
||||
if (orphanedGitStack) {
|
||||
await deleteGitStack(orphanedGitStack.id);
|
||||
await deleteGitStackFiles(orphanedGitStack.id, orphanedGitStack.stackName, orphanedGitStack.environmentId);
|
||||
if (deleteFiles) await deleteGitStackFiles(orphanedGitStack.id, orphanedGitStack.stackName, orphanedGitStack.environmentId);
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* Stage a stack's files onto a REMOTE `direct` daemon's host filesystem.
|
||||
*
|
||||
* A `direct` env has no agent and no shared FS with Dockhand, so a `docker compose up`
|
||||
* over TCP resolves relative binds (`./config.yaml`, `./data`) against a path the remote
|
||||
* daemon can't see - it auto-creates empty dirs. This function puts the stack dir onto the
|
||||
* remote host under `remoteDir` (compose points there via --project-directory), using a
|
||||
* throwaway helper container + the Docker put-archive API - the same mechanism the backup
|
||||
* helper uses to write to a remote host. After this runs, the relative binds resolve.
|
||||
*
|
||||
* The tar is STREAMED from disk (O(1) RAM) via the same primitive the backup helper uses:
|
||||
* a stack dir with a large relative bind (a few-GB ./data) must not buffer in memory.
|
||||
* DIRECT streams (transportCanStream is true for everything but hawser-edge, which never
|
||||
* reaches this code - staging is direct-only).
|
||||
*
|
||||
* DIRECT ONLY. Socket shares the FS; Hawser has an agent that writes files itself.
|
||||
*/
|
||||
import { runContainerWithStreaming } from './docker';
|
||||
import { buildTarStream, type TarFileSource } from './backups/tar';
|
||||
import { putContainerArchiveStreaming } from './backups/put-archive-stream';
|
||||
import { readdirSync } from 'node:fs';
|
||||
import { join, relative, sep } from 'node:path';
|
||||
|
||||
const STAGER_IMAGE = 'alpine:latest';
|
||||
|
||||
export interface StageResult {
|
||||
staged: number;
|
||||
remoteDir: string;
|
||||
}
|
||||
|
||||
/** Walk a stack dir into TarFileSource[] (disk paths, no in-RAM contents), so the tar
|
||||
* streams lazily. `.git` is skipped. There is deliberately NO per-file size cap: the tar
|
||||
* streams from disk with O(1) RAM, so a large relative-bind file (a big ./data config)
|
||||
* must stage in full - a cap would defeat the whole point of streaming. */
|
||||
function collectSources(rootDir: string): TarFileSource[] {
|
||||
const sources: TarFileSource[] = [];
|
||||
const walk = (dir: string) => {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.name === '.git') continue;
|
||||
const full = join(dir, entry.name);
|
||||
if (entry.isDirectory()) { walk(full); continue; }
|
||||
if (!entry.isFile()) continue;
|
||||
// archivePath is POSIX-relative to rootDir (tar entries under remoteDir).
|
||||
sources.push({ diskPath: full, archivePath: relative(rootDir, full).split(sep).join('/') });
|
||||
}
|
||||
};
|
||||
walk(rootDir);
|
||||
return sources;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream the stack dir at `localDir` onto the remote host at `remoteDir` via a helper
|
||||
* container. Throws on failure so the caller fails loudly instead of deploying against
|
||||
* empty dirs (a silent no-op deploy).
|
||||
*
|
||||
* @param envId the direct env whose daemon hosts the files
|
||||
* @param remoteDir absolute path on the remote host (validated by the API layer)
|
||||
* @param localDir the local stack dir to stage (its files land under remoteDir 1:1)
|
||||
*/
|
||||
export async function stageStackDirOnRemote(
|
||||
envId: number,
|
||||
remoteDir: string,
|
||||
localDir: string
|
||||
): Promise<StageResult> {
|
||||
const sources = collectSources(localDir);
|
||||
if (sources.length === 0) return { staged: 0, remoteDir };
|
||||
|
||||
const name = `dockhand-stage-${envId}-${Math.floor(Date.now() / 1000)}`;
|
||||
|
||||
await runContainerWithStreaming({
|
||||
image: STAGER_IMAGE,
|
||||
// The container only needs to exist for the put-archive in beforeStart; the bind mount
|
||||
// itself creates remoteDir on the remote host, then it exits immediately.
|
||||
cmd: ['sh', '-c', `mkdir -p '${remoteDir.replace(/'/g, "'\\''")}'`],
|
||||
binds: [`${remoteDir}:${remoteDir}:rw`],
|
||||
name,
|
||||
envId,
|
||||
timeout: 300000,
|
||||
labels: { 'io.dockhand.role': 'stackfile-stager' },
|
||||
// After create, before start: the bind mount already exists, so a streaming put-archive
|
||||
// lands the files on the remote host FS under remoteDir with O(1) RAM.
|
||||
beforeStart: async (containerId: string) => {
|
||||
const tar = buildTarStream(sources);
|
||||
const { streamed } = await putContainerArchiveStreaming(containerId, remoteDir, tar, envId);
|
||||
// direct always streams; a false here would mean a transport we don't stage on.
|
||||
if (!streamed) throw new Error(`remote staging: transport for env ${envId} cannot stream the stack dir`);
|
||||
},
|
||||
});
|
||||
|
||||
return { staged: sources.length, remoteDir };
|
||||
}
|
||||
@@ -95,6 +95,9 @@ export interface BackupOptions {
|
||||
excludeCaches?: boolean;
|
||||
webhookSuccess?: string;
|
||||
webhookFailure?: string;
|
||||
/** Stack-dir entries the user DEselected in the "Stack files on the host" picker; excluded
|
||||
* from the stack-dir capture (never load-bearing compose/.env - the server re-guards). */
|
||||
excludedStackFiles?: string[];
|
||||
}
|
||||
|
||||
export function parseRetention(retention: string | RetentionPolicy | null | undefined): RetentionPolicy {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { authorize } from '$lib/server/authorize';
|
||||
import { requireBackups } from '$lib/server/backups/route-guards';
|
||||
import { runRestore } from '$lib/server/backups';
|
||||
import { validateRestoreRequest, type RestoreMode } from '$lib/server/backups/validate';
|
||||
import { createJobResponse } from '$lib/server/sse';
|
||||
@@ -10,9 +11,8 @@ import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
|
||||
export const POST: RequestHandler = async (event) => {
|
||||
const { request, cookies } = event;
|
||||
const auth = await authorize(cookies);
|
||||
if (auth.authEnabled && !await auth.can('backups', 'manage')) {
|
||||
return json({ error: 'Permission denied' }, { status: 403 });
|
||||
}
|
||||
const rbacDenied = await requireBackups(auth, 'manage');
|
||||
if (rbacDenied) return rbacDenied;
|
||||
|
||||
const body = await request.json();
|
||||
const mode: RestoreMode = body.mode === 'in-place' ? 'in-place' : 'new-location';
|
||||
@@ -25,6 +25,9 @@ export const POST: RequestHandler = async (event) => {
|
||||
// The client sends false to bring the stack up without them. Only a stack redeploy
|
||||
// consumes this; container restores ignore it.
|
||||
const restoreSecrets = body.restoreSecrets !== false;
|
||||
// New-location STACK restore: skip the captured compose/.env (data-only restore),
|
||||
// and/or ADOPT the restored stack into Dockhand afterwards (opt-in). Both default off.
|
||||
const skipStackFiles = body.skipStackFiles === true;
|
||||
|
||||
// Authorization gates run BEFORE request validation, so an unauthorized caller
|
||||
// never learns anything about the request shape.
|
||||
@@ -76,6 +79,7 @@ export const POST: RequestHandler = async (event) => {
|
||||
postRestore,
|
||||
volumeDestinations,
|
||||
restoreSecrets,
|
||||
skipStackFiles,
|
||||
},
|
||||
access,
|
||||
// Stream progress to the client (the restore modal's log). Without this the
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { authorize } from '$lib/server/authorize';
|
||||
import { requireBackups } from '$lib/server/backups/route-guards';
|
||||
import { probeStackDir } from '$lib/server/backups';
|
||||
|
||||
// GET /api/backup/stack-dir-listing?target=<stack>&env=<id>
|
||||
// Probe the TARGET host and list the actual contents of the stack directory, so the
|
||||
// create/edit backup dialog can show the resolved host path + let the user pick which
|
||||
// stack-dir entries to back up. Returns { kind:'listed', hostPath, entries } or
|
||||
// { kind:'unknown', reason } (never throws for an operational failure).
|
||||
export const GET: RequestHandler = async ({ url, cookies }) => {
|
||||
const auth = await authorize(cookies);
|
||||
const denied = await requireBackups(auth, 'view');
|
||||
if (denied) return denied;
|
||||
|
||||
const targetName = url.searchParams.get('target');
|
||||
if (!targetName) return json({ error: 'target is required' }, { status: 400 });
|
||||
const envParam = url.searchParams.get('env');
|
||||
const environmentId = envParam ? parseInt(envParam) : null;
|
||||
|
||||
if (environmentId != null && auth.isEnterprise && !await auth.canAccessEnvironment(environmentId)) {
|
||||
return json({ error: 'Environment access denied' }, { status: 403 });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await probeStackDir(targetName, environmentId);
|
||||
return json(result);
|
||||
} catch (e) {
|
||||
return json({ kind: 'unknown', reason: e instanceof Error ? e.message : 'failed to probe stack directory' });
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,31 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { authorize } from '$lib/server/authorize';
|
||||
import { requireBackups } from '$lib/server/backups/route-guards';
|
||||
import { previewStackBackupPath } from '$lib/server/backups';
|
||||
|
||||
// GET /api/backup/stack-path?target=<stack>&env=<id>
|
||||
// Preview WHERE a stack's directory will be captured from on the host (the resolved host
|
||||
// bind path), or `unknown` with a reason - so the create-schedule dialog can show it up front
|
||||
// and warn before scheduling a backup that would hard-fail the stack-dir probe.
|
||||
export const GET: RequestHandler = async ({ url, cookies }) => {
|
||||
const auth = await authorize(cookies);
|
||||
const denied = await requireBackups(auth, 'view');
|
||||
if (denied) return denied;
|
||||
|
||||
const targetName = url.searchParams.get('target');
|
||||
if (!targetName) return json({ error: 'target is required' }, { status: 400 });
|
||||
const envParam = url.searchParams.get('env');
|
||||
const environmentId = envParam ? parseInt(envParam) : null;
|
||||
|
||||
if (environmentId != null && auth.isEnterprise && !await auth.canAccessEnvironment(environmentId)) {
|
||||
return json({ error: 'Environment access denied' }, { status: 403 });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await previewStackBackupPath(targetName, environmentId);
|
||||
return json(result);
|
||||
} catch (e) {
|
||||
return json({ kind: 'unknown', reason: e instanceof Error ? e.message : 'failed to resolve stack path' });
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,53 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { authorize } from '$lib/server/authorize';
|
||||
import { getEnvironment, getEnvSetting, setEnvSetting } from '$lib/server/db';
|
||||
|
||||
// Per-env "remote stacks directory" for `direct` environments. A direct daemon has no
|
||||
// agent and no shared filesystem with Dockhand, so relative bind files (./config.yaml,
|
||||
// ./data) never reach it. When this is set, Dockhand stages the stack files onto the
|
||||
// remote host under this dir before `docker compose up`. Empty/unset = current behavior.
|
||||
// Stored via the generic per-env settings store (no schema column).
|
||||
|
||||
export const GET: RequestHandler = async ({ params, cookies }) => {
|
||||
const auth = await authorize(cookies);
|
||||
if (auth.authEnabled && !(await auth.can('environments', 'view'))) {
|
||||
return json({ error: 'Permission denied' }, { status: 403 });
|
||||
}
|
||||
const id = parseInt(params.id);
|
||||
const env = await getEnvironment(id);
|
||||
if (!env) return json({ error: 'Environment not found' }, { status: 404 });
|
||||
|
||||
const stored = await getEnvSetting('remote_stacks_dir', id);
|
||||
const remoteStacksDir = stored && typeof stored === 'string' && stored.trim() !== '' ? stored : null;
|
||||
return json({ remoteStacksDir });
|
||||
};
|
||||
|
||||
export const POST: RequestHandler = async ({ params, request, cookies }) => {
|
||||
const auth = await authorize(cookies);
|
||||
if (auth.authEnabled && !(await auth.can('environments', 'edit'))) {
|
||||
return json({ error: 'Permission denied' }, { status: 403 });
|
||||
}
|
||||
const id = parseInt(params.id);
|
||||
const env = await getEnvironment(id);
|
||||
if (!env) return json({ error: 'Environment not found' }, { status: 404 });
|
||||
|
||||
const data = await request.json().catch(() => ({}));
|
||||
const raw = data?.remoteStacksDir;
|
||||
|
||||
// null / '' clears the setting (reverts to current behavior).
|
||||
if (raw === null || raw === undefined || raw === '') {
|
||||
await setEnvSetting('remote_stacks_dir', '', id);
|
||||
return json({ success: true, remoteStacksDir: null });
|
||||
}
|
||||
if (typeof raw !== 'string') {
|
||||
return json({ error: 'remoteStacksDir must be a string or null' }, { status: 400 });
|
||||
}
|
||||
// Must be an absolute path on the remote host; reject relative or traversal-y input.
|
||||
const value = raw.trim();
|
||||
if (!value.startsWith('/') || value.includes('..')) {
|
||||
return json({ error: 'remoteStacksDir must be an absolute path with no ".."' }, { status: 400 });
|
||||
}
|
||||
await setEnvSetting('remote_stacks_dir', value, id);
|
||||
return json({ success: true, remoteStacksDir: value });
|
||||
};
|
||||
@@ -13,6 +13,7 @@
|
||||
import { getRepoTypeIcon, formatCron, runBackupAction, type BackupFormState } from '$lib/utils/backup';
|
||||
import { toast } from 'svelte-sonner';
|
||||
import VolumePicker from '$lib/components/backup/VolumePicker.svelte';
|
||||
import StackFilesPicker from '$lib/components/backup/StackFilesPicker.svelte';
|
||||
import DestinationPicker from '$lib/components/backup/DestinationPicker.svelte';
|
||||
import { normalizeMounts, volumesForStack, type VolumeInfo } from '$lib/utils/mounts';
|
||||
|
||||
@@ -47,6 +48,45 @@
|
||||
let allVolumes = $state(true);
|
||||
let selectedVolumes = $state<string[]>([]);
|
||||
|
||||
// Stack files on the host (probe listing) — stack targets only.
|
||||
type StackListing =
|
||||
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
||||
| { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
||||
| { kind: 'helper-failed'; reason: string }
|
||||
| { kind: 'unknown'; reason: string }
|
||||
| null;
|
||||
let stackListing = $state<StackListing>(null);
|
||||
// The probe helper couldn't run on the target - a backup can't run either, so block Save.
|
||||
const stackHelperFailed = $derived(stackListing?.kind === 'helper-failed');
|
||||
let loadingStackListing = $state(false);
|
||||
let excludedStackFiles = $state<string[]>([]);
|
||||
// The user-set "Remote stack path (for backup)" for direct/hawser envs, shown in the picker
|
||||
// tooltip so the user sees THEIR configured path (empty when not set / socket env).
|
||||
let stackRemoteDir = $state<string>('');
|
||||
|
||||
async function fetchStackListing(item: ContainerItem) {
|
||||
if (item.type !== 'stack') { stackListing = null; return; }
|
||||
loadingStackListing = true;
|
||||
stackListing = null;
|
||||
excludedStackFiles = [];
|
||||
stackRemoteDir = '';
|
||||
if (item.envId != null) {
|
||||
fetch(`/api/environments/${item.envId}/remote-stacks-dir`)
|
||||
.then((r) => (r.ok ? r.json() : null))
|
||||
.then((d) => { stackRemoteDir = d?.remoteStacksDir ?? ''; })
|
||||
.catch(() => { /* non-fatal - tooltip just omits the configured path */ });
|
||||
}
|
||||
try {
|
||||
const envQ = item.envId != null ? `&env=${item.envId}` : '';
|
||||
const res = await fetch(`/api/backup/stack-dir-listing?target=${encodeURIComponent(item.name)}${envQ}`);
|
||||
stackListing = await res.json();
|
||||
} catch (e) {
|
||||
stackListing = { kind: 'unknown', reason: e instanceof Error ? e.message : 'probe failed' };
|
||||
} finally {
|
||||
loadingStackListing = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Schedule & Run
|
||||
let saveSchedule = $state(false);
|
||||
let schedule = $state('0 2 * * *');
|
||||
@@ -126,6 +166,7 @@
|
||||
function selectSource(item: ContainerItem) {
|
||||
selectedItem = item;
|
||||
step = 2;
|
||||
fetchStackListing(item);
|
||||
}
|
||||
|
||||
// Save the config, then (when run=true) hand it off to the parent's shared
|
||||
@@ -141,7 +182,8 @@
|
||||
destinationId: selectedDestId,
|
||||
stopBeforeBackup,
|
||||
allVolumes,
|
||||
selectedVolumes
|
||||
selectedVolumes,
|
||||
options: excludedStackFiles.length > 0 ? { excludedStackFiles } : undefined
|
||||
};
|
||||
try {
|
||||
const result = await runBackupAction({
|
||||
@@ -182,7 +224,7 @@
|
||||
</script>
|
||||
|
||||
<Dialog.Root bind:open onOpenChange={(isOpen) => { if (!isOpen && saving) return; open = isOpen; }}>
|
||||
<Dialog.Content class="max-w-3xl h-[70vh] flex flex-col">
|
||||
<Dialog.Content class="max-w-5xl h-[85vh] flex flex-col">
|
||||
<Dialog.Header class="pb-0">
|
||||
<Dialog.Title class="flex items-center gap-2 text-base">
|
||||
<Package class="w-4 h-4" />Create backup
|
||||
@@ -316,6 +358,19 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{#if selectedItem?.type === 'stack'}
|
||||
<StackFilesPicker
|
||||
listing={stackListing}
|
||||
loading={loadingStackListing}
|
||||
connectionType={selectedEnv?.connectionType}
|
||||
envName={selectedEnv?.name}
|
||||
envIcon={selectedEnv?.icon}
|
||||
envId={selectedEnv?.id}
|
||||
configuredStackPath={stackRemoteDir}
|
||||
bind:excludedStackFiles
|
||||
/>
|
||||
{/if}
|
||||
|
||||
{#if selectedItem}
|
||||
<VolumePicker
|
||||
volumes={selectedItem.volumes}
|
||||
@@ -367,12 +422,12 @@
|
||||
<Button variant="outline" size="sm" onclick={() => step = 2} disabled={saving}>Back</Button>
|
||||
<div class="flex-1"></div>
|
||||
{#if saveSchedule}
|
||||
<Button variant="outline" size="sm" onclick={() => saveAndMaybeRun(false)} disabled={saving || scheduleInvalid}>
|
||||
<Button variant="outline" size="sm" onclick={() => saveAndMaybeRun(false)} disabled={saving || scheduleInvalid || stackHelperFailed}>
|
||||
{#if saving}<Loader2 class="w-3.5 h-3.5 mr-1.5 animate-spin" />{:else}<CheckCircle2 class="w-3.5 h-3.5 mr-1.5" />{/if}
|
||||
Save schedule only
|
||||
</Button>
|
||||
{/if}
|
||||
<Button size="sm" onclick={() => saveAndMaybeRun(true)} disabled={saving || (saveSchedule && scheduleInvalid)}>
|
||||
<Button size="sm" onclick={() => saveAndMaybeRun(true)} disabled={saving || (saveSchedule && scheduleInvalid) || stackHelperFailed}>
|
||||
{#if saving}<Loader2 class="w-3.5 h-3.5 mr-1.5 animate-spin" />{:else}<Play class="w-3.5 h-3.5 mr-1.5" />{/if}
|
||||
{saveSchedule ? 'Run & save schedule' : 'Run backup now'}
|
||||
</Button>
|
||||
|
||||
@@ -7,9 +7,10 @@
|
||||
import { Badge } from '$lib/components/ui/badge';
|
||||
import { TogglePill } from '$lib/components/ui/toggle-pill';
|
||||
import CronEditor from '$lib/components/cron-editor.svelte';
|
||||
import { Save, Play, Pause, RefreshCw, CheckCircle, XCircle, ChevronDown, Loader2, Plus, Trash2, Pencil, X, Check, HardDrive, Box, Layers, FileText } from 'lucide-svelte';
|
||||
import { Save, Play, Pause, RefreshCw, CheckCircle, XCircle, ChevronDown, Loader2, Plus, Trash2, Pencil, X, Check, HardDrive, Box, Layers, FileText, FolderOpen, AlertTriangle } from 'lucide-svelte';
|
||||
import RotateCwFadingClock from '$lib/components/icons/RotateCwFadingClock.svelte';
|
||||
import VolumePicker from '$lib/components/backup/VolumePicker.svelte';
|
||||
import StackFilesPicker from '$lib/components/backup/StackFilesPicker.svelte';
|
||||
import DestinationPicker from '$lib/components/backup/DestinationPicker.svelte';
|
||||
import LogConsole from '$lib/components/LogConsole.svelte';
|
||||
import SnapshotsPanel from './SnapshotsPanel.svelte';
|
||||
@@ -23,10 +24,11 @@
|
||||
import { watchJob } from '$lib/utils/sse-fetch';
|
||||
import ConfirmPopover from '$lib/components/ConfirmPopover.svelte';
|
||||
import { getRepoTypeIcon, parseRetention, parseOptions, retentionSummary as getRetentionSummary, formatCron, runBackupAction, classifyJobResult, tagLogLine, fetchBackupExecutions, type BackupAction, type BackupFormState } from '$lib/utils/backup';
|
||||
import { reconcileSelectedVolumeKeys } from '$lib/utils/mounts';
|
||||
|
||||
interface Props {
|
||||
containerName: string;
|
||||
volumes: { name: string; mountPoint: string; mountType?: 'volume' | 'bind' }[];
|
||||
volumes: { key: string; name: string; mountPoint: string; mountType?: 'volume' | 'bind' }[];
|
||||
type?: 'container' | 'stack';
|
||||
environmentId?: number;
|
||||
onConfigSaved?: () => void;
|
||||
@@ -238,9 +240,12 @@
|
||||
editLimitDownload = opts.limitDownload || '';
|
||||
editWebhookSuccess = opts.webhookSuccess || '';
|
||||
editWebhookFailure = opts.webhookFailure || '';
|
||||
excludedStackFiles = Array.isArray(opts.excludedStackFiles) ? opts.excludedStackFiles : [];
|
||||
editStopBefore = cfg.stopBeforeBackup;
|
||||
editAllVolumes = cfg.selectedVolumes === null;
|
||||
editSelectedVolumes = cfg.selectedVolumes ? (typeof cfg.selectedVolumes === 'string' ? JSON.parse(cfg.selectedVolumes) : cfg.selectedVolumes) : [];
|
||||
const storedSel = cfg.selectedVolumes ? (typeof cfg.selectedVolumes === 'string' ? JSON.parse(cfg.selectedVolumes) : cfg.selectedVolumes) : [];
|
||||
// Reconcile a stored selection (may hold legacy names) to the current volume keys.
|
||||
editSelectedVolumes = reconcileSelectedVolumeKeys(volumes, storedSel);
|
||||
showAdvanced = false;
|
||||
}
|
||||
|
||||
@@ -283,9 +288,11 @@
|
||||
if (editWebhookFailure !== (opts.webhookFailure || '')) return true;
|
||||
if (editStopBefore !== cfg.stopBeforeBackup) return true;
|
||||
if (editAllVolumes !== (cfg.selectedVolumes === null)) return true;
|
||||
const origVolumes = cfg.selectedVolumes
|
||||
const storedSel = cfg.selectedVolumes
|
||||
? (typeof cfg.selectedVolumes === 'string' ? JSON.parse(cfg.selectedVolumes) : cfg.selectedVolumes)
|
||||
: [];
|
||||
// Reconcile the baseline to keys too, so a name->key format shift alone isn't seen as dirty.
|
||||
const origVolumes = reconcileSelectedVolumeKeys(volumes, storedSel);
|
||||
if (JSON.stringify([...editSelectedVolumes].sort()) !== JSON.stringify([...origVolumes].sort())) return true;
|
||||
return false;
|
||||
}
|
||||
@@ -314,7 +321,8 @@
|
||||
limitUpload: editLimitUpload || undefined,
|
||||
limitDownload: editLimitDownload || undefined,
|
||||
webhookSuccess: editWebhookSuccess || undefined,
|
||||
webhookFailure: editWebhookFailure || undefined
|
||||
webhookFailure: editWebhookFailure || undefined,
|
||||
excludedStackFiles: excludedStackFiles.length > 0 ? excludedStackFiles : undefined
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -468,7 +476,53 @@
|
||||
return `${sel.length} of ${volumes.length} volumes`;
|
||||
}
|
||||
|
||||
// Resolved host stack-dir path (stacks only): WHERE the helper will read the stack folder
|
||||
// from on the target host. Shown in the create form so the user sees what gets captured -
|
||||
// and a clear warning up front when it can't be located (would hard-fail the backup).
|
||||
let stackPath = $state<{ kind: 'candidate'; hostPath: string; composeFile: string } | { kind: 'unknown'; reason: string } | null>(null);
|
||||
async function fetchStackPath() {
|
||||
if (type !== 'stack') return;
|
||||
try {
|
||||
const params = new URLSearchParams({ target: containerName });
|
||||
// Use the derived envId (prop OR the currently selected environment), the SAME
|
||||
// value the rest of the panel uses for save/run - the bare `environmentId` prop is
|
||||
// often undefined (the modal doesn't pass it), which made the server report
|
||||
// "No environment specified".
|
||||
if (envId != null) params.set('env', String(envId));
|
||||
const res = await fetch(`/api/backup/stack-path?${params}`);
|
||||
if (res.ok) stackPath = await res.json();
|
||||
} catch { /* non-fatal: the path preview is informational */ }
|
||||
}
|
||||
|
||||
// Stack files on the host (probe listing) — supersedes the plain stackPath callout for the
|
||||
// picker, but stackPath is still fetched for the compose-file name shown elsewhere.
|
||||
type StackListing =
|
||||
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
||||
| { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
|
||||
| { kind: 'unknown'; reason: string }
|
||||
| null;
|
||||
let stackListing = $state<StackListing>(null);
|
||||
let loadingStackListing = $state(false);
|
||||
let excludedStackFiles = $state<string[]>([]);
|
||||
async function fetchStackListing() {
|
||||
if (type !== 'stack') { stackListing = null; return; }
|
||||
loadingStackListing = true;
|
||||
try {
|
||||
const params = new URLSearchParams({ target: containerName });
|
||||
if (envId != null) params.set('env', String(envId));
|
||||
const res = await fetch(`/api/backup/stack-dir-listing?${params}`);
|
||||
stackListing = await res.json();
|
||||
} catch (e) {
|
||||
stackListing = { kind: 'unknown', reason: e instanceof Error ? e.message : 'probe failed' };
|
||||
} finally {
|
||||
loadingStackListing = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => { fetchDestinations(); fetchConfigs(); });
|
||||
// Resolve the stack path once envId is known (it's derived from $currentEnvironment, which
|
||||
// may not be ready at mount). Re-runs if the environment changes.
|
||||
$effect(() => { if (type === 'stack' && envId != null) { void fetchStackPath(); void fetchStackListing(); } });
|
||||
</script>
|
||||
|
||||
<!-- Sub-tabs: Schedules (cheap) vs Snapshots. SnapshotsPanel is mounted as soon as
|
||||
@@ -610,6 +664,20 @@
|
||||
<TogglePill bind:checked={editStopBefore} />
|
||||
<Label class="text-xs">Stop {type} during backup</Label>
|
||||
</div>
|
||||
<!-- Stack files on the host: probe the host, show the resolved dir + let the user
|
||||
pick which entries to back up. Shown BEFORE the volume list. -->
|
||||
{#if type === 'stack'}
|
||||
<StackFilesPicker
|
||||
listing={stackListing}
|
||||
loading={loadingStackListing}
|
||||
connectionType={fullEnv?.connectionType}
|
||||
envName={fullEnv?.name}
|
||||
envIcon={fullEnv?.icon}
|
||||
envId={fullEnv?.id}
|
||||
bind:excludedStackFiles
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<!-- Volume picker (all-or-subset toggle + per-volume checkboxes) -->
|
||||
<VolumePicker
|
||||
volumes={volumes}
|
||||
|
||||
@@ -275,6 +275,9 @@
|
||||
let formHost = $state('');
|
||||
let formPort = $state(2375); // Default for direct Docker connection
|
||||
let formProtocol = $state('http');
|
||||
// direct-only: where Dockhand stages compose + relative-bind files on the remote host,
|
||||
// so `./config`/`./data` binds resolve. Empty = current behavior (relative binds unsupported).
|
||||
let formRemoteStacksDir = $state('');
|
||||
let formTlsCa = $state('');
|
||||
let formTlsCert = $state('');
|
||||
let formTlsKey = $state('');
|
||||
@@ -294,6 +297,11 @@
|
||||
let formDiskWarningThreshold = $state(80);
|
||||
let formDiskWarningThresholdGb = $state(50);
|
||||
let formConnectionType = $state<ConnectionType>('socket');
|
||||
// Envs that keep stack files on a REMOTE host, so backup needs a declared stack path:
|
||||
// direct (Dockhand stages there, drives deploy + backup) and hawser (the agent keeps files
|
||||
// at its STACKS_DIR; the path is backup-only, does not steer deploy).
|
||||
const usesStackPath = (ct: ConnectionType) => ct === 'direct' || ct === 'hawser-standard' || ct === 'hawser-edge';
|
||||
const isHawserConn = (ct: ConnectionType) => ct === 'hawser-standard' || ct === 'hawser-edge';
|
||||
let formHawserToken = $state('');
|
||||
let formLabels = $state<string[]>([]);
|
||||
let newLabelInput = $state('');
|
||||
@@ -598,6 +606,7 @@
|
||||
loadImagePruneSettings(environment.id);
|
||||
loadTimezone(environment.id);
|
||||
loadDiskWarningSettings(environment.id);
|
||||
if (usesStackPath(formConnectionType)) loadRemoteStacksDir(environment.id);
|
||||
// Load Hawser token if edge mode
|
||||
if (formConnectionType === 'hawser-edge') {
|
||||
loadHawserToken(environment.id);
|
||||
@@ -607,6 +616,7 @@
|
||||
formHost = '';
|
||||
formPort = 2375;
|
||||
formProtocol = 'http';
|
||||
formRemoteStacksDir = '';
|
||||
formTlsCa = '';
|
||||
formTlsCert = '';
|
||||
formTlsKey = '';
|
||||
@@ -883,6 +893,7 @@
|
||||
if (newEnv?.id) {
|
||||
await saveTimezone(newEnv.id);
|
||||
await saveDiskWarningSettings(newEnv.id);
|
||||
if (usesStackPath(formConnectionType)) await saveRemoteStacksDir(newEnv.id);
|
||||
}
|
||||
onSaved();
|
||||
onClose();
|
||||
@@ -1019,6 +1030,7 @@
|
||||
await saveImagePruneSettings(environment.id);
|
||||
await saveTimezone(environment.id);
|
||||
await saveDiskWarningSettings(environment.id);
|
||||
if (usesStackPath(formConnectionType)) await saveRemoteStacksDir(environment.id);
|
||||
toast.success(`Updated environment: ${formName}`);
|
||||
onSaved();
|
||||
onClose();
|
||||
@@ -1049,6 +1061,30 @@
|
||||
}
|
||||
}
|
||||
|
||||
async function loadRemoteStacksDir(envId: number) {
|
||||
try {
|
||||
const response = await fetch(`/api/environments/${envId}/remote-stacks-dir`);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
formRemoteStacksDir = data.remoteStacksDir ?? '';
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load remote stacks dir:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function saveRemoteStacksDir(envId: number) {
|
||||
try {
|
||||
await fetch(`/api/environments/${envId}/remote-stacks-dir`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ remoteStacksDir: formRemoteStacksDir.trim() || null })
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to save remote stacks dir:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function saveDiskWarningSettings(envId: number) {
|
||||
try {
|
||||
await fetch(`/api/environments/${envId}/disk-warning`, {
|
||||
@@ -1607,7 +1643,7 @@
|
||||
</Tabs.Trigger>
|
||||
</Tabs.List>
|
||||
|
||||
<div class="overflow-y-auto py-4 h-[520px] [scrollbar-gutter:stable] pr-3">
|
||||
<div class="overflow-y-auto py-4 h-[520px] [scrollbar-gutter:stable] pr-5">
|
||||
<!-- General Tab (Connection Settings) -->
|
||||
<Tabs.Content value="general" class="space-y-4 mt-0 h-full">
|
||||
<!-- Name field -->
|
||||
@@ -1911,6 +1947,75 @@
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Stack path for backup (direct = also drives deploy; hawser = backup-only) -->
|
||||
{#if usesStackPath(formConnectionType)}
|
||||
<div class="space-y-2">
|
||||
<div class="flex items-center gap-1.5">
|
||||
<Label for="edit-env-remote-stacks-dir">
|
||||
{isHawserConn(formConnectionType) ? 'Remote stack path (for backup)' : 'Remote stacks directory'}
|
||||
<span class="text-muted-foreground font-normal">(optional)</span>
|
||||
</Label>
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger type="button" class="text-muted-foreground hover:text-foreground">
|
||||
<HelpCircle class="w-3.5 h-3.5" />
|
||||
</Tooltip.Trigger>
|
||||
<Tooltip.Content class="w-80 z-[200]" side="right">
|
||||
{#if isHawserConn(formConnectionType)}
|
||||
<div class="space-y-2">
|
||||
<p class="font-medium">Where the agent keeps stack files</p>
|
||||
<p class="text-muted-foreground">
|
||||
The Hawser agent stores each stack's folder at
|
||||
<code class="bg-muted px-1 rounded"><this path>/<stack></code>
|
||||
on <span class="font-medium text-foreground">its own host</span>. Backup reads
|
||||
the compose and config from there. This does <span class="font-medium text-foreground">not</span>
|
||||
change where deploy or restore write - the agent always uses its own
|
||||
<code class="bg-muted px-1 rounded">STACKS_DIR</code>; this only tells backup where to look.
|
||||
</p>
|
||||
<p class="text-muted-foreground">
|
||||
So set it to MATCH the agent's <code class="bg-muted px-1 rounded">STACKS_DIR</code>
|
||||
(default <code class="bg-muted px-1 rounded">/data/stacks</code>). Leave empty for
|
||||
the default; set it only if the agent runs with a custom one.
|
||||
</p>
|
||||
</div>
|
||||
{:else}
|
||||
<div class="space-y-2">
|
||||
<p class="font-medium">How this path is used</p>
|
||||
<p class="text-muted-foreground">
|
||||
A direct daemon shares no filesystem with Dockhand. When set, Dockhand
|
||||
materialises each stack's whole folder - compose, <code class="bg-muted px-1 rounded">.env</code>,
|
||||
and relative-bind files (<code class="bg-muted px-1 rounded">./config</code>,
|
||||
<code class="bg-muted px-1 rounded">./data</code>) - into
|
||||
<code class="bg-muted px-1 rounded"><this path>/<stack></code>
|
||||
<span class="font-medium text-foreground">on the remote host</span>, and runs compose there
|
||||
via <code class="bg-muted px-1 rounded">--project-directory</code>.
|
||||
</p>
|
||||
<p class="text-muted-foreground">
|
||||
This applies to normal stacks and git sync alike, so relative binds resolve
|
||||
and every stack is backupable. Leave empty to use only named volumes or
|
||||
absolute paths.
|
||||
</p>
|
||||
</div>
|
||||
{/if}
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
</div>
|
||||
<Input
|
||||
id="edit-env-remote-stacks-dir"
|
||||
bind:value={formRemoteStacksDir}
|
||||
placeholder={isHawserConn(formConnectionType) ? '/data/stacks' : '/opt/dockhand/stacks'}
|
||||
/>
|
||||
<p class="text-xs text-muted-foreground">
|
||||
{#if isHawserConn(formConnectionType)}
|
||||
Absolute path on the agent's host where it keeps stack folders. Used only to back up
|
||||
each stack's compose and config. Leave empty for the default <code class="bg-muted px-1 rounded">/data/stacks</code>.
|
||||
{:else}
|
||||
Absolute path on the remote host where Dockhand stages each stack's files before deploying.
|
||||
Set it for stacks with relative binds, or to keep stacks backupable on a remote direct daemon.
|
||||
{/if}
|
||||
</p>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Direct connection settings -->
|
||||
{#if formConnectionType === 'direct'}
|
||||
<div class="grid grid-cols-2 gap-4">
|
||||
|
||||
Reference in new issue
Block a user