read-side backup snapshot endpoints use job polling

This commit is contained in:
jarek committed 2026-08-13 10:20:30 +02:00
1 parent 315c795058
commit 610e46dc19
15 files changed
+224 -165

No files matched your search

+18
View File
@@ -68,3 +68,21 @@ export function createJobResponse(
return json({ jobId: job.id }); return json({ jobId: job.id });
} }
/**
* Job-polling wrapper for a read endpoint that just returns DATA (no progress lines):
* run `fn`, deliver its value as the single `result`, and turn a thrown error into a
* `{ error }` result (HTTP 200 + a failed-shaped payload the client reads back via
* readJobResponse). Collapses the identical createJobResponse+try/catch boilerplate every
* backup read endpoint (browse/preview/metadata/diff) had. The proxy sees {jobId} at once,
* so a slow restic op can't be aborted mid-flight at the reverse-proxy's ~15s cap.
*/
export function jobResult<T>(request: Request | undefined, fn: () => Promise<T>): Response {
return createJobResponse(async (send) => {
try {
send('result', await fn());
} catch (error) {
send('result', { error: error instanceof Error ? error.message : String(error) });
}
}, request);
}
+18
View File
@@ -0,0 +1,18 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { authorize } from '$lib/server/authorize';
import { requireBackups } from '$lib/server/backups/route-guards';
import { getInstanceId } from '$lib/server/backups/identity';
// This install's stable instance id. The snapshots list used to ride it on an
// X-Dockhand-Instance response header, but that list is now job-polled (a proxy would
// abort the sync `restic snapshots` at ~15s) and headers don't survive polling. The
// client fetches the id here once and matches it against each snapshot's
// dockhand:instance= tag to tell own snapshots from foreign orphans (#1351).
export const GET: RequestHandler = async ({ cookies }) => {
const auth = await authorize(cookies);
const denied = await requireBackups(auth, 'view');
if (denied) return denied;
return json({ instanceId: await getInstanceId() });
};
@@ -5,6 +5,7 @@ import { requireBackups } from '$lib/server/backups/route-guards';
import { previewSnapshot, previewRestoreTargets } from '$lib/server/backups'; import { previewSnapshot, previewRestoreTargets } from '$lib/server/backups';
import { validateSnapshotId } from '$lib/server/docker-validation'; import { validateSnapshotId } from '$lib/server/docker-validation';
import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards'; import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
import { jobResult } from '$lib/server/sse';
export const POST: RequestHandler = async ({ request, cookies }) => { export const POST: RequestHandler = async ({ request, cookies }) => {
const auth = await authorize(cookies); const auth = await authorize(cookies);
@@ -26,7 +27,11 @@ export const POST: RequestHandler = async ({ request, cookies }) => {
const denied = await guardSnapshotEnvAccess(auth, body.destinationId, body.snapshotId); const denied = await guardSnapshotEnvAccess(auth, body.destinationId, body.snapshotId);
if (denied) return denied; if (denied) return denied;
try { // Job-polling: the preview runs restic (verify-ownership + ls/dump) and, with a mode,
// per-volume probe helpers - all slow enough that a reverse proxy would abort the sync
// request at ~15s and SIGTERM the restic mid-op (the "wrong password / signal terminated"
// red herrings). Returning {jobId} immediately keeps the connection alive; the client polls.
return jobResult(request, async () => {
const access = { isEnterprise: auth.isEnterprise, canAccessEnvironment: (id: number) => auth.canAccessEnvironment(id) }; const access = { isEnterprise: auth.isEnterprise, canAccessEnvironment: (id: number) => auth.canAccessEnvironment(id) };
const preview = await previewSnapshot(body.destinationId, body.snapshotId, access); const preview = await previewSnapshot(body.destinationId, body.snapshotId, access);
// When the caller supplies a restore mode, ALSO resolve the exact on-disk targets and probe // When the caller supplies a restore mode, ALSO resolve the exact on-disk targets and probe
@@ -44,11 +49,8 @@ export const POST: RequestHandler = async ({ request, cookies }) => {
mergeStackFiles: body.mergeStackFiles, mergeStackFiles: body.mergeStackFiles,
volumes: body.volumes, volumes: body.volumes,
}, access); }, access);
return json({ ...preview, targets }); return { ...preview, targets };
} }
return json(preview); return preview;
} catch (error) { });
const errorMsg = error instanceof Error ? error.message : String(error);
return json({ error: errorMsg }, { status: 500 });
}
}; };
+24 -37
View File
@@ -5,8 +5,9 @@ import { requireBackups } from '$lib/server/backups/route-guards';
import { getBackupConfig, getBackupDestinations } from '$lib/server/db'; import { getBackupConfig, getBackupDestinations } from '$lib/server/db';
import { listSnapshots } from '$lib/server/backups'; import { listSnapshots } from '$lib/server/backups';
import { filterSnapshotsByEnvAccess } from '$lib/server/backups/route-guards'; import { filterSnapshotsByEnvAccess } from '$lib/server/backups/route-guards';
import { jobResult } from '$lib/server/sse';
export const GET: RequestHandler = async ({ url, cookies }) => { export const GET: RequestHandler = async ({ url, cookies, request }) => {
const auth = await authorize(cookies); const auth = await authorize(cookies);
const denied = await requireBackups(auth, 'view'); const denied = await requireBackups(auth, 'view');
if (denied) return denied; if (denied) return denied;
@@ -30,13 +31,16 @@ export const GET: RequestHandler = async ({ url, cookies }) => {
// Narrow to THIS config's identity by its stable config-id tag so a sibling // Narrow to THIS config's identity by its stable config-id tag so a sibling
// config sharing the same destination + targetName isn't mixed in. // config sharing the same destination + targetName isn't mixed in.
if (allDests) { if (allDests) {
// Search ALL destinations for this config's snapshots // Search ALL destinations for this config's snapshots.
try { // Job-polling: `restic snapshots` across every destination is the slowest read here
// and a reverse proxy would abort the sync request at ~15s. The result stays a bare
// snapshot array (unchanged contract - many callers/tests treat it as an array); the
// old X-Incomplete-Destinations header is now log-only.
return jobResult(request, async () => {
const destinations = await getBackupDestinations(); const destinations = await getBackupDestinations();
// Bound each destination's listing so one slow or unreachable repo // Bound each destination's listing so one slow or unreachable repo
// can't stall the whole response for minutes (restic's own timeout is // can't stall the whole response for minutes (restic's own timeout is
// several minutes). A timed-out destination is treated as a failed // several minutes). A timed-out destination is treated as a failed one.
// one — surfaced as a partial result via X-Incomplete-Destinations.
const PER_DEST_MS = Number(process.env.SNAPSHOT_LIST_TIMEOUT_MS ?? 30_000); const PER_DEST_MS = Number(process.env.SNAPSHOT_LIST_TIMEOUT_MS ?? 30_000);
const withTimeout = <T>(p: Promise<T>, name: string): Promise<T> => const withTimeout = <T>(p: Promise<T>, name: string): Promise<T> =>
Promise.race([ Promise.race([
@@ -50,10 +54,8 @@ export const GET: RequestHandler = async ({ url, cookies }) => {
); );
const allSnapshots: any[] = []; const allSnapshots: any[] = [];
const seen = new Set<string>(); const seen = new Set<string>();
// (audit #29) Don't silently drop destinations that errored. Collect // (audit #29) Don't silently drop destinations that errored. Collect the
// the failures and surface them via a response header (keeping the body // failures and surface a count so a partial result doesn't look complete.
// a plain array for compatibility), plus a server log — so a partial
// result doesn't look complete.
const failed: string[] = []; const failed: string[] = [];
results.forEach((result, i) => { results.forEach((result, i) => {
if (result.status === 'fulfilled') { if (result.status === 'fulfilled') {
@@ -72,48 +74,33 @@ export const GET: RequestHandler = async ({ url, cookies }) => {
allSnapshots.sort((a, b) => new Date(b.time).getTime() - new Date(a.time).getTime()); allSnapshots.sort((a, b) => new Date(b.time).getTime() - new Date(a.time).getTime());
if (failed.length > 0) { if (failed.length > 0) {
console.warn(`[Backup] Snapshot search incomplete — ${failed.length} destination(s) failed: ${failed.join('; ')}`); console.warn(`[Backup] Snapshot search incomplete — ${failed.length} destination(s) failed: ${failed.join('; ')}`);
return json(allSnapshots, { headers: { 'X-Incomplete-Destinations': String(failed.length) } });
} }
return json(allSnapshots); return allSnapshots;
} catch (error) { });
const errorMsg = error instanceof Error ? error.message : String(error);
return json({ error: errorMsg }, { status: 500 });
}
} }
// Default: only search the config's current destination // Default: only search the config's current destination.
try { return jobResult(request, () => listSnapshots(config.destinationId, config.id));
const snapshots = await listSnapshots(config.destinationId, config.id);
return json(snapshots);
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
return json({ error: errorMsg }, { status: 500 });
}
} else if (destIdParam) { } else if (destIdParam) {
const destinationId = parseInt(destIdParam); const destinationId = parseInt(destIdParam);
if (isNaN(destinationId)) return json({ error: 'Invalid destinationId' }, { status: 400 }); if (isNaN(destinationId)) return json({ error: 'Invalid destinationId' }, { status: 400 });
try { // Job-polling: `restic snapshots` would otherwise be aborted at the proxy's ~15s cap.
// Returns a bare snapshot array (unchanged contract). The per-snapshot dockhand:instance=
// tag identifies the writing install; the client learns its OWN id from
// GET /api/backup/instance (the old X-Dockhand-Instance header can't ride job polling).
return jobResult(request, async () => {
// No target filter — return all snapshots in this destination (incl. snapshots // No target filter — return all snapshots in this destination (incl. snapshots
// from OTHER Dockhand instances sharing/copied into the repo). The response carries // from OTHER Dockhand instances sharing/copied into the repo).
// this install's instance id so the client can tell own snapshots from foreign
// orphans without relying on configid (which collides across instances - #1351).
const { getInstanceId } = await import('$lib/server/backups/identity');
const thisInstanceId = await getInstanceId();
const snapshots = await listSnapshots(destinationId); const snapshots = await listSnapshots(destinationId);
// (HIGH #8) On enterprise, a non-admin must not see snapshots belonging to // (HIGH #8) On enterprise, a non-admin must not see snapshots belonging to
// environments they can't access — the ids/paths alone are disclosure. Drop // environments they can't access — the ids/paths alone are disclosure. Drop
// any snapshot whose owning env (from its dockhand:env tag) isn't accessible. // any snapshot whose owning env (from its dockhand:env tag) isn't accessible.
const headers = { 'X-Dockhand-Instance': thisInstanceId };
if (auth.isEnterprise) { if (auth.isEnterprise) {
const filtered = await filterSnapshotsByEnvAccess(auth, snapshots); return await filterSnapshotsByEnvAccess(auth, snapshots);
return json(filtered, { headers });
} }
return json(snapshots, { headers }); return snapshots;
} catch (error) { });
const errorMsg = error instanceof Error ? error.message : String(error);
return json({ error: errorMsg }, { status: 500 });
}
} else { } else {
return json({ error: 'configId or destinationId parameter is required' }, { status: 400 }); return json({ error: 'configId or destinationId parameter is required' }, { status: 400 });
} }
@@ -5,8 +5,9 @@ import { authorize } from '$lib/server/authorize';
import { requireBackups } from '$lib/server/backups/route-guards'; import { requireBackups } from '$lib/server/backups/route-guards';
import { browseSnapshot } from '$lib/server/backups'; import { browseSnapshot } from '$lib/server/backups';
import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards'; import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
import { jobResult } from '$lib/server/sse';
export const GET: RequestHandler = async ({ params, url, cookies }) => { export const GET: RequestHandler = async ({ params, url, cookies, request }) => {
const auth = await authorize(cookies); const auth = await authorize(cookies);
const denied = await requireBackups(auth, 'view'); const denied = await requireBackups(auth, 'view');
if (denied) return denied; if (denied) return denied;
@@ -38,11 +39,10 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => {
return json({ error: 'Environment access denied' }, { status: 403 }); return json({ error: 'Environment access denied' }, { status: 403 });
} }
try { // Job-polling: `restic ls` behind a reverse proxy would abort at ~15s and SIGTERM restic
// (surfacing a misleading "wrong password / signal terminated"). Return {jobId} at once.
return jobResult(request, async () => {
const entries = await browseSnapshot(destinationId, snapshotId, path); const entries = await browseSnapshot(destinationId, snapshotId, path);
return json({ entries, path }); return { entries, path };
} catch (error) { });
const errorMsg = error instanceof Error ? error.message : String(error);
return json({ error: errorMsg }, { status: 500 });
}
}; };
@@ -6,8 +6,9 @@ import { requireBackups } from '$lib/server/backups/route-guards';
import { dumpSnapshotFile, dumpSnapshotFileBytes, dumpSnapshotArchive } from '$lib/server/backups'; import { dumpSnapshotFile, dumpSnapshotFileBytes, dumpSnapshotArchive } from '$lib/server/backups';
import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards'; import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
import { parseSnapshotLayout, redactSnapshotLayout } from '$lib/server/backups/snapshot-layout'; import { parseSnapshotLayout, redactSnapshotLayout } from '$lib/server/backups/snapshot-layout';
import { jobResult } from '$lib/server/sse';
export const GET: RequestHandler = async ({ params, url, cookies }) => { export const GET: RequestHandler = async ({ params, url, cookies, request }) => {
const auth = await authorize(cookies); const auth = await authorize(cookies);
const denied = await requireBackups(auth, 'view'); const denied = await requireBackups(auth, 'view');
if (denied) return denied; if (denied) return denied;
@@ -58,35 +59,35 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => {
if (download) { if (download) {
return json({ error: 'metadata.json cannot be downloaded raw; use the snapshot metadata endpoint (secrets are redacted there)' }, { status: 403 }); return json({ error: 'metadata.json cannot be downloaded raw; use the snapshot metadata endpoint (secrets are redacted there)' }, { status: 403 });
} }
try { // Job-polling: `restic dump` behind a reverse proxy would abort at ~15s.
return jobResult(request, async () => {
const raw = await dumpSnapshotFile(destinationId, snapshotId, '/metadata/metadata.json'); const raw = await dumpSnapshotFile(destinationId, snapshotId, '/metadata/metadata.json');
const layout = parseSnapshotLayout(raw); const layout = parseSnapshotLayout(raw);
if (!layout) return json({ error: 'metadata unreadable' }, { status: 404 }); if (!layout) return { error: 'metadata unreadable' };
return json({ content: JSON.stringify(redactSnapshotLayout(layout), null, 2) }); return { content: JSON.stringify(redactSnapshotLayout(layout), null, 2) };
} catch (error) { });
const errorMsg = error instanceof Error ? error.message : String(error);
return json({ error: errorMsg }, { status: 500 });
}
} }
// Sanitize filename for Content-Disposition (strip quotes, backslashes, control chars) // Sanitize filename for Content-Disposition (strip quotes, backslashes, control chars)
const sanitizeFilename = (name: string) => name.replace(/["\\\x00-\x1f]/g, '_'); const sanitizeFilename = (name: string) => name.replace(/["\\\x00-\x1f]/g, '_');
try { // Binary downloads are window.open navigations, not fetch()es — they can't poll a job,
if (download && isDir) { // so they stay synchronous. (A raw byte/tar stream also can't be JSON-wrapped.) These
// Binary tar stream — serve the raw bytes untouched (a UTF-8 round-trip // stream restic's output, so the proxy sees bytes flowing and won't idle-abort them.
// would corrupt any non-ASCII byte in the archive). if (download) {
const tarData = await dumpSnapshotArchive(destinationId, snapshotId, path); try {
const filename = sanitizeFilename((path.split('/').filter(Boolean).pop() || 'archive') + '.tar'); if (isDir) {
return new Response(new Uint8Array(tarData), { // Binary tar stream — serve the raw bytes untouched (a UTF-8 round-trip
headers: { // would corrupt any non-ASCII byte in the archive).
'Content-Type': 'application/x-tar', const tarData = await dumpSnapshotArchive(destinationId, snapshotId, path);
'Content-Disposition': `attachment; filename="${filename}"` const filename = sanitizeFilename((path.split('/').filter(Boolean).pop() || 'archive') + '.tar');
} return new Response(new Uint8Array(tarData), {
}); headers: {
} 'Content-Type': 'application/x-tar',
'Content-Disposition': `attachment; filename="${filename}"`
if (download) { }
});
}
// A file download may be binary — serve raw bytes, not a decoded string. // A file download may be binary — serve raw bytes, not a decoded string.
const bytes = await dumpSnapshotFileBytes(destinationId, snapshotId, path); const bytes = await dumpSnapshotFileBytes(destinationId, snapshotId, path);
const filename = sanitizeFilename(path.split('/').pop() || 'file'); const filename = sanitizeFilename(path.split('/').pop() || 'file');
@@ -96,13 +97,16 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => {
'Content-Disposition': `attachment; filename="${filename}"` 'Content-Disposition': `attachment; filename="${filename}"`
} }
}); });
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
return json({ error: errorMsg }, { status: 500 });
} }
// Inline preview — text only.
const content = await dumpSnapshotFile(destinationId, snapshotId, path);
return json({ content });
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
return json({ error: errorMsg }, { status: 500 });
} }
// Inline preview (text only) — job-polling: `restic dump` would otherwise be aborted
// at the reverse-proxy's ~15s cap.
return jobResult(request, async () => {
const content = await dumpSnapshotFile(destinationId, snapshotId, path);
return { content };
});
}; };
@@ -6,8 +6,9 @@ import { requireBackups } from '$lib/server/backups/route-guards';
import { getSnapshotMetadata } from '$lib/server/backups'; import { getSnapshotMetadata } from '$lib/server/backups';
import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards'; import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
import { redactSnapshotLayout } from '$lib/server/backups/snapshot-layout'; import { redactSnapshotLayout } from '$lib/server/backups/snapshot-layout';
import { jobResult } from '$lib/server/sse';
export const GET: RequestHandler = async ({ params, url, cookies }) => { export const GET: RequestHandler = async ({ params, url, cookies, request }) => {
const auth = await authorize(cookies); const auth = await authorize(cookies);
const denied = await requireBackups(auth, 'view'); const denied = await requireBackups(auth, 'view');
if (denied) return denied; if (denied) return denied;
@@ -26,14 +27,12 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => {
const envDenied = await guardSnapshotEnvAccess(auth, destinationId, snapshotId); const envDenied = await guardSnapshotEnvAccess(auth, destinationId, snapshotId);
if (envDenied) return envDenied; if (envDenied) return envDenied;
try { // Job-polling: reading metadata runs `restic dump` which a proxy would abort at ~15s.
return jobResult(request, async () => {
const metadata = await getSnapshotMetadata(destinationId, snapshotId); const metadata = await getSnapshotMetadata(destinationId, snapshotId);
if (!metadata) return json({ error: 'No metadata available' }, { status: 404 }); if (!metadata) return { error: 'No metadata available' };
// Single redaction point: strips stack.secrets values AND container inspect // Single redaction point: strips stack.secrets values AND container inspect
// Config.Env/Labels (plaintext secrets) before anything reaches the client. // Config.Env/Labels (plaintext secrets) before anything reaches the client.
return json(redactSnapshotLayout(metadata)); return redactSnapshotLayout(metadata);
} catch (error) { });
const msg = error instanceof Error ? error.message : String(error);
return json({ error: msg }, { status: 500 });
}
}; };
@@ -5,8 +5,9 @@ import { requireBackups } from '$lib/server/backups/route-guards';
import { diffSnapshots } from '$lib/server/backups'; import { diffSnapshots } from '$lib/server/backups';
import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards'; import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
import { validateSnapshotId } from '$lib/server/docker-validation'; import { validateSnapshotId } from '$lib/server/docker-validation';
import { jobResult } from '$lib/server/sse';
export const GET: RequestHandler = async ({ url, cookies }) => { export const GET: RequestHandler = async ({ url, cookies, request }) => {
const auth = await authorize(cookies); const auth = await authorize(cookies);
const denied = await requireBackups(auth, 'view'); const denied = await requireBackups(auth, 'view');
if (denied) return denied; if (denied) return denied;
@@ -31,11 +32,6 @@ export const GET: RequestHandler = async ({ url, cookies }) => {
const deniedB = await guardSnapshotEnvAccess(auth, destinationId, snapB); const deniedB = await guardSnapshotEnvAccess(auth, destinationId, snapB);
if (deniedB) return deniedB; if (deniedB) return deniedB;
try { // Job-polling: `restic diff` runs two restic reads a proxy would abort at ~15s.
const result = await diffSnapshots(destinationId, snapA, snapB); return jobResult(request, () => diffSnapshots(destinationId, snapA, snapB));
return json(result);
} catch (error) {
const msg = error instanceof Error ? error.message : String(error);
return json({ error: msg }, { status: 500 });
}
}; };
+29 -17
View File
@@ -25,7 +25,7 @@
import RotateCwFadingClock from '$lib/components/icons/RotateCwFadingClock.svelte'; import RotateCwFadingClock from '$lib/components/icons/RotateCwFadingClock.svelte';
import { formatDateTime, formatRelativeTime } from '$lib/stores/settings'; import { formatDateTime, formatRelativeTime } from '$lib/stores/settings';
import { currentEnvironment } from '$lib/stores/environment'; import { currentEnvironment } from '$lib/stores/environment';
import { watchJob } from '$lib/utils/sse-fetch'; import { watchJob, readJobResponse } from '$lib/utils/sse-fetch';
import { getRepoTypeIcon, formatCron, retentionSummary, classifyJobResult, tagLogLine } from '$lib/utils/backup'; import { getRepoTypeIcon, formatCron, retentionSummary, classifyJobResult, tagLogLine } from '$lib/utils/backup';
import SnapshotBrowser from '../containers/SnapshotBrowser.svelte'; import SnapshotBrowser from '../containers/SnapshotBrowser.svelte';
import RestoreModal from '../containers/RestoreModal.svelte'; import RestoreModal from '../containers/RestoreModal.svelte';
@@ -328,6 +328,14 @@
// one of these belongs to a shown config; anything else is orphaned. // one of these belongs to a shown config; anything else is orphaned.
const liveConfigIds = new Set(configs.map(c => c.id)); const liveConfigIds = new Set(configs.map(c => c.id));
// This install's id, fetched once: a snapshot is a LOCAL config's only when THIS
// instance wrote it (configid collides across installs - #1351). Formerly read from
// the snapshots list's X-Dockhand-Instance header, which job polling can't carry.
const thisInstance = await fetch('/api/backup/instance')
.then(r => r.ok ? r.json() : { instanceId: '' })
.then(d => d.instanceId || '')
.catch(() => '');
// Query each destination once, then split its snapshots two ways: PER CONFIG // Query each destination once, then split its snapshots two ways: PER CONFIG
// (by dockhand:configid) for the shown rows, and PER NAME for the orphan rows // (by dockhand:configid) for the shown rows, and PER NAME for the orphan rows
// (snapshots whose config was deleted). The right-column count must match the // (snapshots whose config was deleted). The right-column count must match the
@@ -340,13 +348,12 @@
// rest keep filling in counts progressively behind the scenes. // rest keep filling in counts progressively behind the scenes.
await Promise.allSettled(destinations.map(async (dest) => { await Promise.allSettled(destinations.map(async (dest) => {
try { try {
const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`); // Job-polling so a slow `restic snapshots` behind a proxy isn't aborted at ~15s.
if (!res.ok) return; const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`, {
// This install id: a snapshot belongs to a LOCAL config only when THIS instance wrote it. headers: { Accept: 'text/event-stream' }
// configid collides across instances (a fresh install reuses 1,2,3...), so a foreign });
// snapshot with configid=1 must be an orphan, not mis-bucketed into local config #1 (#1351). const d = await readJobResponse(res);
const thisInstance = res.headers.get("X-Dockhand-Instance") || ""; if (!d || d.error) return;
const d = await res.json();
const snaps: any[] = d.snapshots ?? d; const snaps: any[] = d.snapshots ?? d;
if (!Array.isArray(snaps)) return; if (!Array.isArray(snaps)) return;
@@ -406,16 +413,17 @@
loadingSnapshots = newLoading; loadingSnapshots = newLoading;
try { try {
// Query snapshots for this config's destination only (not allDestinations) // Query snapshots for this config's destination only (not allDestinations).
// Job-polling so a slow `restic snapshots` behind a proxy isn't aborted at ~15s.
const snapUrl = config.isOrphan const snapUrl = config.isOrphan
? `/api/backup/snapshots?destinationId=${config.destinationId}` ? `/api/backup/snapshots?destinationId=${config.destinationId}`
: `/api/backup/snapshots?configId=${config.id}`; : `/api/backup/snapshots?configId=${config.id}`;
const [snapRes, execRes] = await Promise.all([ const [snapJson, execRes] = await Promise.all([
fetch(snapUrl), fetch(snapUrl, { headers: { Accept: 'text/event-stream' } }).then(readJobResponse),
config.isOrphan ? Promise.resolve(new Response('{"executions":[]}')) : fetch(`/api/schedules/executions?scheduleType=backup&scheduleId=${config.id}&limit=50`) config.isOrphan ? Promise.resolve(new Response('{"executions":[]}')) : fetch(`/api/schedules/executions?scheduleType=backup&scheduleId=${config.id}&limit=50`)
]); ]);
const snapJson = await snapRes.json(); const snapError = snapJson?.error;
let snapData = snapJson.snapshots ?? snapJson; let snapData = snapJson?.snapshots ?? snapJson;
// For orphans, filter by target name (they can span destinations). // For orphans, filter by target name (they can span destinations).
if (config.isOrphan && Array.isArray(snapData)) { if (config.isOrphan && Array.isArray(snapData)) {
snapData = snapData.filter((s: any) => (s.tags || []).some((t: string) => t === `dockhand:name=${config.targetName}`)); snapData = snapData.filter((s: any) => (s.tags || []).some((t: string) => t === `dockhand:name=${config.targetName}`));
@@ -433,11 +441,11 @@
})); }));
} }
const newMap = new Map(snapshotsMap); const newMap = new Map(snapshotsMap);
if (snapRes.ok) { if (!snapError) {
newMap.set(key, Array.isArray(snapData) ? snapData : []); newMap.set(key, Array.isArray(snapData) ? snapData : []);
} else { } else {
newMap.set(key, []); newMap.set(key, []);
toast.error(cleanErrorMessage(snapData.error || 'Failed to load snapshots')); toast.error(cleanErrorMessage(snapError || 'Failed to load snapshots'));
} }
snapshotsMap = newMap; snapshotsMap = newMap;
@@ -784,13 +792,17 @@
<p class="text-sm text-muted-foreground">Configure backups on individual containers or stacks via their edit modal.</p> <p class="text-sm text-muted-foreground">Configure backups on individual containers or stacks via their edit modal.</p>
</div> </div>
{:else} {:else}
<!-- virtualScroll is OFF here on purpose: it assumes a fixed rowHeight, but an
expanded config renders a variable-height snapshot list, so the fixed-height
spacers mis-size the scroll area (flicker + endless scroll). Backup configs
number in the dozens, so plain rendering - the browser measures real heights -
is correct and fast. -->
<DataGrid <DataGrid
data={filteredConfigs} data={filteredConfigs}
keyField="key" keyField="key"
gridId="backups" gridId="backups"
loading={loading} loading={loading}
virtualScroll={true} virtualScroll={false}
rowHeight={33}
sortState={{ field: sortField, direction: sortDirection }} sortState={{ field: sortField, direction: sortDirection }}
onSortChange={(state) => { sortField = state.field; sortDirection = state.direction; }} onSortChange={(state) => { sortField = state.field; sortDirection = state.direction; }}
bind:expandedKeys={expandedKeys} bind:expandedKeys={expandedKeys}
+12 -12
View File
@@ -3,6 +3,7 @@
import { Badge } from '$lib/components/ui/badge'; import { Badge } from '$lib/components/ui/badge';
import { Button } from '$lib/components/ui/button'; import { Button } from '$lib/components/ui/button';
import { Plus, Minus, FileEdit, FileCheck, Loader2, ArrowRight, FolderOpen, Container, Info } from 'lucide-svelte'; import { Plus, Minus, FileEdit, FileCheck, Loader2, ArrowRight, FolderOpen, Container, Info } from 'lucide-svelte';
import { readJobResponse } from '$lib/utils/sse-fetch';
import { formatDateTime } from '$lib/stores/settings'; import { formatDateTime } from '$lib/stores/settings';
interface Props { interface Props {
@@ -43,13 +44,11 @@
metaA = null; metaA = null;
metaB = null; metaB = null;
try { try {
const res = await fetch(`/api/backup/snapshots/diff?destinationId=${destinationId}&snapshotA=${snapshotA.id}&snapshotB=${snapshotB.id}`); // Job-polling so the two `restic diff` reads behind a proxy aren't aborted at ~15s.
if (!res.ok) { const res = await fetch(`/api/backup/snapshots/diff?destinationId=${destinationId}&snapshotA=${snapshotA.id}&snapshotB=${snapshotB.id}`, { headers: { Accept: 'text/event-stream' } });
const data = await res.json(); const data = await readJobResponse(res);
error = data.error || `HTTP ${res.status}`; if (data?.error) { error = data.error; return; }
return; diff = data;
}
diff = await res.json();
} catch (err) { } catch (err) {
error = err instanceof Error ? err.message : String(err); error = err instanceof Error ? err.message : String(err);
} finally { } finally {
@@ -61,12 +60,13 @@
if (metaA && metaB) return; if (metaA && metaB) return;
metaLoading = true; metaLoading = true;
try { try {
const [resA, resB] = await Promise.all([ // Job-polling for both metadata dumps (slow `restic dump` behind a proxy).
fetch(`/api/backup/snapshots/${snapshotA.id}/metadata?destinationId=${destinationId}`), const [dataA, dataB] = await Promise.all([
fetch(`/api/backup/snapshots/${snapshotB.id}/metadata?destinationId=${destinationId}`) fetch(`/api/backup/snapshots/${snapshotA.id}/metadata?destinationId=${destinationId}`, { headers: { Accept: 'text/event-stream' } }).then(readJobResponse),
fetch(`/api/backup/snapshots/${snapshotB.id}/metadata?destinationId=${destinationId}`, { headers: { Accept: 'text/event-stream' } }).then(readJobResponse)
]); ]);
metaA = resA.ok ? await resA.json() : null; metaA = dataA?.error ? null : dataA;
metaB = resB.ok ? await resB.json() : null; metaB = dataB?.error ? null : dataB;
} catch {} } catch {}
metaLoading = false; metaLoading = false;
} }
+18 -8
View File
@@ -1,5 +1,6 @@
<script lang="ts"> <script lang="ts">
import { onMount, onDestroy } from 'svelte'; import { onMount, onDestroy } from 'svelte';
import { readJobResponse } from '$lib/utils/sse-fetch';
import { Button } from '$lib/components/ui/button'; import { Button } from '$lib/components/ui/button';
import * as Table from '$lib/components/ui/table'; import * as Table from '$lib/components/ui/table';
import * as Dialog from '$lib/components/ui/dialog'; import * as Dialog from '$lib/components/ui/dialog';
@@ -400,9 +401,15 @@
try { try {
let res: Response; let res: Response;
let data: any;
if (isSnapshotMode) { if (isSnapshotMode) {
const params = new URLSearchParams({ destinationId: String(destinationId), path: filePath }); const params = new URLSearchParams({ destinationId: String(destinationId), path: filePath });
res = await fetch(`/api/backup/snapshots/${snapshotId}/dump?${params}`); // Job-polling (readJobResponse) so a slow `restic dump` behind a proxy isn't aborted at ~15s.
res = await fetch(`/api/backup/snapshots/${snapshotId}/dump?${params}`, {
headers: { Accept: 'text/event-stream' }
});
data = await readJobResponse(res);
if (data?.error) throw new Error(data.error);
} else { } else {
const params = new URLSearchParams({ path: filePath }); const params = new URLSearchParams({ path: filePath });
if (envId) params.set('env', envId.toString()); if (envId) params.set('env', envId.toString());
@@ -411,11 +418,10 @@
} else { } else {
res = await fetch(`/api/containers/${effectiveContainerId}/files/content?${params}`); res = await fetch(`/api/containers/${effectiveContainerId}/files/content?${params}`);
} }
} data = await res.json();
const data = await res.json(); if (!res.ok) {
throw new Error(data.error || 'Failed to read file');
if (!res.ok) { }
throw new Error(data.error || 'Failed to read file');
} }
viewingFile = { viewingFile = {
@@ -740,8 +746,12 @@
if (isSnapshotMode) { if (isSnapshotMode) {
const snapshotParams = new URLSearchParams({ destinationId: String(destinationId), path }); const snapshotParams = new URLSearchParams({ destinationId: String(destinationId), path });
res = await fetch(`/api/backup/snapshots/${snapshotId}/browse?${snapshotParams}`); // Job-polling (readJobResponse) so a slow `restic ls` behind a proxy isn't aborted at ~15s.
data = await res.json(); res = await fetch(`/api/backup/snapshots/${snapshotId}/browse?${snapshotParams}`, { headers: { Accept: 'text/event-stream' } });
data = await readJobResponse(res);
// A jobified error comes back as { error } with HTTP 200, so surface it explicitly
// (the shared `if (!res.ok)` gate below can't see it).
if (data?.error) throw new Error(data.error);
} else if (isVolumeMode) { } else if (isVolumeMode) {
res = await fetch(`/api/volumes/${encodeURIComponent(volumeName!)}/browse?${params}`); res = await fetch(`/api/volumes/${encodeURIComponent(volumeName!)}/browse?${params}`);
data = await res.json(); data = await res.json();
+14 -11
View File
@@ -9,7 +9,7 @@
import { RotateCcw, AlertTriangle, Loader2, HardDrive, Folder, Clock, Play, CheckCircle2, XCircle, Server, PackagePlus, Ban, Rocket, Box, Layers, HelpCircle, Info, KeyRound, FileX } from 'lucide-svelte'; import { RotateCcw, AlertTriangle, Loader2, HardDrive, Folder, Clock, Play, CheckCircle2, XCircle, Server, PackagePlus, Ban, Rocket, Box, Layers, HelpCircle, Info, KeyRound, FileX } from 'lucide-svelte';
import * as Tooltip from '$lib/components/ui/tooltip'; import * as Tooltip from '$lib/components/ui/tooltip';
import { toast } from 'svelte-sonner'; import { toast } from 'svelte-sonner';
import { watchJob } from '$lib/utils/sse-fetch'; import { watchJob, readJobResponse } from '$lib/utils/sse-fetch';
import { environments } from '$lib/stores/environment'; import { environments } from '$lib/stores/environment';
import { formatDateTime } from '$lib/stores/settings'; import { formatDateTime } from '$lib/stores/settings';
import EnvironmentIcon from '$lib/components/EnvironmentIcon.svelte'; import EnvironmentIcon from '$lib/components/EnvironmentIcon.svelte';
@@ -346,16 +346,18 @@
loading = true; loading = true;
error = ''; error = '';
try { try {
// Job-polling (server returns {jobId}; readJobResponse polls to the result) so a slow
// restic verify/read behind a reverse proxy can't be aborted at ~15s.
const res = await fetch('/api/backup/restore/preview', { const res = await fetch('/api/backup/restore/preview', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json', Accept: 'text/event-stream' },
body: JSON.stringify({ destinationId, snapshotId, environmentId }) body: JSON.stringify({ destinationId, snapshotId, environmentId })
}); });
if (!res.ok) { const data = await readJobResponse(res);
error = (await res.json()).error || 'Failed to read the snapshot'; if (data?.error) {
error = data.error || 'Failed to read the snapshot';
return; return;
} }
const data = await res.json();
const types: Record<string, 'volume' | 'bind'> = data.volumeTypes || {}; const types: Record<string, 'volume' | 'bind'> = data.volumeTypes || {};
const sources: Record<string, string> = data.volumeSources || {}; const sources: Record<string, string> = data.volumeSources || {};
// Pre-fill each destination 1:1 from the snapshot's original location: a // Pre-fill each destination 1:1 from the snapshot's original location: a
@@ -398,9 +400,11 @@
const volumeDestinations = mode === 'new-location' const volumeDestinations = mode === 'new-location'
? selectedRows.map((v) => ({ volume: v.name, kind: v.destKind, target: v.dest.trim() })) ? selectedRows.map((v) => ({ volume: v.name, kind: v.destKind, target: v.dest.trim() }))
: undefined; : undefined;
// Job-polling: the target probe runs a helper container per volume on the target host
// (slowest read path) - a sync request would abort at the proxy's ~15s.
const res = await fetch('/api/backup/restore/preview', { const res = await fetch('/api/backup/restore/preview', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json', Accept: 'text/event-stream' },
body: JSON.stringify({ body: JSON.stringify({
destinationId, snapshotId, mode, environmentId: envId, destinationId, snapshotId, mode, environmentId: envId,
targetType: targetIsStack ? 'stack' : 'container', targetType: targetIsStack ? 'stack' : 'container',
@@ -409,14 +413,13 @@
volumes: selectedRows.map((v) => v.name), volumes: selectedRows.map((v) => v.name),
}) })
}); });
if (seq !== targetPreviewSeq) return; // a newer request superseded this one const data = await readJobResponse(res);
if (!res.ok) { if (seq !== targetPreviewSeq) return; // a newer request superseded this one (checked AFTER the poll)
if (data?.error) {
targetPreview = null; targetPreview = null;
const msg = await res.json().then((d) => d?.error).catch(() => null); targetPreviewError = data.error || 'Could not check the target paths on the environment.';
targetPreviewError = msg || 'Could not check the target paths on the environment.';
return; return;
} }
const data = await res.json();
targetPreview = data.targets ?? null; targetPreview = data.targets ?? null;
} catch { } catch {
if (seq === targetPreviewSeq) { targetPreview = null; targetPreviewError = 'Could not reach the server to check the target paths.'; } if (seq === targetPreviewSeq) { targetPreview = null; targetPreviewError = 'Could not reach the server to check the target paths.'; }
+12 -10
View File
@@ -1,5 +1,6 @@
<script lang="ts"> <script lang="ts">
import * as Dialog from '$lib/components/ui/dialog'; import * as Dialog from '$lib/components/ui/dialog';
import { readJobResponse } from '$lib/utils/sse-fetch';
import { Badge } from '$lib/components/ui/badge'; import { Badge } from '$lib/components/ui/badge';
import { import {
FolderOpen, Box, Layers, Network, HardDrive, Tag, Variable, FolderOpen, Box, Layers, Network, HardDrive, Tag, Variable,
@@ -68,9 +69,12 @@
destinationRepo = dest.repository || ''; destinationRepo = dest.repository || '';
} }
// Get snapshot time + owning environment from the snapshot's tags. // Get snapshot time + owning environment from the snapshot's tags.
const snapRes = await fetch(`/api/backup/snapshots?destinationId=${destinationId}`); // Job-polling so a slow `restic snapshots` behind a proxy isn't aborted at ~15s.
if (snapRes.ok) { const snapRes = await fetch(`/api/backup/snapshots?destinationId=${destinationId}`, {
const d = await snapRes.json(); headers: { Accept: 'text/event-stream' }
});
const d = await readJobResponse(snapRes);
if (d && !d.error) {
const snaps = d.snapshots ?? d; const snaps = d.snapshots ?? d;
const snap = (Array.isArray(snaps) ? snaps : []).find((s: any) => s.id === snapshotId || s.shortId === snapshotId?.slice(0, 8)); const snap = (Array.isArray(snaps) ? snaps : []).find((s: any) => s.id === snapshotId || s.shortId === snapshotId?.slice(0, 8));
if (snap) { if (snap) {
@@ -91,13 +95,11 @@
metadataLoading = true; metadataLoading = true;
metadataError = ''; metadataError = '';
try { try {
const res = await fetch(`/api/backup/snapshots/${snapshotId}/metadata?destinationId=${destinationId}`); // Job-polling so a slow `restic dump` behind a proxy isn't aborted at ~15s.
if (res.ok) { const res = await fetch(`/api/backup/snapshots/${snapshotId}/metadata?destinationId=${destinationId}`, { headers: { Accept: 'text/event-stream' } });
metadata = await res.json(); const data = await readJobResponse(res);
} else { if (data?.error) metadataError = data.error || 'No metadata available';
try { metadataError = (await res.json()).error || 'No metadata available'; } else metadata = data;
catch { metadataError = 'No metadata available'; }
}
} catch { metadataError = 'Failed to load metadata'; } } catch { metadataError = 'Failed to load metadata'; }
finally { metadataLoading = false; } finally { metadataLoading = false; }
} }
+7 -3
View File
@@ -6,6 +6,7 @@
import { LoadingState } from '$lib/components/ui/loading-state'; import { LoadingState } from '$lib/components/ui/loading-state';
import { formatDateTime, formatRelativeTime } from '$lib/stores/settings'; import { formatDateTime, formatRelativeTime } from '$lib/stores/settings';
import { formatBytes } from '$lib/utils/format'; import { formatBytes } from '$lib/utils/format';
import { readJobResponse } from '$lib/utils/sse-fetch';
import { getRepoTypeIcon } from '$lib/utils/backup'; import { getRepoTypeIcon } from '$lib/utils/backup';
import SnapshotBrowser from './SnapshotBrowser.svelte'; import SnapshotBrowser from './SnapshotBrowser.svelte';
import RestoreModal from './RestoreModal.svelte'; import RestoreModal from './RestoreModal.svelte';
@@ -111,9 +112,12 @@
// many (or one unreachable) repos the list fills in live rather than after a long wait. // many (or one unreachable) repos the list fills in live rather than after a long wait.
await Promise.all(destinations.map(async (dest) => { await Promise.all(destinations.map(async (dest) => {
try { try {
const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`); // Job-polling so a slow `restic snapshots` behind a proxy isn't aborted at ~15s.
if (!res.ok) return; const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`, {
const data = await res.json(); headers: { Accept: 'text/event-stream' }
});
const data = await readJobResponse(res);
if (data?.error) return;
const snaps: any[] = data.snapshots ?? data; const snaps: any[] = data.snapshots ?? data;
const mine = snaps const mine = snaps
.filter((s) => (s.tags || []).some((t: string) => t === nameTag)) .filter((s) => (s.tags || []).some((t: string) => t === nameTag))
+11 -7
View File
@@ -26,7 +26,7 @@
import { canAccess } from '$lib/stores/auth'; import { canAccess } from '$lib/stores/auth';
import DestinationModal from './DestinationModal.svelte'; import DestinationModal from './DestinationModal.svelte';
import BackupLogModal from '../../backups/BackupLogModal.svelte'; import BackupLogModal from '../../backups/BackupLogModal.svelte';
import { watchJob } from '$lib/utils/sse-fetch'; import { watchJob, readJobResponse } from '$lib/utils/sse-fetch';
import RotatePasswordModal from './RotatePasswordModal.svelte'; import RotatePasswordModal from './RotatePasswordModal.svelte';
import { EmptyState } from '$lib/components/ui/empty-state'; import { EmptyState } from '$lib/components/ui/empty-state';
@@ -370,13 +370,17 @@
browseOpen = true; browseOpen = true;
browseLoading = true; browseLoading = true;
try { try {
// List all snapshots in this destination (no configId filter) // List all snapshots in this destination (no configId filter). Job-polling so a
const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`); // slow `restic snapshots` behind a proxy isn't aborted at ~15s.
if (res.ok) { const d = await res.json(); browseSnapshots = d.snapshots ?? d; } const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`, {
else { headers: { Accept: 'text/event-stream' }
const data = await res.json(); });
const errMsg = data.error || 'Failed to list snapshots'; const d = await readJobResponse(res);
if (d?.error) {
const errMsg = d.error || 'Failed to list snapshots';
try { const p = JSON.parse(errMsg); toast.error(p.message || errMsg); } catch { toast.error(errMsg); } try { const p = JSON.parse(errMsg); toast.error(p.message || errMsg); } catch { toast.error(errMsg); }
} else {
browseSnapshots = d.snapshots ?? d;
} }
} catch { toast.error('Failed to list snapshots'); } } catch { toast.error('Failed to list snapshots'); }
finally { browseLoading = false; } finally { browseLoading = false; }