improved restic repository operations log (prune, check, repair index etc.)

This commit is contained in:
jarek committed 2026-08-09 10:31:28 +02:00
1 parent 89b260758e
commit 16451d95d7
3 files changed
+224 -85

No files matched your search

+39 -17
View File
@@ -1,5 +1,5 @@
/**
* backups/repo.ts — repository lifecycle operations that talk only to the restic
* backups/repo.ts - repository lifecycle operations that talk only to the restic
* repo (no container volumes): initialise, check integrity, prune, unlock, and a
* reachability test. Each is a thin, explicit wrapper over the injected Restic
* runner that turns restic's exit code + output into a structured result.
@@ -14,7 +14,26 @@ import { cleanErrorMsg, timingSafeStrEqual } from './security';
/** The restic surface repo.ts needs (injected). */
export interface ResticLocal {
runLocal(destination: any, args: string[], tier?: 'interactive' | 'data'): Promise<ResticRun>;
runLocal(
destination: any,
args: string[],
tier?: 'interactive' | 'data',
stream?: { onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void }
): Promise<ResticRun>;
}
/** Live-output callback for a repo op: each restic line as it arrives. Optional -
* omit it and the op runs with buffered output only. */
export type OnProgress = (line: string) => void;
/** Split a raw restic chunk into lines and forward each. Kept local so repo.ts stays
* dependency-free; mirrors formatResticLines' line-splitting for the maintenance ops. */
function forwardChunk(onProgress: OnProgress | undefined, chunk: string): void {
if (!onProgress) return;
for (const line of chunk.split(/\r?\n/)) {
const t = line.trim();
if (t) onProgress(t);
}
}
/** A repo operation result: ok + human output, or a classified failure. */
@@ -69,39 +88,42 @@ export async function testRepository(restic: ResticLocal, destination: any): Pro
/** Integrity check. `readDataSubset` (e.g. '5%') re-reads pack data to prove
* restorability; omit for a structure-only check. Runs on the long timeout tier
* because a data check on a large cloud repo far exceeds the interactive limit. */
export async function checkRepository(restic: ResticLocal, destination: any, readDataSubset?: string): Promise<RepoResult> {
export async function checkRepository(restic: ResticLocal, destination: any, readDataSubset?: string, onProgress?: OnProgress): Promise<RepoResult> {
const args = ['check', '--no-lock'];
if (readDataSubset) args.push('--read-data-subset', readDataSubset);
const run = await restic.runLocal(destination, args, 'data');
const stream = onProgress && { onStdout: (c: string) => forwardChunk(onProgress, c), onStderr: (c: string) => forwardChunk(onProgress, c) };
const run = await restic.runLocal(destination, args, 'data', stream || undefined);
return toResult(run);
}
/** Prune the repository (reclaim unreferenced data). Long timeout tier. */
export async function pruneRepository(restic: ResticLocal, destination: any, maxUnused?: string): Promise<RepoResult> {
export async function pruneRepository(restic: ResticLocal, destination: any, maxUnused?: string, onProgress?: OnProgress): Promise<RepoResult> {
const args = ['prune', '--retry-lock', '5m'];
if (maxUnused) args.push('--max-unused', maxUnused);
const run = await restic.runLocal(destination, args, 'data');
const stream = onProgress && { onStdout: (c: string) => forwardChunk(onProgress, c), onStderr: (c: string) => forwardChunk(onProgress, c) };
const run = await restic.runLocal(destination, args, 'data', stream || undefined);
return toResult(run);
}
/**
* Remove repository locks left by a crashed operation.
*
* `removeAll` (default true) uses `restic unlock --remove-all` — clears EVERY lock,
* `removeAll` (default true) uses `restic unlock --remove-all` - clears EVERY lock,
* including one it cannot prove stale (an orphan whose helper-container hostname restic
* won't age out for 30 min). That is what the EXPLICIT user "Unlock" button needs, and
* it is safe only when no other op runs against the repo — the operator's call.
* it is safe only when no other op runs against the repo - the operator's call.
*
* `removeAll = false` is a plain `restic unlock`: it reaps ONLY locks restic can prove
* stale and LEAVES a live foreign lock intact. AUTOMATIC callers (scheduled maintenance
* auto-unlock) MUST use this — the per-repo serializer is per-instance and cannot see a
* auto-unlock) MUST use this - the per-repo serializer is per-instance and cannot see a
* separate instance's live lock on a shared repo, so a blind --remove-all there would
* silently wipe another instance's in-flight backup lock (same hazard backup-service
* deliberately avoids). #1313-adjacent shared-repo data-safety fix.
* silently wipe another instance's in-flight backup lock. #1313-adjacent shared-repo
* data-safety fix.
*/
export async function unlockRepository(restic: ResticLocal, destination: any, removeAll = true): Promise<RepoResult> {
export async function unlockRepository(restic: ResticLocal, destination: any, removeAll = true, onProgress?: OnProgress): Promise<RepoResult> {
const args = removeAll ? ['unlock', '--remove-all'] : ['unlock'];
const run = await restic.runLocal(destination, args);
const stream = onProgress && { onStdout: (c: string) => forwardChunk(onProgress, c), onStderr: (c: string) => forwardChunk(onProgress, c) };
const run = await restic.runLocal(destination, args, 'interactive', stream || undefined);
return toResult(run);
}
@@ -122,7 +144,7 @@ export type RotateResult =
| { ok: true }
| { ok: false; error: string; dbOutOfSync?: boolean };
/** A decrypted destination — carries the plaintext current password for the
/** A decrypted destination - carries the plaintext current password for the
* constant-time pre-check. */
interface DecryptedDestination {
id: number;
@@ -149,12 +171,12 @@ export interface RotatePorts {
* 1. Reject a weak or unchanged new password before doing anything.
* 2. Constant-time compare the supplied current password against the stored one,
* so response timing can't leak how much of a guess is right.
* 3. Run `restic key passwd` with the new password in a 0600 tmpfile — never on
* 3. Run `restic key passwd` with the new password in a 0600 tmpfile - never on
* argv, which leaks via /proc/<pid>/cmdline. The tmpfile is shredded on every
* exit path. The key change is serialized on the destination.
* 4. Persist the new password. If restic already swapped the key (step 3
* succeeded) but this write fails, the repo expects the new password and the
* DB still holds the old one — a critical out-of-sync state we flag distinctly
* DB still holds the old one - a critical out-of-sync state we flag distinctly
* so the caller can prompt the operator to fix it.
*/
export async function rotateDestinationPassword(
@@ -197,7 +219,7 @@ export async function rotateDestinationPassword(
}
// restic accepted the new password and removed the old key. From now on the
// repo only accepts newPassword — persist it immediately.
// repo only accepts newPassword - persist it immediately.
try {
await ports.updatePassword(destinationId, newPassword);
} catch (err) {
@@ -1,16 +1,17 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { authorize } from '$lib/server/authorize';
import { requireBackups } from '$lib/server/backups/route-guards';
import { auditBackupDestination } from '$lib/server/audit';
import { getBackupDestination, getBackupConfigs } from '$lib/server/db';
import { runRepoTask } from '$lib/server/backups';
import { createJobResponse } from '$lib/server/sse';
export const POST: RequestHandler = async (event) => {
const { params, request, cookies } = event;
const auth = await authorize(cookies);
if (auth.authEnabled && !await auth.can('backups', 'manage')) {
return json({ error: 'Permission denied' }, { status: 403 });
}
const denied = await requireBackups(auth, 'manage');
if (denied) return denied;
const destId = parseInt(params.id);
if (isNaN(destId)) return json({ error: 'Invalid destination ID' }, { status: 400 });
@@ -41,20 +42,33 @@ export const POST: RequestHandler = async (event) => {
}
}
try {
// `task` is validated against the allowed list above.
const result = await runRepoTask(destId, task as import('$lib/server/backups').RepoTask);
// 'prune' and 'check' map to canonical audit actions; the rest land on
// 'update' so they're still recoverable from the log.
const action = task === 'prune' ? 'prune' : task === 'check' ? 'verify' : 'update';
// 'prune' and 'check' map to canonical audit actions; the rest land on
// 'update' so they're still recoverable from the log.
const action = task === 'prune' ? 'prune' : task === 'check' ? 'verify' : 'update';
// Stream restic output live into the shared job/log modal (same channel as
// backup/restore). Guards above (auth, env-scope, allowlist) already ran; only
// the restic run is wrapped. `createJobResponse` keeps the backward-compat
// synchronous JSON path for `Accept: application/json` callers (tests/CLI).
return createJobResponse(async (send) => {
let result: Awaited<ReturnType<typeof runRepoTask>>;
try {
// `task` is validated against the allowed list above.
result = await runRepoTask(destId, task as import('$lib/server/backups').RepoTask, {
onProgress: (message) => send('progress', { message })
});
} catch (error) {
// A genuine throw (unexpected). Audit as failed, then rethrow so the job
// ends in 'error' state.
const msg = error instanceof Error ? error.message : String(error);
await auditBackupDestination(event, 'update', destId, dest.name, { task, success: false, error: msg });
throw error instanceof Error ? error : new Error(msg);
}
// runRepoTask can resolve with { success: false, error } WITHOUT throwing
// (e.g. repo unreachable / needs init). Derive the audited success and
// error from the result rather than hardcoding success (audit #53).
// (e.g. repo unreachable / needs init). Derive the audited success/error
// from the result rather than hardcoding success (audit #53).
await auditBackupDestination(event, action, destId, dest.name, { task, success: result.success, error: result.error });
return json(result, { status: result.success ? 200 : 500 });
} catch (error) {
const msg = error instanceof Error ? error.message : String(error);
await auditBackupDestination(event, 'update', destId, dest.name, { task, success: false, error: msg });
return json({ error: msg }, { status: 500 });
}
send('result', result);
if (!result.success) throw new Error(result.error || `${task} failed`);
}, request);
};
+154 -51
View File
@@ -4,6 +4,7 @@
import { Button } from '$lib/components/ui/button';
import { Badge } from '$lib/components/ui/badge';
import { Input } from '$lib/components/ui/input';
import * as Select from '$lib/components/ui/select';
import { DataGrid } from '$lib/components/data-grid';
import { Plus, Trash2, Pencil, HardDrive, Server, CheckCircle, XCircle, AlertCircle, Wifi, Database, RefreshCw, Search, FolderSync, Archive, Loader2, Save, CircleHelp, Unlock, PackageCheck, Eraser, BarChart3, Wrench, FolderCheck, KeyRound } from 'lucide-svelte';
import * as Tooltip from '$lib/components/ui/tooltip';
@@ -18,12 +19,14 @@
import { LoadingState } from '$lib/components/ui/loading-state';
import { getRepoTypeIcon, getRepoTypeLabel } from '$lib/utils/backup';
import { shouldSaveBackupImage } from '$lib/utils/backup-image';
import { summarizeTestResults, formatTestSummary, type TestOutcome } from '$lib/utils/backup-test-summary';
import SnapshotBrowser from '../../containers/SnapshotBrowser.svelte';
import type { Component } from 'svelte';
import ConfirmPopover from '$lib/components/ConfirmPopover.svelte';
import { canAccess } from '$lib/stores/auth';
import DestinationModal from './DestinationModal.svelte';
import VerifyModal from './VerifyModal.svelte';
import BackupLogModal from '../../backups/BackupLogModal.svelte';
import { watchJob } from '$lib/utils/sse-fetch';
import RotatePasswordModal from './RotatePasswordModal.svelte';
import { EmptyState } from '$lib/components/ui/empty-state';
@@ -56,12 +59,21 @@
let confirmDeleteId = $state<number | null>(null);
let confirmAction = $state<{ destId: number; task: string } | null>(null);
let searchQuery = $state('');
// Sort + status filter for the destinations grid.
let destSort = $state<{ field: string; direction: 'asc' | 'desc' }>({ field: 'name', direction: 'asc' });
let statusFilter = $state<'all' | 'success' | 'needs_init' | 'failed'>('all');
let testingId = $state<number | null>(null);
let testingAll = $state(false);
let verifyModalOpen = $state(false);
let verifyDestId = $state(0);
let verifyDestName = $state('');
// Shared live-log modal for every restic maintenance action (verify/check/prune/
// repair/unlock) — the same BackupLogModal used by backup/restore, so all
// destination actions share one elegant log UI instead of a raw toast dump.
let actionLogOpen = $state(false);
let actionLogTitle = $state('');
let actionLogIcon = $state<any>(undefined);
let actionLogStatus = $state<'running' | 'success' | 'error'>('running');
let actionLogLogs = $state<string[]>([]);
let actionLogError = $state('');
let rotateModalOpen = $state(false);
let rotateDestId = $state(0);
let rotateDestName = $state('');
@@ -97,8 +109,6 @@
async function testAllDestinations() {
testingAll = true;
let passed = 0;
let failed = 0;
// Clear statuses so UI shows spinners
for (const dest of destinations) {
@@ -107,32 +117,33 @@
destinations = [...destinations];
repoStats = new Map();
// Run all tests + stats in parallel, update UI incrementally
await Promise.allSettled(destinations.map(async (dest) => {
// Run all tests + stats in parallel, update UI incrementally. Collect the
// three-way outcome per destination; a needs-init repo is NOT a failure.
const outcomes = await Promise.all(destinations.map(async (dest): Promise<TestOutcome> => {
try {
const res = await fetch(`/api/backup/destinations/${dest.id}/test`, { method: 'POST' });
const data = await res.json();
if (data.success) {
passed++;
dest.lastTestStatus = 'success';
destinations = [...destinations];
await fetchRepoStats(dest.id);
} else {
failed++;
dest.lastTestStatus = data.status === 'needs_init' ? 'needs_init' : 'failed';
dest.lastTestError = data.error || null;
destinations = [...destinations];
return 'success';
}
const status: TestOutcome = data.status === 'needs_init' ? 'needs_init' : 'failed';
dest.lastTestStatus = status;
dest.lastTestError = data.error || null;
destinations = [...destinations];
return status;
} catch {
failed++;
dest.lastTestStatus = 'failed';
destinations = [...destinations];
return 'failed';
}
}));
testingAll = false;
if (failed === 0) toast.success(`All ${passed} destinations tested & stats collected`);
else toast.error(`${failed} failed, ${passed} passed`);
const summary = formatTestSummary(summarizeTestResults(outcomes));
toast[summary.severity](summary.text);
}
// Backup helper image setting
@@ -173,29 +184,90 @@
let initializingId = $state<number | null>(null);
let runningTask = $state<{ destId: number; task: string } | null>(null);
const REPO_TASK_TITLES: Record<string, string> = {
unlock: 'Unlock repository',
prune: 'Prune unused data',
check: 'Check integrity',
'repair-index': 'Repair index',
'repair-snapshots': 'Repair snapshots'
};
async function runRepoTask(destId: number, task: string) {
runningTask = { destId, task };
try {
const res = await fetch(`/api/backup/destinations/${destId}/task`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ task })
});
const data = await res.json();
if (data.success) {
if (task === 'stats' && data.stats) {
// `stats` is a silent read that only populates the cells — no log modal.
if (task === 'stats') {
try {
const res = await fetch(`/api/backup/destinations/${destId}/task`, {
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ task })
});
const data = await res.json();
if (data.success && data.stats) {
const m = new Map(repoStats);
m.set(destId, data.stats);
repoStats = m;
} else if (!data.success) {
toast.error(data.error || 'Stats failed');
}
toast.success(data.output || `${task} completed`);
} catch (err: any) {
toast.error(err.message || 'Stats failed');
} finally {
runningTask = null;
}
return;
}
const dest = destinations.find((d) => d.id === destId);
await streamRepoAction(
`${REPO_TASK_TITLES[task] || task}${dest ? ` - ${dest.name}` : ''}`,
`/api/backup/destinations/${destId}/task`,
{ task },
dest?.repository
);
runningTask = null;
}
function verifyDestination(dest: any) {
streamRepoAction(`Verify data integrity - ${dest.name}`, `/api/backup/destinations/${dest.id}/verify`, {}, dest.repository);
}
/** Run a restic maintenance action, streaming its output live into the shared
* BackupLogModal (identical UX to backup/restore logs). */
async function streamRepoAction(title: string, url: string, body: Record<string, unknown>, repository?: string) {
actionLogTitle = title;
actionLogIcon = repository ? getRepoTypeIcon(repository) : undefined;
actionLogStatus = 'running';
actionLogLogs = [];
actionLogError = '';
actionLogOpen = true;
try {
const res = await fetch(url, {
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body)
});
const data = await res.json();
if (data.jobId) {
const result = await watchJob(data.jobId, (line) => {
const d = line.data as any;
if (line.event === 'progress' && d?.message) {
actionLogLogs = [...actionLogLogs, d.message];
}
}) as any;
if (result?.success === false) {
actionLogStatus = 'error';
actionLogError = result.error || 'Action failed';
} else {
actionLogStatus = 'success';
}
} else if (data.error) {
actionLogStatus = 'error';
actionLogError = data.error;
} else {
toast.error(data.error || `${task} failed`);
// Backward-compat JSON path (Accept negotiated to one-shot): show result.
actionLogStatus = data.success ? 'success' : 'error';
if (!data.success) actionLogError = data.error || 'Action failed';
}
} catch (err: any) {
toast.error(err.message || `${task} failed`);
} finally {
runningTask = null;
actionLogStatus = 'error';
actionLogError = err.message || 'Action failed';
}
}
@@ -321,15 +393,31 @@
};
}
const filteredDestinations = $derived(
searchQuery.trim()
? destinations.filter(d =>
d.name.toLowerCase().includes(searchQuery.toLowerCase()) ||
d.repository.toLowerCase().includes(searchQuery.toLowerCase()) ||
getTypeLabel(d.repository).toLowerCase().includes(searchQuery.toLowerCase())
)
: destinations
);
const filteredDestinations = $derived.by(() => {
const q = searchQuery.trim().toLowerCase();
let rows = destinations.filter(d => {
if (q && !(
d.name.toLowerCase().includes(q) ||
d.repository.toLowerCase().includes(q) ||
getTypeLabel(d.repository).toLowerCase().includes(q)
)) return false;
if (statusFilter !== 'all' && (d.lastTestStatus ?? '') !== statusFilter) return false;
return true;
});
// Sort key per column. 'status' orders by a fixed rank so like statuses group.
const statusRank: Record<string, number> = { success: 0, needs_init: 1, failed: 2, '': 3 };
const key = (d: Destination) =>
destSort.field === 'type' ? getTypeLabel(d.repository) :
destSort.field === 'repository' ? d.repository :
destSort.field === 'status' ? String(statusRank[d.lastTestStatus ?? ''] ?? 3) :
d.name;
rows = [...rows].sort((a, b) => {
const ka = key(a).toLowerCase(), kb = key(b).toLowerCase();
const cmp = ka < kb ? -1 : ka > kb ? 1 : 0;
return destSort.direction === 'asc' ? cmp : -cmp;
});
return rows;
});
async function fetchData() {
loading = true;
@@ -449,6 +537,17 @@
class="pl-9 h-8 w-64 text-sm"
/>
</div>
<Select.Root type="single" value={statusFilter} onValueChange={(v) => statusFilter = (v as typeof statusFilter) ?? 'all'}>
<Select.Trigger class="h-8 w-40 text-sm">
{statusFilter === 'success' ? 'Initialized' : statusFilter === 'needs_init' ? 'Needs init' : statusFilter === 'failed' ? 'Failed' : 'All statuses'}
</Select.Trigger>
<Select.Content>
<Select.Item value="all">All statuses</Select.Item>
<Select.Item value="success"><CheckCircle class="w-3.5 h-3.5 text-green-500 mr-1.5 inline" />Initialized</Select.Item>
<Select.Item value="needs_init"><AlertCircle class="w-3.5 h-3.5 text-amber-500 mr-1.5 inline" />Needs init</Select.Item>
<Select.Item value="failed"><XCircle class="w-3.5 h-3.5 text-destructive mr-1.5 inline" />Failed</Select.Item>
</Select.Content>
</Select.Root>
<Badge variant="secondary" class="text-xs">{destinations.length} destination{destinations.length !== 1 ? 's' : ''}</Badge>
</div>
<div class="flex gap-2">
@@ -483,15 +582,16 @@
gridId="backupDestinations"
loading={loading}
onRowClick={(dest) => openModal(dest)}
onSortChange={(s) => destSort = { field: s.field, direction: s.direction }}
class="border-none"
wrapperClass="border rounded-lg"
>
{#snippet cell(column, dest)}
{#if column.id === 'type'}
{@const TypeIcon = getTypeIcon(dest.repository)}
<div class="flex items-center gap-1.5" title={getTypeLabel(dest.repository)}>
<TypeIcon class="w-4 h-4 text-muted-foreground" />
<span class="text-xs text-muted-foreground">{getTypeLabel(dest.repository)}</span>
<div class="flex items-center gap-1.5 min-w-0" title={getTypeLabel(dest.repository)}>
<TypeIcon class="w-4 h-4 shrink-0 text-muted-foreground" />
<span class="text-xs text-muted-foreground truncate">{getTypeLabel(dest.repository)}</span>
</div>
{:else if column.id === 'name'}
<span class="font-medium text-sm">{dest.name}</span>
@@ -575,12 +675,12 @@
<button type="button" class="p-0.5 rounded hover:bg-muted transition-colors opacity-70 hover:opacity-100" onclick={() => runRepoTask(dest.id, 'check')} disabled={runningTask?.destId === dest.id} title="Check integrity">
{#if runningTask?.destId === dest.id && runningTask.task === 'check'}<Loader2 class="grid-action-icon text-muted-foreground animate-spin" />{:else}<PackageCheck class="grid-action-icon grid-action-info text-muted-foreground" />{/if}
</button>
<button type="button" class="p-0.5 rounded hover:bg-muted transition-colors opacity-70 hover:opacity-100" onclick={() => { verifyDestId = dest.id; verifyDestName = dest.name; verifyModalOpen = true; }} title="Verify data integrity">
<button type="button" class="p-0.5 rounded hover:bg-muted transition-colors opacity-70 hover:opacity-100" onclick={() => verifyDestination(dest)} title="Verify data integrity">
<FolderCheck class="grid-action-icon grid-action-info text-muted-foreground" />
</button>
<ConfirmPopover
open={confirmAction?.destId === dest.id && confirmAction.task === 'unlock'}
action="Unlock" itemType="repository" itemName={dest.name} confirmText="Unlock" position="left"
action="Unlock" itemType="repository" itemName={dest.name} confirmText="Unlock" position="left" title="Unlock repository"
onConfirm={() => { confirmAction = null; runRepoTask(dest.id, 'unlock'); }}
onOpenChange={(o) => confirmAction = o ? { destId: dest.id, task: 'unlock' } : null}
>
@@ -590,7 +690,7 @@
</ConfirmPopover>
<ConfirmPopover
open={confirmAction?.destId === dest.id && confirmAction.task === 'prune'}
action="Prune" itemType="unused data from" itemName={dest.name} confirmText="Prune" variant="destructive" position="left"
action="Prune" itemType="unused data from" itemName={dest.name} confirmText="Prune" variant="destructive" position="left" title="Prune unused data"
onConfirm={() => { confirmAction = null; runRepoTask(dest.id, 'prune'); }}
onOpenChange={(o) => confirmAction = o ? { destId: dest.id, task: 'prune' } : null}
>
@@ -600,7 +700,7 @@
</ConfirmPopover>
<ConfirmPopover
open={confirmAction?.destId === dest.id && confirmAction.task === 'repair'}
action="Repair" itemType="index for" itemName={dest.name} confirmText="Repair" variant="destructive" position="left"
action="Repair" itemType="index for" itemName={dest.name} confirmText="Repair" variant="destructive" position="left" title="Repair index"
onConfirm={() => { confirmAction = null; runRepoTask(dest.id, 'repair-index'); }}
onOpenChange={(o) => confirmAction = o ? { destId: dest.id, task: 'repair' } : null}
>
@@ -775,10 +875,13 @@
targetName={snapshotBrowseName}
/>
<VerifyModal
bind:open={verifyModalOpen}
destinationId={verifyDestId}
destinationName={verifyDestName}
<BackupLogModal
bind:open={actionLogOpen}
title={actionLogTitle}
icon={actionLogIcon}
status={actionLogStatus}
logs={actionLogLogs}
error={actionLogError}
/>
<RotatePasswordModal