From 16451d95d7006508e82a08783b02f7c74527b8b3 Mon Sep 17 00:00:00 2001 From: jarek Date: Sun, 9 Aug 2026 10:31:28 +0200 Subject: [PATCH] improved restic repository operations log (prune, check, repair index etc.) --- src/lib/server/backups/repo.ts | 56 +++-- .../backup/destinations/[id]/task/+server.ts | 48 ++-- src/routes/settings/backups/BackupsTab.svelte | 205 +++++++++++++----- 3 files changed, 224 insertions(+), 85 deletions(-) diff --git a/src/lib/server/backups/repo.ts b/src/lib/server/backups/repo.ts index 4a74f58..76d4409 100644 --- a/src/lib/server/backups/repo.ts +++ b/src/lib/server/backups/repo.ts @@ -1,5 +1,5 @@ /** - * backups/repo.ts — repository lifecycle operations that talk only to the restic + * backups/repo.ts - repository lifecycle operations that talk only to the restic * repo (no container volumes): initialise, check integrity, prune, unlock, and a * reachability test. Each is a thin, explicit wrapper over the injected Restic * runner that turns restic's exit code + output into a structured result. @@ -14,7 +14,26 @@ import { cleanErrorMsg, timingSafeStrEqual } from './security'; /** The restic surface repo.ts needs (injected). */ export interface ResticLocal { - runLocal(destination: any, args: string[], tier?: 'interactive' | 'data'): Promise; + runLocal( + destination: any, + args: string[], + tier?: 'interactive' | 'data', + stream?: { onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void } + ): Promise; +} + +/** Live-output callback for a repo op: each restic line as it arrives. Optional - + * omit it and the op runs with buffered output only. */ +export type OnProgress = (line: string) => void; + +/** Split a raw restic chunk into lines and forward each. Kept local so repo.ts stays + * dependency-free; mirrors formatResticLines' line-splitting for the maintenance ops. */ +function forwardChunk(onProgress: OnProgress | undefined, chunk: string): void { + if (!onProgress) return; + for (const line of chunk.split(/\r?\n/)) { + const t = line.trim(); + if (t) onProgress(t); + } } /** A repo operation result: ok + human output, or a classified failure. */ @@ -69,39 +88,42 @@ export async function testRepository(restic: ResticLocal, destination: any): Pro /** Integrity check. `readDataSubset` (e.g. '5%') re-reads pack data to prove * restorability; omit for a structure-only check. Runs on the long timeout tier * because a data check on a large cloud repo far exceeds the interactive limit. */ -export async function checkRepository(restic: ResticLocal, destination: any, readDataSubset?: string): Promise { +export async function checkRepository(restic: ResticLocal, destination: any, readDataSubset?: string, onProgress?: OnProgress): Promise { const args = ['check', '--no-lock']; if (readDataSubset) args.push('--read-data-subset', readDataSubset); - const run = await restic.runLocal(destination, args, 'data'); + const stream = onProgress && { onStdout: (c: string) => forwardChunk(onProgress, c), onStderr: (c: string) => forwardChunk(onProgress, c) }; + const run = await restic.runLocal(destination, args, 'data', stream || undefined); return toResult(run); } /** Prune the repository (reclaim unreferenced data). Long timeout tier. */ -export async function pruneRepository(restic: ResticLocal, destination: any, maxUnused?: string): Promise { +export async function pruneRepository(restic: ResticLocal, destination: any, maxUnused?: string, onProgress?: OnProgress): Promise { const args = ['prune', '--retry-lock', '5m']; if (maxUnused) args.push('--max-unused', maxUnused); - const run = await restic.runLocal(destination, args, 'data'); + const stream = onProgress && { onStdout: (c: string) => forwardChunk(onProgress, c), onStderr: (c: string) => forwardChunk(onProgress, c) }; + const run = await restic.runLocal(destination, args, 'data', stream || undefined); return toResult(run); } /** * Remove repository locks left by a crashed operation. * - * `removeAll` (default true) uses `restic unlock --remove-all` — clears EVERY lock, + * `removeAll` (default true) uses `restic unlock --remove-all` - clears EVERY lock, * including one it cannot prove stale (an orphan whose helper-container hostname restic * won't age out for 30 min). That is what the EXPLICIT user "Unlock" button needs, and - * it is safe only when no other op runs against the repo — the operator's call. + * it is safe only when no other op runs against the repo - the operator's call. * * `removeAll = false` is a plain `restic unlock`: it reaps ONLY locks restic can prove * stale and LEAVES a live foreign lock intact. AUTOMATIC callers (scheduled maintenance - * auto-unlock) MUST use this — the per-repo serializer is per-instance and cannot see a + * auto-unlock) MUST use this - the per-repo serializer is per-instance and cannot see a * separate instance's live lock on a shared repo, so a blind --remove-all there would - * silently wipe another instance's in-flight backup lock (same hazard backup-service - * deliberately avoids). #1313-adjacent shared-repo data-safety fix. + * silently wipe another instance's in-flight backup lock. #1313-adjacent shared-repo + * data-safety fix. */ -export async function unlockRepository(restic: ResticLocal, destination: any, removeAll = true): Promise { +export async function unlockRepository(restic: ResticLocal, destination: any, removeAll = true, onProgress?: OnProgress): Promise { const args = removeAll ? ['unlock', '--remove-all'] : ['unlock']; - const run = await restic.runLocal(destination, args); + const stream = onProgress && { onStdout: (c: string) => forwardChunk(onProgress, c), onStderr: (c: string) => forwardChunk(onProgress, c) }; + const run = await restic.runLocal(destination, args, 'interactive', stream || undefined); return toResult(run); } @@ -122,7 +144,7 @@ export type RotateResult = | { ok: true } | { ok: false; error: string; dbOutOfSync?: boolean }; -/** A decrypted destination — carries the plaintext current password for the +/** A decrypted destination - carries the plaintext current password for the * constant-time pre-check. */ interface DecryptedDestination { id: number; @@ -149,12 +171,12 @@ export interface RotatePorts { * 1. Reject a weak or unchanged new password before doing anything. * 2. Constant-time compare the supplied current password against the stored one, * so response timing can't leak how much of a guess is right. - * 3. Run `restic key passwd` with the new password in a 0600 tmpfile — never on + * 3. Run `restic key passwd` with the new password in a 0600 tmpfile - never on * argv, which leaks via /proc//cmdline. The tmpfile is shredded on every * exit path. The key change is serialized on the destination. * 4. Persist the new password. If restic already swapped the key (step 3 * succeeded) but this write fails, the repo expects the new password and the - * DB still holds the old one — a critical out-of-sync state we flag distinctly + * DB still holds the old one - a critical out-of-sync state we flag distinctly * so the caller can prompt the operator to fix it. */ export async function rotateDestinationPassword( @@ -197,7 +219,7 @@ export async function rotateDestinationPassword( } // restic accepted the new password and removed the old key. From now on the - // repo only accepts newPassword — persist it immediately. + // repo only accepts newPassword - persist it immediately. try { await ports.updatePassword(destinationId, newPassword); } catch (err) { diff --git a/src/routes/api/backup/destinations/[id]/task/+server.ts b/src/routes/api/backup/destinations/[id]/task/+server.ts index a8bb902..c4dfd02 100644 --- a/src/routes/api/backup/destinations/[id]/task/+server.ts +++ b/src/routes/api/backup/destinations/[id]/task/+server.ts @@ -1,16 +1,17 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import { authorize } from '$lib/server/authorize'; +import { requireBackups } from '$lib/server/backups/route-guards'; import { auditBackupDestination } from '$lib/server/audit'; import { getBackupDestination, getBackupConfigs } from '$lib/server/db'; import { runRepoTask } from '$lib/server/backups'; +import { createJobResponse } from '$lib/server/sse'; export const POST: RequestHandler = async (event) => { const { params, request, cookies } = event; const auth = await authorize(cookies); - if (auth.authEnabled && !await auth.can('backups', 'manage')) { - return json({ error: 'Permission denied' }, { status: 403 }); - } + const denied = await requireBackups(auth, 'manage'); + if (denied) return denied; const destId = parseInt(params.id); if (isNaN(destId)) return json({ error: 'Invalid destination ID' }, { status: 400 }); @@ -41,20 +42,33 @@ export const POST: RequestHandler = async (event) => { } } - try { - // `task` is validated against the allowed list above. - const result = await runRepoTask(destId, task as import('$lib/server/backups').RepoTask); - // 'prune' and 'check' map to canonical audit actions; the rest land on - // 'update' so they're still recoverable from the log. - const action = task === 'prune' ? 'prune' : task === 'check' ? 'verify' : 'update'; + // 'prune' and 'check' map to canonical audit actions; the rest land on + // 'update' so they're still recoverable from the log. + const action = task === 'prune' ? 'prune' : task === 'check' ? 'verify' : 'update'; + + // Stream restic output live into the shared job/log modal (same channel as + // backup/restore). Guards above (auth, env-scope, allowlist) already ran; only + // the restic run is wrapped. `createJobResponse` keeps the backward-compat + // synchronous JSON path for `Accept: application/json` callers (tests/CLI). + return createJobResponse(async (send) => { + let result: Awaited>; + try { + // `task` is validated against the allowed list above. + result = await runRepoTask(destId, task as import('$lib/server/backups').RepoTask, { + onProgress: (message) => send('progress', { message }) + }); + } catch (error) { + // A genuine throw (unexpected). Audit as failed, then rethrow so the job + // ends in 'error' state. + const msg = error instanceof Error ? error.message : String(error); + await auditBackupDestination(event, 'update', destId, dest.name, { task, success: false, error: msg }); + throw error instanceof Error ? error : new Error(msg); + } // runRepoTask can resolve with { success: false, error } WITHOUT throwing - // (e.g. repo unreachable / needs init). Derive the audited success and - // error from the result rather than hardcoding success (audit #53). + // (e.g. repo unreachable / needs init). Derive the audited success/error + // from the result rather than hardcoding success (audit #53). await auditBackupDestination(event, action, destId, dest.name, { task, success: result.success, error: result.error }); - return json(result, { status: result.success ? 200 : 500 }); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - await auditBackupDestination(event, 'update', destId, dest.name, { task, success: false, error: msg }); - return json({ error: msg }, { status: 500 }); - } + send('result', result); + if (!result.success) throw new Error(result.error || `${task} failed`); + }, request); }; diff --git a/src/routes/settings/backups/BackupsTab.svelte b/src/routes/settings/backups/BackupsTab.svelte index 7cee933..531cb70 100644 --- a/src/routes/settings/backups/BackupsTab.svelte +++ b/src/routes/settings/backups/BackupsTab.svelte @@ -4,6 +4,7 @@ import { Button } from '$lib/components/ui/button'; import { Badge } from '$lib/components/ui/badge'; import { Input } from '$lib/components/ui/input'; + import * as Select from '$lib/components/ui/select'; import { DataGrid } from '$lib/components/data-grid'; import { Plus, Trash2, Pencil, HardDrive, Server, CheckCircle, XCircle, AlertCircle, Wifi, Database, RefreshCw, Search, FolderSync, Archive, Loader2, Save, CircleHelp, Unlock, PackageCheck, Eraser, BarChart3, Wrench, FolderCheck, KeyRound } from 'lucide-svelte'; import * as Tooltip from '$lib/components/ui/tooltip'; @@ -18,12 +19,14 @@ import { LoadingState } from '$lib/components/ui/loading-state'; import { getRepoTypeIcon, getRepoTypeLabel } from '$lib/utils/backup'; import { shouldSaveBackupImage } from '$lib/utils/backup-image'; + import { summarizeTestResults, formatTestSummary, type TestOutcome } from '$lib/utils/backup-test-summary'; import SnapshotBrowser from '../../containers/SnapshotBrowser.svelte'; import type { Component } from 'svelte'; import ConfirmPopover from '$lib/components/ConfirmPopover.svelte'; import { canAccess } from '$lib/stores/auth'; import DestinationModal from './DestinationModal.svelte'; - import VerifyModal from './VerifyModal.svelte'; + import BackupLogModal from '../../backups/BackupLogModal.svelte'; + import { watchJob } from '$lib/utils/sse-fetch'; import RotatePasswordModal from './RotatePasswordModal.svelte'; import { EmptyState } from '$lib/components/ui/empty-state'; @@ -56,12 +59,21 @@ let confirmDeleteId = $state(null); let confirmAction = $state<{ destId: number; task: string } | null>(null); let searchQuery = $state(''); + // Sort + status filter for the destinations grid. + let destSort = $state<{ field: string; direction: 'asc' | 'desc' }>({ field: 'name', direction: 'asc' }); + let statusFilter = $state<'all' | 'success' | 'needs_init' | 'failed'>('all'); let testingId = $state(null); let testingAll = $state(false); - let verifyModalOpen = $state(false); - let verifyDestId = $state(0); - let verifyDestName = $state(''); + // Shared live-log modal for every restic maintenance action (verify/check/prune/ + // repair/unlock) — the same BackupLogModal used by backup/restore, so all + // destination actions share one elegant log UI instead of a raw toast dump. + let actionLogOpen = $state(false); + let actionLogTitle = $state(''); + let actionLogIcon = $state(undefined); + let actionLogStatus = $state<'running' | 'success' | 'error'>('running'); + let actionLogLogs = $state([]); + let actionLogError = $state(''); let rotateModalOpen = $state(false); let rotateDestId = $state(0); let rotateDestName = $state(''); @@ -97,8 +109,6 @@ async function testAllDestinations() { testingAll = true; - let passed = 0; - let failed = 0; // Clear statuses so UI shows spinners for (const dest of destinations) { @@ -107,32 +117,33 @@ destinations = [...destinations]; repoStats = new Map(); - // Run all tests + stats in parallel, update UI incrementally - await Promise.allSettled(destinations.map(async (dest) => { + // Run all tests + stats in parallel, update UI incrementally. Collect the + // three-way outcome per destination; a needs-init repo is NOT a failure. + const outcomes = await Promise.all(destinations.map(async (dest): Promise => { try { const res = await fetch(`/api/backup/destinations/${dest.id}/test`, { method: 'POST' }); const data = await res.json(); if (data.success) { - passed++; dest.lastTestStatus = 'success'; destinations = [...destinations]; await fetchRepoStats(dest.id); - } else { - failed++; - dest.lastTestStatus = data.status === 'needs_init' ? 'needs_init' : 'failed'; - dest.lastTestError = data.error || null; - destinations = [...destinations]; + return 'success'; } + const status: TestOutcome = data.status === 'needs_init' ? 'needs_init' : 'failed'; + dest.lastTestStatus = status; + dest.lastTestError = data.error || null; + destinations = [...destinations]; + return status; } catch { - failed++; dest.lastTestStatus = 'failed'; destinations = [...destinations]; + return 'failed'; } })); testingAll = false; - if (failed === 0) toast.success(`All ${passed} destinations tested & stats collected`); - else toast.error(`${failed} failed, ${passed} passed`); + const summary = formatTestSummary(summarizeTestResults(outcomes)); + toast[summary.severity](summary.text); } // Backup helper image setting @@ -173,29 +184,90 @@ let initializingId = $state(null); let runningTask = $state<{ destId: number; task: string } | null>(null); + const REPO_TASK_TITLES: Record = { + unlock: 'Unlock repository', + prune: 'Prune unused data', + check: 'Check integrity', + 'repair-index': 'Repair index', + 'repair-snapshots': 'Repair snapshots' + }; + async function runRepoTask(destId: number, task: string) { runningTask = { destId, task }; - try { - const res = await fetch(`/api/backup/destinations/${destId}/task`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ task }) - }); - const data = await res.json(); - if (data.success) { - if (task === 'stats' && data.stats) { + // `stats` is a silent read that only populates the cells — no log modal. + if (task === 'stats') { + try { + const res = await fetch(`/api/backup/destinations/${destId}/task`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ task }) + }); + const data = await res.json(); + if (data.success && data.stats) { const m = new Map(repoStats); m.set(destId, data.stats); repoStats = m; + } else if (!data.success) { + toast.error(data.error || 'Stats failed'); } - toast.success(data.output || `${task} completed`); + } catch (err: any) { + toast.error(err.message || 'Stats failed'); + } finally { + runningTask = null; + } + return; + } + + const dest = destinations.find((d) => d.id === destId); + await streamRepoAction( + `${REPO_TASK_TITLES[task] || task}${dest ? ` - ${dest.name}` : ''}`, + `/api/backup/destinations/${destId}/task`, + { task }, + dest?.repository + ); + runningTask = null; + } + + function verifyDestination(dest: any) { + streamRepoAction(`Verify data integrity - ${dest.name}`, `/api/backup/destinations/${dest.id}/verify`, {}, dest.repository); + } + + /** Run a restic maintenance action, streaming its output live into the shared + * BackupLogModal (identical UX to backup/restore logs). */ + async function streamRepoAction(title: string, url: string, body: Record, repository?: string) { + actionLogTitle = title; + actionLogIcon = repository ? getRepoTypeIcon(repository) : undefined; + actionLogStatus = 'running'; + actionLogLogs = []; + actionLogError = ''; + actionLogOpen = true; + try { + const res = await fetch(url, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) + }); + const data = await res.json(); + if (data.jobId) { + const result = await watchJob(data.jobId, (line) => { + const d = line.data as any; + if (line.event === 'progress' && d?.message) { + actionLogLogs = [...actionLogLogs, d.message]; + } + }) as any; + if (result?.success === false) { + actionLogStatus = 'error'; + actionLogError = result.error || 'Action failed'; + } else { + actionLogStatus = 'success'; + } + } else if (data.error) { + actionLogStatus = 'error'; + actionLogError = data.error; } else { - toast.error(data.error || `${task} failed`); + // Backward-compat JSON path (Accept negotiated to one-shot): show result. + actionLogStatus = data.success ? 'success' : 'error'; + if (!data.success) actionLogError = data.error || 'Action failed'; } } catch (err: any) { - toast.error(err.message || `${task} failed`); - } finally { - runningTask = null; + actionLogStatus = 'error'; + actionLogError = err.message || 'Action failed'; } } @@ -321,15 +393,31 @@ }; } - const filteredDestinations = $derived( - searchQuery.trim() - ? destinations.filter(d => - d.name.toLowerCase().includes(searchQuery.toLowerCase()) || - d.repository.toLowerCase().includes(searchQuery.toLowerCase()) || - getTypeLabel(d.repository).toLowerCase().includes(searchQuery.toLowerCase()) - ) - : destinations - ); + const filteredDestinations = $derived.by(() => { + const q = searchQuery.trim().toLowerCase(); + let rows = destinations.filter(d => { + if (q && !( + d.name.toLowerCase().includes(q) || + d.repository.toLowerCase().includes(q) || + getTypeLabel(d.repository).toLowerCase().includes(q) + )) return false; + if (statusFilter !== 'all' && (d.lastTestStatus ?? '') !== statusFilter) return false; + return true; + }); + // Sort key per column. 'status' orders by a fixed rank so like statuses group. + const statusRank: Record = { success: 0, needs_init: 1, failed: 2, '': 3 }; + const key = (d: Destination) => + destSort.field === 'type' ? getTypeLabel(d.repository) : + destSort.field === 'repository' ? d.repository : + destSort.field === 'status' ? String(statusRank[d.lastTestStatus ?? ''] ?? 3) : + d.name; + rows = [...rows].sort((a, b) => { + const ka = key(a).toLowerCase(), kb = key(b).toLowerCase(); + const cmp = ka < kb ? -1 : ka > kb ? 1 : 0; + return destSort.direction === 'asc' ? cmp : -cmp; + }); + return rows; + }); async function fetchData() { loading = true; @@ -449,6 +537,17 @@ class="pl-9 h-8 w-64 text-sm" /> + statusFilter = (v as typeof statusFilter) ?? 'all'}> + + {statusFilter === 'success' ? 'Initialized' : statusFilter === 'needs_init' ? 'Needs init' : statusFilter === 'failed' ? 'Failed' : 'All statuses'} + + + All statuses + Initialized + Needs init + Failed + + {destinations.length} destination{destinations.length !== 1 ? 's' : ''}
@@ -483,15 +582,16 @@ gridId="backupDestinations" loading={loading} onRowClick={(dest) => openModal(dest)} + onSortChange={(s) => destSort = { field: s.field, direction: s.direction }} class="border-none" wrapperClass="border rounded-lg" > {#snippet cell(column, dest)} {#if column.id === 'type'} {@const TypeIcon = getTypeIcon(dest.repository)} -
- - {getTypeLabel(dest.repository)} +
+ + {getTypeLabel(dest.repository)}
{:else if column.id === 'name'} {dest.name} @@ -575,12 +675,12 @@ - { confirmAction = null; runRepoTask(dest.id, 'unlock'); }} onOpenChange={(o) => confirmAction = o ? { destId: dest.id, task: 'unlock' } : null} > @@ -590,7 +690,7 @@ { confirmAction = null; runRepoTask(dest.id, 'prune'); }} onOpenChange={(o) => confirmAction = o ? { destId: dest.id, task: 'prune' } : null} > @@ -600,7 +700,7 @@ { confirmAction = null; runRepoTask(dest.id, 'repair-index'); }} onOpenChange={(o) => confirmAction = o ? { destId: dest.id, task: 'repair' } : null} > @@ -775,10 +875,13 @@ targetName={snapshotBrowseName} /> -