This commit is contained in:
jarek committed 2026-07-11 15:31:52 +02:00
1 parent 685132ff89
commit c718ac9a84
101 files changed
+5097 -642

No files matched your search

+16
View File
@@ -247,6 +247,22 @@ export async function listUserTokens(userId: number) {
.where(eq(table.userId, userId));
}
/**
* Fleet-wide API token stats for the metrics endpoint (no secrets returned):
* total configured, how many are expired, and how many have never been used.
*/
export async function getApiTokenStats(): Promise<{ total: number; expired: number; neverUsed: number }> {
const table = await getApiTokensTable();
const rows = await db.select({ lastUsed: table.lastUsed, expiresAt: table.expiresAt }).from(table);
const now = Date.now();
let expired = 0, neverUsed = 0;
for (const r of rows) {
if (r.expiresAt && new Date(r.expiresAt).getTime() < now) expired++;
if (!r.lastUsed) neverUsed++;
}
return { total: rows.length, expired, neverUsed };
}
/**
* Revoke (delete) a token. Owner or admin can revoke.
*/
+22
View File
@@ -8,6 +8,7 @@
* - dockhand.url=<url> — Custom clickable URL displayed alongside container ports
* - dockhand.port.<hostPort>.url=<url> — Override the click URL for a specific published port
* - dockhand.order=<int> — Controls display order within a stack (lower = first, default 0)
* - dockhand.adopt=false — Prevent this stack from being adopted (any container in the stack)
*
* All label values are case-insensitive and accept: true/yes/1 and false/no/0.
* The opt-out model means labels override DB settings (label wins).
@@ -20,6 +21,7 @@ export const DOCKHAND_LABELS = {
NOTIFY: 'dockhand.notify',
URL: 'dockhand.url',
ORDER: 'dockhand.order',
ADOPT: 'dockhand.adopt',
} as const;
const TRUTHY_VALUES = new Set(['true', 'yes', '1']);
@@ -77,6 +79,26 @@ export function isNotifyDisabledByLabel(labels: Record<string, string> | undefin
return value === false; // explicitly disabled
}
/**
* Check if a single container opts out of stack adoption.
* Returns true if dockhand.adopt is explicitly set to false/no/0.
* Default (no label): adoptable (opt-out model).
*/
export function isAdoptDisabledByLabel(labels: Record<string, string> | undefined | null): boolean {
return parseLabelBool(getLabel(labels, DOCKHAND_LABELS.ADOPT)) === false;
}
/**
* Check if a stack should be excluded from adoption. A stack is unadoptable when
* ANY of its containers carries dockhand.adopt=false. Only enforceable while the
* stack is running (the label lives on containers, not the compose file).
*/
export function isStackUnadoptable(
containerLabels: Array<Record<string, string> | undefined | null>
): boolean {
return containerLabels.some(isAdoptDisabledByLabel);
}
/**
* Get the custom URL from dockhand.url label.
* Returns the URL string if set, or undefined.
+19
View File
@@ -0,0 +1,19 @@
/**
* Build a safe `Content-Disposition: attachment` header value (RFC 6266).
*
* The filename is often derived from a user-influenced path, so it must never
* be interpolated raw — a `"` would break out of the quoted string and CR/LF
* could attempt header injection. We emit both:
* - an ASCII-sanitized `filename="..."` fallback (quotes/control chars stripped)
* - a percent-encoded `filename*=UTF-8''...` for the real (possibly non-ASCII) name
*/
export function attachmentContentDisposition(filename: string): string {
const fallback = (filename || 'download')
// drop control chars (incl. CR/LF) and characters that break the quoted form
.replace(/[\x00-\x1f\x7f"\\]/g, '')
.trim() || 'download';
const encoded = encodeURIComponent(filename || 'download');
return `attachment; filename="${fallback}"; filename*=UTF-8''${encoded}`;
}
+25
View File
@@ -0,0 +1,25 @@
/** Shared CSV helpers for export endpoints. */
/**
* Escape a value for a CSV cell:
* - Neutralize spreadsheet formula injection: a cell whose first character is
* `=`, `+`, `-`, `@`, tab or CR is executed as a formula by Excel/Sheets.
* Exported vuln reports carry scanner/image-derived strings (an image tag or
* package name could start with `=`), so we prefix such cells with a single
* quote to force them to be treated as text.
* - Quote per RFC-4180 when the value contains a comma, quote, CR or LF.
*/
export function escapeCSV(value: string | number | null | undefined): string {
if (value === null || value === undefined) return '';
let str = String(value);
if (/^[=+\-@\t\r]/.test(str)) str = `'${str}`;
if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\r')) {
return `"${str.replace(/"/g, '""')}"`;
}
return str;
}
/** Build a CSV document from a header row and pre-escaped/raw cell rows. */
export function rowsToCSV(headers: string[], rows: (string | number | null | undefined)[][]): string {
return [headers.join(','), ...rows.map((row) => row.map(escapeCSV).join(','))].join('\n');
}
+165 -4
View File
@@ -79,6 +79,7 @@ import {
import type { AllGridPreferences, GridId, GridColumnPreferences } from '$lib/types';
import { encrypt, decrypt } from './encryption.js';
import { parseEnvInterpolation } from './env-interpolation';
import { invalidateVulnerabilitiesCache } from './vulnerabilities-cache';
// Re-export for backwards compatibility
export { db, isPostgres, isSqlite };
@@ -491,6 +492,31 @@ export async function resetAllGridPreferences(userId?: number): Promise<void> {
await deleteSetting(key);
}
// =============================================================================
// SIDEBAR MENU PREFERENCES
// =============================================================================
export interface SidebarPreferences {
order: string[];
hidden: string[];
}
export async function getSidebarPreferences(userId?: number): Promise<SidebarPreferences> {
const key = userId ? `user:${userId}:sidebar_preferences` : 'sidebar_preferences';
const value = await getSetting(key);
return value || { order: [], hidden: [] };
}
export async function setSidebarPreferences(prefs: SidebarPreferences, userId?: number): Promise<void> {
const key = userId ? `user:${userId}:sidebar_preferences` : 'sidebar_preferences';
await setSetting(key, prefs);
}
export async function deleteSidebarPreferences(userId?: number): Promise<void> {
const key = userId ? `user:${userId}:sidebar_preferences` : 'sidebar_preferences';
await deleteSetting(key);
}
// =============================================================================
// ENVIRONMENT PUBLIC IPS (for port links)
// =============================================================================
@@ -2998,6 +3024,9 @@ export async function saveVulnerabilityScan(data: {
vulnerabilities: JSON.stringify(data.vulnerabilities),
error: data.error ?? null
}).returning();
// A new scan makes the dashboard's cached findings stale — drop them so every
// writer (routes + schedulers) refreshes it, no separate call to remember.
invalidateVulnerabilitiesCache(data.environmentId ?? undefined);
return getVulnerabilityScan(result[0].id) as Promise<VulnerabilityScanData>;
}
@@ -3041,9 +3070,47 @@ export async function getLatestScanForImage(
} as VulnerabilityScanData;
}
export async function getScansForImage(imageId: string, limit = 10): Promise<VulnerabilityScanData[]> {
/**
* Delete all previous scan rows for a specific image + scanner in an environment.
* Used by "scan all" to replace an image's prior scan rather than accumulate rows.
* Returns the number of rows removed.
*/
export async function deleteScansForImageScanner(
imageId: string,
scanner: 'grype' | 'trivy',
environmentId?: number | null
): Promise<number> {
const conditions = [
eq(vulnerabilityScans.imageId, imageId),
eq(vulnerabilityScans.scanner, scanner)
];
if (environmentId === null || environmentId === undefined) {
conditions.push(isNull(vulnerabilityScans.environmentId));
} else {
conditions.push(eq(vulnerabilityScans.environmentId, environmentId));
}
const result = await db.delete(vulnerabilityScans).where(and(...conditions)).returning({ id: vulnerabilityScans.id });
return result.length;
}
export async function getScansForImage(
imageId: string,
environmentId?: number | null,
limit = 10
): Promise<VulnerabilityScanData[]> {
// Scope by environment so a caller can't read another environment's scans for
// the same image SHA (the vulnerability_scans table is per-environment). When
// environmentId is omitted (undefined) all environments are returned — callers
// exposed to untrusted input MUST pass a concrete env.
const conditions = [eq(vulnerabilityScans.imageId, imageId)];
if (environmentId !== undefined) {
conditions.push(environmentId === null
? isNull(vulnerabilityScans.environmentId)
: eq(vulnerabilityScans.environmentId, environmentId));
}
const results = await db.select().from(vulnerabilityScans)
.where(eq(vulnerabilityScans.imageId, imageId))
.where(and(...conditions))
.orderBy(desc(vulnerabilityScans.scannedAt))
.limit(limit);
@@ -3135,11 +3202,56 @@ export async function getAllLatestScans(environmentId?: number | null): Promise<
})) as VulnerabilityScanData[];
}
/**
* Scan freshness for the metrics endpoint: how stale the OLDEST scan is
* (surfaces environments whose scans have gone stale) and the average scan
* duration (surfaces slow scanners). Ages/durations in seconds; nulls when there
* are no scans.
*
* A pure SQL aggregate over only scanned_at / scan_duration — it deliberately
* does NOT touch the large `vulnerabilities` JSON blob (unlike getAllLatestScans),
* so it stays cheap when called per-environment on every scrape.
*/
export async function getScanFreshness(environmentId?: number | null): Promise<{
scans: number; oldestAgeSeconds: number | null; avgDurationSeconds: number | null;
}> {
const envCond = environmentId === undefined
? undefined
: environmentId === null
? isNull(vulnerabilityScans.environmentId)
: eq(vulnerabilityScans.environmentId, environmentId);
const rows = await db
.select({
n: sql<number>`count(*)`,
oldest: sql<string | null>`min(${vulnerabilityScans.scannedAt})`,
avgDur: sql<number | null>`avg(${vulnerabilityScans.scanDuration})`
})
.from(vulnerabilityScans)
.where(envCond);
const r = rows[0];
const n = Number(r?.n ?? 0);
if (n === 0) return { scans: 0, oldestAgeSeconds: null, avgDurationSeconds: null };
const oldestT = r?.oldest ? new Date(r.oldest).getTime() : NaN;
const oldestAgeSeconds = Number.isNaN(oldestT) ? null : Math.max(0, Math.round((Date.now() - oldestT) / 1000));
const avgDurationSeconds = r?.avgDur != null ? Math.round(Number(r.avgDur) / 1000) : null;
return { scans: n, oldestAgeSeconds, avgDurationSeconds };
}
export async function deleteOldScans(keepDays = 30): Promise<number> {
const cutoffDate = new Date(Date.now() - keepDays * 24 * 60 * 60 * 1000).toISOString();
await db.delete(vulnerabilityScans)
const countResult = await db.select({ count: sql<number>`count(*)` })
.from(vulnerabilityScans)
.where(sql`scanned_at < ${cutoffDate}`);
return 0;
const count = Number(countResult[0]?.count ?? 0);
if (count > 0) {
await db.delete(vulnerabilityScans)
.where(sql`scanned_at < ${cutoffDate}`);
}
return count;
}
// =============================================================================
@@ -4052,6 +4164,55 @@ export async function getScheduleExecutions(filters: ScheduleExecutionFilters =
};
}
/**
* Scheduled-task health for the metrics endpoint: execution counts by
* (scheduleType, status), and per-type age (seconds) of the last run and last
* SUCCESSFUL run. The last-run age detects a scheduler that stopped firing; the
* last-success age detects one that fires but keeps failing. Best-effort.
*/
export async function getScheduleStats(): Promise<{
byTypeStatus: Array<{ type: string; status: string; count: number }>;
lastRunSecondsByType: Record<string, number>;
lastSuccessSecondsByType: Record<string, number>;
}> {
const counts = await db
.select({
type: scheduleExecutions.scheduleType,
status: scheduleExecutions.status,
count: sql<number>`count(*)`
})
.from(scheduleExecutions)
.groupBy(scheduleExecutions.scheduleType, scheduleExecutions.status);
// Most-recent triggeredAt per type (any status) and per type for successes.
const lastAny = await db
.select({ type: scheduleExecutions.scheduleType, ts: sql<string>`max(${scheduleExecutions.triggeredAt})` })
.from(scheduleExecutions)
.groupBy(scheduleExecutions.scheduleType);
const lastOk = await db
.select({ type: scheduleExecutions.scheduleType, ts: sql<string>`max(${scheduleExecutions.triggeredAt})` })
.from(scheduleExecutions)
.where(eq(scheduleExecutions.status, 'success'))
.groupBy(scheduleExecutions.scheduleType);
const now = Date.now();
const ageMap = (rows: Array<{ type: string; ts: string | null }>): Record<string, number> => {
const out: Record<string, number> = {};
for (const r of rows) {
if (!r.ts) continue;
const t = new Date(r.ts).getTime();
if (!Number.isNaN(t)) out[r.type] = Math.max(0, Math.round((now - t) / 1000));
}
return out;
};
return {
byTypeStatus: counts.map((c) => ({ type: c.type, status: c.status, count: Number(c.count) })),
lastRunSecondsByType: ageMap(lastAny),
lastSuccessSecondsByType: ageMap(lastOk)
};
}
export async function getLastExecutionForSchedule(
scheduleType: ScheduleType,
scheduleId: number
+84
View File
@@ -1041,3 +1041,87 @@ export function getPostgresConnectionInfo(): { host: string; port: string } | nu
return null;
}
}
/** High-churn tables worth exposing a row count for (they grow and affect tuning). */
const METRICS_TABLES = [
'container_events', 'vulnerability_scans', 'audit_logs', 'sessions',
'host_metrics', 'schedule_executions', 'pending_container_updates'
];
export interface DatabaseStats {
type: 'postgres' | 'sqlite';
/** Engine version string, e.g. "3.45.1" (sqlite) or "16.2" (postgres). */
version: string;
/** On-disk size in bytes (sqlite: page_count*page_size; postgres: pg_database_size). 0 if unknown. */
sizeBytes: number;
/** Row counts for the high-churn tables (missing tables omitted). */
rowCounts: Record<string, number>;
/**
* Engine-specific extra numeric gauges, keyed by name:
* - sqlite: wal_bytes (WAL file size), freelist_bytes (reclaimable), cache_size_pages
* - postgres: connections (current backends), max_connections
*/
extra: Record<string, number>;
}
/**
* Cheap DB stats for the metrics endpoint: engine + version, on-disk size, row
* counts of the tables that actually grow, plus engine-specific tuning gauges.
* Never throws — returns best-effort data so a scrape can't fail on a DB hiccup.
*/
export async function getDatabaseStats(): Promise<DatabaseStats> {
const type = isPostgres ? 'postgres' : 'sqlite';
let sizeBytes = 0;
let version = 'unknown';
const rowCounts: Record<string, number> = {};
const extra: Record<string, number> = {};
try {
if (isPostgres) {
const size = await rawClient`SELECT pg_database_size(current_database()) AS bytes`;
sizeBytes = Number(size[0]?.bytes ?? 0);
try {
const v = await rawClient`SHOW server_version`;
version = String(v[0]?.server_version ?? 'unknown').split(' ')[0];
} catch { /* ignore */ }
try {
const c = await rawClient`SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname = current_database()`;
extra.connections = Number(c[0]?.n ?? 0);
const mc = await rawClient`SELECT setting::int AS n FROM pg_settings WHERE name = 'max_connections'`;
extra.max_connections = Number(mc[0]?.n ?? 0);
} catch { /* ignore */ }
for (const table of METRICS_TABLES) {
try {
// Identifiers can't be parameterized; table names are a fixed const list.
const r = await rawClient.unsafe(`SELECT count(*)::int AS n FROM ${table}`);
rowCounts[table] = Number(r[0]?.n ?? 0);
} catch { /* table may not exist on older schemas */ }
}
} else {
const pc = rawClient.prepare('PRAGMA page_count').get() as { page_count?: number };
const ps = rawClient.prepare('PRAGMA page_size').get() as { page_size?: number };
const pageSize = ps?.page_size ?? 0;
sizeBytes = (pc?.page_count ?? 0) * pageSize;
try {
version = String((rawClient.prepare('SELECT sqlite_version() AS v').get() as { v?: string })?.v ?? 'unknown');
} catch { /* ignore */ }
try {
// freelist_count is a read-only header read (reclaimable space). NB: do
// NOT read WAL size via `PRAGMA wal_checkpoint` — that pragma actively
// CHECKPOINTS the WAL (a write), which would contend on every scrape.
const fl = rawClient.prepare('PRAGMA freelist_count').get() as { freelist_count?: number };
if (fl?.freelist_count != null) extra.freelist_bytes = fl.freelist_count * pageSize;
} catch { /* ignore */ }
for (const table of METRICS_TABLES) {
try {
const r = rawClient.prepare(`SELECT count(*) AS n FROM ${table}`).get() as { n?: number };
rowCounts[table] = Number(r?.n ?? 0);
} catch { /* table may not exist */ }
}
}
} catch {
/* best-effort */
}
return { type, version, sizeBytes, rowCounts, extra };
}
+16 -7
View File
@@ -1035,6 +1035,8 @@ export interface ContainerInfo {
id: string;
name: string;
image: string;
/** sha256 image ID (container.ImageID) — matches vulnerability scan.imageId. */
imageId: string;
state: string;
status: string;
created: number;
@@ -1161,6 +1163,7 @@ export async function listContainers(all = true, envId?: number | null): Promise
id: container.Id,
name: container.Names[0]?.replace(/^\//, '') || 'unnamed',
image: container.Image,
imageId: container.ImageID || '',
state: container.State,
status: container.Status,
created: container.Created,
@@ -2611,8 +2614,8 @@ export async function listImages(envId?: number | null): Promise<ImageInfo[]> {
/**
* Diagnostic log for registry auth headers (#1105).
* Logs lengths and boundary char codes — never the plaintext credentials.
* Header prefix is safe to log: it encodes the start of `{"username":"...`.
* Logs only whether credentials were present and the target registry — no
* credential-derived material (lengths, byte values, or header bytes).
*/
function logAuthDiagnostics(
tag: string,
@@ -2622,12 +2625,9 @@ function logAuthDiagnostics(
password: string,
authHeader: string
): void {
const userLast = username.length ? username.charCodeAt(username.length - 1).toString(16) : 'na';
const pwLast = password.length ? password.charCodeAt(password.length - 1).toString(16) : 'na';
console.log(
`${tag} auth: registry=${registry} user(len=${username.length},last=0x${userLast}) ` +
`pw(len=${password.length},last=0x${pwLast}) serveraddress=${serveraddress} ` +
`authHeader(len=${authHeader.length},prefix=${authHeader.slice(0, 16)})`
`${tag} auth: registry=${registry} serveraddress=${serveraddress} ` +
`hasUser=${username.length > 0} hasPassword=${password.length > 0} hasAuthHeader=${authHeader.length > 0}`
);
}
@@ -2720,6 +2720,10 @@ export async function pullImage(imageName: string, onProgress?: (data: any) => v
const decoder = new TextDecoder();
let buffer = '';
// Docker streams pull failures as {"error":...} lines inside a 200 response.
// Capture the last one and throw after the stream ends so callers see the
// failure (and onProgress still fires for the error line).
let streamError: string | null = null;
while (true) {
const { done, value } = await reader.read();
@@ -2735,6 +2739,7 @@ export async function pullImage(imageName: string, onProgress?: (data: any) => v
const data = JSON.parse(line);
if (data.error || data.errorDetail) {
console.error(`[Pull] stream error: ${line}`);
streamError = data.errorDetail?.message || data.error || 'Image pull failed';
}
if (onProgress) onProgress(data);
} catch {
@@ -2743,6 +2748,10 @@ export async function pullImage(imageName: string, onProgress?: (data: any) => v
}
}
}
if (streamError) {
throw new Error(`Failed to pull image: ${streamError}`);
}
}
export async function removeImage(id: string, force = false, envId?: number | null) {
+66
View File
@@ -0,0 +1,66 @@
/** Shared helpers for the vulnerability export endpoints (env-wide + per-image). */
import { findingsToSarif } from '$lib/utils/sarif';
import type { Finding } from '$lib/utils/vulnerability';
/** JSON error response with the given status. */
export function jsonError(message: string, status: number): Response {
return new Response(JSON.stringify({ error: message }), {
status,
headers: { 'Content-Type': 'application/json' }
});
}
/** Make a value safe for use in a download filename. */
export function slugify(name: string, fallback = 'export'): string {
return name.trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || fallback;
}
/**
* Build the export download response for json | csv | sarif. The caller supplies
* the format-specific JSON body and a CSV renderer; sarif is generated from the
* shared findings so both endpoints emit identical SARIF.
*/
export function exportResponse(opts: {
format: string;
filenameBase: string;
findings: Finding[];
jsonBody: unknown;
toCSV: (findings: Finding[]) => string;
}): Response {
const { format, filenameBase, findings, jsonBody, toCSV } = opts;
let content: string;
let contentType: string;
let ext: string;
switch (format) {
case 'csv':
content = toCSV(findings);
contentType = 'text/csv';
ext = 'csv';
break;
case 'sarif':
content = JSON.stringify(findingsToSarif(findings), null, 2);
contentType = 'application/sarif+json';
ext = 'sarif';
break;
case 'json':
default:
content = JSON.stringify(jsonBody, null, 2);
contentType = 'application/json';
ext = 'json';
break;
}
// Defensively strip anything that could break out of the quoted filename or
// inject a header, regardless of what the caller passed (today it's slug+date).
const safeBase = filenameBase.replace(/[^\w.-]+/g, '-').replace(/^-+|-+$/g, '') || 'export';
return new Response(content, {
status: 200,
headers: {
'Content-Type': contentType,
'Content-Disposition': `attachment; filename="${safeBase}.${ext}"`
}
});
}
+75
View File
@@ -0,0 +1,75 @@
/**
* Guard for the filesystem-browser endpoints (`/api/system/files*`).
*
* Dockhand runs in a container, so there is no host filesystem to leak — users
* are allowed to browse the container freely (they need this to find/adopt
* external stacks). The only things that must never be readable are Dockhand's
* own secrets:
* - the database directory ($DATA_DIR/db, contains dockhand.db)
* - the encryption key file ($DATA_DIR/.encryption_key)
* - /proc — /proc/<pid>/environ exposes process env vars, including
* DATABASE_URL (Postgres credentials) and ENCRYPTION_KEY. The whole tree is
* blocked because /proc/self, /proc/1, etc. all leak the same secrets and
* nothing legitimate is browsed there.
*
* isProtectedPath resolves symlinks (via the nearest existing ancestor) so a
* symlink pointing into a protected location can't be used to bypass the check.
*/
import { realpathSync } from 'node:fs';
import { resolve, dirname, join, basename, sep } from 'node:path';
const KEY_FILE_NAME = '.encryption_key';
function getDataDir(): string {
return process.env.DATA_DIR || './data';
}
/** Absolute paths Dockhand must never expose through the file browser. */
function protectedPaths(): { dbDir: string; keyFile: string } {
const dataDir = resolve(getDataDir());
return {
dbDir: join(dataDir, 'db'),
keyFile: join(dataDir, KEY_FILE_NAME)
};
}
/**
* Resolve `p` to an absolute, symlink-free path. If `p` (or part of it) does not
* exist yet, realpath the deepest existing ancestor and re-append the missing
* tail — this still defeats a symlinked ancestor.
*/
function safeResolve(p: string): string {
let current = resolve(p);
const tail: string[] = [];
// Walk up until we hit a path that exists, realpath it, then rejoin the tail.
while (true) {
try {
const real = realpathSync(current);
return tail.length ? join(real, ...tail.reverse()) : real;
} catch {
const parent = dirname(current);
if (parent === current) {
// Reached the root without an existing ancestor; return as-is.
return resolve(p);
}
tail.push(basename(current));
current = parent;
}
}
}
function isInside(child: string, parent: string): boolean {
return child === parent || child.startsWith(parent + sep);
}
/**
* Returns true if `requestedPath` resolves to Dockhand's DB directory, the
* encryption-key file, or anywhere under /proc (process env / cmdline leak
* credentials). Such paths must be hidden from the file browser.
*/
export function isProtectedPath(requestedPath: string): boolean {
const { dbDir, keyFile } = protectedPaths();
const resolved = safeResolve(requestedPath);
return isInside(resolved, dbDir) || resolved === keyFile || isInside(resolved, '/proc');
}
+34 -9
View File
@@ -425,6 +425,9 @@ async function computeSyncDeletionPlan(options: {
/**
* Persist the manifest after a deploy and log the per-file change summary.
* Called only after a successful deploy (locally applied or agent-confirmed).
* Progress popovers show the plan-based change table before the deploy
* instead (#1260); this summary (with real apply results) goes to the
* server log only.
*/
async function finalizeDeletionSync(options: {
stackId: number;
@@ -434,9 +437,8 @@ async function finalizeDeletionSync(options: {
newFiles: Record<string, string>;
plan: DeletionPlan;
applyResult: DeletionApplyResult | undefined;
onLine?: (line: string) => void; // extra sink (job output)
}): Promise<void> {
const { stackId, logPrefix, previousManifest, newCommitFull, newFiles, plan, applyResult, onLine } = options;
const { stackId, logPrefix, previousManifest, newCommitFull, newFiles, plan, applyResult } = options;
// No apply result means deletions were requested but nothing reported back
// (defensive — executors always return one). Logged as skips; skips are final.
@@ -455,10 +457,6 @@ async function finalizeDeletionSync(options: {
for (const line of tableLines.slice(1)) {
console.log(`${logPrefix} ${line}`);
}
if (onLine) {
onLine(`File changes: ${tableLines[0]}`);
for (const line of tableLines.slice(1)) onLine(line);
}
const nextManifest = buildNextManifest(newCommitFull, newFiles);
await updateGitStack(stackId, { syncedFiles: serializeManifest(nextManifest) });
@@ -1458,6 +1456,23 @@ export async function deployGitStackWithProgress(
cleanupSshKey(credential);
// Show the git file changes BEFORE the deploy starts, so the user sees
// what changed while the deploy runs and the deploy start/result lines
// stay together (#1260). Removals reflect the deletion plan here;
// apply-stage divergences (rare) are reported after the deploy.
const changeTable = formatChangeTable(
buildSyncChangeSummary(
deletionData.previousManifest.files,
deletionData.newFiles,
{ deleted: deletionData.plan.toDelete.map((f) => f.path), skipped: [] },
deletionData.plan.skipped
)
);
onProgress({ status: 'deploying', message: `File changes: ${changeTable[0]}`, step: 5, totalSteps });
for (const line of changeTable.slice(1)) {
onProgress({ status: 'deploying', message: line, step: 5, totalSteps });
}
// Step 5: Deploying stack
// Uses `docker compose up -d --remove-orphans` which only recreates changed services
onProgress({ status: 'deploying', message: `Deploying ${gitStack.stackName}...`, step: 5, totalSteps });
@@ -1494,7 +1509,8 @@ export async function deployGitStackWithProgress(
if (result.success) {
// Deletion sync: persist manifest + log per-file change summary.
// onLine feeds the per-file change table into the deploy progress popover.
// The change table was already shown before the deploy (#1260);
// report only apply-stage divergences from the plan here.
await finalizeDeletionSync({
stackId,
logPrefix,
@@ -1502,10 +1518,19 @@ export async function deployGitStackWithProgress(
newCommitFull: newCommit,
newFiles: deletionData.newFiles,
plan: deletionData.plan,
applyResult: result.deletion,
onLine: (line) => onProgress({ status: 'deploying', message: line, step: 5, totalSteps })
applyResult: result.deletion
});
const applySkips = (result.deletion?.skipped ?? []).filter((s) => s.reason !== 'already-absent');
for (const skip of applySkips) {
onProgress({
status: 'deploying',
message: `Kept "${skip.path}" — ${skipReasonMessage(skip.reason)}`,
step: 5,
totalSteps
});
}
// Record the stack source with resolved compose path for consistency
const stackDir = await getStackDir(gitStack.stackName, gitStack.environmentId);
const resolvedComposePath = join(stackDir, progressComposeFileName);
+4
View File
@@ -1183,6 +1183,10 @@ async function handleHawserWsMessage(ws: any, msg: any, connId: string, remoteIp
return;
}
// Auth succeeded — clear any prior failure cooldown for this IP so a
// legitimate agent (e.g. after a token rotation) isn't locked out.
hawserAuthFailCache.delete(rateLimitKey);
// Throttle reconnection storms (successful auth but broken Docker = rapid reconnect loop)
const throttle = recordReconnection(result.environmentId);
if (!throttle.allowed) {
+22
View File
@@ -12,6 +12,8 @@ export interface Job {
result?: unknown;
createdAt: number;
updatedAt: number;
/** Set when a client requests cancellation; long-running jobs should poll this. */
cancelRequested?: boolean;
}
const jobs = new Map<string, Job>();
@@ -32,6 +34,17 @@ export function getJob(id: string): Job | undefined {
return jobs.get(id);
}
/** Live job counts by status — for the metrics endpoint. */
export function getJobStats(): { running: number; done: number; error: number; total: number } {
let running = 0, done = 0, error = 0;
for (const job of jobs.values()) {
if (job.status === 'running') running++;
else if (job.status === 'done') done++;
else if (job.status === 'error') error++;
}
return { running, done, error, total: jobs.size };
}
export function appendLine(job: Job, line: JobLine): void {
job.lines.push(line);
job.updatedAt = Date.now();
@@ -49,6 +62,15 @@ export function failJob(job: Job, error: string): void {
job.updatedAt = Date.now();
}
/** Request cancellation of a running job. The job's operation must poll job.cancelRequested. */
export function cancelJob(id: string): boolean {
const job = jobs.get(id);
if (!job || job.status !== 'running') return false;
job.cancelRequested = true;
job.updatedAt = Date.now();
return true;
}
// Cleanup jobs older than 10 minutes that are no longer running
const CLEANUP_INTERVAL_MS = 60_000;
const JOB_TTL_MS = 10 * 60_000;
+349
View File
@@ -0,0 +1,349 @@
/**
* Prometheus metrics for Dockhand (#339), gated by `EXPORT_METRICS=true`.
*
* Three families of series:
* 1. Per-environment DOCKER STATE — containers by state/health, images (count +
* bytes), volumes, networks, stacks, vulnerabilities by severity, and event
* counts. Gathered fresh (per scrape) but behind a short TTL cache so a tight
* scrape interval can't hammer Docker; a dead environment is skipped, not fatal.
* 2. Per-environment RESOURCE usage — cpu/memory. NOT re-collected here: these are
* surfaced from the Go collector's samples via metrics-store, so we add no load.
* 3. Dockhand INTERNALS — build/uptime, process memory (prom-client defaults),
* hawser edge agents, jobs, scan queue, cache occupancy, scheduler, database.
*
* The registry and all metric objects are created once (module singletons). Each
* scrape refreshes gauge values; prom-client renders the exposition text.
*/
import { Registry, collectDefaultMetrics, Gauge } from 'prom-client';
import {
getEnvironments, getContainerEventStats, getPendingContainerUpdates,
getUsers, getRegistries, getScanFreshness, getScheduleStats,
getGitRepositories, getConfigSets
} from '$lib/server/db';
import { getApiTokenStats } from '$lib/server/api-tokens';
import { listContainers, listImages, listVolumes, listNetworks } from '$lib/server/docker';
import { listComposeStacks } from '$lib/server/stacks';
import { getVulnerabilitiesMeta } from '$lib/server/vulnerabilities';
import { getVulnerabilitiesCacheStats } from '$lib/server/vulnerabilities-cache';
import { getLatestMetric } from '$lib/server/metrics-store';
import { getServerUptime } from '$lib/server/uptime';
import { getJobStats } from '$lib/server/jobs';
import { getScannerStats } from '$lib/server/scanner';
import { getSchedulerStats } from '$lib/server/scheduler';
import { edgeConnections } from '$lib/server/hawser';
import { getDatabaseStats } from '$lib/server/db/drizzle';
export const METRICS_ENABLED = process.env.EXPORT_METRICS === 'true';
/** How long a gathered snapshot is reused before a scrape re-collects (ms). */
const COLLECT_TTL_MS = 15_000;
/** Hard cap on a full gather so a slow/hung environment can never stall a scrape. */
const COLLECT_TIMEOUT_MS = 10_000;
/** Max environments gathered concurrently — bounds the Docker/DB call burst so a
* scrape never monopolizes those resources against live user traffic. */
const ENV_CONCURRENCY = 4;
const PREFIX = 'dockhand_';
const appVersion = typeof __APP_VERSION__ !== 'undefined' ? (__APP_VERSION__ ?? 'unknown') : 'unknown';
// ---------------------------------------------------------------------------
// Registry + metric definitions (created once).
// ---------------------------------------------------------------------------
const registry = new Registry();
// Free Node/process metrics: heap, RSS, GC, event-loop lag, open FDs, etc.
collectDefaultMetrics({ register: registry, prefix: PREFIX });
const g = (name: string, help: string, labelNames: string[] = []) =>
new Gauge({ name: PREFIX + name, help, labelNames, registers: [registry] });
// --- Build / process ---
const buildInfo = g('build_info', 'Dockhand build info; value is always 1.', ['version']);
const uptimeSeconds = g('uptime_seconds', 'Seconds since the Dockhand server started.');
// --- Per-environment state ---
const envUp = g('env_up', '1 if the environment responded to the last scrape, else 0.', ['env', 'env_id', 'connection_type']);
const containers = g('containers', 'Containers by lifecycle state.', ['env', 'env_id', 'state']);
const containersHealth = g('containers_health', 'Containers by health status.', ['env', 'env_id', 'health']);
const containersRestarts = g('container_restarts_total', 'Sum of container restart counts.', ['env', 'env_id']);
const containersTotal = g('containers_count', 'Total containers (all states).', ['env', 'env_id']);
const updatesAvailable = g('updates_available', 'Containers with a pending image update.', ['env', 'env_id']);
const imagesTotal = g('images_count', 'Number of images.', ['env', 'env_id']);
const imagesDangling = g('images_dangling', 'Untagged (dangling) images.', ['env', 'env_id']);
const imageBytes = g('image_bytes', 'Total on-disk size of images in bytes.', ['env', 'env_id']);
const volumesTotal = g('volumes_count', 'Number of volumes.', ['env', 'env_id']);
const networksTotal = g('networks_count', 'Number of networks.', ['env', 'env_id']);
const stacksTotal = g('stacks_count', 'Number of compose stacks.', ['env', 'env_id']);
const vulnerabilities = g('vulnerabilities', 'Vulnerability findings by severity.', ['env', 'env_id', 'severity']);
const vulnerabilitiesTotal = g('vulnerabilities_count', 'Total vulnerability findings.', ['env', 'env_id']);
const imagesScanned = g('images_scanned', 'Images with at least one persisted scan.', ['env', 'env_id']);
const eventsTotal = g('container_events_total', 'Recorded container events (all time).', ['env', 'env_id']);
const eventsToday = g('container_events_today', 'Recorded container events since local midnight.', ['env', 'env_id']);
const eventsByAction = g('container_events_by_action', 'Recorded container events by action type.', ['env', 'env_id', 'action']);
const scanOldestAge = g('scan_oldest_age_seconds', 'Age of the oldest current scan (staleness).', ['env', 'env_id']);
const scanAvgDuration = g('scan_avg_duration_seconds', 'Average scan duration.', ['env', 'env_id']);
// --- Per-environment resources (from the Go collector, via metrics-store) ---
const envCpu = g('env_cpu_percent', 'Host CPU usage percent (from the collector).', ['env', 'env_id']);
const envMemUsed = g('env_memory_used_bytes', 'Host memory used in bytes (from the collector).', ['env', 'env_id']);
const envMemTotal = g('env_memory_total_bytes', 'Host memory total in bytes (from the collector).', ['env', 'env_id']);
// --- Internals ---
const environmentsTotal = g('environments', 'Configured environments.');
const usersTotal = g('users', 'Configured users.');
const registriesTotal = g('registries', 'Configured image registries.');
const gitReposTotal = g('git_repositories', 'Configured git repositories.');
const configSetsTotal = g('config_sets', 'Saved config sets.');
const apiTokens = g('api_tokens', 'API tokens by state.', ['state']);
const scheduleExec = g('schedule_executions', 'Scheduled task executions by type and status.', ['type', 'status']);
const scheduleLastRun = g('schedule_last_run_seconds', 'Age of the last execution, by schedule type.', ['type']);
const scheduleLastSuccess = g('schedule_last_success_seconds', 'Age of the last SUCCESSFUL execution, by schedule type.', ['type']);
const hawserAgents = g('hawser_agents_connected', 'Connected hawser edge agents.');
const hawserPending = g('hawser_pending_requests', 'In-flight requests across all edge agents.');
const hawserAgentInfo = g('hawser_agent_info', 'Connected edge agent info; value is connection age in seconds.', ['env_id', 'agent', 'agent_version', 'docker_version', 'hostname']);
const jobs = g('jobs', 'Background SSE jobs by status.', ['status']);
const scanQueue = g('scan_queue', 'Vulnerability scan queue depth.', ['kind']);
const cacheEntries = g('vuln_cache_entries', 'Vulnerability aggregation cache occupancy.', ['kind']);
const schedulerRunning = g('scheduler_running', '1 if the scheduler is running, else 0.');
const schedulerJobs = g('scheduler_active_jobs', 'Active scheduled cron jobs.');
const dbSizeBytes = g('database_size_bytes', 'On-disk database size in bytes.', ['type']);
const dbInfo = g('database_info', 'Database engine info; value is always 1.', ['type', 'version']);
const dbRows = g('database_rows', 'Row counts for high-churn tables.', ['table']);
const dbExtra = g('database_stat', 'Engine-specific DB gauges (postgres connections, sqlite wal/freelist bytes, ...).', ['type', 'stat']);
// ---------------------------------------------------------------------------
// Collection.
// ---------------------------------------------------------------------------
let lastCollectAt = 0;
let inflight: Promise<void> | null = null;
/** Gather one environment's Docker state, tolerating a dead host. */
async function collectEnvironment(env: { id: number; name: string; connectionType?: string | null }): Promise<void> {
const labels = { env: env.name, env_id: String(env.id) };
const upLabels = { ...labels, connection_type: env.connectionType ?? 'socket' };
// Emit env_up=0 up front so the series ALWAYS exists for every configured env,
// even if this gather times out before the calls below settle — otherwise a
// down env would have no series at all and `dockhand_env_up == 0` alerts would
// never fire. Flipped to 1 only once Docker actually responds.
envUp.set(upLabels, 0);
// Resource usage from the collector's last sample — cheap, never touches Docker.
const point = getLatestMetric(env.id);
if (point) {
envCpu.set(labels, point.cpuPercent);
envMemUsed.set(labels, point.memoryUsed);
envMemTotal.set(labels, point.memoryTotal);
}
// One failing call marks the env down but doesn't abort the others.
let up = 1;
const [cs, imgs, vols, nets, stacks, vuln, events, updates, freshness] = await Promise.allSettled([
listContainers(true, env.id),
listImages(env.id),
listVolumes(env.id),
listNetworks(env.id),
listComposeStacks(env.id),
getVulnerabilitiesMeta(env.id),
getContainerEventStats(env.id),
getPendingContainerUpdates(env.id),
getScanFreshness(env.id)
]);
if (cs.status === 'fulfilled') {
const byState: Record<string, number> = {};
const byHealth: Record<string, number> = {};
let restarts = 0;
for (const c of cs.value) {
byState[c.state] = (byState[c.state] ?? 0) + 1;
if (c.health) byHealth[c.health] = (byHealth[c.health] ?? 0) + 1;
restarts += c.restartCount ?? 0;
}
for (const [state, n] of Object.entries(byState)) containers.set({ ...labels, state }, n);
for (const [health, n] of Object.entries(byHealth)) containersHealth.set({ ...labels, health }, n);
containersTotal.set(labels, cs.value.length);
containersRestarts.set(labels, restarts);
} else up = 0; // ANY failure of a core Docker call means the env isn't fully reachable
if (imgs.status === 'fulfilled') {
imagesTotal.set(labels, imgs.value.length);
// Dangling = no usable repo tag (untagged layers left after rebuilds/pulls).
const dangling = imgs.value.filter((i) => {
const tags = i.tags ?? [];
return tags.length === 0 || tags.every((t) => t === '<none>:<none>' || t.startsWith('<none>'));
}).length;
imagesDangling.set(labels, dangling);
imageBytes.set(labels, imgs.value.reduce((sum, i) => sum + (i.size ?? 0), 0));
} else up = 0;
if (vols.status === 'fulfilled') volumesTotal.set(labels, vols.value.length);
else up = 0;
if (nets.status === 'fulfilled') networksTotal.set(labels, nets.value.length);
else up = 0;
if (stacks.status === 'fulfilled') stacksTotal.set(labels, stacks.value.length);
if (updates.status === 'fulfilled') updatesAvailable.set(labels, updates.value.length);
if (vuln.status === 'fulfilled') {
const s = vuln.value.summary;
vulnerabilities.set({ ...labels, severity: 'critical' }, s.critical);
vulnerabilities.set({ ...labels, severity: 'high' }, s.high);
vulnerabilities.set({ ...labels, severity: 'medium' }, s.medium);
vulnerabilities.set({ ...labels, severity: 'low' }, s.low);
vulnerabilitiesTotal.set(labels, s.total);
imagesScanned.set(labels, s.imagesScanned);
}
if (events.status === 'fulfilled') {
eventsTotal.set(labels, events.value.total);
eventsToday.set(labels, events.value.today);
for (const [action, n] of Object.entries(events.value.byAction ?? {})) {
eventsByAction.set({ ...labels, action }, n);
}
}
if (freshness.status === 'fulfilled') {
if (freshness.value.oldestAgeSeconds !== null) scanOldestAge.set(labels, freshness.value.oldestAgeSeconds);
if (freshness.value.avgDurationSeconds !== null) scanAvgDuration.set(labels, freshness.value.avgDurationSeconds);
}
envUp.set(upLabels, up);
}
/** Refresh Dockhand's own internal gauges — all cheap in-memory reads + one DB stat. */
async function collectInternals(envCount: number): Promise<void> {
buildInfo.set({ version: appVersion }, 1);
uptimeSeconds.set(getServerUptime());
environmentsTotal.set(envCount);
try { usersTotal.set((await getUsers()).length); } catch { /* auth may be off */ }
try { registriesTotal.set((await getRegistries()).length); } catch { /* best-effort */ }
try { gitReposTotal.set((await getGitRepositories()).length); } catch { /* best-effort */ }
try { configSetsTotal.set((await getConfigSets()).length); } catch { /* best-effort */ }
try {
const t = await getApiTokenStats();
apiTokens.set({ state: 'total' }, t.total);
apiTokens.set({ state: 'expired' }, t.expired);
apiTokens.set({ state: 'never_used' }, t.neverUsed);
} catch { /* best-effort */ }
try {
scheduleExec.reset(); scheduleLastRun.reset(); scheduleLastSuccess.reset();
const s = await getScheduleStats();
for (const r of s.byTypeStatus) scheduleExec.set({ type: r.type, status: r.status }, r.count);
for (const [type, age] of Object.entries(s.lastRunSecondsByType)) scheduleLastRun.set({ type }, age);
for (const [type, age] of Object.entries(s.lastSuccessSecondsByType)) scheduleLastSuccess.set({ type }, age);
} catch { /* best-effort */ }
hawserAgents.set(edgeConnections.size);
hawserAgentInfo.reset();
let pending = 0;
const nowMs = Date.now();
for (const [envId, conn] of edgeConnections) {
pending += conn.pendingRequests.size;
hawserAgentInfo.set({
env_id: String(envId),
agent: conn.agentName || conn.agentId,
agent_version: conn.agentVersion || 'unknown',
docker_version: conn.dockerVersion || 'unknown',
hostname: conn.hostname || 'unknown'
}, Math.max(0, Math.round((nowMs - conn.connectedAt.getTime()) / 1000)));
}
hawserPending.set(pending);
const j = getJobStats();
jobs.set({ status: 'running' }, j.running);
jobs.set({ status: 'done' }, j.done);
jobs.set({ status: 'error' }, j.error);
const sc = getScannerStats();
scanQueue.set({ kind: 'in_progress' }, sc.inProgress);
scanQueue.set({ kind: 'locked' }, sc.locked);
const cache = getVulnerabilitiesCacheStats();
cacheEntries.set({ kind: 'envs' }, cache.envs);
cacheEntries.set({ kind: 'views' }, cache.views);
cacheEntries.set({ kind: 'inflight' }, cache.inflight);
const sched = getSchedulerStats();
schedulerRunning.set(sched.running ? 1 : 0);
schedulerJobs.set(sched.activeJobs);
try {
const db = await getDatabaseStats();
dbSizeBytes.set({ type: db.type }, db.sizeBytes);
dbInfo.set({ type: db.type, version: db.version }, 1);
for (const [table, n] of Object.entries(db.rowCounts)) dbRows.set({ table }, n);
for (const [stat, n] of Object.entries(db.extra)) dbExtra.set({ type: db.type, stat }, n);
} catch { /* best-effort */ }
}
/**
* Reset per-env gauges before a fresh gather so a removed environment (or a
* container state that disappeared) doesn't leave a stale series lingering.
*/
function resetPerEnvGauges(): void {
for (const m of [
envUp, containers, containersHealth, containersTotal, containersRestarts, updatesAvailable,
imagesTotal, imagesDangling, imageBytes, volumesTotal, networksTotal, stacksTotal,
vulnerabilities, vulnerabilitiesTotal, imagesScanned, eventsTotal, eventsToday, eventsByAction,
scanOldestAge, scanAvgDuration, envCpu, envMemUsed, envMemTotal
]) m.reset();
}
/**
* Gather everything. Resolves `true` if the full gather completed within the
* timeout, `false` if it timed out. Bounding the wait protects the live UI from
* a hung environment; the caller uses the return value so a timed-out (partial)
* gather is NOT cached as if it were fresh — the next scrape re-collects.
*/
/** Run `worker` over `items` with at most `limit` in flight at once. */
async function mapLimit<T>(items: T[], limit: number, worker: (item: T) => Promise<void>): Promise<void> {
let i = 0;
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
while (i < items.length) {
const item = items[i++];
await worker(item).catch(() => { /* isolated per item */ });
}
});
await Promise.all(runners);
}
async function collectAll(): Promise<boolean> {
const envs = await getEnvironments();
resetPerEnvGauges();
const TIMED_OUT = Symbol('timeout');
const deadline = new Promise<typeof TIMED_OUT>((resolve) => setTimeout(() => resolve(TIMED_OUT), COLLECT_TIMEOUT_MS));
// Gather envs in bounded batches (not all at once) so the Docker/DB call burst
// can't monopolize resources against live traffic; internals run alongside.
const gather = Promise.all([
mapLimit(envs, ENV_CONCURRENCY, collectEnvironment),
collectInternals(envs.length).catch(() => {})
]).then(() => true as const);
const result = await Promise.race([gather, deadline]);
return result !== TIMED_OUT;
}
/**
* Render the exposition text. Collection is refreshed at most once per
* COLLECT_TTL_MS; concurrent scrapes within the window share one gather. A
* gather that times out does NOT advance the cache clock, so the next scrape
* retries rather than serving the partial snapshot for a full TTL.
*/
export async function renderMetrics(): Promise<string> {
const now = Date.now();
if (now - lastCollectAt >= COLLECT_TTL_MS) {
if (!inflight) {
inflight = collectAll()
.then((completed) => { if (completed) lastCollectAt = Date.now(); })
.catch((e) => { console.error('[metrics] collection failed:', e); })
.finally(() => { inflight = null; });
}
await inflight;
}
return registry.metrics();
}
export const metricsContentType = registry.contentType;
+2 -2
View File
@@ -14,7 +14,7 @@
*
* Setup docs: https://github.com/caronc/apprise-api
*/
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendApprise(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
const isSecure = appriseUrl.startsWith('apprises');
@@ -65,7 +65,7 @@ export async function sendApprise(appriseUrl: string, payload: NotificationPaylo
if (format) body.format = format; // text | markdown | html
try {
const response = await fetch(`${baseUrl}/notify/${encodeURIComponent(key)}`, {
const response = await notificationFetch(`${baseUrl}/notify/${encodeURIComponent(key)}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
+2 -1
View File
@@ -12,6 +12,7 @@
* ?group=, ?icon=, ?url=, ?badge=N, ?copy=, ?subtitle=,
* ?volume=, ?ttl=, ?call=1, ?autoCopy=1, ?isArchive=1, ?action=none
*/
import { notificationFetch } from './shared';
import type { NotificationPayload, NotificationResult } from './shared';
export async function sendBark(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
@@ -93,7 +94,7 @@ export async function sendBark(appriseUrl: string, payload: NotificationPayload)
}
try {
const response = await fetch(`${baseUrl}/push`, {
const response = await notificationFetch(`${baseUrl}/push`, {
method: 'POST',
headers: { 'Content-Type': 'application/json; charset=utf-8' },
body: JSON.stringify(body)
+2 -2
View File
@@ -1,5 +1,5 @@
/** Discord webhook notifications. discord:// or discords://. */
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendDiscord(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
// discord://webhook_id/webhook_token or discords://...
@@ -7,7 +7,7 @@ export async function sendDiscord(appriseUrl: string, payload: NotificationPaylo
const titleWithEnv = payload.environmentName ? `${payload.title} [${payload.environmentName}]` : payload.title;
try {
const response = await fetch(url, {
const response = await notificationFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
@@ -1,12 +1,12 @@
/** Generic JSON webhook. json:// or jsons:// (HTTPS). */
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendGenericWebhook(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
// json://hostname/path or jsons://hostname/path
const url = appriseUrl.replace(/^jsons?:\/\//, appriseUrl.startsWith('jsons') ? 'https://' : 'http://');
try {
const response = await fetch(url, {
const response = await notificationFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
+2 -2
View File
@@ -1,6 +1,6 @@
/** Gotify. gotify:// or gotifys:// (HTTPS). */
import { buildGotifyUrl } from '$lib/utils/notification-parsers';
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendGotify(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
const parsed = buildGotifyUrl(appriseUrl);
@@ -12,7 +12,7 @@ export async function sendGotify(appriseUrl: string, payload: NotificationPayloa
const defaultPriority = payload.type === 'error' ? 8 : payload.type === 'warning' ? 5 : 2;
try {
const response = await fetch(parsed.url, {
const response = await notificationFetch(parsed.url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
+2 -2
View File
@@ -1,5 +1,5 @@
/** Mattermost incoming webhook. mmost:// or mmosts:// (HTTPS). */
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendMattermost(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
// mmost://[botname@]hostname[:port][/path]/token or mmosts://...
@@ -37,7 +37,7 @@ export async function sendMattermost(appriseUrl: string, payload: NotificationPa
}
try {
const response = await fetch(url, {
const response = await notificationFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
+10 -3
View File
@@ -1,5 +1,5 @@
/** ntfy.sh + self-hosted ntfy. ntfy:// or ntfys:// (HTTPS). */
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
// HTTP header values must be ByteString (every char ≤ 0xFF). Reject anything
// else so we never blow up fetch() with a 65533 (U+FFFD) replacement char.
@@ -35,7 +35,7 @@ export async function sendNtfy(appriseUrl: string, payload: NotificationPayload)
// ntfy://user:pass@host/topic (custom server with basic auth)
// ntfy://token@host/topic (custom server with bearer token)
// ntfy://host/topic?auth=TOKEN (raw token, e.g. tk_..., or base64-encoded "Bearer <token>")
// Query params: ?tags=ship,whale &title=Custom &priority=5
// Query params: ?tags=ship,whale &title=Custom &priority=5 &email=me@example.com
// ntfys:// variants for HTTPS
const isSecure = appriseUrl.startsWith('ntfys');
const path = appriseUrl.replace(/^ntfys?:\/\//, '');
@@ -47,6 +47,7 @@ export async function sendNtfy(appriseUrl: string, payload: NotificationPayload)
let queryTags: string | null = null;
let queryTitle: string | null = null;
let queryPriority: string | null = null;
let queryEmail: string | null = null;
let cleanPath = path;
const qIndex = path.indexOf('?');
if (qIndex !== -1) {
@@ -55,6 +56,7 @@ export async function sendNtfy(appriseUrl: string, payload: NotificationPayload)
queryTags = params.get('tags');
queryTitle = params.get('title');
queryPriority = params.get('priority');
queryEmail = params.get('email');
cleanPath = path.substring(0, qIndex);
}
@@ -95,8 +97,13 @@ export async function sendNtfy(appriseUrl: string, payload: NotificationPayload)
headers['Authorization'] = authHeader;
}
// ?email=<address> → ntfy's Email header so ntfy forwards the message as email (#1231)
if (queryEmail && isHeaderSafe(queryEmail)) {
headers['Email'] = queryEmail;
}
try {
const response = await fetch(url, {
const response = await notificationFetch(url, {
method: 'POST',
headers,
body: payload.message
+2 -2
View File
@@ -1,5 +1,5 @@
/** Pushover. pushover://user_key/api_token. */
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendPushover(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
const match = appriseUrl.match(/^pushover:\/\/([^/]+)\/(.+)/);
@@ -12,7 +12,7 @@ export async function sendPushover(appriseUrl: string, payload: NotificationPayl
const titleWithEnv = payload.environmentName ? `${payload.title} [${payload.environmentName}]` : payload.title;
try {
const response = await fetch(url, {
const response = await notificationFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
+19
View File
@@ -19,6 +19,25 @@ export interface NotificationResult {
error?: string;
}
/**
* Timeout for an outbound notification request, overridable via
* NOTIFICATION_TIMEOUT_MS. Falls back to 10s if unset or invalid (a bad value
* must not silently disable the timeout).
*/
const NOTIFICATION_TIMEOUT_MS = (() => {
const parsed = Number(process.env.NOTIFICATION_TIMEOUT_MS);
return Number.isFinite(parsed) && parsed > 0 ? parsed : 10_000;
})();
/**
* fetch() with a per-request timeout. Notification destinations are
* user-configured (and a potential SSRF target); without a timeout a hung
* endpoint pins the request indefinitely and blocks the rest of the dispatch.
*/
export function notificationFetch(input: string | URL, init: RequestInit = {}): Promise<Response> {
return fetch(input, { ...init, signal: init.signal ?? AbortSignal.timeout(NOTIFICATION_TIMEOUT_MS) });
}
/** Drain a response body to release the underlying socket/TLS connection. */
export async function drainResponse(response: Response): Promise<void> {
if (!response.bodyUsed) {
+2 -2
View File
@@ -11,7 +11,7 @@
* numbers (numeric, '+' gets added) or group IDs (signal-cli's "group.<base64>"
* form, passed through untouched).
*/
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendSignal(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
const isSecure = appriseUrl.startsWith('signals');
@@ -49,7 +49,7 @@ export async function sendSignal(appriseUrl: string, payload: NotificationPayloa
const baseUrl = `${isSecure ? 'https' : 'http'}://${hostPort}`;
try {
const response = await fetch(`${baseUrl}/v2/send`, {
const response = await notificationFetch(`${baseUrl}/v2/send`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
+2 -2
View File
@@ -1,5 +1,5 @@
/** Slack incoming webhook. slack:// or slacks:// or a raw hooks.slack.com URL. */
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendSlack(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
// slack://token_a/token_b/token_c or webhook URL
@@ -14,7 +14,7 @@ export async function sendSlack(appriseUrl: string, payload: NotificationPayload
const envTag = payload.environmentName ? ` \`${payload.environmentName}\`` : '';
try {
const response = await fetch(url, {
const response = await notificationFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
+2 -2
View File
@@ -1,6 +1,6 @@
/** Telegram bot. tgram://bot_token/chat_id[:topic_id]. */
import { escapeTelegramMarkdown, parseTelegramUrl } from '$lib/utils/notification-parsers';
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendTelegram(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
const parsed = parseTelegramUrl(appriseUrl);
@@ -16,7 +16,7 @@ export async function sendTelegram(appriseUrl: string, payload: NotificationPayl
const envTag = payload.environmentName ? ` [${escapeTelegramMarkdown(payload.environmentName)}]` : '';
try {
const response = await fetch(url, {
const response = await notificationFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
+2 -2
View File
@@ -1,6 +1,6 @@
/** Microsoft Power Automate Workflows (e.g. Microsoft Teams). workflows://. */
import { parseWorkflowsUrl, buildWorkflowsHttpUrl } from '$lib/utils/notification-parsers';
import { drainResponse, type NotificationPayload, type NotificationResult } from './shared';
import { notificationFetch, drainResponse, type NotificationPayload, type NotificationResult } from './shared';
export async function sendWorkflows(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
const parsed = parseWorkflowsUrl(appriseUrl);
@@ -12,7 +12,7 @@ export async function sendWorkflows(appriseUrl: string, payload: NotificationPay
const titleWithEnv = payload.environmentName ? `${payload.title} [${payload.environmentName}]` : payload.title;
try {
const response = await fetch(url, {
const response = await notificationFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
+33 -2
View File
@@ -115,13 +115,19 @@ async function withScannerLock<T>(scannerType: string, fn: () => Promise<T>): Pr
// Key: "{scannerType}:{imageName}", Value: Promise that resolves to the scan result
const inProgressScans = new Map<string, Promise<string>>();
/** Scanner queue depth — for the metrics endpoint. `inProgress` = distinct
* image scans running/deduped; `locked` = scanner types holding the serial lock. */
export function getScannerStats(): { inProgress: number; locked: number } {
return { inProgress: inProgressScans.size, locked: scannerLocks.size };
}
// Default CLI arguments for scanners (image name is substituted for {image})
export const DEFAULT_GRYPE_ARGS = '-o json -v {image}';
export const DEFAULT_TRIVY_ARGS = 'image --format json {image}';
// Pinned scanner images — avoid :latest after the March 2026 Trivy supply chain attack
export const DEFAULT_GRYPE_IMAGE = 'anchore/grype:v0.110.0';
export const DEFAULT_TRIVY_IMAGE = 'aquasec/trivy:0.69.3';
export const DEFAULT_GRYPE_IMAGE = 'anchore/grype:v0.115.0';
export const DEFAULT_TRIVY_IMAGE = 'aquasec/trivy:0.71.2';
export interface VulnerabilitySeverity {
critical: number;
@@ -154,6 +160,31 @@ export interface ScanResult {
error?: string;
}
/**
* Convert a ScanResult into the row shape expected by saveVulnerabilityScan().
* Note: `vulnerabilities` is passed as the raw array — saveVulnerabilityScan()
* does the JSON.stringify. (The previous inline copy of this helper stringified
* here too, which double-encoded the column; readers must tolerate both.)
*/
export function scanResultToDbFormat(result: ScanResult, envId?: number | null) {
return {
environmentId: envId ?? null,
imageId: result.imageId || result.imageName, // Fallback to imageName if imageId is undefined
imageName: result.imageName,
scanner: result.scanner,
scannedAt: result.scannedAt,
scanDuration: result.scanDuration,
criticalCount: result.summary.critical,
highCount: result.summary.high,
mediumCount: result.summary.medium,
lowCount: result.summary.low,
negligibleCount: result.summary.negligible,
unknownCount: result.summary.unknown,
vulnerabilities: result.vulnerabilities,
error: result.error ?? null
};
}
export interface ScanProgress {
stage: 'checking' | 'pulling-scanner' | 'scanning' | 'parsing' | 'complete' | 'error';
message: string;
+10 -2
View File
@@ -184,6 +184,11 @@ export async function startScheduler(): Promise<void> {
/**
* Stop the scheduler service and cleanup all jobs.
*/
/** Scheduler state — for the metrics endpoint. */
export function getSchedulerStats(): { running: boolean; activeJobs: number } {
return { running: isRunning, activeJobs: activeJobs.size };
}
export function stopScheduler(): void {
if (!isRunning) return;
@@ -368,8 +373,11 @@ export async function registerSchedule(
// Get timezone for this environment
const timezone = environmentId ? await getEnvironmentTimezone(environmentId) : 'UTC';
// Create new Cron instance with timezone
const job = new Cron(cronExpression, { timezone, legacyMode: false }, async () => {
// Create new Cron instance with timezone.
// protect: skip a scheduled tick if the previous run is still in progress
// (prevents a slow update/sync from overlapping itself — duplicate
// container recreation, concurrent git pull on the same stack dir).
const job = new Cron(cronExpression, { timezone, legacyMode: false, protect: true }, async () => {
// Defensive check: verify schedule still exists and is enabled
if (type === 'container_update') {
const setting = await getAutoUpdateSettingById(scheduleId);
@@ -0,0 +1,122 @@
/**
* Vulnerability block decision with a fresh current-image scan.
*
* Kept separate from update-utils.ts so that the pure decision logic
* (shouldBlockUpdate, combineScanSummaries) stays free of DB/scanner runtime
* deps and remains unit-testable. This module is allowed to touch the DB and
* the scanner.
*/
import type { VulnerabilityCriteria } from '../../db';
import { type VulnerabilitySeverity, scanImage } from '../../scanner';
import { getCombinedScanForImage, saveVulnerabilityScan } from '../../db';
import { shouldBlockUpdate, combineScanSummaries } from './update-utils';
/**
* Resolve the block decision for a new image, handling the 'more_than_current'
* criteria robustly.
*
* For 'more_than_current' the new image's vuln count is compared against the
* CURRENT image's count. The current count comes from the scan cache, which can
* be (a) stale — vuln DBs grow, so an old cached scan understates the current
* image and falsely makes the new image look worse (#1022), or (b) missing — no
* scan exists, so the comparison is skipped and a genuinely more-vulnerable
* update slips through. Both are fixed here: when the cached comparison would
* block (or there is no cache), the current image is RE-SCANNED fresh and the
* comparison is redone against up-to-date numbers.
*
* For all other criteria this is a thin wrapper around shouldBlockUpdate.
*
* @param newSummary combined scan summary of the new image
* @param currentImageId sha256 image id of the currently-running image
* @param envId environment id (for scanner settings + cache scoping)
* @param criteria the configured vulnerability criteria
* @param log log sink for human-readable progress
*/
export async function resolveBlockDecision(
newSummary: VulnerabilitySeverity,
currentImageId: string,
envId: number | null | undefined,
criteria: VulnerabilityCriteria,
log: (msg: string) => void
): Promise<{ blocked: boolean; reason: string }> {
if (criteria !== 'more_than_current') {
return shouldBlockUpdate(criteria, newSummary);
}
const total = (s: VulnerabilitySeverity) => s.critical + s.high + s.medium + s.low;
const newTotal = total(newSummary);
// 1. Start from the cached scan of the current image, if any.
let currentSummary: VulnerabilitySeverity | undefined;
let currentFromCache = false;
try {
const cached = await getCombinedScanForImage(currentImageId, envId ?? null);
if (cached) {
currentSummary = cached;
currentFromCache = true;
log(`more_than_current: current image cached scan = ${total(cached)} vulns (${cached.critical}C/${cached.high}H/${cached.medium}M/${cached.low}L)`);
} else {
log(`more_than_current: no cached scan for current image`);
}
} catch (err: any) {
log(`more_than_current: cache lookup failed (${err.message})`);
}
const wouldBlock = currentSummary !== undefined && newTotal > total(currentSummary);
// 2. Re-scan the current image when the cached comparison would block, or
// when there is no cached scan at all. A fresh scan of the SAME current
// image is the only trustworthy basis for blocking (#1022).
if (!currentFromCache || wouldBlock) {
log(
currentFromCache
? `more_than_current: cached comparison would block (new ${newTotal} > current ${total(currentSummary!)}) — re-scanning current image to confirm`
: `more_than_current: scanning current image (${currentImageId.substring(0, 19)}) for comparison`
);
try {
const results = await scanImage(currentImageId, envId ?? undefined, (p) => {
if (p.message) log(` [${p.scanner || 'scan'}] ${p.message}`);
});
if (results.length > 0) {
const fresh = combineScanSummaries(results.map((r) => ({ summary: r.summary })));
log(`more_than_current: current image fresh scan = ${total(fresh)} vulns (${fresh.critical}C/${fresh.high}H/${fresh.medium}M/${fresh.low}L)`);
currentSummary = fresh;
// Persist so the next cycle starts from a current value.
for (const r of results) {
try {
await saveVulnerabilityScan({
environmentId: envId ?? null,
imageId: currentImageId,
imageName: r.imageName,
scanner: r.scanner,
scannedAt: r.scannedAt,
scanDuration: r.scanDuration,
criticalCount: r.summary.critical,
highCount: r.summary.high,
mediumCount: r.summary.medium,
lowCount: r.summary.low,
negligibleCount: r.summary.negligible,
unknownCount: r.summary.unknown,
vulnerabilities: r.vulnerabilities,
error: r.error ?? null
});
} catch { /* ignore save errors */ }
}
} else {
log(`more_than_current: current image scan returned no results`);
}
} catch (err: any) {
log(`more_than_current: current image scan failed (${err.message})`);
}
}
const decision = shouldBlockUpdate('more_than_current', newSummary, currentSummary);
const curTotal = currentSummary ? total(currentSummary) : 'unknown';
log(
decision.blocked
? `more_than_current: BLOCKED — new ${newTotal} > current ${curTotal}`
: `more_than_current: allowed — new ${newTotal} <= current ${curTotal}${currentSummary === undefined ? ' (current count unavailable; not blocking)' : ''}`
);
return decision;
}
@@ -15,8 +15,7 @@ import {
createScheduleExecution,
updateScheduleExecution,
appendScheduleExecutionLog,
saveVulnerabilityScan,
getCombinedScanForImage
saveVulnerabilityScan
} from '../../db';
import {
pullImage,
@@ -37,7 +36,8 @@ import {
} from '../../docker';
import { getScannerSettings, scanImage, type ScanResult, type VulnerabilitySeverity } from '../../scanner';
import { sendEventNotification } from '../../notifications';
import { parseImageNameAndTag, shouldBlockUpdate, combineScanSummaries, isSystemContainer, isPodmanInfraContainer } from './update-utils';
import { parseImageNameAndTag, combineScanSummaries, isSystemContainer, isPodmanInfraContainer } from './update-utils';
import { resolveBlockDecision } from './block-decision';
import { isUpdateDisabledByLabel, isHiddenByLabel } from '../../container-labels';
// =============================================================================
@@ -150,54 +150,16 @@ async function scanAndCheckBlock(ctx: ScanContext): Promise<ScanOutcome> {
}
}
// Handle 'more_than_current' criteria - need to get/scan current image
let currentScanSummary: VulnerabilitySeverity | undefined;
if (vulnerabilityCriteria === 'more_than_current') {
log(`Looking up cached scan for current image...`);
try {
const cachedScan = await getCombinedScanForImage(currentImageId, envId ?? null);
if (cachedScan) {
currentScanSummary = cachedScan;
log(`Cached scan: ${currentScanSummary.critical} critical, ${currentScanSummary.high} high`);
} else {
log(`No cached scan found, scanning current image...`);
const currentScanResults = await scanImage(currentImageId, envId, (progress) => {
const tag = progress.scanner ? `[${progress.scanner}]` : '[scan]';
if (progress.message) log(`${tag} ${progress.message}`);
});
if (currentScanResults.length > 0) {
currentScanSummary = combineScanSummaries(currentScanResults);
log(`Current image: ${currentScanSummary.critical} critical, ${currentScanSummary.high} high`);
// Save for future use
for (const result of currentScanResults) {
try {
await saveVulnerabilityScan({
environmentId: envId ?? null,
imageId: currentImageId,
imageName: result.imageName,
scanner: result.scanner,
scannedAt: result.scannedAt,
scanDuration: result.scanDuration,
criticalCount: result.summary.critical,
highCount: result.summary.high,
mediumCount: result.summary.medium,
lowCount: result.summary.low,
negligibleCount: result.summary.negligible,
unknownCount: result.summary.unknown,
vulnerabilities: result.vulnerabilities,
error: result.error ?? null
});
} catch { /* ignore */ }
}
}
}
} catch (cacheError: any) {
log(`Warning: Could not get current scan: ${cacheError.message}`);
}
}
// Check if update should be blocked
const { blocked, reason } = shouldBlockUpdate(vulnerabilityCriteria, scanSummary, currentScanSummary);
// Decide whether to block. For 'more_than_current' this re-scans the current
// image so the comparison uses up-to-date numbers (#1022) and works on a
// cold cache.
const { blocked, reason } = await resolveBlockDecision(
scanSummary,
currentImageId,
envId,
vulnerabilityCriteria,
log
);
if (blocked) {
log(`UPDATE BLOCKED: ${reason}`);
@@ -339,13 +301,9 @@ export async function runContainerUpdate(
return;
}
// Get the full container config to extract the image name (tag)
const inspectData = await inspectContainer(container.id, envId) as any;
const imageNameFromConfig = inspectData.Config?.Image;
// Podman pod-infra containers may have an empty Config.Image — silently
// skip them before the "Could not determine image name" failure (#1221).
if (isPodmanInfraContainer(imageNameFromConfig, inspectData.Config?.Labels)) {
// Podman pod-infra containers (#1221): silently skip before any inspect /
// image-resolution that would fail with "Could not determine image name".
if (isPodmanInfraContainer(containerName)) {
log(`Skipping Podman pod-infra container`);
await updateScheduleExecution(execution.id, {
status: 'skipped',
@@ -356,6 +314,10 @@ export async function runContainerUpdate(
return;
}
// Get the full container config to extract the image name (tag)
const inspectData = await inspectContainer(container.id, envId) as any;
const imageNameFromConfig = inspectData.Config?.Image;
if (!imageNameFromConfig) {
log(`Could not determine image name from container config`);
await updateScheduleExecution(execution.id, {
@@ -30,7 +30,8 @@ import {
} from '../../docker';
import { sendEventNotification } from '../../notifications';
import { getScannerSettings, scanImage, type VulnerabilitySeverity } from '../../scanner';
import { parseImageNameAndTag, shouldBlockUpdate, combineScanSummaries, isSystemContainer, isPodmanInfraContainer } from './update-utils';
import { parseImageNameAndTag, combineScanSummaries, isSystemContainer, isPodmanInfraContainer } from './update-utils';
import { resolveBlockDecision } from './block-decision';
import { isUpdateDisabledByLabel, isHiddenByLabel } from '../../container-labels';
import { recreateContainer } from './container-update';
@@ -110,7 +111,7 @@ export async function runEnvUpdateCheckJob(
const allContainers = await listContainers(true, environmentId);
// Skip hidden + Podman pod-infra (#1083, #1221)
const containers = allContainers.filter(
(c) => !isHiddenByLabel(c.labels) && !isPodmanInfraContainer(c.image, c.labels)
(c) => !isHiddenByLabel(c.labels) && !isPodmanInfraContainer(c.name)
);
const hiddenCount = allContainers.length - containers.length;
await log(`Found ${containers.length} containers${hiddenCount ? ` (${hiddenCount} hidden/infra)` : ''}`);
@@ -324,8 +325,16 @@ export async function runEnvUpdateCheckJob(
} catch { /* ignore save errors */ }
}
// Check if blocked
const { blocked, reason } = shouldBlockUpdate(vulnerabilityCriteria, scanSummary, undefined);
// Decide whether to block. For 'more_than_current' this
// re-scans the current image so the comparison uses
// up-to-date numbers (#1022) and works on a cold cache.
const { blocked, reason } = await resolveBlockDecision(
scanSummary,
update.currentImageId,
environmentId,
vulnerabilityCriteria,
(m) => { void log(` ${m}`); }
);
if (blocked) {
scanBlocked = true;
blockReason = reason;
+8 -12
View File
@@ -128,19 +128,15 @@ export function isSystemContainer(imageName: string): SystemContainerType | null
}
/**
* Podman creates a hidden "pod-infra" container for every pod it runs (#1221).
* The image (`localhost/podman-pause`) is generated locally and never published
* to any registry, so update checks always fail noisily. The label is the
* authoritative signal — but Podman may set Config.Image empty for these
* containers, so the name fallback catches that too.
* Podman creates a hidden "pod-infra" container per pod, always named
* `<pod-id-or-name>-infra` (#1221). The infra image is locally generated
* and can be overridden via --infra-image, so neither image nor label is
* universal — the name suffix is the only reliable signal.
*
* Anchored to the end so user containers like "my-infrastructure" don't match.
*/
export function isPodmanInfraContainer(
imageName: string | undefined,
labels: Record<string, string> | undefined
): boolean {
if (labels?.['io.podman.annotations.infra'] === 'true') return true;
if (imageName && /podman-pause/i.test(imageName)) return true;
return false;
export function isPodmanInfraContainer(containerName: string | undefined): boolean {
return !!containerName && /[-_]infra$/i.test(containerName);
}
/**
+3 -3
View File
@@ -18,7 +18,7 @@ export { prefersJSON, sseToJSON } from '$lib/server/sse-parser';
* can reconstruct the same event stream semantics used by the old SSE flow.
*/
export function createJobResponse(
operation: (send: (event: string, data: unknown) => void) => Promise<void>,
operation: (send: (event: string, data: unknown) => void, isCancelled: () => boolean) => Promise<void>,
request?: Request
): Response {
// Backward compat: synchronous JSON path for explicit application/json callers
@@ -31,7 +31,7 @@ export function createJobResponse(
resultData = data;
};
try {
await operation(send);
await operation(send, () => false);
} catch (error) {
resultData = { success: false, error: String(error) };
}
@@ -51,7 +51,7 @@ export function createJobResponse(
appendLine(job, { event, data });
};
operation(send)
operation(send, () => job.cancelRequested === true)
.then(() => {
const resultLine = job.lines.findLast((l) => l.event === 'result');
completeJob(job, resultLine?.data ?? { success: true });
+23 -1
View File
@@ -34,6 +34,7 @@ export interface DiscoveredStack {
sourceDir: string;
serviceCount?: number; // Number of services defined in compose file
runningOn?: RunningStackInfo[];
unadoptable?: boolean; // a running container carries dockhand.adopt=false (#998)
}
export interface ScanResult {
@@ -201,6 +202,20 @@ export async function adoptStack(
stack: DiscoveredStack,
environmentId: number
): Promise<{ success: boolean; adoptedName?: string; error?: string }> {
// Defense in depth: re-check the live running stack for dockhand.adopt=false so a
// forged/stale request can't bypass the UI filter (#998). Only enforceable while
// the stack is running, since the label lives on containers.
try {
const { listComposeStacks } = await import('./stacks.js');
const { isStackUnadoptable } = await import('./container-labels.js');
const running = (await listComposeStacks(environmentId)).find((s) => s.name === stack.name);
if (running && isStackUnadoptable(running.containerDetails?.map((c) => c.labels) ?? [])) {
return { success: false, error: 'Stack is marked not adoptable (dockhand.adopt=false)' };
}
} catch {
// Environment offline / unreachable — fall through; nothing to enforce against.
}
// Get all existing stack sources to check for duplicates
const existingSources = await getStackSources();
@@ -466,6 +481,7 @@ export async function detectRunningStacks(
// Dynamic imports to avoid circular dependencies
const { listComposeStacks } = await import('./stacks.js');
const { getEnvironments } = await import('./db.js');
const { isStackUnadoptable } = await import('./container-labels.js');
// Get all environments
const environments = await getEnvironments();
@@ -476,6 +492,8 @@ export async function detectRunningStacks(
// Build map of stack name -> running info across all environments
const runningStacksMap = new Map<string, RunningStackInfo[]>();
// Stack names that opt out of adoption via dockhand.adopt=false on any container (#998)
const unadoptableStacks = new Set<string>();
// Query each environment in parallel for running stacks
await Promise.all(
@@ -490,6 +508,9 @@ export async function detectRunningStacks(
containerCount: stack.containers?.length || 0
});
runningStacksMap.set(stack.name, existing);
if (isStackUnadoptable(stack.containerDetails?.map((c) => c.labels) ?? [])) {
unadoptableStacks.add(stack.name);
}
}
} catch (error) {
if (error instanceof DockerConnectionError) {
@@ -504,6 +525,7 @@ export async function detectRunningStacks(
// Attach running info to discovered stacks by matching name
return discovered.map((stack) => ({
...stack,
runningOn: runningStacksMap.get(stack.name)
runningOn: runningStacksMap.get(stack.name),
unadoptable: unadoptableStacks.has(stack.name)
}));
}
+17 -1
View File
@@ -33,6 +33,7 @@ import {
getStackSources,
deleteStackEnvVars,
removePendingContainerUpdate,
getPendingContainerUpdates,
deleteAutoUpdateSchedule,
getAutoUpdateSetting,
getStackSourceByComposePath
@@ -1740,6 +1741,18 @@ export async function listComposeStacks(envId?: number | null): Promise<ComposeS
const containers = await listContainers(true, envId);
const stacks = new Map<string, Set<string>>();
// Container IDs with pending image updates (populated by manual/scheduled update checks).
// Used to flag stacks that contain at least one outdated container.
const pendingUpdateIds = new Set<string>();
if (typeof envId === 'number') {
try {
const pending = await getPendingContainerUpdates(envId);
pending.forEach((p) => pendingUpdateIds.add(p.containerId));
} catch {
// Non-fatal: stacks just won't show update markers
}
}
containers.forEach((container) => {
const projectLabel = container.labels['com.docker.compose.project'];
if (projectLabel) {
@@ -1796,7 +1809,8 @@ export async function listComposeStacks(envId?: number | null): Promise<ComposeS
restartCount: c.restartCount || 0,
exitCode: c.exitCode,
created: c.created,
labels: c.labels || {}
labels: c.labels || {},
updateAvailable: pendingUpdateIds.has(c.id)
};
})
.sort((a, b) => {
@@ -1810,6 +1824,8 @@ export async function listComposeStacks(envId?: number | null): Promise<ComposeS
name,
containers: Array.from(containerIds),
containerDetails,
updatesAvailable: stackContainers.some((c) => pendingUpdateIds.has(c.id)),
updateCount: stackContainers.filter((c) => pendingUpdateIds.has(c.id)).length,
status:
activeTotal === 0
? 'stopped'
+43
View File
@@ -0,0 +1,43 @@
/**
* Shared access check for the vulnerability endpoints (grid / count / export /
* per-image export). Parses `env`, then enforces the same two guards every
* endpoint needs: images:view RBAC, and (enterprise) environment scoping.
*
* Returns the parsed envIdNum plus a `denied` reason when access is refused —
* the caller renders it with whatever error helper it already uses (json vs
* jsonError), so response shape stays each endpoint's choice.
*/
import { authorize } from '$lib/server/authorize';
import type { Cookies } from '@sveltejs/kit';
export interface VulnAccess {
envIdNum: number | undefined;
/** Whether app auth is enabled (a tenant boundary exists to enforce). */
authEnabled: boolean;
/** Present when access is refused; { message, status } for the caller to render. */
denied?: { message: string; status: number };
}
export async function authorizeVulnAccess(cookies: Cookies, url: URL): Promise<VulnAccess> {
const auth = await authorize(cookies);
// Parse env strictly: only a non-negative integer counts as a concrete env.
// A present-but-garbage value is a client error, not "all environments".
const envParam = url.searchParams.get('env');
let envIdNum: number | undefined;
if (envParam !== null) {
const n = Number(envParam);
if (!Number.isInteger(n) || n < 0) {
return { envIdNum: undefined, authEnabled: auth.authEnabled, denied: { message: 'Invalid env', status: 400 } };
}
envIdNum = n;
}
if (auth.authEnabled && !(await auth.can('images', 'view', envIdNum))) {
return { envIdNum, authEnabled: auth.authEnabled, denied: { message: 'Permission denied', status: 403 } };
}
if (envIdNum !== undefined && auth.isEnterprise && !(await auth.canAccessEnvironment(envIdNum))) {
return { envIdNum, authEnabled: auth.authEnabled, denied: { message: 'Access denied to this environment', status: 403 } };
}
return { envIdNum, authEnabled: auth.authEnabled };
}
+75
View File
@@ -0,0 +1,75 @@
/**
* Short-TTL per-environment cache for the aggregated vulnerability findings.
*
* Lives in its own module (importing only client-safe types) so that db.ts can
* invalidate it from inside saveVulnerabilityScan WITHOUT a circular import
* (vulnerabilities.ts imports db.ts). That way every writer of a scan row
* automatically refreshes the dashboard — no caller can forget.
*/
import { EMPTY_SUMMARY } from '../utils/vulnerability';
import type { Finding, VulnerabilitySummary } from '$lib/utils/vulnerability';
export const CACHE_TTL_MS = 30_000;
/** Per-env cap on memoized filter/sort views (bounds server memory). */
export const MAX_VIEWS = 8;
/** Cap on cached environments — bounds total memory when many envs are viewed.
* Least-recently-inserted env entry is evicted past this. */
export const MAX_ENVS = 16;
export interface AggregatedVulnerabilities {
findings: Finding[];
summary: VulnerabilitySummary;
}
export interface VulnerabilitiesMeta {
total: number;
summary: VulnerabilitySummary;
options: { images: string[]; containers: string[]; stacks: string[] };
}
/** Empty meta for the no-environment case — shared so the shape isn't re-typed. */
export const EMPTY_META: VulnerabilitiesMeta = {
total: 0,
summary: EMPTY_SUMMARY,
options: { images: [], containers: [], stacks: [] }
};
export interface CacheEntry {
at: number;
data: AggregatedVulnerabilities;
/** Memoized filtered+sorted arrays within this cache window, keyed by query. */
views: Map<string, Finding[]>;
/** Memoized meta (distinct filter options) — computed once per window. */
meta?: VulnerabilitiesMeta;
}
// Keyed by envId. The null-environment ("local"/default) is keyed as 0 since the
// dashboard aggregation only ever runs for a concrete numeric env id.
export const aggregateCache = new Map<number, CacheEntry>();
/** Collapse concurrent cold-start requests into one aggregation. */
export const inflight = new Map<number, Promise<AggregatedVulnerabilities>>();
/**
* Drop cached findings. Pass an env id to clear just that environment; pass
* nothing to clear all (e.g. a broad change where the affected env is unknown).
*/
/** Cache occupancy — for the metrics endpoint. `envs` = cached environments,
* `views` = total memoized filter/sort views, `inflight` = cold aggregations running. */
export function getVulnerabilitiesCacheStats(): { envs: number; views: number; inflight: number } {
let views = 0;
for (const entry of aggregateCache.values()) views += entry.views.size;
return { envs: aggregateCache.size, views, inflight: inflight.size };
}
export function invalidateVulnerabilitiesCache(envIdNum?: number | null): void {
if (envIdNum === undefined || envIdNum === null) {
aggregateCache.clear();
// Also drop in-flight aggregations: one racing a scan-save would otherwise
// resolve to pre-scan data and get installed with a fresh TTL, masking the
// new scan for a full window. Dropping it forces the next reader to re-run.
inflight.clear();
} else {
aggregateCache.delete(envIdNum);
inflight.delete(envIdNum);
}
}
+290
View File
@@ -0,0 +1,290 @@
/**
* Shared aggregation for the Vulnerabilities dashboard: flattens the latest
* persisted scans into per-CVE findings, filters out images no longer present
* on the host, and enriches each finding with the containers/stacks using it.
*
* Used by both GET /api/vulnerabilities (grid) and GET /api/vulnerabilities/export.
*/
import { getAllLatestScans } from '$lib/server/db';
import { listImages, listContainers, DockerConnectionError, EnvironmentNotFoundError } from '$lib/server/docker';
import { flattenScansToFindings, filterFindings, sortFindings } from '$lib/utils/vulnerability';
import type { Finding, FindingContainer, VulnerabilitySummary, ScanRow, FindingFilter, SortField } from '$lib/utils/vulnerability';
import {
aggregateCache, inflight, CACHE_TTL_MS, MAX_VIEWS, MAX_ENVS, invalidateVulnerabilitiesCache,
type CacheEntry, type AggregatedVulnerabilities, type VulnerabilitiesMeta
} from '$lib/server/vulnerabilities-cache';
// Re-export the shared client/server helpers so existing importers keep working.
export type { Finding, FindingContainer, VulnerabilitySummary, ScanRow };
export type { AggregatedVulnerabilities, VulnerabilitiesMeta };
export { flattenScansToFindings, invalidateVulnerabilitiesCache };
export async function aggregateVulnerabilities(envIdNum: number): Promise<AggregatedVulnerabilities> {
const scans = await getAllLatestScans(envIdNum);
// Best-effort live Docker state — total image count (for "Images scanned: N/M")
// and running containers used to enrich each finding. Never throw on Docker down.
let existingImageIds: Set<string> | null = null;
let totalImages = 0;
const containersByImage = new Map<string, FindingContainer[]>();
const stacksByImage = new Map<string, Set<string>>();
try {
const images = await listImages(envIdNum);
totalImages = images.length;
existingImageIds = new Set(images.map((img) => img.id));
} catch (error) {
if (!(error instanceof DockerConnectionError) && !(error instanceof EnvironmentNotFoundError)) {
console.error('Error listing images for vulnerability summary:', error);
}
}
try {
const containers = await listContainers(true, envIdNum);
for (const c of containers) {
if (!c.imageId) continue;
const list = containersByImage.get(c.imageId) ?? [];
list.push({ id: c.id, name: c.name });
containersByImage.set(c.imageId, list);
const stack = c.labels?.['com.docker.compose.project'];
if (stack) {
const set = stacksByImage.get(c.imageId) ?? new Set<string>();
set.add(stack);
stacksByImage.set(c.imageId, set);
}
}
} catch (error) {
if (!(error instanceof DockerConnectionError) && !(error instanceof EnvironmentNotFoundError)) {
console.error('Error listing containers for vulnerability enrichment:', error);
}
}
const scannedImageIds = new Set<string>();
let critical = 0, high = 0, medium = 0, low = 0;
// Keep only scans for images still present on the host (scanned <= total).
const liveScans = existingImageIds
? scans.filter((s) => existingImageIds!.has(s.imageId))
: scans;
for (const s of liveScans) scannedImageIds.add(s.imageId);
const findings = flattenScansToFindings(liveScans, { containersByImage, stacksByImage });
for (const f of findings) {
switch ((f.severity || '').toLowerCase()) {
case 'critical': critical++; break;
case 'high': high++; break;
case 'medium': medium++; break;
case 'low': low++; break;
}
}
// If Docker was unreachable we couldn't filter stale scans; fall back to the raw
// distinct scanned count and use it as the total so the ratio stays sane.
if (!existingImageIds) {
totalImages = scannedImageIds.size;
}
return {
findings,
summary: {
total: findings.length,
critical,
high,
medium,
low,
imagesScanned: scannedImageIds.size,
totalImages
}
};
}
// ---------------------------------------------------------------------------
// Paged access with a short-TTL cache of the flattened+enriched result.
//
// Findings are exploded from per-scan JSON blobs, so we can't page at the SQL
// layer — every request must flatten the full set. To keep paging/scrolling
// cheap, the full AggregatedVulnerabilities is cached per environment for a
// few seconds and reused across page requests. This bounds *browser* memory
// (only a page is sent to the client) without re-flattening on every scroll.
// ---------------------------------------------------------------------------
async function getCacheEntry(envIdNum: number): Promise<CacheEntry> {
const cached = aggregateCache.get(envIdNum);
if (cached && Date.now() - cached.at < CACHE_TTL_MS) return cached;
if (cached) aggregateCache.delete(envIdNum); // release the expired findings array
let promise = inflight.get(envIdNum);
if (!promise) {
promise = aggregateVulnerabilities(envIdNum).finally(() => inflight.delete(envIdNum));
inflight.set(envIdNum, promise);
}
const data = await promise;
// A concurrent cold caller may have already installed an entry for this same
// aggregation — reuse it so both share one views/meta cache.
const existing = aggregateCache.get(envIdNum);
if (existing && existing.data === data) return existing;
const entry: CacheEntry = { at: Date.now(), data, views: new Map() };
aggregateCache.set(envIdNum, entry);
// Bound *cache* memory across environments. This is a performance cache only —
// evicting an env never hides its vulns: the next request for it just
// re-aggregates from the DB. Past the cap, drop an expired entry if there is
// one (free win), else the oldest, never the entry we just created.
if (aggregateCache.size > MAX_ENVS) {
const now = Date.now();
let victim: number | undefined;
for (const [id, e] of aggregateCache) {
if (id === envIdNum) continue;
if (now - e.at >= CACHE_TTL_MS) { victim = id; break; } // prefer an expired entry
if (victim === undefined) victim = id; // fall back to the oldest (insertion order)
}
if (victim !== undefined) aggregateCache.delete(victim);
}
return entry;
}
async function getAggregatedCached(envIdNum: number): Promise<AggregatedVulnerabilities> {
return (await getCacheEntry(envIdNum)).data;
}
export interface VulnerabilitiesQuery extends FindingFilter {
sort?: SortField;
dir?: 'asc' | 'desc';
limit?: number;
offset?: number;
}
export interface PagedVulnerabilities {
findings: Finding[];
total: number;
/** Severity counts + total for the FILTERED set (so the header pills track the
* active filters). imagesScanned/totalImages stay env-wide (not per-filter). */
summary: VulnerabilitySummary;
}
/** Split a comma-separated multi-value filter param into a trimmed, non-empty list. */
function parseList(url: URL, key: string): string[] {
const raw = url.searchParams.get(key);
return raw ? raw.split(',').map((s) => s.trim()).filter(Boolean) : [];
}
/**
* Parse the shared filter/sort/paging query params used by the grid and export
* endpoints, so the parsing lives in one place (a new filter can't be added to
* one endpoint and forgotten in another).
*/
export function parseVulnerabilitiesQuery(url: URL): VulnerabilitiesQuery {
const limit = url.searchParams.get('limit');
const offset = url.searchParams.get('offset');
const sort = url.searchParams.get('sort');
return {
q: url.searchParams.get('q') ?? '',
severities: parseList(url, 'severity'),
images: parseList(url, 'image'),
containers: parseList(url, 'container'),
stacks: parseList(url, 'stack'),
sort: (sort as SortField) || undefined,
dir: url.searchParams.get('dir') === 'desc' ? 'desc' : 'asc',
limit: limit ? parseInt(limit) : undefined,
offset: offset ? parseInt(offset) : undefined
};
}
function viewKey(q: VulnerabilitiesQuery): string {
return [
q.sort ?? '', q.dir ?? '',
(q.q ?? '').toLowerCase().trim(),
(q.severities ?? []).join('|'),
(q.images ?? []).join('|'),
(q.containers ?? []).join('|'),
(q.stacks ?? []).join('|')
].join('¬');
}
/**
* The full filtered+sorted finding set for a query, memoized within the cache
* window so repeated page/scroll requests reuse the filter+sort work. Shared by
* the paged grid endpoint and the export endpoint so their ordering can't drift.
*/
export async function getFilteredSortedFindings(envIdNum: number, query: VulnerabilitiesQuery): Promise<Finding[]> {
const entry = await getCacheEntry(envIdNum);
const key = viewKey(query);
const cachedView = entry.views.get(key);
if (cachedView) {
// LRU touch: move to newest so it survives eviction.
entry.views.delete(key);
entry.views.set(key, cachedView);
return cachedView;
}
const filtered = filterFindings(entry.data.findings, query);
// filterFindings returns the input BY REFERENCE when no filter is active, and an
// unsorted view keeps that reference — storing the master findings array as a
// memoized view would make any accidental in-place mutation of a view corrupt
// the shared cache. Copy when the view would alias the master. (sortFindings
// already returns a fresh array.)
const view = query.sort
? sortFindings(filtered, query.sort, query.dir ?? 'asc')
: (filtered === entry.data.findings ? filtered.slice() : filtered);
// Bound the memo (rapid filter/sort changes would otherwise accumulate a full
// array per distinct query for the cache window). Evict the oldest.
if (entry.views.size >= MAX_VIEWS) {
const oldest = entry.views.keys().next().value;
if (oldest !== undefined) entry.views.delete(oldest);
}
entry.views.set(key, view);
return view;
}
/** Filter + sort the full set, then return one page. `total` is the filtered count. */
export async function getVulnerabilitiesPage(envIdNum: number, query: VulnerabilitiesQuery): Promise<PagedVulnerabilities> {
const view = await getFilteredSortedFindings(envIdNum, query);
const offset = Math.max(0, query.offset ?? 0);
const limit = query.limit ?? view.length;
// Severity breakdown of the FILTERED view so the header pills reflect the
// active filters (imagesScanned/totalImages are env-wide, taken from meta).
let critical = 0, high = 0, medium = 0, low = 0;
for (const f of view) {
switch ((f.severity || '').toLowerCase()) {
case 'critical': critical++; break;
case 'high': high++; break;
case 'medium': medium++; break;
case 'low': low++; break;
}
}
const meta = await getVulnerabilitiesMeta(envIdNum);
return {
findings: view.slice(offset, offset + limit),
total: view.length,
summary: {
total: view.length,
critical, high, medium, low,
imagesScanned: meta.summary.imagesScanned,
totalImages: meta.summary.totalImages
}
};
}
/** Totals + distinct filter-dropdown values across the full (unfiltered) set.
* Memoized on the cache entry — computed once per 30s window, not per request. */
export async function getVulnerabilitiesMeta(envIdNum: number): Promise<VulnerabilitiesMeta> {
const entry = await getCacheEntry(envIdNum);
if (entry.meta) return entry.meta;
const { findings, summary } = entry.data;
const images = new Set<string>();
const containers = new Set<string>();
const stacks = new Set<string>();
for (const f of findings) {
images.add(f.imageName);
for (const c of f.containers ?? []) containers.add(c.name);
for (const s of f.stacks ?? []) stacks.add(s);
}
const sorted = (s: Set<string>) => Array.from(s).sort();
entry.meta = {
total: summary.total,
summary,
options: { images: sorted(images), containers: sorted(containers), stacks: sorted(stacks) }
};
return entry.meta;
}