This commit is contained in:
jarek committed 2026-03-02 07:59:58 +01:00
1 parent 4b430340db
commit 9c451aedf9
120 files changed
+7777 -6046

No files matched your search

+4 -2
View File
@@ -1,5 +1,5 @@
import { json } from '@sveltejs/kit';
import { listImages, EnvironmentNotFoundError } from '$lib/server/docker';
import { listImages, EnvironmentNotFoundError, DockerConnectionError } from '$lib/server/docker';
import { authorize } from '$lib/server/authorize';
import { hasEnvironments } from '$lib/server/db';
import type { RequestHandler } from './$types';
@@ -32,7 +32,9 @@ export const GET: RequestHandler = async ({ url, cookies }) => {
if (error instanceof EnvironmentNotFoundError) {
return json({ error: 'Environment not found' }, { status: 404 });
}
console.error('Error listing images:', error);
if (!(error instanceof DockerConnectionError)) {
console.error('Error listing images:', error);
}
// Return empty array instead of error to allow UI to load
return json([]);
}
+56 -107
View File
@@ -1,11 +1,12 @@
import { json } from '@sveltejs/kit';
import { pullImage } from '$lib/server/docker';
import { pullImage, buildRegistryAuthHeader } from '$lib/server/docker';
import type { RequestHandler } from './$types';
import { getScannerSettings, scanImage } from '$lib/server/scanner';
import { saveVulnerabilityScan, getEnvironment } from '$lib/server/db';
import { authorize } from '$lib/server/authorize';
import { auditImage } from '$lib/server/audit';
import { sendEdgeStreamRequest, isEdgeConnected } from '$lib/server/hawser';
import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs';
/**
* Check if environment is edge mode
@@ -73,49 +74,26 @@ export const POST: RequestHandler = async (event) => {
// Check if this is an edge environment
const edgeCheck = await isEdgeMode(envId);
const encoder = new TextEncoder();
let controllerClosed = false;
let controller: ReadableStreamDefaultController<Uint8Array>;
let heartbeatInterval: ReturnType<typeof setInterval> | null = null;
let cancelEdgeStream: (() => void) | null = null;
// Job pattern: create job, run in background, return jobId immediately
const job = createJob();
const safeEnqueue = (data: string) => {
if (!controllerClosed) {
try {
controller.enqueue(encoder.encode(data));
} catch {
controllerClosed = true;
}
}
};
const cleanup = () => {
if (heartbeatInterval) {
clearInterval(heartbeatInterval);
heartbeatInterval = null;
}
if (cancelEdgeStream) {
cancelEdgeStream();
cancelEdgeStream = null;
}
controllerClosed = true;
const sendData = (data: unknown) => {
appendLine(job, { data });
};
/**
* Handle scan-on-pull after image is pulled
*/
const handleScanOnPull = async () => {
// Skip if caller explicitly requested no scan (e.g., CreateContainerModal handles scanning separately)
if (skipScanOnPull) return;
const { scanner } = await getScannerSettings(envId);
// Scan if scanning is enabled (scanner !== 'none')
if (scanner !== 'none') {
safeEnqueue(`data: ${JSON.stringify({ status: 'scanning', message: 'Starting vulnerability scan...' })}\n\n`);
sendData({ status: 'scanning', message: 'Starting vulnerability scan...' });
try {
const results = await scanImage(image, envId, (progress) => {
safeEnqueue(`data: ${JSON.stringify({ status: 'scan-progress', ...progress })}\n\n`);
sendData({ status: 'scan-progress', ...progress });
});
for (const result of results) {
@@ -138,128 +116,99 @@ export const POST: RequestHandler = async (event) => {
}
const totalVulns = results.reduce((sum, r) => sum + r.vulnerabilities.length, 0);
safeEnqueue(`data: ${JSON.stringify({
sendData({
status: 'scan-complete',
message: `Scan complete - found ${totalVulns} vulnerabilities`,
results
})}\n\n`);
});
} catch (scanError) {
console.error('Scan-on-pull failed:', scanError);
safeEnqueue(`data: ${JSON.stringify({
sendData({
status: 'scan-error',
error: scanError instanceof Error ? scanError.message : String(scanError)
})}\n\n`);
});
}
}
};
const stream = new ReadableStream({
async start(ctrl) {
controller = ctrl;
// Run operation in background
(async () => {
console.log(`Starting pull for image: ${image}${edgeCheck.isEdge ? ' (edge mode)' : ''}`);
// Start heartbeat to keep connection alive through Traefik (10s idle timeout)
heartbeatInterval = setInterval(() => {
safeEnqueue(`: keepalive\n\n`);
}, 5000);
if (edgeCheck.isEdge && edgeCheck.environmentId) {
if (!isEdgeConnected(edgeCheck.environmentId)) {
sendData({ status: 'error', error: 'Edge agent not connected' });
failJob(job, 'Edge agent not connected');
return;
}
console.log(`Starting pull for image: ${image}${edgeCheck.isEdge ? ' (edge mode)' : ''}`);
// Handle edge mode with streaming
if (edgeCheck.isEdge && edgeCheck.environmentId) {
if (!isEdgeConnected(edgeCheck.environmentId)) {
safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: 'Edge agent not connected' })}\n\n`);
cleanup();
controller.close();
return;
}
const pullUrl = buildPullUrl(image);
const pullUrl = buildPullUrl(image);
const authHeaders = await buildRegistryAuthHeader(image);
await new Promise<void>((resolve) => {
const { cancel } = sendEdgeStreamRequest(
edgeCheck.environmentId,
edgeCheck.environmentId!,
'POST',
pullUrl,
{
onData: (data: string) => {
// Data is base64 encoded JSON lines from Docker
try {
const decoded = Buffer.from(data, 'base64').toString('utf-8');
// Docker sends newline-delimited JSON
const lines = decoded.split('\n').filter(line => line.trim());
const lines = decoded.split('\n').filter((line) => line.trim());
for (const line of lines) {
try {
const progress = JSON.parse(line);
safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`);
sendData(JSON.parse(line));
} catch {
// Ignore parse errors for partial lines
}
}
} catch {
// If not base64, try as-is
try {
const progress = JSON.parse(data);
safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`);
sendData(JSON.parse(data));
} catch {
// Ignore
}
}
},
onEnd: async () => {
safeEnqueue(`data: ${JSON.stringify({ status: 'complete' })}\n\n`);
// Handle scan-on-pull
sendData({ status: 'complete' });
await handleScanOnPull();
cleanup();
controller.close();
completeJob(job, { status: 'complete' });
resolve();
},
onError: (error: string) => {
console.error('Edge pull error:', error);
safeEnqueue(`data: ${JSON.stringify({ status: 'error', error })}\n\n`);
cleanup();
controller.close();
sendData({ status: 'error', error });
failJob(job, error);
resolve();
}
}
},
undefined,
authHeaders
);
cancelEdgeStream = cancel;
} else {
// Non-edge mode: use existing pullImage function
try {
await pullImage(image, (progress) => {
const data = JSON.stringify(progress) + '\n';
safeEnqueue(`data: ${data}\n\n`);
}, envId);
// Store cancel reference (not used currently but available)
void cancel;
});
} else {
try {
await pullImage(image, (progress) => {
sendData(progress);
}, envId);
safeEnqueue(`data: ${JSON.stringify({ status: 'complete' })}\n\n`);
// Handle scan-on-pull
await handleScanOnPull();
cleanup();
controller.close();
} catch (error) {
console.error('Error pulling image:', error);
safeEnqueue(`data: ${JSON.stringify({
status: 'error',
error: String(error)
})}\n\n`);
cleanup();
controller.close();
}
sendData({ status: 'complete' });
await handleScanOnPull();
completeJob(job, { status: 'complete' });
} catch (error) {
console.error('Error pulling image:', error);
const errMsg = String(error);
sendData({ status: 'error', error: errMsg });
failJob(job, errMsg);
}
},
cancel() {
cleanup();
}
})().catch((err) => {
failJob(job, err instanceof Error ? err.message : String(err));
});
return new Response(stream, {
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'X-Accel-Buffering': 'no'
}
});
return json({ jobId: job.id });
};
+73 -153
View File
@@ -5,6 +5,8 @@ import { getRegistry, getEnvironment } from '$lib/server/db';
import { authorize } from '$lib/server/authorize';
import { auditImage } from '$lib/server/audit';
import { sendEdgeStreamRequest, isEdgeConnected } from '$lib/server/hawser';
import { prefersJSON } from '$lib/server/sse';
import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs';
/**
* Check if environment is edge mode
@@ -119,35 +121,6 @@ export const POST: RequestHandler = async (event) => {
// Check if this is an edge environment
const edgeCheck = await isEdgeMode(envIdNum);
// Stream the push progress
const encoder = new TextEncoder();
let controllerClosed = false;
let controller: ReadableStreamDefaultController<Uint8Array>;
let heartbeatInterval: ReturnType<typeof setInterval> | null = null;
let cancelEdgeStream: (() => void) | null = null;
const safeEnqueue = (data: string) => {
if (!controllerClosed) {
try {
controller.enqueue(encoder.encode(data));
} catch {
controllerClosed = true;
}
}
};
const cleanup = () => {
if (heartbeatInterval) {
clearInterval(heartbeatInterval);
heartbeatInterval = null;
}
if (cancelEdgeStream) {
cancelEdgeStream();
cancelEdgeStream = null;
}
controllerClosed = true;
};
const formatError = (error: any): string => {
const errorMessage = error.message || error || '';
let userMessage = errorMessage || 'Failed to push image';
@@ -163,140 +136,87 @@ export const POST: RequestHandler = async (event) => {
return userMessage;
};
const stream = new ReadableStream({
async start(ctrl) {
controller = ctrl;
// Core push logic — emit callback receives progress data objects
async function runPush(emit: (data: unknown) => void): Promise<void> {
emit({ status: 'tagging', message: 'Tagging image...' });
await tagImage(imageId, repo, tag, envIdNum);
emit({ status: 'pushing', message: 'Pushing to registry...' });
// Start heartbeat to keep connection alive through Traefik (10s idle timeout)
heartbeatInterval = setInterval(() => {
safeEnqueue(`: keepalive\n\n`);
}, 5000);
if (edgeCheck.isEdge && edgeCheck.environmentId) {
if (!isEdgeConnected(edgeCheck.environmentId)) {
emit({ status: 'error', error: 'Edge agent not connected' });
return;
}
try {
// Send tagging status
safeEnqueue(`data: ${JSON.stringify({ status: 'tagging', message: 'Tagging image...' })}\n\n`);
const authHeader = Buffer.from(JSON.stringify(authConfig)).toString('base64');
// Tag the image with the target registry
await tagImage(imageId, repo, tag, envIdNum);
// Send pushing status
safeEnqueue(`data: ${JSON.stringify({ status: 'pushing', message: 'Pushing to registry...' })}\n\n`);
// Handle edge mode with streaming
if (edgeCheck.isEdge && edgeCheck.environmentId) {
if (!isEdgeConnected(edgeCheck.environmentId)) {
safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: 'Edge agent not connected' })}\n\n`);
cleanup();
controller.close();
return;
}
// Create X-Registry-Auth header
const authHeader = Buffer.from(JSON.stringify(authConfig)).toString('base64');
const { cancel } = sendEdgeStreamRequest(
edgeCheck.environmentId,
'POST',
`/images/${encodeURIComponent(targetTag)}/push`,
{
onData: (data: string) => {
// Data is base64 encoded JSON lines from Docker
try {
const decoded = Buffer.from(data, 'base64').toString('utf-8');
const lines = decoded.split('\n').filter(line => line.trim());
for (const line of lines) {
try {
const progress = JSON.parse(line);
if (progress.error) {
safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: formatError(progress.error) })}\n\n`);
} else {
safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`);
}
} catch {
// Ignore parse errors for partial lines
}
}
} catch {
// If not base64, try as-is
await new Promise<void>((resolve, reject) => {
sendEdgeStreamRequest(
edgeCheck.environmentId!,
'POST',
`/images/${encodeURIComponent(targetTag)}/push`,
{
onData: (data: string) => {
try {
const decoded = Buffer.from(data, 'base64').toString('utf-8');
for (const line of decoded.split('\n').filter((l) => l.trim())) {
try {
const progress = JSON.parse(data);
if (progress.error) {
safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: formatError(progress.error) })}\n\n`);
} else {
safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`);
}
} catch {
// Ignore
}
const progress = JSON.parse(line);
emit(progress.error ? { status: 'error', error: formatError(progress.error) } : progress);
} catch { /* ignore partial lines */ }
}
},
onEnd: async () => {
// Audit log
await auditImage(event, 'push', imageId, imageName || targetTag, envIdNum, { targetTag, registry: registry.name });
safeEnqueue(`data: ${JSON.stringify({
status: 'complete',
message: `Image pushed to ${targetTag}`,
targetTag
})}\n\n`);
cleanup();
controller.close();
},
onError: (error: string) => {
console.error('Edge push error:', error);
safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: formatError(error) })}\n\n`);
cleanup();
controller.close();
} catch {
try {
const progress = JSON.parse(data);
emit(progress.error ? { status: 'error', error: formatError(progress.error) } : progress);
} catch { /* ignore */ }
}
},
undefined,
{ 'X-Registry-Auth': authHeader }
);
cancelEdgeStream = cancel;
} else {
// Non-edge mode: use existing pushImage function
await pushImage(targetTag, authConfig, (progress) => {
safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`);
}, envIdNum);
// Audit log
await auditImage(event, 'push', imageId, imageName || targetTag, envIdNum, { targetTag, registry: registry.name });
// Send completion message
safeEnqueue(`data: ${JSON.stringify({
status: 'complete',
message: `Image pushed to ${targetTag}`,
targetTag
})}\n\n`);
cleanup();
controller.close();
}
} catch (error: any) {
console.error('Error pushing image:', error);
safeEnqueue(`data: ${JSON.stringify({
status: 'error',
error: formatError(error)
})}\n\n`);
cleanup();
controller.close();
}
},
cancel() {
cleanup();
onEnd: async () => {
await auditImage(event, 'push', imageId, imageName || targetTag, envIdNum, { targetTag, registry: registry.name });
emit({ status: 'complete', message: `Image pushed to ${targetTag}`, targetTag });
resolve();
},
onError: (error: string) => {
console.error('Edge push error:', error);
emit({ status: 'error', error: formatError(error) });
reject(new Error(error));
}
},
undefined,
{ 'X-Registry-Auth': authHeader }
);
});
} else {
await pushImage(targetTag, authConfig, (progress) => emit(progress), envIdNum);
await auditImage(event, 'push', imageId, imageName || targetTag, envIdNum, { targetTag, registry: registry.name });
emit({ status: 'complete', message: `Image pushed to ${targetTag}`, targetTag });
}
});
}
return new Response(stream, {
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'X-Accel-Buffering': 'no'
// Sync path for API clients sending Accept: application/json only
if (prefersJSON(request)) {
try {
let lastEvent: unknown = null;
await runPush((data) => { lastEvent = data; });
return json(lastEvent || { success: true });
} catch (error: any) {
return json({ status: 'error', error: formatError(error) }, { status: 500 });
}
});
}
// Job pattern: return jobId immediately, push runs in background
const job = createJob();
(async () => {
try {
await runPush((data) => appendLine(job, { data }));
completeJob(job, job.lines[job.lines.length - 1]?.data ?? { success: true });
} catch (error: any) {
appendLine(job, { data: { status: 'error', error: formatError(error) } });
failJob(job, error.message);
}
})();
return json({ jobId: job.id });
} catch (error: any) {
console.error('Error setting up push:', error);
return json({ error: error.message || 'Failed to push image' }, { status: 500 });
+36 -63
View File
@@ -2,6 +2,7 @@ import { json, type RequestHandler } from '@sveltejs/kit';
import { scanImage, type ScanProgress, type ScanResult } from '$lib/server/scanner';
import { saveVulnerabilityScan, getLatestScanForImage } from '$lib/server/db';
import { authorize } from '$lib/server/authorize';
import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs';
// Helper to convert ScanResult to database format
function scanResultToDbFormat(result: ScanResult, envId?: number) {
@@ -23,7 +24,7 @@ function scanResultToDbFormat(result: ScanResult, envId?: number) {
};
}
// POST - Start a scan (returns SSE stream for progress)
// POST - Start a scan (returns { jobId } for progress polling)
export const POST: RequestHandler = async ({ request, url, cookies }) => {
const auth = await authorize(cookies);
@@ -42,75 +43,47 @@ export const POST: RequestHandler = async ({ request, url, cookies }) => {
return json({ error: 'Image name is required' }, { status: 400 });
}
// Create a readable stream for SSE
const stream = new ReadableStream({
async start(controller) {
const encoder = new TextEncoder();
let controllerClosed = false;
// Job pattern: create job, run in background, return jobId immediately
const job = createJob();
const sendProgress = (progress: ScanProgress) => {
if (controllerClosed) return;
try {
const data = `data: ${JSON.stringify(progress)}\n\n`;
controller.enqueue(encoder.encode(data));
} catch {
controllerClosed = true;
}
};
const sendProgress = (progress: ScanProgress) => {
appendLine(job, { data: progress });
};
// Send SSE keepalive comments every 5s to prevent Traefik timeout
const keepaliveInterval = setInterval(() => {
if (controllerClosed) return;
try {
controller.enqueue(encoder.encode(`: keepalive\n\n`));
} catch {
controllerClosed = true;
}
}, 5000);
(async () => {
try {
const results = await scanImage(imageName, envId, sendProgress, forceScannerType);
try {
const results = await scanImage(imageName, envId, sendProgress, forceScannerType);
// Save results to database
for (const result of results) {
await saveVulnerabilityScan(scanResultToDbFormat(result, envId));
}
// Send final complete message with all results
sendProgress({
stage: 'complete',
message: `Scan complete - found ${results.reduce((sum, r) => sum + r.vulnerabilities.length, 0)} vulnerabilities`,
progress: 100,
result: results[0],
results: results // Include all scanner results
});
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
sendProgress({
stage: 'error',
message: `Scan failed: ${errorMsg}`,
error: errorMsg
});
} finally {
clearInterval(keepaliveInterval);
if (!controllerClosed) {
try {
controller.close();
} catch {
// Already closed
}
}
// Save results to database
for (const result of results) {
await saveVulnerabilityScan(scanResultToDbFormat(result, envId));
}
// Send final complete message with all results
const completeProgress: ScanProgress = {
stage: 'complete',
message: `Scan complete - found ${results.reduce((sum, r) => sum + r.vulnerabilities.length, 0)} vulnerabilities`,
progress: 100,
result: results[0],
results: results // Include all scanner results
};
sendProgress(completeProgress);
completeJob(job, completeProgress);
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
const errorProgress: ScanProgress = {
stage: 'error',
message: `Scan failed: ${errorMsg}`,
error: errorMsg
};
sendProgress(errorProgress);
failJob(job, errorMsg);
}
})().catch((err) => {
failJob(job, err instanceof Error ? err.message : String(err));
});
return new Response(stream, {
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive'
}
});
return json({ jobId: job.id });
};
// GET - Get cached scan results for an image