Initial commit
This commit is contained in:
commit
62e3c6439e
552 files changed
+104858
No files matched your search
@@ -0,0 +1,509 @@
|
||||
/**
|
||||
* Environment Update Check Task
|
||||
*
|
||||
* Checks all containers in an environment for available image updates.
|
||||
* Can optionally auto-update containers when updates are found.
|
||||
*/
|
||||
|
||||
import type { ScheduleTrigger, VulnerabilityCriteria } from '../../db';
|
||||
import {
|
||||
getEnvUpdateCheckSettings,
|
||||
getEnvironment,
|
||||
createScheduleExecution,
|
||||
updateScheduleExecution,
|
||||
appendScheduleExecutionLog,
|
||||
saveVulnerabilityScan,
|
||||
clearPendingContainerUpdates,
|
||||
addPendingContainerUpdate,
|
||||
removePendingContainerUpdate
|
||||
} from '../../db';
|
||||
import {
|
||||
listContainers,
|
||||
inspectContainer,
|
||||
checkImageUpdateAvailable,
|
||||
pullImage,
|
||||
stopContainer,
|
||||
removeContainer,
|
||||
createContainer,
|
||||
getTempImageTag,
|
||||
isDigestBasedImage,
|
||||
getImageIdByTag,
|
||||
removeTempImage,
|
||||
tagImage
|
||||
} from '../../docker';
|
||||
import { sendEventNotification } from '../../notifications';
|
||||
import { getScannerSettings, scanImage, type VulnerabilitySeverity } from '../../scanner';
|
||||
import { parseImageNameAndTag, shouldBlockUpdate, combineScanSummaries, isDockhandContainer } from './update-utils';
|
||||
|
||||
interface UpdateInfo {
|
||||
containerId: string;
|
||||
containerName: string;
|
||||
imageName: string;
|
||||
currentImageId: string;
|
||||
currentDigest?: string;
|
||||
newDigest?: string;
|
||||
}
|
||||
|
||||
// Track running update checks to prevent concurrent execution
|
||||
const runningUpdateChecks = new Set<number>();
|
||||
|
||||
/**
|
||||
* Execute environment update check job.
|
||||
* @param environmentId - The environment ID to check
|
||||
* @param triggeredBy - What triggered this execution
|
||||
*/
|
||||
export async function runEnvUpdateCheckJob(
|
||||
environmentId: number,
|
||||
triggeredBy: ScheduleTrigger = 'cron'
|
||||
): Promise<void> {
|
||||
// Prevent concurrent execution for the same environment
|
||||
if (runningUpdateChecks.has(environmentId)) {
|
||||
console.log(`[EnvUpdateCheck] Environment ${environmentId} update check already running, skipping`);
|
||||
return;
|
||||
}
|
||||
|
||||
runningUpdateChecks.add(environmentId);
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
// Get environment info
|
||||
const env = await getEnvironment(environmentId);
|
||||
if (!env) {
|
||||
console.error(`[EnvUpdateCheck] Environment ${environmentId} not found`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Get settings
|
||||
const config = await getEnvUpdateCheckSettings(environmentId);
|
||||
if (!config) {
|
||||
console.error(`[EnvUpdateCheck] No settings found for environment ${environmentId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Create execution record
|
||||
const execution = await createScheduleExecution({
|
||||
scheduleType: 'env_update_check',
|
||||
scheduleId: environmentId,
|
||||
environmentId,
|
||||
entityName: `Update: ${env.name}`,
|
||||
triggeredBy,
|
||||
status: 'running'
|
||||
});
|
||||
|
||||
await updateScheduleExecution(execution.id, {
|
||||
startedAt: new Date().toISOString()
|
||||
});
|
||||
|
||||
const log = async (message: string) => {
|
||||
console.log(`[EnvUpdateCheck] ${message}`);
|
||||
await appendScheduleExecutionLog(execution.id, `[${new Date().toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
try {
|
||||
await log(`Starting update check for environment: ${env.name}`);
|
||||
await log(`Auto-update mode: ${config.autoUpdate ? 'ON' : 'OFF'}`);
|
||||
|
||||
// Clear pending updates at the start - we'll re-add as we discover updates
|
||||
await clearPendingContainerUpdates(environmentId);
|
||||
|
||||
// Get all containers in this environment
|
||||
const containers = await listContainers(true, environmentId);
|
||||
await log(`Found ${containers.length} containers`);
|
||||
|
||||
const updatesAvailable: UpdateInfo[] = [];
|
||||
let checkedCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
// Check each container for updates
|
||||
for (const container of containers) {
|
||||
try {
|
||||
const inspectData = await inspectContainer(container.id, environmentId) as any;
|
||||
const imageName = inspectData.Config?.Image;
|
||||
const currentImageId = inspectData.Image;
|
||||
|
||||
if (!imageName) {
|
||||
await log(` [${container.name}] Skipping - no image name found`);
|
||||
continue;
|
||||
}
|
||||
|
||||
checkedCount++;
|
||||
await log(` Checking: ${container.name} (${imageName})`);
|
||||
|
||||
const result = await checkImageUpdateAvailable(imageName, currentImageId, environmentId);
|
||||
|
||||
if (result.isLocalImage) {
|
||||
await log(` Local image - skipping update check`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (result.error) {
|
||||
await log(` Error: ${result.error}`);
|
||||
errorCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (result.hasUpdate) {
|
||||
updatesAvailable.push({
|
||||
containerId: container.id,
|
||||
containerName: container.name,
|
||||
imageName,
|
||||
currentImageId,
|
||||
currentDigest: result.currentDigest,
|
||||
newDigest: result.registryDigest
|
||||
});
|
||||
// Add to pending table immediately - will be removed on successful update
|
||||
await addPendingContainerUpdate(environmentId, container.id, container.name, imageName);
|
||||
await log(` UPDATE AVAILABLE`);
|
||||
await log(` Current: ${result.currentDigest?.substring(0, 24) || 'unknown'}...`);
|
||||
await log(` New: ${result.registryDigest?.substring(0, 24) || 'unknown'}...`);
|
||||
} else {
|
||||
await log(` Up to date`);
|
||||
}
|
||||
} catch (err: any) {
|
||||
await log(` [${container.name}] Error: ${err.message}`);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
await log('');
|
||||
await log('=== SUMMARY ===');
|
||||
await log(`Total containers: ${containers.length}`);
|
||||
await log(`Checked: ${checkedCount}`);
|
||||
await log(`Updates available: ${updatesAvailable.length}`);
|
||||
await log(`Errors: ${errorCount}`);
|
||||
|
||||
if (updatesAvailable.length === 0) {
|
||||
await log('All containers are up to date');
|
||||
// Pending updates already cleared at start, nothing to add
|
||||
await updateScheduleExecution(execution.id, {
|
||||
status: 'success',
|
||||
completedAt: new Date().toISOString(),
|
||||
duration: Date.now() - startTime,
|
||||
details: {
|
||||
updatesFound: 0,
|
||||
containersChecked: checkedCount,
|
||||
errors: errorCount
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Build notification message with details
|
||||
const updateList = updatesAvailable
|
||||
.map(u => {
|
||||
const currentShort = u.currentDigest?.substring(0, 12) || 'unknown';
|
||||
const newShort = u.newDigest?.substring(0, 12) || 'unknown';
|
||||
return `- ${u.containerName} (${u.imageName})\n ${currentShort}... -> ${newShort}...`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
if (config.autoUpdate) {
|
||||
// Auto-update mode: actually update the containers with safe-pull flow
|
||||
await log('');
|
||||
await log('=== AUTO-UPDATE MODE ===');
|
||||
|
||||
// Get scanner settings and vulnerability criteria
|
||||
const scannerSettings = await getScannerSettings(environmentId);
|
||||
const vulnerabilityCriteria = (config.vulnerabilityCriteria || 'never') as VulnerabilityCriteria;
|
||||
// Scan if scanning is enabled (scanner !== 'none')
|
||||
// The vulnerabilityCriteria only controls whether to BLOCK updates, not whether to SCAN
|
||||
const shouldScan = scannerSettings.scanner !== 'none';
|
||||
|
||||
await log(`Vulnerability criteria: ${vulnerabilityCriteria}`);
|
||||
if (shouldScan) {
|
||||
await log(`Scanner: ${scannerSettings.scanner} (scan enabled)`);
|
||||
}
|
||||
await log(`Updating ${updatesAvailable.length} containers...`);
|
||||
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
let blockedCount = 0;
|
||||
const updatedContainers: string[] = [];
|
||||
const failedContainers: string[] = [];
|
||||
const blockedContainers: { name: string; reason: string; scannerResults?: { scanner: string; critical: number; high: number; medium: number; low: number }[] }[] = [];
|
||||
|
||||
for (const update of updatesAvailable) {
|
||||
// Skip Dockhand container - cannot update itself
|
||||
if (isDockhandContainer(update.imageName)) {
|
||||
await log(`\n[${update.containerName}] Skipping - cannot auto-update Dockhand itself`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
await log(`\nUpdating: ${update.containerName}`);
|
||||
|
||||
// Get full container config
|
||||
const inspectData = await inspectContainer(update.containerId, environmentId) as any;
|
||||
const wasRunning = inspectData.State.Running;
|
||||
const containerConfig = inspectData.Config;
|
||||
const hostConfig = inspectData.HostConfig;
|
||||
|
||||
// SAFE-PULL FLOW
|
||||
if (shouldScan && !isDigestBasedImage(update.imageName)) {
|
||||
const tempTag = getTempImageTag(update.imageName);
|
||||
await log(` Safe-pull with temp tag: ${tempTag}`);
|
||||
|
||||
// Step 1: Pull new image
|
||||
await log(` Pulling ${update.imageName}...`);
|
||||
await pullImage(update.imageName, () => {}, environmentId);
|
||||
|
||||
// Step 2: Get new image ID
|
||||
const newImageId = await getImageIdByTag(update.imageName, environmentId);
|
||||
if (!newImageId) {
|
||||
throw new Error('Failed to get new image ID after pull');
|
||||
}
|
||||
await log(` New image: ${newImageId.substring(0, 19)}`);
|
||||
|
||||
// Step 3: SAFETY - Restore original tag to old image
|
||||
const [oldRepo, oldTag] = parseImageNameAndTag(update.imageName);
|
||||
await tagImage(update.currentImageId, oldRepo, oldTag, environmentId);
|
||||
await log(` Restored original tag to safe image`);
|
||||
|
||||
// Step 4: Tag new image with temp suffix
|
||||
const [tempRepo, tempTagName] = parseImageNameAndTag(tempTag);
|
||||
await tagImage(newImageId, tempRepo, tempTagName, environmentId);
|
||||
|
||||
// Step 5: Scan temp image
|
||||
await log(` Scanning for vulnerabilities...`);
|
||||
let scanBlocked = false;
|
||||
let blockReason = '';
|
||||
let currentScannerResults: { scanner: string; critical: number; high: number; medium: number; low: number }[] = [];
|
||||
|
||||
// Collect scan logs to log after scan completes
|
||||
const scanLogs: string[] = [];
|
||||
|
||||
try {
|
||||
const scanResults = await scanImage(tempTag, environmentId, (progress) => {
|
||||
if (progress.message) {
|
||||
scanLogs.push(` [${progress.scanner || 'scan'}] ${progress.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Log collected scan messages
|
||||
for (const scanLog of scanLogs) {
|
||||
await log(scanLog);
|
||||
}
|
||||
|
||||
if (scanResults.length > 0) {
|
||||
const scanSummary = combineScanSummaries(scanResults);
|
||||
await log(` Scan: ${scanSummary.critical} critical, ${scanSummary.high} high, ${scanSummary.medium} medium, ${scanSummary.low} low`);
|
||||
|
||||
// Capture per-scanner results for blocking info
|
||||
currentScannerResults = scanResults.map(r => ({
|
||||
scanner: r.scanner,
|
||||
critical: r.summary.critical,
|
||||
high: r.summary.high,
|
||||
medium: r.summary.medium,
|
||||
low: r.summary.low
|
||||
}));
|
||||
|
||||
// Save scan results
|
||||
for (const result of scanResults) {
|
||||
try {
|
||||
await saveVulnerabilityScan({
|
||||
environmentId,
|
||||
imageId: newImageId,
|
||||
imageName: result.imageName,
|
||||
scanner: result.scanner,
|
||||
scannedAt: result.scannedAt,
|
||||
scanDuration: result.scanDuration,
|
||||
criticalCount: result.summary.critical,
|
||||
highCount: result.summary.high,
|
||||
mediumCount: result.summary.medium,
|
||||
lowCount: result.summary.low,
|
||||
negligibleCount: result.summary.negligible,
|
||||
unknownCount: result.summary.unknown,
|
||||
vulnerabilities: result.vulnerabilities,
|
||||
error: result.error ?? null
|
||||
});
|
||||
} catch { /* ignore save errors */ }
|
||||
}
|
||||
|
||||
// Check if blocked
|
||||
const { blocked, reason } = shouldBlockUpdate(vulnerabilityCriteria, scanSummary, undefined);
|
||||
if (blocked) {
|
||||
scanBlocked = true;
|
||||
blockReason = reason;
|
||||
}
|
||||
}
|
||||
} catch (scanErr: any) {
|
||||
await log(` Scan failed: ${scanErr.message}`);
|
||||
scanBlocked = true;
|
||||
blockReason = `Scan failed: ${scanErr.message}`;
|
||||
}
|
||||
|
||||
if (scanBlocked) {
|
||||
// BLOCKED - Remove temp image
|
||||
await log(` UPDATE BLOCKED: ${blockReason}`);
|
||||
await removeTempImage(newImageId, environmentId);
|
||||
await log(` Removed blocked image - container stays safe`);
|
||||
blockedCount++;
|
||||
blockedContainers.push({
|
||||
name: update.containerName,
|
||||
reason: blockReason,
|
||||
scannerResults: currentScannerResults.length > 0 ? currentScannerResults : undefined
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// APPROVED - Re-tag to original
|
||||
await log(` Scan passed, re-tagging...`);
|
||||
await tagImage(newImageId, oldRepo, oldTag, environmentId);
|
||||
try {
|
||||
await removeTempImage(tempTag, environmentId);
|
||||
} catch { /* ignore cleanup errors */ }
|
||||
} else {
|
||||
// Simple pull (no scanning or digest-based image)
|
||||
await log(` Pulling ${update.imageName}...`);
|
||||
await pullImage(update.imageName, () => {}, environmentId);
|
||||
}
|
||||
|
||||
// Stop container if running
|
||||
if (wasRunning) {
|
||||
await log(` Stopping...`);
|
||||
await stopContainer(update.containerId, environmentId);
|
||||
}
|
||||
|
||||
// Remove old container
|
||||
await log(` Removing old container...`);
|
||||
await removeContainer(update.containerId, true, environmentId);
|
||||
|
||||
// Prepare port bindings
|
||||
const ports: { [key: string]: { HostPort: string } } = {};
|
||||
if (hostConfig.PortBindings) {
|
||||
for (const [containerPort, bindings] of Object.entries(hostConfig.PortBindings)) {
|
||||
if (bindings && (bindings as any[]).length > 0) {
|
||||
ports[containerPort] = { HostPort: (bindings as any[])[0].HostPort || '' };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create new container
|
||||
await log(` Creating new container...`);
|
||||
const newContainer = await createContainer({
|
||||
name: update.containerName,
|
||||
image: update.imageName,
|
||||
ports,
|
||||
volumeBinds: hostConfig.Binds || [],
|
||||
env: containerConfig.Env || [],
|
||||
labels: containerConfig.Labels || {},
|
||||
cmd: containerConfig.Cmd || undefined,
|
||||
restartPolicy: hostConfig.RestartPolicy?.Name || 'no',
|
||||
networkMode: hostConfig.NetworkMode || undefined
|
||||
}, environmentId);
|
||||
|
||||
// Start if was running
|
||||
if (wasRunning) {
|
||||
await log(` Starting...`);
|
||||
await newContainer.start();
|
||||
}
|
||||
|
||||
await log(` Updated successfully`);
|
||||
successCount++;
|
||||
updatedContainers.push(update.containerName);
|
||||
// Remove from pending table - successfully updated
|
||||
await removePendingContainerUpdate(environmentId, update.containerId);
|
||||
} catch (err: any) {
|
||||
await log(` FAILED: ${err.message}`);
|
||||
failCount++;
|
||||
failedContainers.push(update.containerName);
|
||||
}
|
||||
}
|
||||
|
||||
await log('');
|
||||
await log(`=== UPDATE COMPLETE ===`);
|
||||
await log(`Updated: ${successCount}`);
|
||||
await log(`Blocked: ${blockedCount}`);
|
||||
await log(`Failed: ${failCount}`);
|
||||
|
||||
// Send notifications
|
||||
if (blockedCount > 0) {
|
||||
await sendEventNotification('auto_update_blocked', {
|
||||
title: `${blockedCount} update(s) blocked in ${env.name}`,
|
||||
message: blockedContainers.map(c => `- ${c.name}: ${c.reason}`).join('\n'),
|
||||
type: 'warning'
|
||||
}, environmentId);
|
||||
}
|
||||
|
||||
const notificationMessage = successCount > 0
|
||||
? `Updated ${successCount} container(s) in ${env.name}:\n${updatedContainers.map(c => `- ${c}`).join('\n')}${blockedCount > 0 ? `\n\nBlocked (${blockedCount}):\n${blockedContainers.map(c => `- ${c.name}`).join('\n')}` : ''}${failCount > 0 ? `\n\nFailed (${failCount}):\n${failedContainers.map(c => `- ${c}`).join('\n')}` : ''}`
|
||||
: blockedCount > 0 ? `All updates blocked in ${env.name}` : `Update failed for all containers in ${env.name}`;
|
||||
|
||||
await sendEventNotification('batch_update_success', {
|
||||
title: successCount > 0 ? `Containers updated in ${env.name}` : blockedCount > 0 ? `Updates blocked in ${env.name}` : `Container updates failed in ${env.name}`,
|
||||
message: notificationMessage,
|
||||
type: successCount > 0 && failCount === 0 && blockedCount === 0 ? 'success' : successCount > 0 ? 'warning' : 'error'
|
||||
}, environmentId);
|
||||
|
||||
// Blocked/failed containers stay in pending table (successfully updated ones were removed)
|
||||
|
||||
await updateScheduleExecution(execution.id, {
|
||||
status: failCount > 0 && successCount === 0 && blockedCount === 0 ? 'failed' : 'success',
|
||||
completedAt: new Date().toISOString(),
|
||||
duration: Date.now() - startTime,
|
||||
details: {
|
||||
mode: 'auto_update',
|
||||
updatesFound: updatesAvailable.length,
|
||||
containersChecked: checkedCount,
|
||||
errors: errorCount,
|
||||
autoUpdate: true,
|
||||
vulnerabilityCriteria,
|
||||
summary: { checked: checkedCount, updated: successCount, blocked: blockedCount, failed: failCount },
|
||||
containers: [
|
||||
...updatedContainers.map(name => ({ name, status: 'updated' as const })),
|
||||
...blockedContainers.map(c => ({ name: c.name, status: 'blocked' as const, blockReason: c.reason, scannerResults: c.scannerResults })),
|
||||
...failedContainers.map(name => ({ name, status: 'failed' as const }))
|
||||
],
|
||||
updated: successCount,
|
||||
blocked: blockedCount,
|
||||
failed: failCount,
|
||||
blockedContainers
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// Check-only mode: just send notification
|
||||
await log('');
|
||||
await log('Check-only mode - sending notification about available updates');
|
||||
// Pending updates already added as we discovered them
|
||||
|
||||
await sendEventNotification('updates_detected', {
|
||||
title: `Container updates available in ${env.name}`,
|
||||
message: `${updatesAvailable.length} update(s) available:\n${updateList}`,
|
||||
type: 'info'
|
||||
}, environmentId);
|
||||
|
||||
await updateScheduleExecution(execution.id, {
|
||||
status: 'success',
|
||||
completedAt: new Date().toISOString(),
|
||||
duration: Date.now() - startTime,
|
||||
details: {
|
||||
mode: 'notify_only',
|
||||
updatesFound: updatesAvailable.length,
|
||||
containersChecked: checkedCount,
|
||||
errors: errorCount,
|
||||
autoUpdate: false,
|
||||
summary: { checked: checkedCount, updated: 0, blocked: 0, failed: 0 },
|
||||
containers: updatesAvailable.map(u => ({
|
||||
name: u.containerName,
|
||||
status: 'checked' as const,
|
||||
imageName: u.imageName,
|
||||
currentDigest: u.currentDigest,
|
||||
newDigest: u.newDigest
|
||||
}))
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (error: any) {
|
||||
await log(`Error: ${error.message}`);
|
||||
await updateScheduleExecution(execution.id, {
|
||||
status: 'failed',
|
||||
completedAt: new Date().toISOString(),
|
||||
duration: Date.now() - startTime,
|
||||
errorMessage: error.message
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
runningUpdateChecks.delete(environmentId);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user