v1.0.19
This commit is contained in:
jarek committed
1 parent
4b430340db
commit
9c451aedf9
120 files changed
+7777
-6046
No files matched your search
+99
-62
@@ -2,16 +2,16 @@
|
||||
* Core Authentication Module
|
||||
*
|
||||
* Security features:
|
||||
* - Argon2id password hashing via Bun.password (memory-hard, timing-attack resistant)
|
||||
* - Argon2id password hashing via argon2 (memory-hard, timing-attack resistant)
|
||||
* - Cryptographically secure 32-byte random session tokens
|
||||
* - HttpOnly cookies (prevents XSS from reading tokens)
|
||||
* - Secure flag (HTTPS only in production)
|
||||
* - Secure flag (protocol-aware: x-forwarded-proto or COOKIE_SECURE env var, default off)
|
||||
* - SameSite=Strict (CSRF protection)
|
||||
*/
|
||||
|
||||
import os from 'node:os';
|
||||
import { secureRandomBytes, usingFallback } from './crypto-fallback';
|
||||
import { argon2id, argon2Verify } from 'hash-wasm';
|
||||
import { createHash } from 'node:crypto';
|
||||
import argon2 from 'argon2';
|
||||
import type { Cookies } from '@sveltejs/kit';
|
||||
import {
|
||||
getAuthSettings,
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
} from './db';
|
||||
import { Client as LdapClient } from 'ldapts';
|
||||
import { isEnterprise } from './license';
|
||||
import { secureRandomBytes } from './crypto-fallback';
|
||||
|
||||
// Session cookie name
|
||||
const SESSION_COOKIE_NAME = 'dockhand_session';
|
||||
@@ -98,42 +99,25 @@ export interface LoginResult {
|
||||
// Password Hashing (Argon2id)
|
||||
// ============================================
|
||||
|
||||
// Argon2id parameters (matching Bun.password defaults)
|
||||
// Argon2id parameters
|
||||
const ARGON2_MEMORY_COST = 65536; // 64 MB in kibibytes
|
||||
const ARGON2_TIME_COST = 3; // 3 iterations
|
||||
const ARGON2_PARALLELISM = 1; // Single-threaded
|
||||
const ARGON2_HASH_LENGTH = 32; // 256-bit output
|
||||
const ARGON2_SALT_LENGTH = 16; // 128-bit salt
|
||||
|
||||
/**
|
||||
* Hash a password using Argon2id
|
||||
*
|
||||
* On modern kernels (>=3.17): Uses Bun's native password API (faster)
|
||||
* On old kernels (<3.17): Uses hash-wasm (WASM-based, no getrandom dependency)
|
||||
*
|
||||
* Argon2id is the recommended variant - resistant to both side-channel and GPU attacks
|
||||
* Uses the argon2 npm package (C binding) for native performance.
|
||||
* Returns PHC format: $argon2id$v=19$m=65536,t=3,p=1$...
|
||||
*/
|
||||
export async function hashPassword(password: string): Promise<string> {
|
||||
// On old kernels, Bun.password.hash() crashes because it internally uses getrandom()
|
||||
// Use hash-wasm as a fallback which is pure WASM and doesn't depend on the syscall
|
||||
if (usingFallback()) {
|
||||
const salt = secureRandomBytes(ARGON2_SALT_LENGTH);
|
||||
return argon2id({
|
||||
password,
|
||||
salt,
|
||||
iterations: ARGON2_TIME_COST,
|
||||
parallelism: ARGON2_PARALLELISM,
|
||||
memorySize: ARGON2_MEMORY_COST,
|
||||
hashLength: ARGON2_HASH_LENGTH,
|
||||
outputType: 'encoded' // Returns PHC format: $argon2id$v=19$m=65536,t=3,p=1$...
|
||||
});
|
||||
}
|
||||
|
||||
// Modern kernels: use Bun's native implementation (faster)
|
||||
return Bun.password.hash(password, {
|
||||
algorithm: 'argon2id',
|
||||
return argon2.hash(password, {
|
||||
type: argon2.argon2id,
|
||||
memoryCost: ARGON2_MEMORY_COST,
|
||||
timeCost: ARGON2_TIME_COST
|
||||
timeCost: ARGON2_TIME_COST,
|
||||
parallelism: ARGON2_PARALLELISM,
|
||||
hashLength: ARGON2_HASH_LENGTH
|
||||
});
|
||||
}
|
||||
|
||||
@@ -141,18 +125,13 @@ export async function hashPassword(password: string): Promise<string> {
|
||||
* Verify a password against a hash
|
||||
* Uses constant-time comparison internally
|
||||
*
|
||||
* Both Bun.password and hash-wasm use the same PHC format, so hashes are compatible
|
||||
* The argon2 npm package uses standard PHC format, compatible with existing hashes
|
||||
*/
|
||||
export async function verifyPassword(password: string, hash: string): Promise<boolean> {
|
||||
try {
|
||||
// On old kernels, use hash-wasm for verification
|
||||
if (usingFallback()) {
|
||||
return await argon2Verify({ password, hash });
|
||||
}
|
||||
|
||||
// Modern kernels: use Bun's native implementation
|
||||
return await Bun.password.verify(password, hash);
|
||||
} catch {
|
||||
return await argon2.verify(hash, password);
|
||||
} catch (e) {
|
||||
console.error('[Auth] argon2.verify() threw unexpectedly:', e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -169,14 +148,43 @@ function generateSessionToken(): string {
|
||||
return secureRandomBytes(32).toString('base64url');
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether to set the Secure flag on the session cookie.
|
||||
*
|
||||
* Priority:
|
||||
* 1. COOKIE_SECURE env var ('true' / 'false') — explicit override
|
||||
* 2. x-forwarded-proto header — set by Traefik / Nginx / Caddy
|
||||
* 3. false — default, matches v1.0.18 Bun behavior
|
||||
*
|
||||
* Defaulting to false (not NODE_ENV) is intentional: Dockhand is commonly
|
||||
* run over plain HTTP in homelabs. Setting Secure unconditionally in production
|
||||
* causes a login loop when there is no HTTPS reverse proxy, because browsers
|
||||
* silently discard Secure cookies on HTTP connections.
|
||||
*
|
||||
* Users behind an HTTPS reverse proxy get Secure cookies automatically via
|
||||
* x-forwarded-proto. Users who terminate TLS in the app itself can opt in
|
||||
* with COOKIE_SECURE=true.
|
||||
*/
|
||||
function isSecureContext(request?: Request): boolean {
|
||||
if (process.env.COOKIE_SECURE === 'false') return false;
|
||||
if (process.env.COOKIE_SECURE === 'true') return true;
|
||||
if (request) {
|
||||
const proto = request.headers.get('x-forwarded-proto');
|
||||
if (proto === 'https') return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new session for a user
|
||||
* @param provider - Auth provider: 'local', or provider name like 'Keycloak', 'Azure AD', etc.
|
||||
* @param request - Optional incoming request (used to detect HTTPS via x-forwarded-proto)
|
||||
*/
|
||||
export async function createUserSession(
|
||||
userId: number,
|
||||
provider: string,
|
||||
cookies: Cookies
|
||||
cookies: Cookies,
|
||||
request?: Request
|
||||
): Promise<Session> {
|
||||
// Clean up expired sessions periodically
|
||||
await deleteExpiredSessions();
|
||||
@@ -198,7 +206,7 @@ export async function createUserSession(
|
||||
const session = await dbCreateSession(sessionId, userId, provider, expiresAt);
|
||||
|
||||
// Set secure cookie
|
||||
setSessionCookie(cookies, sessionId, sessionTimeout);
|
||||
setSessionCookie(cookies, sessionId, sessionTimeout, request);
|
||||
|
||||
// Update user's last login time
|
||||
await updateUser(userId, { lastLogin: new Date().toISOString() });
|
||||
@@ -209,11 +217,11 @@ export async function createUserSession(
|
||||
/**
|
||||
* Set the session cookie with secure attributes
|
||||
*/
|
||||
function setSessionCookie(cookies: Cookies, sessionId: string, maxAge: number): void {
|
||||
function setSessionCookie(cookies: Cookies, sessionId: string, maxAge: number, request?: Request): void {
|
||||
cookies.set(SESSION_COOKIE_NAME, sessionId, {
|
||||
path: '/',
|
||||
httpOnly: true, // Prevents XSS attacks from reading cookie
|
||||
secure: process.env.NODE_ENV === 'production', // HTTPS only in production
|
||||
secure: isSecureContext(request), // Protocol-aware: checks x-forwarded-proto or NODE_ENV
|
||||
sameSite: 'strict', // CSRF protection
|
||||
maxAge: maxAge // Session timeout in seconds
|
||||
});
|
||||
@@ -563,6 +571,19 @@ export async function authenticateLdap(
|
||||
return { success: false, error: 'Invalid username or password' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape special characters in an LDAP filter value (RFC 4515).
|
||||
* Prevents LDAP injection via wildcards or control characters.
|
||||
*/
|
||||
function escapeLdapFilterValue(value: string): string {
|
||||
return value
|
||||
.replace(/\\/g, '\\5c')
|
||||
.replace(/\*/g, '\\2a')
|
||||
.replace(/\(/g, '\\28')
|
||||
.replace(/\)/g, '\\29')
|
||||
.replace(/\0/g, '\\00');
|
||||
}
|
||||
|
||||
/**
|
||||
* Try authentication against a specific LDAP configuration
|
||||
*/
|
||||
@@ -585,8 +606,10 @@ async function tryLdapAuth(
|
||||
await client.bind(config.bindDn, config.bindPassword);
|
||||
}
|
||||
|
||||
// Search for the user
|
||||
const filter = config.userFilter.replace('{{username}}', username);
|
||||
// Escape the username before interpolating into the LDAP filter (RFC 4515)
|
||||
// to prevent LDAP injection via wildcards or special characters.
|
||||
const safeUsername = escapeLdapFilterValue(username);
|
||||
const filter = config.userFilter.replace('{{username}}', safeUsername);
|
||||
const { searchEntries } = await client.search(config.baseDn, {
|
||||
scope: 'sub',
|
||||
filter: filter,
|
||||
@@ -599,9 +622,11 @@ async function tryLdapAuth(
|
||||
]
|
||||
});
|
||||
|
||||
// Use a single generic error for both "not found" and "wrong password"
|
||||
// to avoid leaking whether a username exists via response content or timing.
|
||||
if (searchEntries.length === 0) {
|
||||
await client.unbind();
|
||||
return { success: false, error: 'User not found' };
|
||||
return { success: false, error: 'Invalid username or password' };
|
||||
}
|
||||
|
||||
const userEntry = searchEntries[0];
|
||||
@@ -666,11 +691,11 @@ async function tryLdapAuth(
|
||||
if (adminRole) {
|
||||
const hasAdminRole = await userHasAdminRole(user.id);
|
||||
if (shouldBeAdmin && !hasAdminRole) {
|
||||
// Assign Admin role
|
||||
await assignUserRole(user.id, adminRole.id, null);
|
||||
} else if (!shouldBeAdmin && hasAdminRole && config.adminGroup) {
|
||||
// Remove Admin role if user is no longer in admin group
|
||||
await removeUserRole(user.id, adminRole.id);
|
||||
}
|
||||
// Note: We don't remove Admin role if not in LDAP group anymore
|
||||
// to prevent accidental lockouts (same behavior as before)
|
||||
}
|
||||
|
||||
// Process role mappings (Enterprise feature)
|
||||
@@ -683,14 +708,29 @@ async function tryLdapAuth(
|
||||
const userExistingRoles = await getUserRoles(user.id);
|
||||
const existingRoleIds = new Set(userExistingRoles.map(r => r.roleId));
|
||||
|
||||
for (const mapping of roleMappings) {
|
||||
// Skip if user already has this role
|
||||
if (existingRoleIds.has(mapping.roleId)) continue;
|
||||
// All role IDs referenced in mappings (these are LDAP-managed)
|
||||
const mappedRoleIds = new Set(roleMappings.map(m => m.roleId));
|
||||
|
||||
// Check if user is a member of the LDAP group
|
||||
// Determine which mapped roles user should have based on current group membership
|
||||
const shouldHaveRoleIds = new Set<number>();
|
||||
for (const mapping of roleMappings) {
|
||||
const isInGroup = await checkLdapGroupMembership(config, userDn, mapping.groupDn);
|
||||
if (isInGroup) {
|
||||
await assignUserRole(user.id, mapping.roleId, undefined);
|
||||
shouldHaveRoleIds.add(mapping.roleId);
|
||||
}
|
||||
}
|
||||
|
||||
// Add roles user should have but doesn't
|
||||
for (const roleId of shouldHaveRoleIds) {
|
||||
if (!existingRoleIds.has(roleId)) {
|
||||
await assignUserRole(user.id, roleId, undefined);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove mapped roles user has but shouldn't
|
||||
for (const roleId of mappedRoleIds) {
|
||||
if (existingRoleIds.has(roleId) && !shouldHaveRoleIds.has(roleId)) {
|
||||
await removeUserRole(user.id, roleId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -744,8 +784,9 @@ async function checkLdapGroupMembership(
|
||||
let groupFilter: string;
|
||||
|
||||
if (config.groupFilter) {
|
||||
// User provided custom filter
|
||||
searchBase = config.groupBaseDn || groupDnOrName;
|
||||
// User provided custom filter - use group DN directly when it's a full DN
|
||||
// to avoid searching all groups under groupBaseDn
|
||||
searchBase = isFullDn ? groupDnOrName : (config.groupBaseDn || groupDnOrName);
|
||||
groupFilter = config.groupFilter
|
||||
.replace('{{username}}', userDn)
|
||||
.replace('{{user_dn}}', userDn)
|
||||
@@ -765,7 +806,7 @@ async function checkLdapGroupMembership(
|
||||
}
|
||||
|
||||
const { searchEntries } = await client.search(searchBase, {
|
||||
scope: isFullDn && !config.groupFilter ? 'base' : 'sub',
|
||||
scope: isFullDn ? 'base' : 'sub',
|
||||
filter: groupFilter,
|
||||
sizeLimit: 1
|
||||
});
|
||||
@@ -825,9 +866,7 @@ function generateBackupCodes(): string[] {
|
||||
async function hashBackupCode(code: string): Promise<string> {
|
||||
// Normalize: uppercase, remove spaces and dashes
|
||||
const normalized = code.toUpperCase().replace(/[\s-]/g, '');
|
||||
const hasher = new Bun.CryptoHasher('sha256');
|
||||
hasher.update(normalized);
|
||||
return hasher.digest('hex');
|
||||
return createHash('sha256').update(normalized).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1172,9 +1211,7 @@ async function getOidcDiscovery(issuerUrl: string): Promise<OidcDiscoveryDocumen
|
||||
*/
|
||||
function generatePkce(): { codeVerifier: string; codeChallenge: string } {
|
||||
const codeVerifier = secureRandomBytes(32).toString('base64url');
|
||||
const hasher = new Bun.CryptoHasher('sha256');
|
||||
hasher.update(codeVerifier);
|
||||
const codeChallenge = hasher.digest('base64url') as string;
|
||||
const codeChallenge = createHash('sha256').update(codeVerifier).digest('base64url');
|
||||
return { codeVerifier, codeChallenge };
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* Crypto Fallback for Old Linux Kernels
|
||||
*
|
||||
* The getrandom() syscall was added in Linux 3.17. On older kernels (like 3.10.x),
|
||||
* Bun's built-in crypto functions will fail with "getrandom() failed to provide entropy".
|
||||
* Node.js crypto functions may fail with "getrandom() failed to provide entropy".
|
||||
*
|
||||
* This module provides fallback implementations that read from /dev/urandom directly
|
||||
* when running on kernels older than 3.17.
|
||||
|
||||
+34
-40
@@ -7,7 +7,6 @@
|
||||
|
||||
import {
|
||||
db,
|
||||
rawClient,
|
||||
isPostgres,
|
||||
isSqlite,
|
||||
eq,
|
||||
@@ -200,6 +199,10 @@ export async function deleteEnvironment(id: number): Promise<boolean> {
|
||||
const env = await getEnvironment(id);
|
||||
if (!env) return false;
|
||||
|
||||
// Clean up in-memory metrics
|
||||
const { clearEnvironmentMetrics } = await import('./metrics-store.js');
|
||||
clearEnvironmentMetrics(id);
|
||||
|
||||
// Clean up related records that don't have cascade delete defined
|
||||
try {
|
||||
await db.delete(hostMetrics).where(eq(hostMetrics.environmentId, id));
|
||||
@@ -576,45 +579,28 @@ export async function saveHostMetric(
|
||||
memoryPercent: number,
|
||||
memoryUsed: number,
|
||||
memoryTotal: number,
|
||||
environmentId?: number
|
||||
environmentId?: number,
|
||||
_skipEnvCheck = false
|
||||
): Promise<void> {
|
||||
// Verify environment exists before inserting (avoids FK violations on deleted envs)
|
||||
if (environmentId) {
|
||||
const env = await getEnvironment(environmentId);
|
||||
if (!env) return;
|
||||
}
|
||||
|
||||
await db.insert(hostMetrics).values({
|
||||
environmentId: environmentId || null,
|
||||
cpuPercent,
|
||||
memoryPercent,
|
||||
memoryUsed,
|
||||
memoryTotal
|
||||
});
|
||||
|
||||
// Cleanup old metrics (keep last 24 hours)
|
||||
const cutoff24h = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
||||
await db.delete(hostMetrics).where(sql`timestamp < ${cutoff24h}`);
|
||||
// Delegated to in-memory ring buffer (no DB writes)
|
||||
if (!environmentId) return;
|
||||
const { pushMetric } = await import('./metrics-store.js');
|
||||
pushMetric(environmentId, cpuPercent, memoryPercent, memoryUsed, memoryTotal);
|
||||
}
|
||||
|
||||
export async function getHostMetrics(limit = 60, environmentId?: number): Promise<HostMetric[]> {
|
||||
if (environmentId) {
|
||||
return db.select().from(hostMetrics)
|
||||
.where(eq(hostMetrics.environmentId, environmentId))
|
||||
.orderBy(desc(hostMetrics.timestamp))
|
||||
.limit(limit);
|
||||
const { getMetricsHistory } = await import('./metrics-store.js');
|
||||
// getMetricsHistory returns oldest-first, but callers expect newest-first
|
||||
return getMetricsHistory(environmentId, limit).reverse();
|
||||
}
|
||||
return db.select().from(hostMetrics)
|
||||
.orderBy(desc(hostMetrics.timestamp))
|
||||
.limit(limit);
|
||||
const { getAllMetrics } = await import('./metrics-store.js');
|
||||
return getAllMetrics(limit);
|
||||
}
|
||||
|
||||
export async function getLatestHostMetrics(environmentId: number): Promise<HostMetric | null> {
|
||||
const results = await db.select().from(hostMetrics)
|
||||
.where(eq(hostMetrics.environmentId, environmentId))
|
||||
.orderBy(desc(hostMetrics.timestamp))
|
||||
.limit(1);
|
||||
return results[0] ?? null;
|
||||
const { getLatestMetric } = await import('./metrics-store.js');
|
||||
return getLatestMetric(environmentId);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
@@ -3269,20 +3255,23 @@ export interface ContainerEventResult {
|
||||
offset: number;
|
||||
}
|
||||
|
||||
export async function logContainerEvent(data: ContainerEventCreateData): Promise<ContainerEventData> {
|
||||
// Timestamp is already an ISO-8601 string from event-subprocess
|
||||
// Both SQLite and PostgreSQL schemas use mode: 'string' so we pass it directly
|
||||
const result = await db.insert(containerEvents).values({
|
||||
export async function logContainerEvent(
|
||||
data: ContainerEventCreateData
|
||||
): Promise<ContainerEventData> {
|
||||
const attrs = data.actorAttributes ? JSON.stringify(data.actorAttributes) : null;
|
||||
|
||||
const [inserted] = await db.insert(containerEvents).values({
|
||||
environmentId: data.environmentId ?? null,
|
||||
containerId: data.containerId,
|
||||
containerName: data.containerName ?? null,
|
||||
image: data.image ?? null,
|
||||
action: data.action,
|
||||
actorAttributes: data.actorAttributes ? JSON.stringify(data.actorAttributes) : null,
|
||||
actorAttributes: attrs,
|
||||
timestamp: data.timestamp
|
||||
}).returning();
|
||||
}).returning({ id: containerEvents.id });
|
||||
|
||||
return getContainerEvent(result[0].id) as Promise<ContainerEventData>;
|
||||
const event = await getContainerEvent(inserted.id);
|
||||
return event!;
|
||||
}
|
||||
|
||||
export async function getContainerEvent(id: number): Promise<ContainerEventData | undefined> {
|
||||
@@ -4502,11 +4491,16 @@ export async function setStackEnvVars(
|
||||
));
|
||||
}
|
||||
|
||||
// Insert new vars
|
||||
// Insert new vars (deduplicate by key - last entry wins)
|
||||
if (variables.length > 0) {
|
||||
const seen = new Map<string, { key: string; value: string; isSecret?: boolean }>();
|
||||
for (const v of variables) {
|
||||
seen.set(v.key, v);
|
||||
}
|
||||
const deduped = Array.from(seen.values());
|
||||
const now = new Date().toISOString();
|
||||
await db.insert(stackEnvironmentVariables).values(
|
||||
variables.map(v => ({
|
||||
deduped.map(v => ({
|
||||
stackName,
|
||||
environmentId,
|
||||
key: v.key,
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
/**
|
||||
* Database Connection Module
|
||||
*
|
||||
* Provides a unified database connection using Bun's SQL API.
|
||||
* Supports both SQLite (default) and PostgreSQL (via DATABASE_URL).
|
||||
*/
|
||||
|
||||
import { SQL } from 'bun';
|
||||
import { existsSync, mkdirSync, readFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Database configuration
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
const dataDir = process.env.DATA_DIR || './data';
|
||||
|
||||
// Detect database type
|
||||
export const isPostgres = databaseUrl && (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://'));
|
||||
export const isSqlite = !isPostgres;
|
||||
|
||||
/**
|
||||
* Read a SQL file from the appropriate sql directory.
|
||||
*/
|
||||
function readSql(filename: string): string {
|
||||
const sqlDir = isPostgres ? 'postgres' : 'sqlite';
|
||||
return readFileSync(join(__dirname, sqlDir, 'sql', filename), 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate PostgreSQL connection URL format.
|
||||
*/
|
||||
function validatePostgresUrl(url: string): void {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
|
||||
if (parsed.protocol !== 'postgres:' && parsed.protocol !== 'postgresql:') {
|
||||
exitWithError(`Invalid protocol "${parsed.protocol}". Expected "postgres:" or "postgresql:"`, url);
|
||||
}
|
||||
|
||||
if (!parsed.hostname) {
|
||||
exitWithError('Missing hostname in DATABASE_URL', url);
|
||||
}
|
||||
|
||||
if (!parsed.pathname || parsed.pathname === '/') {
|
||||
exitWithError('Missing database name in DATABASE_URL', url);
|
||||
}
|
||||
} catch {
|
||||
exitWithError('Invalid URL format', url);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Print connection error and exit.
|
||||
*/
|
||||
function exitWithError(error: string, url?: string): never {
|
||||
console.error('\n' + '='.repeat(70));
|
||||
console.error('DATABASE CONNECTION ERROR');
|
||||
console.error('='.repeat(70));
|
||||
console.error(`\nError: ${error}`);
|
||||
|
||||
if (url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.password) parsed.password = '***';
|
||||
console.error(`\nProvided URL: ${parsed.toString()}`);
|
||||
} catch {
|
||||
console.error(`\nProvided URL: ${url.replace(/:[^:@]+@/, ':***@')}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.error('\n' + '-'.repeat(70));
|
||||
console.error('DATABASE_URL format:');
|
||||
console.error('-'.repeat(70));
|
||||
console.error('\n postgres://USER:PASSWORD@HOST:PORT/DATABASE');
|
||||
console.error('\nExamples:');
|
||||
console.error(' postgres://dockhand:secret@localhost:5432/dockhand');
|
||||
console.error(' postgres://admin:p4ssw0rd@192.168.1.100:5432/dockhand');
|
||||
console.error(' postgresql://user:pass@db.example.com/mydb?sslmode=require');
|
||||
console.error('\n' + '-'.repeat(70));
|
||||
console.error('To use SQLite instead, remove the DATABASE_URL environment variable.');
|
||||
console.error('='.repeat(70) + '\n');
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the database connection.
|
||||
*/
|
||||
function createConnection(): SQL {
|
||||
if (isPostgres) {
|
||||
// Validate PostgreSQL URL
|
||||
validatePostgresUrl(databaseUrl!);
|
||||
|
||||
console.log('Connecting to PostgreSQL database...');
|
||||
try {
|
||||
const sql = new SQL(databaseUrl!);
|
||||
return sql;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
exitWithError(`Failed to connect to PostgreSQL: ${message}`, databaseUrl);
|
||||
}
|
||||
} else {
|
||||
// SQLite: Ensure db directory exists
|
||||
const dbDir = join(dataDir, 'db');
|
||||
if (!existsSync(dbDir)) {
|
||||
mkdirSync(dbDir, { recursive: true });
|
||||
}
|
||||
|
||||
const dbPath = join(dbDir, 'dockhand.db');
|
||||
console.log(`Using SQLite database at: ${dbPath}`);
|
||||
|
||||
const sql = new SQL(`sqlite://${dbPath}`);
|
||||
|
||||
// Enable WAL mode for better performance
|
||||
sql.run('PRAGMA journal_mode = WAL');
|
||||
|
||||
return sql;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the database schema.
|
||||
*/
|
||||
async function initializeSchema(sql: SQL): Promise<void> {
|
||||
try {
|
||||
// Create schema (tables)
|
||||
await sql.run(readSql('schema.sql'));
|
||||
|
||||
// Create indexes
|
||||
await sql.run(readSql('indexes.sql'));
|
||||
|
||||
// Insert seed data
|
||||
await sql.run(readSql('seed.sql'));
|
||||
|
||||
// Update system roles
|
||||
await sql.run(readSql('system-roles.sql'));
|
||||
|
||||
// Run maintenance
|
||||
await sql.run(readSql('maintenance.sql'));
|
||||
|
||||
console.log(`Database initialized successfully (${isPostgres ? 'PostgreSQL' : 'SQLite'})`);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error('Failed to initialize database schema:', message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Create and export the database connection
|
||||
export const sql = createConnection();
|
||||
|
||||
// Initialize schema (runs async but we handle it)
|
||||
initializeSchema(sql).catch((error) => {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
console.error('[DB] Database initialization failed:', errorMsg);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
/**
|
||||
* Helper to convert SQLite integer booleans to JS booleans.
|
||||
* PostgreSQL returns actual booleans, SQLite returns 0/1.
|
||||
*/
|
||||
export function toBool(value: any): boolean {
|
||||
if (typeof value === 'boolean') return value;
|
||||
return Boolean(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to convert JS boolean to database value.
|
||||
* PostgreSQL uses boolean, SQLite uses 0/1.
|
||||
*/
|
||||
export function fromBool(value: boolean): boolean | number {
|
||||
return isPostgres ? value : (value ? 1 : 0);
|
||||
}
|
||||
@@ -208,11 +208,11 @@ function readMigrationJournal(migrationsFolder: string): MigrationJournal | null
|
||||
async function getAppliedMigrations(client: any, postgres: boolean): Promise<AppliedMigration[]> {
|
||||
try {
|
||||
if (postgres) {
|
||||
// PostgreSQL using Bun.sql - note the 'drizzle' schema
|
||||
// PostgreSQL using postgres-js - note the 'drizzle' schema
|
||||
const result = await client`SELECT hash, created_at FROM drizzle.__drizzle_migrations ORDER BY id`;
|
||||
return result.map((r: any) => ({ hash: r.hash, createdAt: r.created_at }));
|
||||
} else {
|
||||
// SQLite using bun:sqlite
|
||||
// SQLite using better-sqlite3
|
||||
const stmt = client.prepare('SELECT hash, created_at FROM __drizzle_migrations ORDER BY id');
|
||||
return stmt.all().map((r: any) => ({ hash: r.hash, createdAt: r.created_at }));
|
||||
}
|
||||
@@ -484,10 +484,10 @@ async function runMigrations(
|
||||
// Run migrations
|
||||
try {
|
||||
if (postgres) {
|
||||
const { migrate } = await import('drizzle-orm/bun-sql/migrator');
|
||||
const { migrate } = await import('drizzle-orm/postgres-js/migrator');
|
||||
await migrate(database, { migrationsFolder });
|
||||
} else {
|
||||
const { migrate } = await import('drizzle-orm/bun-sqlite/migrator');
|
||||
const { migrate } = await import('drizzle-orm/better-sqlite3/migrator');
|
||||
await migrate(database, { migrationsFolder });
|
||||
}
|
||||
|
||||
@@ -605,7 +605,7 @@ async function initializeDatabase() {
|
||||
logHeader('DATABASE INITIALIZATION');
|
||||
|
||||
if (isPostgres) {
|
||||
// PostgreSQL via postgres-js (more stable than bun:sql for concurrent queries)
|
||||
// PostgreSQL via postgres-js
|
||||
validatePostgresUrl(config.databaseUrl!);
|
||||
|
||||
logInfo(`Database: PostgreSQL`);
|
||||
@@ -634,7 +634,7 @@ async function initializeDatabase() {
|
||||
handleMigrationFailure(result.error, true);
|
||||
}
|
||||
} else {
|
||||
// SQLite via bun:sqlite
|
||||
// SQLite via better-sqlite3
|
||||
const dbDir = join(config.dataDir, 'db');
|
||||
if (!existsSync(dbDir)) {
|
||||
mkdirSync(dbDir, { recursive: true });
|
||||
@@ -645,8 +645,8 @@ async function initializeDatabase() {
|
||||
logInfo(`Database: SQLite`);
|
||||
logInfo(`Path: ${dbPath}`);
|
||||
|
||||
const { drizzle } = await import('drizzle-orm/bun-sqlite');
|
||||
const { Database } = await import('bun:sqlite');
|
||||
const { drizzle } = await import('drizzle-orm/better-sqlite3');
|
||||
const Database = (await import('better-sqlite3')).default;
|
||||
|
||||
// Import SQLite schema
|
||||
schema = await import('./schema/index.js');
|
||||
@@ -655,11 +655,11 @@ async function initializeDatabase() {
|
||||
rawClient = new Database(dbPath);
|
||||
|
||||
// Enable WAL mode for better performance and concurrency
|
||||
rawClient.exec('PRAGMA journal_mode = WAL');
|
||||
rawClient.pragma('journal_mode = WAL');
|
||||
// Synchronous NORMAL is a good balance between safety and speed
|
||||
rawClient.exec('PRAGMA synchronous = NORMAL');
|
||||
rawClient.pragma('synchronous = NORMAL');
|
||||
// Increase busy timeout to handle concurrent access (5 seconds)
|
||||
rawClient.exec('PRAGMA busy_timeout = 5000');
|
||||
rawClient.pragma('busy_timeout = 5000');
|
||||
|
||||
db = drizzle({ client: rawClient, schema });
|
||||
logSuccess('SQLite database opened');
|
||||
|
||||
+638
-157
File diff suppressed because it is too large.
Load diff
@@ -2,17 +2,25 @@
|
||||
* Container Event Emitter
|
||||
*
|
||||
* Shared EventEmitter for broadcasting container events to SSE clients.
|
||||
* Events are emitted by the subprocess-manager when it receives them from the event-subprocess.
|
||||
* Events are emitted by the collection worker when processing Docker events.
|
||||
*
|
||||
* IMPORTANT: Uses globalThis to ensure a single instance across all module imports.
|
||||
* In Vite dev mode and SvelteKit production builds, server modules can be loaded
|
||||
* multiple times (HMR, chunking), creating separate EventEmitter instances.
|
||||
* Using globalThis guarantees emitters and listeners share the same object.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'node:events';
|
||||
|
||||
// Event emitter for broadcasting new events to SSE clients
|
||||
// Used by:
|
||||
// - subprocess-manager.ts: emits events received from event-subprocess via IPC
|
||||
// - api/activity/events/+server.ts: listens for events to broadcast via SSE
|
||||
export const containerEventEmitter = new EventEmitter();
|
||||
const GLOBAL_KEY = '__dockhand_container_event_emitter__';
|
||||
|
||||
// Allow up to 100 concurrent SSE listeners (default is 10)
|
||||
// This prevents MaxListenersExceededWarning with many dashboard clients
|
||||
containerEventEmitter.setMaxListeners(100);
|
||||
// Ensure single instance via globalThis
|
||||
if (!(globalThis as any)[GLOBAL_KEY]) {
|
||||
const emitter = new EventEmitter();
|
||||
// Allow up to 100 concurrent SSE listeners (default is 10)
|
||||
// This prevents MaxListenersExceededWarning with many dashboard clients
|
||||
emitter.setMaxListeners(100);
|
||||
(globalThis as any)[GLOBAL_KEY] = emitter;
|
||||
}
|
||||
|
||||
export const containerEventEmitter: EventEmitter = (globalThis as any)[GLOBAL_KEY];
|
||||
+67
-50
@@ -1,5 +1,7 @@
|
||||
import { existsSync, mkdirSync, rmSync, chmodSync } from 'node:fs';
|
||||
import { existsSync, mkdirSync, rmSync, chmodSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { join, resolve, dirname, basename, relative } from 'node:path';
|
||||
import { spawn as nodeSpawn, spawnSync } from 'node:child_process';
|
||||
import type { ChildProcess } from 'node:child_process';
|
||||
import {
|
||||
getGitRepository,
|
||||
getGitCredential,
|
||||
@@ -14,6 +16,26 @@ import {
|
||||
} from './db';
|
||||
import { deployStack, getStackDir } from './stacks';
|
||||
|
||||
/**
|
||||
* Collect stdout, stderr and exit code from a spawned process.
|
||||
*/
|
||||
function collectProcess(proc: ChildProcess): Promise<{ exitCode: number; stdout: string; stderr: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const stdoutChunks: Buffer[] = [];
|
||||
const stderrChunks: Buffer[] = [];
|
||||
proc.stdout?.on('data', (chunk: Buffer) => stdoutChunks.push(chunk));
|
||||
proc.stderr?.on('data', (chunk: Buffer) => stderrChunks.push(chunk));
|
||||
proc.on('error', reject);
|
||||
proc.on('close', (code) => {
|
||||
resolve({
|
||||
exitCode: code ?? 1,
|
||||
stdout: Buffer.concat(stdoutChunks).toString(),
|
||||
stderr: Buffer.concat(stderrChunks).toString()
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Directory for storing cloned repositories
|
||||
const dataDir = process.env.DATA_DIR || './data';
|
||||
const GIT_REPOS_DIR = resolve(process.env.GIT_REPOS_DIR || join(dataDir, 'git-repos'));
|
||||
@@ -79,7 +101,7 @@ async function ensurePasswdEntry(env: GitEnv): Promise<void> {
|
||||
if (uid === undefined || uid === 0) return; // root or not available
|
||||
|
||||
try {
|
||||
const passwd = await Bun.file('/etc/passwd').text();
|
||||
const passwd = readFileSync('/etc/passwd', 'utf-8');
|
||||
const uidStr = `:${uid}:`;
|
||||
if (passwd.split('\n').some(line => {
|
||||
const parts = line.split(':');
|
||||
@@ -100,17 +122,17 @@ async function ensurePasswdEntry(env: GitEnv): Promise<void> {
|
||||
// Create temp passwd/group with the missing entry
|
||||
try {
|
||||
const gid = process.getgid?.() ?? uid;
|
||||
const passwd = await Bun.file('/etc/passwd').text();
|
||||
const group = await Bun.file('/etc/group').text();
|
||||
const passwd = readFileSync('/etc/passwd', 'utf-8');
|
||||
const group = readFileSync('/etc/group', 'utf-8');
|
||||
|
||||
const passwdEntry = `dockhand:x:${uid}:${gid}:Dockhand:/home/dockhand:/bin/sh`;
|
||||
await Bun.write(TMP_PASSWD, passwd.trimEnd() + '\n' + passwdEntry + '\n');
|
||||
writeFileSync(TMP_PASSWD, passwd.trimEnd() + '\n' + passwdEntry + '\n');
|
||||
|
||||
const gidExists = group.split('\n').some(line => line.split(':')[2] === String(gid));
|
||||
if (gidExists) {
|
||||
await Bun.write(TMP_GROUP, group);
|
||||
writeFileSync(TMP_GROUP, group);
|
||||
} else {
|
||||
await Bun.write(TMP_GROUP, group.trimEnd() + '\n' + `dockhand:x:${gid}:\n`);
|
||||
writeFileSync(TMP_GROUP, group.trimEnd() + '\n' + `dockhand:x:${gid}:\n`);
|
||||
}
|
||||
|
||||
_nssWrapperNeeded = true;
|
||||
@@ -135,8 +157,10 @@ async function buildGitEnv(credential: GitCredential | null): Promise<GitEnv> {
|
||||
await ensurePasswdEntry(env);
|
||||
|
||||
if (credential?.authType === 'ssh' && credential.sshPrivateKey) {
|
||||
// Create a temporary SSH key file (use absolute path so SSH can find it)
|
||||
const sshKeyPath = resolve(join(GIT_REPOS_DIR, `.ssh-key-${credential.id}`));
|
||||
// Write SSH key to /tmp instead of data volume — some filesystems (TrueNAS ZFS,
|
||||
// NFS, CIFS) silently ignore chmod, leaving the key group-readable (e.g. 0670).
|
||||
// SSH refuses keys that are accessible by others. /tmp is always a proper filesystem.
|
||||
const sshKeyPath = `/tmp/.ssh-key-${credential.id}`;
|
||||
|
||||
// Ensure SSH key ends with a newline (newer SSH versions are strict about this)
|
||||
let keyContent = credential.sshPrivateKey;
|
||||
@@ -144,18 +168,19 @@ async function buildGitEnv(credential: GitCredential | null): Promise<GitEnv> {
|
||||
keyContent += '\n';
|
||||
}
|
||||
|
||||
await Bun.write(sshKeyPath, keyContent);
|
||||
writeFileSync(sshKeyPath, keyContent);
|
||||
// Ensure SSH key has correct permissions (0600 = owner read/write only)
|
||||
// Bun.write's mode option doesn't always work reliably, so use chmodSync
|
||||
// writeFileSync's mode option doesn't always work reliably, so use chmodSync
|
||||
chmodSync(sshKeyPath, 0o600);
|
||||
|
||||
// If key has a passphrase, decrypt it in-place so SSH can use it non-interactively
|
||||
if (credential.sshPassphrase) {
|
||||
const result = Bun.spawnSync([
|
||||
'ssh-keygen', '-p', '-f', sshKeyPath,
|
||||
'-P', credential.sshPassphrase, '-N', ''
|
||||
], { env, stderr: 'pipe' });
|
||||
if (result.exitCode !== 0) {
|
||||
const result = spawnSync(
|
||||
'ssh-keygen',
|
||||
['-p', '-f', sshKeyPath, '-P', credential.sshPassphrase, '-N', ''],
|
||||
{ env, stdio: ['pipe', 'pipe', 'pipe'] }
|
||||
);
|
||||
if (result.status !== 0) {
|
||||
const stderr = result.stderr.toString().trim();
|
||||
console.warn(`[git] Failed to decrypt SSH key: ${stderr}`);
|
||||
}
|
||||
@@ -173,7 +198,7 @@ async function buildGitEnv(credential: GitCredential | null): Promise<GitEnv> {
|
||||
|
||||
function cleanupSshKey(credential: GitCredential | null): void {
|
||||
if (credential?.authType === 'ssh') {
|
||||
const sshKeyPath = resolve(join(GIT_REPOS_DIR, `.ssh-key-${credential.id}`));
|
||||
const sshKeyPath = `/tmp/.ssh-key-${credential.id}`;
|
||||
try {
|
||||
if (existsSync(sshKeyPath)) {
|
||||
rmSync(sshKeyPath);
|
||||
@@ -207,21 +232,15 @@ function buildRepoUrl(url: string, credential: GitCredential | null): string {
|
||||
|
||||
async function execGit(args: string[], cwd: string, env: GitEnv): Promise<{ stdout: string; stderr: string; code: number }> {
|
||||
try {
|
||||
const proc = Bun.spawn(['git', ...args], {
|
||||
const proc = nodeSpawn('git', args, {
|
||||
cwd,
|
||||
env,
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe'
|
||||
stdio: ['pipe', 'pipe', 'pipe']
|
||||
});
|
||||
|
||||
const [stdout, stderr] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text()
|
||||
]);
|
||||
const result = await collectProcess(proc);
|
||||
|
||||
const code = await proc.exited;
|
||||
|
||||
return { stdout: stdout.trim(), stderr: stderr.trim(), code };
|
||||
return { stdout: result.stdout.trim(), stderr: result.stderr.trim(), code: result.exitCode };
|
||||
} catch (err: any) {
|
||||
return { stdout: '', stderr: err.message, code: 1 };
|
||||
}
|
||||
@@ -350,8 +369,6 @@ async function testRepositoryConnection(options: {
|
||||
env
|
||||
);
|
||||
|
||||
cleanupSshKey(credential);
|
||||
|
||||
if (result.code !== 0) {
|
||||
console.error('[Git] Connection test failed:', result.stderr);
|
||||
return { success: false, error: cleanGitError(result.stderr) };
|
||||
@@ -366,7 +383,6 @@ async function testRepositoryConnection(options: {
|
||||
process.cwd(),
|
||||
env
|
||||
);
|
||||
cleanupSshKey(credential);
|
||||
|
||||
if (allBranchesResult.code !== 0) {
|
||||
return { success: false, error: cleanGitError(allBranchesResult.stderr) };
|
||||
@@ -400,8 +416,9 @@ async function testRepositoryConnection(options: {
|
||||
lastCommit
|
||||
};
|
||||
} catch (error: any) {
|
||||
cleanupSshKey(credential);
|
||||
return { success: false, error: error.message };
|
||||
} finally {
|
||||
cleanupSshKey(credential);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -519,7 +536,7 @@ export async function syncRepository(repoId: number): Promise<SyncResult> {
|
||||
throw new Error(`Compose file not found: ${repo.composePath}`);
|
||||
}
|
||||
|
||||
const composeContent = await Bun.file(composePath).text();
|
||||
const composeContent = readFileSync(composePath, 'utf-8');
|
||||
|
||||
// Update repository status
|
||||
await updateGitRepository(repoId, {
|
||||
@@ -798,7 +815,7 @@ export async function syncGitStack(stackId: number): Promise<SyncResult> {
|
||||
throw new Error(`Compose file not found: ${gitStack.composePath}`);
|
||||
}
|
||||
|
||||
const composeContent = await Bun.file(composePath).text();
|
||||
const composeContent = readFileSync(composePath, 'utf-8');
|
||||
console.log(`${logPrefix} Compose content length:`, composeContent.length, 'chars');
|
||||
console.log(`${logPrefix} Compose content:`);
|
||||
console.log(composeContent);
|
||||
@@ -819,7 +836,7 @@ export async function syncGitStack(stackId: number): Promise<SyncResult> {
|
||||
if (existsSync(envFilePath)) {
|
||||
try {
|
||||
console.log(`${logPrefix} Reading env file...`);
|
||||
envFileContent = await Bun.file(envFilePath).text();
|
||||
envFileContent = readFileSync(envFilePath, 'utf-8');
|
||||
envFileVars = parseEnvFileContent(envFileContent, gitStack.stackName);
|
||||
console.log(`${logPrefix} Env file parsed, vars count:`, Object.keys(envFileVars).length);
|
||||
|
||||
@@ -1142,7 +1159,7 @@ export async function deployGitStackWithProgress(
|
||||
throw new Error(`Compose file not found: ${gitStack.composePath}`);
|
||||
}
|
||||
|
||||
const composeContent = await Bun.file(composePath).text();
|
||||
const composeContent = readFileSync(composePath, 'utf-8');
|
||||
|
||||
// Determine the compose directory (for copying all files)
|
||||
const composeDir = dirname(composePath);
|
||||
@@ -1153,7 +1170,7 @@ export async function deployGitStackWithProgress(
|
||||
const envFilePath = join(repoPath, gitStack.envFilePath);
|
||||
if (existsSync(envFilePath)) {
|
||||
try {
|
||||
const envContent = await Bun.file(envFilePath).text();
|
||||
const envContent = readFileSync(envFilePath, 'utf-8');
|
||||
envFileVars = parseEnvFileContent(envContent, gitStack.stackName);
|
||||
} catch (err) {
|
||||
// Log but don't fail - env file is optional
|
||||
@@ -1251,12 +1268,11 @@ export async function listGitStackEnvFiles(stackId: number): Promise<{ files: st
|
||||
const maxDepth = 3;
|
||||
|
||||
// Use find to locate all .env* files
|
||||
const proc = Bun.spawn(['find', repoPath, '-maxdepth', String(maxDepth), '-type', 'f', '-name', '.env*'], {
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe'
|
||||
const proc = nodeSpawn('find', [repoPath, '-maxdepth', String(maxDepth), '-type', 'f', '-name', '.env*'], {
|
||||
stdio: ['pipe', 'pipe', 'pipe']
|
||||
});
|
||||
const output = await new Response(proc.stdout).text();
|
||||
await proc.exited;
|
||||
const findResult = await collectProcess(proc);
|
||||
const output = findResult.stdout;
|
||||
|
||||
const files = output.trim().split('\n').filter(f => f);
|
||||
const envFiles: string[] = [];
|
||||
@@ -1372,7 +1388,7 @@ export async function readGitStackEnvFile(
|
||||
}
|
||||
|
||||
try {
|
||||
const content = await Bun.file(fullPath).text();
|
||||
const content = readFileSync(fullPath, 'utf-8');
|
||||
const vars = parseEnvFileContent(content);
|
||||
return { vars };
|
||||
} catch (error: any) {
|
||||
@@ -1426,17 +1442,18 @@ export async function previewRepoEnvFiles(options: PreviewEnvOptions): Promise<P
|
||||
const authenticatedUrl = buildRepoUrl(repoUrl, credential as GitCredential | null);
|
||||
|
||||
// Clone with depth 1 (shallow clone for speed)
|
||||
const cloneProc = Bun.spawn(
|
||||
['git', 'clone', '--depth', '1', '--branch', branch, '--single-branch', authenticatedUrl, tempDir],
|
||||
const cloneProc = nodeSpawn(
|
||||
'git',
|
||||
['clone', '--depth', '1', '--branch', branch, '--single-branch', authenticatedUrl, tempDir],
|
||||
{
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
env
|
||||
}
|
||||
);
|
||||
|
||||
const cloneStderr = await new Response(cloneProc.stderr).text();
|
||||
const cloneExitCode = await cloneProc.exited;
|
||||
const cloneResult = await collectProcess(cloneProc);
|
||||
const cloneStderr = cloneResult.stderr;
|
||||
const cloneExitCode = cloneResult.exitCode;
|
||||
|
||||
if (cloneExitCode !== 0) {
|
||||
console.error(`${logPrefix} Clone failed:`, cloneStderr);
|
||||
@@ -1455,7 +1472,7 @@ export async function previewRepoEnvFiles(options: PreviewEnvOptions): Promise<P
|
||||
// Read base .env file if it exists
|
||||
if (existsSync(baseEnvPath)) {
|
||||
console.log(`${logPrefix} Reading .env from: ${baseEnvPath}`);
|
||||
const content = await Bun.file(baseEnvPath).text();
|
||||
const content = readFileSync(baseEnvPath, 'utf-8');
|
||||
const baseVars = parseEnvFileContent(content, 'preview');
|
||||
for (const [key, value] of Object.entries(baseVars)) {
|
||||
vars[key] = value;
|
||||
@@ -1471,7 +1488,7 @@ export async function previewRepoEnvFiles(options: PreviewEnvOptions): Promise<P
|
||||
const additionalEnvPath = join(tempDir, envFilePath);
|
||||
if (existsSync(additionalEnvPath)) {
|
||||
console.log(`${logPrefix} Reading additional env file: ${additionalEnvPath}`);
|
||||
const content = await Bun.file(additionalEnvPath).text();
|
||||
const content = readFileSync(additionalEnvPath, 'utf-8');
|
||||
const additionalVars = parseEnvFileContent(content, 'preview');
|
||||
for (const [key, value] of Object.entries(additionalVars)) {
|
||||
vars[key] = value;
|
||||
|
||||
+315
-19
@@ -5,10 +5,11 @@
|
||||
* Handles request/response correlation, heartbeat tracking, and metrics collection.
|
||||
*/
|
||||
|
||||
import { db, hawserTokens, environments, eq } from './db/drizzle.js';
|
||||
import { logContainerEvent, saveHostMetric, type ContainerEventAction } from './db.js';
|
||||
import { db, hawserTokens, environments, eq, and } from './db/drizzle.js';
|
||||
import { logContainerEvent, type ContainerEventAction } from './db.js';
|
||||
import { containerEventEmitter } from './event-collector.js';
|
||||
import { sendEnvironmentNotification } from './notifications.js';
|
||||
import { pushMetric } from './metrics-store.js';
|
||||
import { secureGetRandomValues, secureRandomUUID } from './crypto-fallback.js';
|
||||
import { hashPassword, verifyPassword } from './auth.js';
|
||||
|
||||
@@ -40,7 +41,7 @@ export interface EdgeConnection {
|
||||
hostname: string;
|
||||
capabilities: string[];
|
||||
connectedAt: Date;
|
||||
lastHeartbeat: Date;
|
||||
lastHeartbeat: number;
|
||||
pendingRequests: Map<string, PendingRequest>;
|
||||
pendingStreamRequests: Map<string, PendingStreamRequest>;
|
||||
lastMetrics?: {
|
||||
@@ -76,6 +77,9 @@ declare global {
|
||||
var __hawserSendMessage: ((envId: number, message: string) => boolean) | undefined;
|
||||
var __hawserHandleContainerEvent: ((envId: number, event: ContainerEventMessage['event']) => Promise<void>) | undefined;
|
||||
var __hawserHandleMetrics: ((envId: number, metrics: MetricsMessage['metrics']) => Promise<void>) | undefined;
|
||||
var __hawserHandleMessage: ((ws: any, msg: any, connId: string) => Promise<void>) | undefined;
|
||||
var __hawserHandleDisconnect: ((ws: any, connId: string) => void) | undefined;
|
||||
var __terminalHandleExecMessage: ((msg: any) => void) | undefined;
|
||||
}
|
||||
export const edgeConnections: Map<number, EdgeConnection> =
|
||||
globalThis.__hawserEdgeConnections ?? (globalThis.__hawserEdgeConnections = new Map());
|
||||
@@ -94,7 +98,7 @@ export function initializeEdgeManager(): void {
|
||||
const timeout = 90 * 1000; // 90 seconds (3 missed heartbeats)
|
||||
|
||||
for (const [envId, conn] of edgeConnections) {
|
||||
if (now - conn.lastHeartbeat.getTime() > timeout) {
|
||||
if (now - conn.lastHeartbeat > timeout) {
|
||||
const pendingCount = conn.pendingRequests.size;
|
||||
const streamCount = conn.pendingStreamRequests.size;
|
||||
console.log(
|
||||
@@ -120,6 +124,21 @@ export function initializeEdgeManager(): void {
|
||||
updateEnvironmentStatus(envId, null);
|
||||
}
|
||||
}
|
||||
|
||||
// Maintain reconnection tracker: reset for stable connections, prune stale entries
|
||||
for (const [envId, tracker] of reconnectTracker) {
|
||||
const conn = edgeConnections.get(envId);
|
||||
if (conn && now - conn.lastHeartbeat < STABLE_THRESHOLD_MS) {
|
||||
// Connection is stable — reset tracker so next reconnect is unthrottled
|
||||
reconnectTracker.delete(envId);
|
||||
} else if (!conn && tracker.timestamps.length > 0) {
|
||||
const lastAttempt = tracker.timestamps[tracker.timestamps.length - 1];
|
||||
if (now - lastAttempt > STALE_TRACKER_MS) {
|
||||
// No connection and no recent attempts — clean up
|
||||
reconnectTracker.delete(envId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 30000);
|
||||
}
|
||||
|
||||
@@ -137,6 +156,7 @@ export function stopEdgeManager(): void {
|
||||
conn.ws.close(1001, 'Server shutdown');
|
||||
}
|
||||
edgeConnections.clear();
|
||||
reconnectTracker.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -189,7 +209,7 @@ export async function handleEdgeContainerEvent(
|
||||
}
|
||||
}
|
||||
|
||||
// Register global handler for patch-build.ts to use
|
||||
// Register global handler for server.js to use
|
||||
globalThis.__hawserHandleContainerEvent = handleEdgeContainerEvent;
|
||||
|
||||
/**
|
||||
@@ -218,14 +238,8 @@ export async function handleEdgeMetrics(
|
||||
? (metrics.memoryUsed / metrics.memoryTotal) * 100
|
||||
: 0;
|
||||
|
||||
// Save to database using the existing function
|
||||
await saveHostMetric(
|
||||
cpuPercent,
|
||||
memoryPercent,
|
||||
metrics.memoryUsed,
|
||||
metrics.memoryTotal,
|
||||
environmentId
|
||||
);
|
||||
// Push to in-memory ring buffer
|
||||
pushMetric(environmentId, cpuPercent, memoryPercent, metrics.memoryUsed, metrics.memoryTotal);
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
console.error('[Hawser] Error saving metrics:', errorMsg);
|
||||
@@ -288,6 +302,13 @@ export async function generateHawserToken(
|
||||
edgeConnections.delete(environmentId);
|
||||
}
|
||||
|
||||
// Revoke all existing active tokens for this environment so the old agent
|
||||
// can no longer reconnect and fight with the new one over the connection slot
|
||||
await db
|
||||
.update(hawserTokens)
|
||||
.set({ isActive: false })
|
||||
.where(and(eq(hawserTokens.environmentId, environmentId), eq(hawserTokens.isActive, true)));
|
||||
|
||||
// Use provided token or generate a new one
|
||||
let token: string;
|
||||
if (rawToken) {
|
||||
@@ -396,6 +417,7 @@ export function handleEdgeConnection(
|
||||
// Reject all pending requests before closing
|
||||
for (const [requestId, pending] of existing.pendingRequests) {
|
||||
console.log(`[Hawser] Rejecting pending request ${requestId} due to connection replacement`);
|
||||
clearTimeout(pending.timeout);
|
||||
pending.reject(new Error('Connection replaced by new agent'));
|
||||
}
|
||||
for (const [requestId, pending] of existing.pendingStreamRequests) {
|
||||
@@ -405,7 +427,12 @@ export function handleEdgeConnection(
|
||||
existing.pendingRequests.clear();
|
||||
existing.pendingStreamRequests.clear();
|
||||
|
||||
existing.ws.close(1000, 'Replaced by new connection');
|
||||
// Immediately destroy TCP socket — no graceful close needed for replaced connections
|
||||
if (typeof existing.ws.terminate === 'function') {
|
||||
existing.ws.terminate();
|
||||
} else {
|
||||
existing.ws.close(1000, 'Replaced by new connection');
|
||||
}
|
||||
}
|
||||
|
||||
const connection: EdgeConnection = {
|
||||
@@ -418,7 +445,7 @@ export function handleEdgeConnection(
|
||||
hostname: hello.hostname,
|
||||
capabilities: hello.capabilities,
|
||||
connectedAt: new Date(),
|
||||
lastHeartbeat: new Date(),
|
||||
lastHeartbeat: Date.now(),
|
||||
pendingRequests: new Map(),
|
||||
pendingStreamRequests: new Map()
|
||||
};
|
||||
@@ -471,7 +498,8 @@ export async function sendEdgeRequest(
|
||||
body?: unknown,
|
||||
headers?: Record<string, string>,
|
||||
streaming = false,
|
||||
timeout = 30000
|
||||
timeout = 30000,
|
||||
isBinary = false
|
||||
): Promise<EdgeResponse> {
|
||||
const connection = edgeConnections.get(environmentId);
|
||||
if (!connection) {
|
||||
@@ -559,7 +587,8 @@ export async function sendEdgeRequest(
|
||||
method,
|
||||
path,
|
||||
headers: headers || {},
|
||||
body: body, // Body is already an object, will be serialized by JSON.stringify(message)
|
||||
body: body,
|
||||
isBinary, // true when body is base64-encoded binary (tar uploads)
|
||||
streaming
|
||||
};
|
||||
|
||||
@@ -753,7 +782,7 @@ export function handleEdgeResponse(environmentId: number, response: ResponseMess
|
||||
export function handleHeartbeat(environmentId: number): void {
|
||||
const connection = edgeConnections.get(environmentId);
|
||||
if (connection) {
|
||||
connection.lastHeartbeat = new Date();
|
||||
connection.lastHeartbeat = Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -824,7 +853,8 @@ export interface RequestMessage {
|
||||
method: string;
|
||||
path: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: unknown; // JSON-serializable object, will be serialized when message is stringified
|
||||
body?: unknown; // JSON-serializable object, or base64 string when isBinary=true
|
||||
isBinary?: boolean; // true when body is base64-encoded binary data (tar uploads)
|
||||
streaming?: boolean;
|
||||
}
|
||||
|
||||
@@ -946,3 +976,269 @@ export type HawserMessage =
|
||||
| ContainerEventMessage
|
||||
| { type: 'ping'; timestamp: number }
|
||||
| { type: 'pong'; timestamp: number };
|
||||
|
||||
// ─── Production WebSocket message handler (used by server.js) ───
|
||||
|
||||
// Maps WebSocket instances to environment IDs for message routing
|
||||
const wsToEnvId = new Map<any, number>();
|
||||
|
||||
// Auth fail cache to prevent brute-force token validation.
|
||||
// Entries are periodically cleaned up to prevent unbounded growth.
|
||||
const hawserAuthFailCache = new Map<string, number>();
|
||||
const HAWSER_AUTH_FAIL_COOLDOWN_MS = 30_000;
|
||||
|
||||
// Periodic cleanup of expired auth fail entries (every 60s)
|
||||
setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [key, timestamp] of hawserAuthFailCache) {
|
||||
if (now - timestamp > HAWSER_AUTH_FAIL_COOLDOWN_MS) {
|
||||
hawserAuthFailCache.delete(key);
|
||||
}
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
// ─── Reconnection storm throttle ───
|
||||
// Tracks per-environment reconnection frequency to detect storms
|
||||
// (e.g., agent can auth but Docker socket is broken → 30s timeout → reconnect loop)
|
||||
interface ReconnectTrackerEntry {
|
||||
timestamps: number[];
|
||||
cooldownUntil: number; // 0 = no cooldown active
|
||||
cooldownLevel: number; // index into COOLDOWN_LEVELS
|
||||
}
|
||||
const reconnectTracker = new Map<number, ReconnectTrackerEntry>();
|
||||
const RECONNECT_WINDOW_MS = 2 * 60 * 1000; // 2-minute sliding window
|
||||
const RECONNECT_BURST = 3; // allow 3 reconnections per window
|
||||
const COOLDOWN_LEVELS_SECS = [30, 60, 120, 300]; // escalating cooldown in seconds
|
||||
const STABLE_THRESHOLD_MS = 5 * 60 * 1000; // stable connection resets tracker
|
||||
const STALE_TRACKER_MS = 10 * 60 * 1000; // clean up stale tracker entries
|
||||
|
||||
/**
|
||||
* Record a reconnection for an environment and check if throttling is needed.
|
||||
* Returns { allowed: true } or { allowed: false, retryAfter: seconds }.
|
||||
*/
|
||||
function recordReconnection(envId: number): { allowed: true } | { allowed: false; retryAfter: number } {
|
||||
const now = Date.now();
|
||||
let entry = reconnectTracker.get(envId);
|
||||
|
||||
if (!entry) {
|
||||
entry = { timestamps: [now], cooldownUntil: 0, cooldownLevel: 0 };
|
||||
reconnectTracker.set(envId, entry);
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
// Check if currently in cooldown
|
||||
if (now < entry.cooldownUntil) {
|
||||
const retryAfter = Math.ceil((entry.cooldownUntil - now) / 1000);
|
||||
return { allowed: false, retryAfter };
|
||||
}
|
||||
|
||||
// Prune timestamps outside the sliding window
|
||||
entry.timestamps = entry.timestamps.filter(ts => now - ts < RECONNECT_WINDOW_MS);
|
||||
entry.timestamps.push(now);
|
||||
|
||||
// Check if burst limit exceeded
|
||||
if (entry.timestamps.length > RECONNECT_BURST) {
|
||||
const level = Math.min(entry.cooldownLevel, COOLDOWN_LEVELS_SECS.length - 1);
|
||||
const cooldownSecs = COOLDOWN_LEVELS_SECS[level];
|
||||
entry.cooldownUntil = now + cooldownSecs * 1000;
|
||||
entry.cooldownLevel = Math.min(entry.cooldownLevel + 1, COOLDOWN_LEVELS_SECS.length - 1);
|
||||
|
||||
console.warn(
|
||||
`[Hawser WS] Reconnection storm detected for env ${envId}: ` +
|
||||
`${entry.timestamps.length} connections in ${RECONNECT_WINDOW_MS / 1000}s. ` +
|
||||
`Cooldown ${cooldownSecs}s (level ${level})`
|
||||
);
|
||||
|
||||
return { allowed: false, retryAfter: cooldownSecs };
|
||||
}
|
||||
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a WebSocket message from a Hawser Edge agent.
|
||||
* Full protocol handler: hello/welcome auth, ping/pong,
|
||||
* response/stream routing, metrics, container events, exec sessions.
|
||||
*
|
||||
* Registered as globalThis.__hawserHandleMessage for server.js to call.
|
||||
*/
|
||||
async function handleHawserWsMessage(ws: any, msg: any, connId: string): Promise<void> {
|
||||
if (msg.type === 'hello') {
|
||||
const remoteAddr = connId;
|
||||
|
||||
// Rate limit auth failures
|
||||
const lastFail = hawserAuthFailCache.get(remoteAddr);
|
||||
if (lastFail && Date.now() - lastFail < HAWSER_AUTH_FAIL_COOLDOWN_MS) {
|
||||
ws.send(JSON.stringify({ type: 'error', message: 'Too many failed attempts' }));
|
||||
ws.close(1008, 'Rate limited');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!msg.token) {
|
||||
ws.send(JSON.stringify({ type: 'error', message: 'No token provided' }));
|
||||
ws.close(1008, 'Missing token');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await validateHawserToken(msg.token);
|
||||
if (!result.valid || !result.environmentId) {
|
||||
console.log(`[Hawser WS] Authentication failed for connection ${connId}`);
|
||||
hawserAuthFailCache.set(remoteAddr, Date.now());
|
||||
ws.send(JSON.stringify({ type: 'error', message: 'Invalid token' }));
|
||||
ws.close(1008, 'Invalid token');
|
||||
return;
|
||||
}
|
||||
|
||||
// Throttle reconnection storms (successful auth but broken Docker = rapid reconnect loop)
|
||||
const throttle = recordReconnection(result.environmentId);
|
||||
if (!throttle.allowed) {
|
||||
console.log(`[Hawser WS] Throttling reconnection for env ${result.environmentId}: retry after ${throttle.retryAfter}s`);
|
||||
ws.send(JSON.stringify({
|
||||
type: 'error',
|
||||
message: `Reconnection throttled. Retry after ${throttle.retryAfter}s.`,
|
||||
retryAfter: throttle.retryAfter
|
||||
}));
|
||||
ws.close(1008, 'Reconnection throttled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Authenticated — register the connection
|
||||
const connection = handleEdgeConnection(ws, result.environmentId, msg);
|
||||
wsToEnvId.set(ws, result.environmentId);
|
||||
|
||||
// Send welcome
|
||||
ws.send(JSON.stringify({
|
||||
type: 'welcome',
|
||||
serverId: 'dockhand',
|
||||
version: HAWSER_PROTOCOL_VERSION
|
||||
}));
|
||||
|
||||
console.log(`[Hawser WS] Agent authenticated: env=${result.environmentId} agent=${msg.agentName || msg.agentId}`);
|
||||
} catch (error: any) {
|
||||
console.error('[Hawser WS] Auth error:', error.message);
|
||||
ws.send(JSON.stringify({ type: 'error', message: 'Authentication failed' }));
|
||||
ws.close(1011, 'Auth error');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// All other messages require an authenticated connection
|
||||
const envId = wsToEnvId.get(ws);
|
||||
if (!envId) {
|
||||
ws.send(JSON.stringify({ type: 'error', message: 'Not authenticated' }));
|
||||
return;
|
||||
}
|
||||
|
||||
const connection = edgeConnections.get(envId);
|
||||
if (!connection) return;
|
||||
|
||||
// Update heartbeat
|
||||
connection.lastHeartbeat = Date.now();
|
||||
|
||||
switch (msg.type) {
|
||||
case 'ping':
|
||||
ws.send(JSON.stringify({ type: 'pong', timestamp: Date.now() }));
|
||||
break;
|
||||
|
||||
case 'pong':
|
||||
break;
|
||||
|
||||
case 'response': {
|
||||
const pending = connection.pendingRequests.get(msg.requestId);
|
||||
if (pending) {
|
||||
clearTimeout(pending.timeout);
|
||||
connection.pendingRequests.delete(msg.requestId);
|
||||
pending.resolve({
|
||||
statusCode: msg.statusCode,
|
||||
headers: msg.headers || {},
|
||||
body: msg.body,
|
||||
isBinary: msg.isBinary
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'stream': {
|
||||
const streamPending = connection.pendingStreamRequests.get(msg.requestId);
|
||||
if (streamPending) {
|
||||
streamPending.onData?.(msg.data);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'stream_end': {
|
||||
const streamReq = connection.pendingStreamRequests.get(msg.requestId);
|
||||
if (streamReq) {
|
||||
connection.pendingStreamRequests.delete(msg.requestId);
|
||||
streamReq.onEnd?.(msg.reason);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'metrics':
|
||||
if (globalThis.__hawserHandleMetrics) {
|
||||
await globalThis.__hawserHandleMetrics(envId, msg.metrics);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'container_event':
|
||||
if (globalThis.__hawserHandleContainerEvent) {
|
||||
await globalThis.__hawserHandleContainerEvent(envId, msg.event);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'exec_ready':
|
||||
case 'exec_output':
|
||||
case 'exec_end':
|
||||
// Forward exec messages to server.js/vite.config.ts via global callback
|
||||
if (globalThis.__terminalHandleExecMessage) {
|
||||
globalThis.__terminalHandleExecMessage(msg);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
console.error(`[Hawser WS] Agent error (env ${envId}): ${msg.message}`);
|
||||
// Forward exec-related errors (identified by requestId) to terminal handler
|
||||
if (msg.requestId && globalThis.__terminalHandleExecMessage) {
|
||||
globalThis.__terminalHandleExecMessage(msg);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle WebSocket disconnect for a Hawser Edge agent.
|
||||
* Receives the actual ws object to correctly identify which connection closed.
|
||||
*/
|
||||
function handleHawserWsDisconnect(disconnectedWs: any, connId: string): void {
|
||||
const envId = wsToEnvId.get(disconnectedWs);
|
||||
if (!envId) {
|
||||
// This ws was never authenticated (e.g., auth failed), nothing to clean up
|
||||
return;
|
||||
}
|
||||
|
||||
const connection = edgeConnections.get(envId);
|
||||
if (connection && connection.ws === disconnectedWs) {
|
||||
console.log(`[Hawser WS] Agent disconnected: env=${envId}`);
|
||||
|
||||
for (const [, pending] of connection.pendingRequests) {
|
||||
clearTimeout(pending.timeout);
|
||||
pending.reject(new Error('Agent disconnected'));
|
||||
}
|
||||
for (const [, pending] of connection.pendingStreamRequests) {
|
||||
pending.onEnd?.('Agent disconnected');
|
||||
}
|
||||
connection.pendingRequests.clear();
|
||||
connection.pendingStreamRequests.clear();
|
||||
|
||||
edgeConnections.delete(envId);
|
||||
updateEnvironmentStatus(envId, null);
|
||||
}
|
||||
|
||||
wsToEnvId.delete(disconnectedWs);
|
||||
}
|
||||
|
||||
// Register global handlers for server.js to call
|
||||
globalThis.__hawserHandleMessage = handleHawserWsMessage;
|
||||
globalThis.__hawserHandleDisconnect = handleHawserWsDisconnect;
|
||||
+83
-11
@@ -20,6 +20,7 @@
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import * as http from 'node:http';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
// Cache the host data dir to avoid repeated API calls
|
||||
@@ -29,6 +30,11 @@ let detectionAttempted = false;
|
||||
// Cache ALL mounts for path translation (not just DATA_DIR)
|
||||
let cachedMounts: Array<{ source: string; destination: string }> | null = null;
|
||||
|
||||
// Cache Dockhand's own Docker access method (detected from container inspect)
|
||||
// Used by scanner to replicate how Dockhand connects to Docker
|
||||
let cachedOwnDockerHost: string | null = null;
|
||||
let cachedOwnNetworkMode: string | null = null;
|
||||
|
||||
/**
|
||||
* Get our own container ID
|
||||
*/
|
||||
@@ -95,25 +101,48 @@ export async function detectHostDataDir(): Promise<string | null> {
|
||||
|
||||
try {
|
||||
// Query Docker API to inspect our own container
|
||||
// Try unix socket first, fall back to TCP if DOCKER_HOST is set
|
||||
const socketPath = process.env.DOCKER_SOCKET || '/var/run/docker.sock';
|
||||
const dockerHost = process.env.DOCKER_HOST;
|
||||
|
||||
// Use fetch with unix socket
|
||||
const response = await fetch(`http://localhost/containers/${containerId}/json`, {
|
||||
// @ts-ignore - Bun supports unix sockets
|
||||
unix: socketPath
|
||||
});
|
||||
const containerInfo = await new Promise<any>((resolvePromise, reject) => {
|
||||
const reqOptions: http.RequestOptions = dockerHost?.startsWith('tcp://')
|
||||
? (() => {
|
||||
const u = new URL(dockerHost.replace('tcp://', 'http://'));
|
||||
return { hostname: u.hostname, port: u.port, path: `/containers/${containerId}/json`, method: 'GET' };
|
||||
})()
|
||||
: { socketPath, path: `/containers/${containerId}/json`, method: 'GET' };
|
||||
|
||||
if (!response.ok) {
|
||||
console.warn(`[HostPath] Failed to inspect container: ${response.status}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const containerInfo = await response.json() as {
|
||||
const req = http.request(reqOptions, (res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
res.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
res.on('end', () => {
|
||||
if (res.statusCode === 200) {
|
||||
try {
|
||||
resolvePromise(JSON.parse(Buffer.concat(chunks).toString('utf-8')));
|
||||
} catch {
|
||||
reject(new Error('Failed to parse container inspect response'));
|
||||
}
|
||||
} else {
|
||||
reject(new Error(`Container inspect failed: ${res.statusCode}`));
|
||||
}
|
||||
});
|
||||
res.on('error', reject);
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.end();
|
||||
}) as {
|
||||
Mounts?: Array<{
|
||||
Type: string;
|
||||
Source: string;
|
||||
Destination: string;
|
||||
}>;
|
||||
Config?: {
|
||||
Env?: string[];
|
||||
};
|
||||
NetworkSettings?: {
|
||||
Networks?: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
|
||||
// Cache ALL mounts for later path translation (used by rewriteComposeVolumePaths)
|
||||
@@ -123,6 +152,30 @@ export async function detectHostDataDir(): Promise<string | null> {
|
||||
}));
|
||||
console.log(`[HostPath] Cached ${cachedMounts.length} mount(s)`);
|
||||
|
||||
// Cache DOCKER_HOST from Dockhand's own env vars (if set)
|
||||
// This tells us how Dockhand was configured to reach Docker
|
||||
const envVars = containerInfo.Config?.Env || [];
|
||||
for (const v of envVars) {
|
||||
if (v.startsWith('DOCKER_HOST=')) {
|
||||
cachedOwnDockerHost = v.substring('DOCKER_HOST='.length);
|
||||
console.log(`[HostPath] Detected own DOCKER_HOST: ${cachedOwnDockerHost}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Cache Dockhand's network (prefer non-default for service discovery)
|
||||
const networks = containerInfo.NetworkSettings?.Networks;
|
||||
if (networks) {
|
||||
const custom = Object.keys(networks).filter(
|
||||
n => n !== 'bridge' && n !== 'none' && n !== 'host'
|
||||
);
|
||||
cachedOwnNetworkMode = custom.length > 0 ? custom[0]
|
||||
: networks.bridge ? 'bridge' : null;
|
||||
if (cachedOwnNetworkMode) {
|
||||
console.log(`[HostPath] Detected own network: ${cachedOwnNetworkMode}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Find the mount for our DATA_DIR
|
||||
const dataMount = containerInfo.Mounts?.find(m => m.Destination === dataDir);
|
||||
|
||||
@@ -157,6 +210,25 @@ export function getHostDataDir(): string | null {
|
||||
return cachedHostDataDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get DOCKER_HOST from Dockhand's own container config (if set).
|
||||
* Returns the TCP address (e.g., "tcp://socket-proxy:2375") or null.
|
||||
* Populated by detectHostDataDir() at startup.
|
||||
*/
|
||||
export function getOwnDockerHost(): string | null {
|
||||
return cachedOwnDockerHost;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the Docker network Dockhand is attached to.
|
||||
* Used to place scanner containers on the same network so they can reach
|
||||
* TCP-based Docker endpoints (e.g., socket proxy).
|
||||
* Populated by detectHostDataDir() at startup.
|
||||
*/
|
||||
export function getOwnNetworkMode(): string | null {
|
||||
return cachedOwnNetworkMode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate a container path to host path
|
||||
*
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
export interface JobLine {
|
||||
event?: string; // 'result', 'progress', etc. — undefined for bare data lines
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
export interface Job {
|
||||
id: string;
|
||||
status: 'running' | 'done' | 'error';
|
||||
lines: JobLine[];
|
||||
result?: unknown;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
const jobs = new Map<string, Job>();
|
||||
|
||||
export function createJob(): Job {
|
||||
const job: Job = {
|
||||
id: randomUUID(),
|
||||
status: 'running',
|
||||
lines: [],
|
||||
createdAt: Date.now(),
|
||||
updatedAt: Date.now()
|
||||
};
|
||||
jobs.set(job.id, job);
|
||||
return job;
|
||||
}
|
||||
|
||||
export function getJob(id: string): Job | undefined {
|
||||
return jobs.get(id);
|
||||
}
|
||||
|
||||
export function appendLine(job: Job, line: JobLine): void {
|
||||
job.lines.push(line);
|
||||
job.updatedAt = Date.now();
|
||||
}
|
||||
|
||||
export function completeJob(job: Job, result: unknown): void {
|
||||
job.result = result;
|
||||
job.status = 'done';
|
||||
job.updatedAt = Date.now();
|
||||
}
|
||||
|
||||
export function failJob(job: Job, error: string): void {
|
||||
job.result = { success: false, error };
|
||||
job.status = 'error';
|
||||
job.updatedAt = Date.now();
|
||||
}
|
||||
|
||||
// Cleanup jobs older than 10 minutes that are no longer running
|
||||
const CLEANUP_INTERVAL_MS = 60_000;
|
||||
const JOB_TTL_MS = 10 * 60_000;
|
||||
|
||||
setInterval(() => {
|
||||
const cutoff = Date.now() - JOB_TTL_MS;
|
||||
for (const [id, job] of jobs) {
|
||||
if (job.status !== 'running' && job.updatedAt < cutoff) {
|
||||
jobs.delete(id);
|
||||
}
|
||||
}
|
||||
}, CLEANUP_INTERVAL_MS);
|
||||
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* In-Memory Metrics Ring Buffer
|
||||
*
|
||||
* Replaces SQLite/PostgreSQL host_metrics storage with a fixed-size
|
||||
* in-memory circular buffer per environment. Uses pre-allocated arrays
|
||||
* with head/count indices to avoid splice()-based eviction which causes
|
||||
* V8 to repeatedly reallocate backing arrays.
|
||||
*
|
||||
* Memory: 16 envs × 360 slots × ~100 bytes ≈ 576 KB
|
||||
*/
|
||||
|
||||
export interface MetricPoint {
|
||||
id: number;
|
||||
cpuPercent: number;
|
||||
memoryPercent: number;
|
||||
memoryUsed: number;
|
||||
memoryTotal: number;
|
||||
environmentId: number | null;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
const MAX_POINTS_PER_ENV = 360; // 1 hour at 10s interval, 3 hours at 30s
|
||||
|
||||
interface RingBuffer {
|
||||
data: (MetricPoint | null)[];
|
||||
head: number; // next write position
|
||||
count: number; // number of valid entries (≤ MAX_POINTS_PER_ENV)
|
||||
}
|
||||
|
||||
// envId → RingBuffer
|
||||
const store = new Map<number, RingBuffer>();
|
||||
|
||||
let nextId = 1;
|
||||
|
||||
/**
|
||||
* Push a new metric data point for an environment.
|
||||
* Overwrites oldest entry when buffer is full (no array reallocation).
|
||||
*/
|
||||
export function pushMetric(
|
||||
envId: number,
|
||||
cpuPercent: number,
|
||||
memoryPercent: number,
|
||||
memoryUsed: number,
|
||||
memoryTotal: number
|
||||
): void {
|
||||
let ring = store.get(envId);
|
||||
if (!ring) {
|
||||
ring = { data: new Array(MAX_POINTS_PER_ENV).fill(null), head: 0, count: 0 };
|
||||
store.set(envId, ring);
|
||||
}
|
||||
|
||||
ring.data[ring.head] = {
|
||||
id: nextId++,
|
||||
cpuPercent,
|
||||
memoryPercent,
|
||||
memoryUsed,
|
||||
memoryTotal,
|
||||
environmentId: envId,
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
ring.head = (ring.head + 1) % MAX_POINTS_PER_ENV;
|
||||
if (ring.count < MAX_POINTS_PER_ENV) ring.count++;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read entries from a ring buffer in oldest-first order.
|
||||
*/
|
||||
function readRing(ring: RingBuffer, limit: number): MetricPoint[] {
|
||||
const count = Math.min(ring.count, limit);
|
||||
if (count === 0) return [];
|
||||
|
||||
const result: MetricPoint[] = new Array(count);
|
||||
// Start reading from the oldest entry
|
||||
const start = (ring.head - ring.count + MAX_POINTS_PER_ENV) % MAX_POINTS_PER_ENV;
|
||||
const skip = ring.count - count;
|
||||
const readFrom = (start + skip) % MAX_POINTS_PER_ENV;
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
result[i] = ring.data[(readFrom + i) % MAX_POINTS_PER_ENV]!;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the most recent metric for an environment.
|
||||
*/
|
||||
export function getLatestMetric(envId: number): MetricPoint | null {
|
||||
const ring = store.get(envId);
|
||||
if (!ring || ring.count === 0) return null;
|
||||
// head points to next write position, so latest is head - 1
|
||||
const idx = (ring.head - 1 + MAX_POINTS_PER_ENV) % MAX_POINTS_PER_ENV;
|
||||
return ring.data[idx];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metrics history for an environment, oldest first.
|
||||
*/
|
||||
export function getMetricsHistory(envId: number, limit = 60): MetricPoint[] {
|
||||
const ring = store.get(envId);
|
||||
if (!ring || ring.count === 0) return [];
|
||||
return readRing(ring, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all metrics (across all environments), newest first, with optional limit.
|
||||
* Used by the global getHostMetrics() fallback when no envId is specified.
|
||||
*/
|
||||
export function getAllMetrics(limit = 60): MetricPoint[] {
|
||||
const all: MetricPoint[] = [];
|
||||
for (const ring of store.values()) {
|
||||
const points = readRing(ring, ring.count);
|
||||
all.push(...points);
|
||||
}
|
||||
// Sort newest first (matching old DB query behavior)
|
||||
all.sort((a, b) => b.id - a.id);
|
||||
return all.slice(0, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all metrics for an environment (e.g., when environment is deleted).
|
||||
*/
|
||||
export function clearEnvironmentMetrics(envId: number): void {
|
||||
store.delete(envId);
|
||||
}
|
||||
@@ -21,6 +21,13 @@ function escapeTelegramMarkdown(text: string): string {
|
||||
.replace(/`/g, '\\`'); // Backtick (code)
|
||||
}
|
||||
|
||||
/** Drain a response body to release the underlying socket/TLS connection. */
|
||||
async function drainResponse(response: Response): Promise<void> {
|
||||
if (!response.bodyUsed) {
|
||||
try { await response.arrayBuffer(); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
export interface NotificationPayload {
|
||||
title: string;
|
||||
message: string;
|
||||
@@ -35,21 +42,47 @@ export interface NotificationResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// SMTP transporter cache — reuses connections instead of creating a new TLS pool per notification.
|
||||
const transporterCache = new Map<string, { transporter: ReturnType<typeof nodemailer.createTransport>; lastUsed: number }>();
|
||||
|
||||
function getOrCreateTransporter(config: SmtpConfig): ReturnType<typeof nodemailer.createTransport> {
|
||||
const key = `${config.host}:${config.port}:${config.secure}:${config.username || ''}`;
|
||||
const cached = transporterCache.get(key);
|
||||
if (cached) {
|
||||
cached.lastUsed = Date.now();
|
||||
return cached.transporter;
|
||||
}
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
secure: config.secure,
|
||||
auth: config.username ? {
|
||||
user: config.username,
|
||||
pass: config.password
|
||||
} : undefined,
|
||||
tls: config.skipTlsVerify ? {
|
||||
rejectUnauthorized: false
|
||||
} : undefined
|
||||
});
|
||||
transporterCache.set(key, { transporter, lastUsed: Date.now() });
|
||||
return transporter;
|
||||
}
|
||||
|
||||
// Clean up idle transporters every 10 minutes
|
||||
setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of transporterCache) {
|
||||
if (now - entry.lastUsed > 10 * 60 * 1000) {
|
||||
entry.transporter.close();
|
||||
transporterCache.delete(key);
|
||||
}
|
||||
}
|
||||
}, 10 * 60 * 1000);
|
||||
|
||||
// Send notification via SMTP
|
||||
async function sendSmtpNotification(config: SmtpConfig, payload: NotificationPayload): Promise<NotificationResult> {
|
||||
try {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
secure: config.secure,
|
||||
auth: config.username ? {
|
||||
user: config.username,
|
||||
pass: config.password
|
||||
} : undefined,
|
||||
tls: config.skipTlsVerify ? {
|
||||
rejectUnauthorized: false
|
||||
} : undefined
|
||||
});
|
||||
const transporter = getOrCreateTransporter(config);
|
||||
|
||||
const envBadge = payload.environmentName
|
||||
? `<span style="display: inline-block; background: #3b82f6; color: white; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-left: 8px;">${payload.environmentName}</span>`
|
||||
@@ -172,6 +205,7 @@ async function sendDiscord(appriseUrl: string, payload: NotificationPayload): Pr
|
||||
const text = await response.text().catch(() => '');
|
||||
return { success: false, error: `Discord error ${response.status}: ${text || response.statusText}` };
|
||||
}
|
||||
await drainResponse(response);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return { success: false, error: `Discord connection failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
@@ -204,6 +238,7 @@ async function sendSlack(appriseUrl: string, payload: NotificationPayload): Prom
|
||||
const text = await response.text().catch(() => '');
|
||||
return { success: false, error: `Slack error ${response.status}: ${text || response.statusText}` };
|
||||
}
|
||||
await drainResponse(response);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return { success: false, error: `Slack connection failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
@@ -211,7 +246,7 @@ async function sendSlack(appriseUrl: string, payload: NotificationPayload): Prom
|
||||
}
|
||||
|
||||
// Mattermost webhook
|
||||
async function sendMattermost(appriseUrl: string, payload: NotificationPayload): Promise<boolean> {
|
||||
async function sendMattermost(appriseUrl: string, payload: NotificationPayload): Promise<NotificationResult> {
|
||||
// mmost://[botname@]hostname[:port][/path]/token or mmosts://...
|
||||
const isSecure = appriseUrl.startsWith('mmosts');
|
||||
const protocol = isSecure ? 'https' : 'http';
|
||||
@@ -230,8 +265,7 @@ async function sendMattermost(appriseUrl: string, payload: NotificationPayload):
|
||||
// The token is the last segment, everything else is hostname[:port][/path]
|
||||
const lastSlashIndex = urlPart.lastIndexOf('/');
|
||||
if (lastSlashIndex === -1) {
|
||||
console.error('[Notifications] Invalid Mattermost URL format. Expected: mmost://[botname@]hostname[:port][/path]/token');
|
||||
return false;
|
||||
return { success: false, error: 'Invalid Mattermost URL format. Expected: mmost://[botname@]hostname[:port][/path]/token' };
|
||||
}
|
||||
|
||||
const token = urlPart.substring(lastSlashIndex + 1);
|
||||
@@ -249,13 +283,22 @@ async function sendMattermost(appriseUrl: string, payload: NotificationPayload):
|
||||
body.username = username;
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
|
||||
return response.ok;
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => '');
|
||||
return { success: false, error: `Mattermost error ${response.status}: ${text || response.statusText}` };
|
||||
}
|
||||
await drainResponse(response);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return { success: false, error: `Mattermost connection failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
}
|
||||
}
|
||||
|
||||
// Telegram
|
||||
@@ -290,6 +333,7 @@ async function sendTelegram(appriseUrl: string, payload: NotificationPayload): P
|
||||
const errorMsg = errorData.description || response.statusText;
|
||||
return { success: false, error: `Telegram error ${response.status}: ${errorMsg}` };
|
||||
}
|
||||
await drainResponse(response);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return { success: false, error: `Telegram connection failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
@@ -328,6 +372,7 @@ async function sendGotify(appriseUrl: string, payload: NotificationPayload): Pro
|
||||
const text = await response.text().catch(() => '');
|
||||
return { success: false, error: `Gotify error ${response.status}: ${text || response.statusText}` };
|
||||
}
|
||||
await drainResponse(response);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return { success: false, error: `Gotify connection failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
@@ -394,6 +439,7 @@ async function sendNtfy(appriseUrl: string, payload: NotificationPayload): Promi
|
||||
const text = await response.text().catch(() => '');
|
||||
return { success: false, error: `ntfy error ${response.status}: ${text || response.statusText}` };
|
||||
}
|
||||
await drainResponse(response);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return { success: false, error: `ntfy connection failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
@@ -428,6 +474,7 @@ async function sendPushover(appriseUrl: string, payload: NotificationPayload): P
|
||||
const text = await response.text().catch(() => '');
|
||||
return { success: false, error: `Pushover error ${response.status}: ${text || response.statusText}` };
|
||||
}
|
||||
await drainResponse(response);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return { success: false, error: `Pushover connection failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
@@ -455,6 +502,7 @@ async function sendGenericWebhook(appriseUrl: string, payload: NotificationPaylo
|
||||
const text = await response.text().catch(() => '');
|
||||
return { success: false, error: `Webhook error ${response.status}: ${text || response.statusText}` };
|
||||
}
|
||||
await drainResponse(response);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return { success: false, error: `Webhook connection failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
/**
|
||||
* RSS Tracker — Per-operation native memory delta tracking
|
||||
*
|
||||
* Measures process.memoryUsage().rss before and after instrumented operations
|
||||
* to identify which background operation is responsible for native memory growth.
|
||||
*
|
||||
* All functions are no-ops when MEMORY_MONITOR !== 'true'.
|
||||
*/
|
||||
|
||||
import v8 from 'node:v8';
|
||||
import { existsSync, mkdirSync, readdirSync, unlinkSync, statSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface CategoryStats {
|
||||
count: number;
|
||||
totalDelta: number;
|
||||
maxDelta: number;
|
||||
// Lifetime cumulative (never reset)
|
||||
lifetimeCount: number;
|
||||
lifetimeTotalDelta: number;
|
||||
}
|
||||
|
||||
interface RssSnapshot {
|
||||
filename: string;
|
||||
timestamp: string;
|
||||
uptimeMin: number;
|
||||
rssMB: number;
|
||||
sizeMB: number;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// State
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const enabled = process.env.MEMORY_MONITOR === 'true';
|
||||
|
||||
const categories = new Map<string, CategoryStats>();
|
||||
let intervalHandle: ReturnType<typeof setInterval> | null = null;
|
||||
let snapshotIntervalHandle: ReturnType<typeof setInterval> | null = null;
|
||||
let periodNumber = 0;
|
||||
let periodStartRss = 0;
|
||||
const startupTime = Date.now();
|
||||
const startupRss = enabled ? process.memoryUsage().rss : 0;
|
||||
|
||||
// Snapshot settings
|
||||
const SNAPSHOT_WARMUP_MS = 5 * 60 * 1000; // 5 min before first snapshot
|
||||
const SNAPSHOT_INTERVAL_MS = parseInt(process.env.SNAPSHOT_INTERVAL || '60', 10) * 60 * 1000;
|
||||
const MAX_SNAPSHOTS = 48;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Core API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Capture RSS before an operation. Returns the RSS value.
|
||||
* No-op (returns 0) when MEMORY_MONITOR is not set.
|
||||
*/
|
||||
export function rssBeforeOp(): number {
|
||||
if (!enabled) return 0;
|
||||
return process.memoryUsage().rss;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record RSS delta after an operation.
|
||||
* No-op when MEMORY_MONITOR is not set.
|
||||
*/
|
||||
export function rssAfterOp(category: string, before: number): void {
|
||||
if (!enabled || before === 0) return;
|
||||
|
||||
const after = process.memoryUsage().rss;
|
||||
const delta = after - before;
|
||||
|
||||
let stats = categories.get(category);
|
||||
if (!stats) {
|
||||
stats = { count: 0, totalDelta: 0, maxDelta: 0, lifetimeCount: 0, lifetimeTotalDelta: 0 };
|
||||
categories.set(category, stats);
|
||||
}
|
||||
|
||||
stats.count++;
|
||||
stats.totalDelta += delta;
|
||||
if (Math.abs(delta) > Math.abs(stats.maxDelta)) stats.maxDelta = delta;
|
||||
|
||||
stats.lifetimeCount++;
|
||||
stats.lifetimeTotalDelta += delta;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current stats summary.
|
||||
*/
|
||||
export function getRssStats() {
|
||||
if (!enabled) return null;
|
||||
|
||||
const mem = process.memoryUsage();
|
||||
const uptimeMs = Date.now() - startupTime;
|
||||
const uptimeHours = uptimeMs / (1000 * 60 * 60);
|
||||
const rssGrowth = mem.rss - startupRss;
|
||||
|
||||
const perCategory: Record<string, {
|
||||
count: number;
|
||||
avgDelta: string;
|
||||
maxDelta: string;
|
||||
totalDelta: string;
|
||||
lifetimeCount: number;
|
||||
lifetimeTotalDelta: string;
|
||||
}> = {};
|
||||
|
||||
for (const [cat, stats] of categories) {
|
||||
perCategory[cat] = {
|
||||
count: stats.count,
|
||||
avgDelta: stats.count > 0 ? fmtBytes(Math.round(stats.totalDelta / stats.count)) : '0',
|
||||
maxDelta: fmtBytes(stats.maxDelta),
|
||||
totalDelta: fmtBytes(stats.totalDelta),
|
||||
lifetimeCount: stats.lifetimeCount,
|
||||
lifetimeTotalDelta: fmtBytes(stats.lifetimeTotalDelta),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: true,
|
||||
periodNumber,
|
||||
rssMB: fmtMB(mem.rss),
|
||||
rssGrowthTotal: fmtBytes(rssGrowth),
|
||||
rssGrowthPerHour: fmtBytes(uptimeHours > 0.01 ? rssGrowth / uptimeHours : 0),
|
||||
uptimeHours: Math.round(uptimeHours * 100) / 100,
|
||||
categories: perCategory,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Periodic logging
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function logPeriodSummary(): void {
|
||||
periodNumber++;
|
||||
const mem = process.memoryUsage();
|
||||
const rssDelta = mem.rss - periodStartRss;
|
||||
const uptimeMs = Date.now() - startupTime;
|
||||
const uptimeHours = uptimeMs / (1000 * 60 * 60);
|
||||
const rssGrowthTotal = mem.rss - startupRss;
|
||||
const rssPerHour = uptimeHours > 0.01 ? rssGrowthTotal / uptimeHours : 0;
|
||||
|
||||
let summary = `[RSS] #${periodNumber} rss=${fmtMB(mem.rss)}(${fmtDelta(rssDelta)}) total=${fmtDelta(rssGrowthTotal)} rate=${fmtBytes(Math.round(rssPerHour))}/h`;
|
||||
|
||||
// Sort categories by absolute totalDelta descending
|
||||
const sorted = [...categories.entries()]
|
||||
.filter(([, s]) => s.count > 0)
|
||||
.sort((a, b) => Math.abs(b[1].totalDelta) - Math.abs(a[1].totalDelta));
|
||||
|
||||
let accountedDelta = 0;
|
||||
for (const [cat, stats] of sorted) {
|
||||
const avg = stats.count > 0 ? Math.round(stats.totalDelta / stats.count) : 0;
|
||||
summary += `\n ${cat.padEnd(14)} n=${String(stats.count).padStart(4)} avg=${fmtDelta(avg).padStart(7)} max=${fmtDelta(stats.maxDelta).padStart(7)} total=${fmtDelta(stats.totalDelta).padStart(7)}`;
|
||||
accountedDelta += stats.totalDelta;
|
||||
}
|
||||
|
||||
const unaccounted = rssDelta - accountedDelta;
|
||||
if (sorted.length > 0) {
|
||||
summary += `\n ${'unaccounted'.padEnd(14)} ${fmtDelta(unaccounted).padStart(7)}`;
|
||||
}
|
||||
|
||||
console.log(summary);
|
||||
|
||||
// Reset per-period counters (keep lifetime)
|
||||
for (const stats of categories.values()) {
|
||||
stats.count = 0;
|
||||
stats.totalDelta = 0;
|
||||
stats.maxDelta = 0;
|
||||
}
|
||||
periodStartRss = mem.rss;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Heap snapshots
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getSnapshotDir(): string {
|
||||
const dataDir = process.env.DATA_DIR || './data';
|
||||
return join(dataDir, 'snapshots');
|
||||
}
|
||||
|
||||
function ensureSnapshotDir(): string {
|
||||
const dir = getSnapshotDir();
|
||||
if (!existsSync(dir)) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
function cleanupOldSnapshots(dir: string): void {
|
||||
try {
|
||||
const files = readdirSync(dir)
|
||||
.filter(f => f.endsWith('.heapsnapshot'))
|
||||
.map(f => ({ name: f, time: statSync(join(dir, f)).mtimeMs }))
|
||||
.sort((a, b) => a.time - b.time);
|
||||
|
||||
while (files.length > MAX_SNAPSHOTS) {
|
||||
const oldest = files.shift()!;
|
||||
try {
|
||||
unlinkSync(join(dir, oldest.name));
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
/**
|
||||
* Dump a V8 heap snapshot to disk. Returns the filename.
|
||||
*/
|
||||
export function dumpHeapSnapshot(): string | null {
|
||||
if (!enabled) return null;
|
||||
|
||||
const dir = ensureSnapshotDir();
|
||||
cleanupOldSnapshots(dir);
|
||||
|
||||
const mem = process.memoryUsage();
|
||||
const uptimeMin = Math.round((Date.now() - startupTime) / 60000);
|
||||
const rssMB = Math.round(mem.rss / (1024 * 1024));
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, '-').replace('T', '-').substring(0, 19);
|
||||
const filename = `heap-${ts}-${uptimeMin}m-${rssMB}mb.heapsnapshot`;
|
||||
const filepath = join(dir, filename);
|
||||
|
||||
try {
|
||||
v8.writeHeapSnapshot(filepath);
|
||||
console.log(`[RSS] Heap snapshot saved: ${filepath}`);
|
||||
return filename;
|
||||
} catch (err) {
|
||||
console.error(`[RSS] Failed to write heap snapshot:`, err instanceof Error ? err.message : String(err));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List saved heap snapshots.
|
||||
*/
|
||||
export function listHeapSnapshots(): RssSnapshot[] {
|
||||
const dir = getSnapshotDir();
|
||||
if (!existsSync(dir)) return [];
|
||||
|
||||
try {
|
||||
return readdirSync(dir)
|
||||
.filter(f => f.endsWith('.heapsnapshot'))
|
||||
.map(f => {
|
||||
const stat = statSync(join(dir, f));
|
||||
// Parse filename: heap-YYYY-MM-DD-HH-MM-SS-{uptime}m-{rss}mb.heapsnapshot
|
||||
const match = f.match(/heap-(.+?)-(\d+)m-(\d+)mb\.heapsnapshot/);
|
||||
return {
|
||||
filename: f,
|
||||
timestamp: stat.mtime.toISOString(),
|
||||
uptimeMin: match ? parseInt(match[2]) : 0,
|
||||
rssMB: match ? parseInt(match[3]) : 0,
|
||||
sizeMB: Math.round(stat.size / (1024 * 1024) * 10) / 10,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function startAutoSnapshots(): void {
|
||||
// First snapshot after warmup
|
||||
setTimeout(() => {
|
||||
dumpHeapSnapshot();
|
||||
|
||||
// Then every SNAPSHOT_INTERVAL_MS
|
||||
snapshotIntervalHandle = setInterval(() => {
|
||||
dumpHeapSnapshot();
|
||||
}, SNAPSHOT_INTERVAL_MS);
|
||||
}, SNAPSHOT_WARMUP_MS);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Lifecycle
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Start the RSS tracker. Call once on startup.
|
||||
*/
|
||||
export function startRssTracker(): void {
|
||||
if (!enabled) return;
|
||||
|
||||
periodStartRss = process.memoryUsage().rss;
|
||||
console.log(`[RSS] Tracker started. Initial RSS: ${fmtMB(periodStartRss)}. Logging every 60s.`);
|
||||
|
||||
intervalHandle = setInterval(logPeriodSummary, 60_000);
|
||||
|
||||
startAutoSnapshots();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the RSS tracker.
|
||||
*/
|
||||
export function stopRssTracker(): void {
|
||||
if (intervalHandle) {
|
||||
clearInterval(intervalHandle);
|
||||
intervalHandle = null;
|
||||
}
|
||||
if (snapshotIntervalHandle) {
|
||||
clearInterval(snapshotIntervalHandle);
|
||||
snapshotIntervalHandle = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Formatting helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function fmtMB(bytes: number): string {
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)}M`;
|
||||
}
|
||||
|
||||
function fmtBytes(bytes: number): string {
|
||||
const abs = Math.abs(bytes);
|
||||
const sign = bytes < 0 ? '-' : '+';
|
||||
if (abs < 1024) return `${sign}${abs}B`;
|
||||
if (abs < 1024 * 1024) return `${sign}${(abs / 1024).toFixed(1)}K`;
|
||||
return `${sign}${(abs / (1024 * 1024)).toFixed(1)}M`;
|
||||
}
|
||||
|
||||
function fmtDelta(bytes: number): string {
|
||||
return fmtBytes(bytes);
|
||||
}
|
||||
+53
-34
@@ -15,7 +15,7 @@ import {
|
||||
} from './docker';
|
||||
import { getEnvironment, getEnvSetting, getSetting } from './db';
|
||||
import { sendEventNotification } from './notifications';
|
||||
import { getHostDockerSocket, getHostDataDir, extractUidFromSocketPath } from './host-path';
|
||||
import { getHostDockerSocket, getHostDataDir, extractUidFromSocketPath, getOwnDockerHost, getOwnNetworkMode } from './host-path';
|
||||
import { resolve } from 'node:path';
|
||||
import { mkdir, chown } from 'node:fs/promises';
|
||||
|
||||
@@ -306,8 +306,18 @@ export function sanitizeJsonString(json: string): string {
|
||||
const ch2 = json[i];
|
||||
if ('"\\\/bfnrtu'.includes(ch2)) {
|
||||
result += ch2;
|
||||
} else if (ch < 0x20) {
|
||||
// Backslash followed by a raw control character (e.g. \ + 0x0A)
|
||||
// The backslash was already added to result — escape it as \\
|
||||
// then also escape the control character
|
||||
result += '\\';
|
||||
if (ch === 0x0A) result += '\\n';
|
||||
else if (ch === 0x0D) result += '\\r';
|
||||
else if (ch === 0x09) result += '\\t';
|
||||
else result += `\\u${ch.toString(16).padStart(4, '0')}`;
|
||||
sanitized++;
|
||||
} else {
|
||||
// Invalid escape like \x, \a, \0, \_ — convert backslash to literal \\
|
||||
// Invalid escape like \x, \a, \e — convert backslash to literal \\
|
||||
result += '\\' + ch2;
|
||||
sanitized++;
|
||||
}
|
||||
@@ -341,7 +351,7 @@ export function sanitizeJsonString(json: string): string {
|
||||
}
|
||||
|
||||
if (sanitized > 0) {
|
||||
console.warn(`[Scanner] Sanitized ${sanitized} control characters in JSON output`);
|
||||
console.warn(`[Scanner] Sanitized ${sanitized} control/escape characters in JSON output`);
|
||||
}
|
||||
|
||||
return result;
|
||||
@@ -359,12 +369,10 @@ function parseGrypeOutput(output: string): { vulnerabilities: Vulnerability[]; s
|
||||
unknown: 0
|
||||
};
|
||||
|
||||
console.log('[Grype] Raw output length:', output.length);
|
||||
console.log('[Grype] Output starts with:', output.slice(0, 200));
|
||||
console.log('[Grype] Output ends with:', JSON.stringify(output.slice(-50)));
|
||||
|
||||
try {
|
||||
const data = JSON.parse(sanitizeJsonString(extractJson(output)));
|
||||
const extracted = extractJson(output);
|
||||
const sanitized = sanitizeJsonString(extracted);
|
||||
const data = JSON.parse(sanitized);
|
||||
|
||||
if (data.matches) {
|
||||
for (const match of data.matches) {
|
||||
@@ -587,42 +595,45 @@ async function runScannerContainerCore(
|
||||
const basePath = scannerType === 'grype' ? '/cache/grype' : '/cache/trivy';
|
||||
const dbPath = basePath;
|
||||
|
||||
// Detect the host Docker socket path based on connection type
|
||||
// For local socket environments, detect the actual host socket path (handles rootless Docker)
|
||||
// For remote environments (hawser/direct with host), scanner runs remotely and uses standard path
|
||||
// Detect how the scanner container should access Docker.
|
||||
// Strategy: mirror Dockhand's own Docker connection when running locally.
|
||||
const env = envId ? await getEnvironment(envId) : undefined;
|
||||
const connectionType = env?.connectionType;
|
||||
|
||||
// Determine if this is a local socket environment:
|
||||
// - connectionType === 'socket' (explicit)
|
||||
// - connectionType is null/undefined (default behavior)
|
||||
// - connectionType === 'direct' but no host specified (legacy local environments)
|
||||
const isLocalSocket = !connectionType ||
|
||||
connectionType === 'socket' ||
|
||||
(connectionType === 'direct' && !env?.host);
|
||||
const isHawser = connectionType === 'hawser-standard' || connectionType === 'hawser-edge';
|
||||
|
||||
let hostSocketPath: string;
|
||||
let hostSocketPath: string | null = null;
|
||||
let rootlessUid: string | undefined;
|
||||
let scannerNetworkMode: string | undefined;
|
||||
let scannerDockerHost: string | undefined;
|
||||
|
||||
if (isLocalSocket) {
|
||||
// Local socket environment - detect host socket path (handles rootless Docker)
|
||||
// Check if Dockhand itself uses TCP to reach Docker (e.g., socket proxy).
|
||||
// Detected at startup from Dockhand's own container inspect data.
|
||||
// This applies to ALL non-hawser environments since the scanner container
|
||||
// runs on the same Docker daemon and needs the same access method.
|
||||
const ownDockerHost = getOwnDockerHost();
|
||||
|
||||
if (!isHawser && ownDockerHost?.startsWith('tcp://')) {
|
||||
// TCP mode: scanner uses the same DOCKER_HOST + network as Dockhand
|
||||
scannerDockerHost = ownDockerHost;
|
||||
scannerNetworkMode = getOwnNetworkMode() ?? undefined;
|
||||
console.log(`[Scanner] TCP mode (from container inspect) - DOCKER_HOST=${scannerDockerHost}, network=${scannerNetworkMode ?? 'default'}`);
|
||||
} else if (isHawser) {
|
||||
// Hawser: scanner runs on remote host, uses remote host's standard Docker socket
|
||||
hostSocketPath = '/var/run/docker.sock';
|
||||
console.log(`[Scanner] Remote scan via Hawser (${connectionType}) - using standard socket path`);
|
||||
} else {
|
||||
// Local socket — detect host socket path (handles rootless Docker)
|
||||
hostSocketPath = getHostDockerSocket();
|
||||
console.log(`[Scanner] Local socket scan (${connectionType || 'default'}) - detected host Docker socket: ${hostSocketPath}`);
|
||||
|
||||
// For user-specific Docker sockets (rootless Docker), detect UID for cache ownership
|
||||
// but do NOT set container user — in rootless Docker, root inside the container
|
||||
// maps to the socket-owning UID on the host via user namespace remapping
|
||||
const uid = extractUidFromSocketPath(hostSocketPath);
|
||||
if (uid) {
|
||||
rootlessUid = uid;
|
||||
console.log(`[Scanner] Rootless Docker detected (UID ${rootlessUid})`);
|
||||
console.log(`[Scanner] Scanner will run as root inside container (maps to UID ${rootlessUid} on host via user namespace)`);
|
||||
}
|
||||
} else {
|
||||
// Remote environment (direct with host/hawser-standard/hawser-edge)
|
||||
// Scanner runs on remote host, uses remote host's standard Docker socket
|
||||
hostSocketPath = '/var/run/docker.sock';
|
||||
console.log(`[Scanner] Remote scan (${connectionType}, host: ${env?.host}) - using standard socket path: ${hostSocketPath}`);
|
||||
}
|
||||
|
||||
// Determine cache storage strategy based on environment
|
||||
@@ -643,10 +654,12 @@ async function runScannerContainerCore(
|
||||
console.log(`[Scanner] Standard mode - using volume: ${volumeName}`);
|
||||
}
|
||||
|
||||
const binds = [
|
||||
`${hostSocketPath}:/var/run/docker.sock:ro`,
|
||||
cacheBind
|
||||
];
|
||||
// Build binds — only include socket mount when using socket mode
|
||||
const binds: string[] = [];
|
||||
if (hostSocketPath) {
|
||||
binds.push(`${hostSocketPath}:/var/run/docker.sock:ro`);
|
||||
}
|
||||
binds.push(cacheBind);
|
||||
|
||||
console.log(`[Scanner] Container bind mounts: ${JSON.stringify(binds)}`);
|
||||
|
||||
@@ -655,6 +668,11 @@ async function runScannerContainerCore(
|
||||
? [`GRYPE_DB_CACHE_DIR=${dbPath}`]
|
||||
: [`TRIVY_CACHE_DIR=${dbPath}`];
|
||||
|
||||
// In TCP mode, pass DOCKER_HOST so scanner connects to Docker via TCP
|
||||
if (scannerDockerHost) {
|
||||
envVars.push(`DOCKER_HOST=${scannerDockerHost}`);
|
||||
}
|
||||
|
||||
console.log(`[Scanner] Running ${scannerType} with cache mounted at ${basePath}`);
|
||||
console.log(`[Scanner] Container command: ${cmd.join(' ')}`);
|
||||
// Run the scanner container with a 10-minute timeout to prevent indefinite hangs
|
||||
@@ -665,6 +683,7 @@ async function runScannerContainerCore(
|
||||
env: envVars,
|
||||
name: `dockhand-${scannerType}-${Date.now()}`,
|
||||
envId,
|
||||
networkMode: scannerNetworkMode,
|
||||
timeout: 600_000, // 10 minutes
|
||||
onStderr: (data) => {
|
||||
// Stream stderr lines for real-time progress output
|
||||
@@ -680,8 +699,8 @@ async function runScannerContainerCore(
|
||||
console.log(`[Scanner] ${scannerType} container completed, output length: ${output.length}`);
|
||||
if (output.length === 0) {
|
||||
console.error(`[Scanner] WARNING: Empty output from ${scannerType} container`);
|
||||
console.error(`[Scanner] This may indicate the scanner couldn't access Docker socket`);
|
||||
console.error(`[Scanner] Host socket path used: ${hostSocketPath}`);
|
||||
console.error(`[Scanner] This may indicate the scanner couldn't access Docker`);
|
||||
console.error(`[Scanner] Docker access: ${scannerDockerHost ? `TCP ${scannerDockerHost}` : `socket ${hostSocketPath}`}`);
|
||||
} else if (output.length < 100) {
|
||||
console.log(`[Scanner] ${scannerType} output preview: ${output}`);
|
||||
}
|
||||
|
||||
@@ -569,9 +569,9 @@ export async function runContainerUpdate(
|
||||
// =============================================================================
|
||||
|
||||
log(`Recreating container with full config passthrough...`);
|
||||
const success = await recreateContainer(containerName, envId, log, imageNameFromConfig);
|
||||
const result = await recreateContainer(containerName, envId, log, imageNameFromConfig);
|
||||
|
||||
if (success) {
|
||||
if (result.success) {
|
||||
await updateAutoUpdateLastUpdated(containerName, envId);
|
||||
log(`Successfully updated container: ${containerName}`);
|
||||
|
||||
@@ -594,7 +594,7 @@ export async function runContainerUpdate(
|
||||
type: 'success'
|
||||
}, envId);
|
||||
} else {
|
||||
throw new Error('Failed to recreate container');
|
||||
throw new Error(result.error || 'Failed to recreate container');
|
||||
}
|
||||
|
||||
} catch (error: any) {
|
||||
@@ -628,14 +628,14 @@ export async function recreateContainer(
|
||||
envId?: number,
|
||||
log?: (msg: string) => void,
|
||||
imageNameOverride?: string
|
||||
): Promise<boolean> {
|
||||
): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
const containers = await listContainers(true, envId);
|
||||
const container = containers.find(c => c.name === containerName);
|
||||
|
||||
if (!container) {
|
||||
log?.(`Container not found: ${containerName}`);
|
||||
return false;
|
||||
return { success: false, error: `Container not found: ${containerName}` };
|
||||
}
|
||||
|
||||
const inspectData = await inspectContainer(container.id, envId) as any;
|
||||
@@ -644,10 +644,10 @@ export async function recreateContainer(
|
||||
log?.(`Recreating container: ${containerName} (image: ${imageName})`);
|
||||
|
||||
await recreateContainerFromInspect(inspectData, imageName, envId, log);
|
||||
return true;
|
||||
return { success: true };
|
||||
} catch (error: any) {
|
||||
log?.(`Failed to recreate container: ${error.message}`);
|
||||
return false;
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,9 +352,9 @@ export async function runEnvUpdateCheckJob(
|
||||
|
||||
// Recreate container with full config passthrough
|
||||
await log(` Recreating container...`);
|
||||
const ok = await recreateContainer(update.containerName, environmentId,
|
||||
const result = await recreateContainer(update.containerName, environmentId,
|
||||
(msg) => { log(` ${msg}`); });
|
||||
if (!ok) throw new Error('Container recreation failed');
|
||||
if (!result.success) throw new Error(result.error || 'Container recreation failed');
|
||||
|
||||
await log(` Updated successfully`);
|
||||
successCount++;
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs';
|
||||
|
||||
/**
|
||||
* Check if the client prefers JSON over SSE.
|
||||
* Returns true when Accept header includes application/json but NOT text/event-stream.
|
||||
*/
|
||||
export function prefersJSON(request?: Request): boolean {
|
||||
const accept = request?.headers.get('accept') || '';
|
||||
return accept.includes('application/json') && !accept.includes('text/event-stream');
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an SSE Response for JSON-preferring clients.
|
||||
*
|
||||
* Consumes the SSE stream using proper event framing (blank-line delimited,
|
||||
* multi-line data joined with \n, CRLF stripped). Returns the `result` event
|
||||
* data as a JSON response, or a fallback if no result event was emitted.
|
||||
*
|
||||
* Usage:
|
||||
* if (prefersJSON(request)) return sseToJSON(buildSSEResponse());
|
||||
* return buildSSEResponse();
|
||||
*/
|
||||
export async function sseToJSON(sseResponse: Response): Promise<Response> {
|
||||
const reader = sseResponse.body!.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let eventType = '';
|
||||
let dataLines: string[] = [];
|
||||
let resultData: unknown = null;
|
||||
|
||||
const dispatch = () => {
|
||||
const data = dataLines.join('\n');
|
||||
const type = eventType || 'message';
|
||||
eventType = '';
|
||||
dataLines = [];
|
||||
if (type === 'result' && data) {
|
||||
try {
|
||||
resultData = JSON.parse(data);
|
||||
} catch {
|
||||
// keep previous resultData
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const parseLine = (rawLine: string) => {
|
||||
const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine;
|
||||
if (line.startsWith(':')) return;
|
||||
if (line === '') { dispatch(); return; }
|
||||
const colon = line.indexOf(':');
|
||||
const field = colon === -1 ? line : line.slice(0, colon);
|
||||
let val = colon === -1 ? '' : line.slice(colon + 1);
|
||||
if (val.startsWith(' ')) val = val.slice(1);
|
||||
if (field === 'event') eventType = val || 'message';
|
||||
else if (field === 'data') dataLines.push(val);
|
||||
};
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop() || '';
|
||||
|
||||
for (const line of lines) parseLine(line);
|
||||
}
|
||||
|
||||
// Flush remaining bytes and process trailing content
|
||||
buffer += decoder.decode();
|
||||
if (buffer) {
|
||||
for (const line of buffer.split('\n')) parseLine(line);
|
||||
}
|
||||
// Final dispatch for servers missing trailing blank line
|
||||
if (dataLines.length > 0) dispatch();
|
||||
} catch {
|
||||
// stream error, return what we have
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
const body = resultData ?? { success: false, error: 'No result' };
|
||||
return new Response(JSON.stringify(body), {
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Job-based response for long-running operations.
|
||||
*
|
||||
* Backward compat: API clients that send `Accept: application/json` (and not
|
||||
* `text/event-stream`) get a synchronous JSON result directly.
|
||||
*
|
||||
* All other clients receive `{ jobId }` immediately. The operation runs in the
|
||||
* background and results accumulate in the job store. Clients poll /api/jobs/{id}.
|
||||
*
|
||||
* The send() callback stores lines with { event, data } so the polling client
|
||||
* can reconstruct the same event stream semantics used by the old SSE flow.
|
||||
*/
|
||||
export function createJobResponse(
|
||||
operation: (send: (event: string, data: unknown) => void) => Promise<void>,
|
||||
request?: Request
|
||||
): Response {
|
||||
// Backward compat: synchronous JSON path for explicit application/json callers
|
||||
if (prefersJSON(request)) {
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
let resultData: unknown = { success: false, error: 'No result' };
|
||||
const send = (_event: string, data: unknown) => {
|
||||
resultData = data;
|
||||
};
|
||||
try {
|
||||
await operation(send);
|
||||
} catch (error) {
|
||||
resultData = { success: false, error: String(error) };
|
||||
}
|
||||
controller.enqueue(encoder.encode(JSON.stringify(resultData)));
|
||||
controller.close();
|
||||
}
|
||||
});
|
||||
return new Response(stream, {
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
}
|
||||
|
||||
// Fire and forget: create job, run operation in background, return jobId immediately
|
||||
const job = createJob();
|
||||
|
||||
const send = (event: string, data: unknown) => {
|
||||
appendLine(job, { event, data });
|
||||
};
|
||||
|
||||
operation(send)
|
||||
.then(() => {
|
||||
const resultLine = job.lines.findLast((l) => l.event === 'result');
|
||||
completeJob(job, resultLine?.data ?? { success: true });
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
failJob(job, err instanceof Error ? err.message : String(err));
|
||||
});
|
||||
|
||||
return json({ jobId: job.id });
|
||||
}
|
||||
@@ -5,7 +5,7 @@
|
||||
* Discovered stacks are editable - compose and .env files are modified in their original location.
|
||||
*/
|
||||
|
||||
import { readdirSync, existsSync, statSync } from 'node:fs';
|
||||
import { readdirSync, existsSync, statSync, readFileSync } from 'node:fs';
|
||||
import { join, basename, dirname, resolve } from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
import { getExternalStackPaths, getStackSources, upsertStackSource, type StackSourceType } from './db';
|
||||
@@ -57,8 +57,7 @@ export function normalizeStackName(name: string): string {
|
||||
*/
|
||||
async function isComposeFile(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
const file = Bun.file(filePath);
|
||||
const content = await file.text();
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
// Basic check for services key - could be more sophisticated
|
||||
return /^services:/m.test(content) || /\nservices:/m.test(content);
|
||||
} catch {
|
||||
@@ -72,8 +71,7 @@ async function isComposeFile(filePath: string): Promise<boolean> {
|
||||
*/
|
||||
async function countServices(filePath: string): Promise<number> {
|
||||
try {
|
||||
const file = Bun.file(filePath);
|
||||
const content = await file.text();
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
const doc = yaml.load(content) as Record<string, unknown> | null;
|
||||
if (doc?.services && typeof doc.services === 'object') {
|
||||
return Object.keys(doc.services).length;
|
||||
|
||||
+96
-49
@@ -7,6 +7,8 @@
|
||||
|
||||
import { existsSync, mkdirSync, rmSync, readdirSync, cpSync, statSync, unlinkSync, renameSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { join, resolve, dirname, basename } from 'node:path';
|
||||
import { spawn as nodeSpawn } from 'node:child_process';
|
||||
import type { ChildProcess } from 'node:child_process';
|
||||
import {
|
||||
getEnvironment,
|
||||
getSecretEnvVarsAsRecord,
|
||||
@@ -178,13 +180,47 @@ async function withStackLock<T>(stackName: string, fn: () => Promise<T>): Promis
|
||||
}
|
||||
}
|
||||
|
||||
// Timeout configuration for compose operations
|
||||
const COMPOSE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
// Timeout configuration for compose operations (configurable via COMPOSE_TIMEOUT env var in seconds)
|
||||
const COMPOSE_TIMEOUT_MS = parseInt(process.env.COMPOSE_TIMEOUT || '900') * 1000; // Default 15 min
|
||||
const COMPOSE_KILL_GRACE_MS = 5000; // 5 seconds grace period before SIGKILL
|
||||
|
||||
/**
|
||||
* Check if content is binary (not valid UTF-8 text).
|
||||
*/
|
||||
const utf8Decoder = new TextDecoder('utf-8', { fatal: true });
|
||||
function isBinaryContent(bytes: Uint8Array): boolean {
|
||||
try {
|
||||
utf8Decoder.decode(bytes);
|
||||
return false;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect stdout/stderr from a child process and wait for it to exit.
|
||||
*/
|
||||
function collectProcess(proc: ChildProcess): Promise<{ exitCode: number; stdout: string; stderr: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const stdoutChunks: Buffer[] = [];
|
||||
const stderrChunks: Buffer[] = [];
|
||||
proc.stdout?.on('data', (chunk: Buffer) => stdoutChunks.push(chunk));
|
||||
proc.stderr?.on('data', (chunk: Buffer) => stderrChunks.push(chunk));
|
||||
proc.on('error', reject);
|
||||
proc.on('close', (code) => {
|
||||
resolve({
|
||||
exitCode: code ?? 1,
|
||||
stdout: Buffer.concat(stdoutChunks).toString(),
|
||||
stderr: Buffer.concat(stderrChunks).toString()
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read all files from a directory as a map of relative path -> content.
|
||||
* Used to send files to Hawser for remote deployments.
|
||||
* Binary files are base64-encoded with a "base64:" prefix to preserve all bytes.
|
||||
*/
|
||||
async function readDirFilesAsMap(dirPath: string): Promise<Record<string, string>> {
|
||||
const files: Record<string, string> = {};
|
||||
@@ -200,9 +236,12 @@ async function readDirFilesAsMap(dirPath: string): Promise<Record<string, string
|
||||
if (entry.name === '.git') continue;
|
||||
await scanDir(fullPath, relPath);
|
||||
} else if (entry.isFile()) {
|
||||
// Read file content
|
||||
const content = await Bun.file(fullPath).text();
|
||||
files[relPath] = content;
|
||||
const bytes = readFileSync(fullPath);
|
||||
if (isBinaryContent(bytes)) {
|
||||
files[relPath] = `base64:${bytes.toString('base64')}`;
|
||||
} else {
|
||||
files[relPath] = new TextDecoder().decode(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -382,7 +421,7 @@ export async function getStackComposeFile(
|
||||
};
|
||||
}
|
||||
|
||||
const content = await Bun.file(source.composePath).text();
|
||||
const content = readFileSync(source.composePath, 'utf-8');
|
||||
const stackDir = dirname(source.composePath);
|
||||
|
||||
// For custom paths, suggest .env next to compose if envPath not set
|
||||
@@ -420,15 +459,14 @@ export async function getStackComposeFile(
|
||||
|
||||
for (const fileName of composeFileNames) {
|
||||
const actualComposePath = join(stackDir, fileName);
|
||||
const file = Bun.file(actualComposePath);
|
||||
if (await file.exists()) {
|
||||
if (existsSync(actualComposePath)) {
|
||||
// Check for .env file in the same directory
|
||||
const envFilePath = join(stackDir, '.env');
|
||||
const envExists = existsSync(envFilePath);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
content: await file.text(),
|
||||
content: readFileSync(actualComposePath, 'utf-8'),
|
||||
stackDir,
|
||||
// Always return the actual resolved paths for display
|
||||
composePath: actualComposePath,
|
||||
@@ -632,7 +670,7 @@ export async function saveStackComposeFile(
|
||||
}
|
||||
}
|
||||
try {
|
||||
await Bun.write(composePath, content);
|
||||
writeFileSync(composePath, content);
|
||||
return { success: true };
|
||||
} catch (err: any) {
|
||||
return { success: false, error: `Failed to save compose file: ${err.message}` };
|
||||
@@ -672,7 +710,7 @@ export async function saveStackComposeFile(
|
||||
}
|
||||
|
||||
try {
|
||||
await Bun.write(composeFile, content);
|
||||
writeFileSync(composeFile, content);
|
||||
return { success: true };
|
||||
} catch (err: any) {
|
||||
return { success: false, error: `Failed to ${create ? 'create' : 'save'} compose file: ${err.message}` };
|
||||
@@ -712,26 +750,23 @@ async function loginToRegistries(dockerHost?: string, logPrefix = '[Stack]'): Pr
|
||||
|
||||
console.log(`${logPrefix} Logging into registry: ${registryHost}`);
|
||||
|
||||
const proc = Bun.spawn(
|
||||
['docker', 'login', '-u', reg.username, '--password-stdin', registryHost],
|
||||
const proc = nodeSpawn(
|
||||
'docker', ['login', '-u', reg.username, '--password-stdin', registryHost],
|
||||
{
|
||||
env: spawnEnv,
|
||||
stdin: 'pipe',
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe'
|
||||
stdio: ['pipe', 'pipe', 'pipe']
|
||||
}
|
||||
);
|
||||
|
||||
// Write password to stdin (Bun's FileSink API)
|
||||
proc.stdin.write(reg.password);
|
||||
proc.stdin.end();
|
||||
// Write password to stdin
|
||||
proc.stdin!.write(reg.password);
|
||||
proc.stdin!.end();
|
||||
|
||||
const exitCode = await proc.exited;
|
||||
const { exitCode, stderr } = await collectProcess(proc);
|
||||
|
||||
if (exitCode === 0) {
|
||||
console.log(`${logPrefix} Successfully logged into ${registryHost}`);
|
||||
} else {
|
||||
const stderr = await new Response(proc.stderr).text();
|
||||
console.error(`${logPrefix} Failed to login to ${registryHost}: ${stderr}`);
|
||||
}
|
||||
} catch (e) {
|
||||
@@ -786,7 +821,7 @@ function findComposeOverrideFile(stackDir: string, composeFileName: string): str
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a docker compose command locally via Bun.spawn.
|
||||
* Execute a docker compose command locally via child_process.spawn.
|
||||
*
|
||||
* @param tlsConfig - TLS configuration for remote Docker connections (certs written to temp files)
|
||||
* @param envVars - Non-secret environment variables (from .env file, passed for backward compat)
|
||||
@@ -834,7 +869,7 @@ async function executeLocalCompose(
|
||||
: (await findStackDir(stackName, envId) || await getStackDir(stackName, envId));
|
||||
mkdirSync(stackDir, { recursive: true });
|
||||
composeFile = join(stackDir, 'compose.yaml');
|
||||
await Bun.write(composeFile, composeContent);
|
||||
writeFileSync(composeFile, composeContent);
|
||||
}
|
||||
|
||||
// Rewrite relative volume paths for host path translation (in memory only, not saved to disk)
|
||||
@@ -908,15 +943,15 @@ async function executeLocalCompose(
|
||||
// Write certs to files (docker-compose expects specific filenames)
|
||||
if (tlsConfig.ca) {
|
||||
const cleanedCa = cleanPem(tlsConfig.ca);
|
||||
if (cleanedCa) await Bun.write(join(tlsCertDir, 'ca.pem'), cleanedCa);
|
||||
if (cleanedCa) writeFileSync(join(tlsCertDir, 'ca.pem'), cleanedCa);
|
||||
}
|
||||
if (tlsConfig.cert) {
|
||||
const cleanedCert = cleanPem(tlsConfig.cert);
|
||||
if (cleanedCert) await Bun.write(join(tlsCertDir, 'cert.pem'), cleanedCert);
|
||||
if (cleanedCert) writeFileSync(join(tlsCertDir, 'cert.pem'), cleanedCert);
|
||||
}
|
||||
if (tlsConfig.key) {
|
||||
const cleanedKey = cleanPem(tlsConfig.key);
|
||||
if (cleanedKey) await Bun.write(join(tlsCertDir, 'key.pem'), cleanedKey);
|
||||
if (cleanedKey) writeFileSync(join(tlsCertDir, 'key.pem'), cleanedKey);
|
||||
}
|
||||
|
||||
// Set Docker TLS environment variables
|
||||
@@ -941,13 +976,13 @@ async function executeLocalCompose(
|
||||
// Also include override file if it exists (needs path translation too)
|
||||
const overrideFile = findComposeOverrideFile(stackDir, basename(composeFile));
|
||||
if (overrideFile) {
|
||||
let overrideContent = await Bun.file(overrideFile).text();
|
||||
let overrideContent = readFileSync(overrideFile, 'utf-8');
|
||||
if (getHostDataDir()) {
|
||||
const rewrite = rewriteComposeVolumePaths(overrideContent, stackDir);
|
||||
if (rewrite.modified) overrideContent = rewrite.content;
|
||||
}
|
||||
tempOverridePath = join(stackDir, '.compose.override.translated.yaml');
|
||||
await Bun.write(tempOverridePath, overrideContent);
|
||||
writeFileSync(tempOverridePath, overrideContent);
|
||||
args.push('-f', tempOverridePath);
|
||||
console.log(`${logPrefix} Including override file (path-translated): ${basename(overrideFile)}`);
|
||||
}
|
||||
@@ -983,7 +1018,7 @@ async function executeLocalCompose(
|
||||
const overrideEnvPath = join(stackDir, '.env.dockhand');
|
||||
const header = '# Auto-generated by Dockhand. Do not edit - changes will be overwritten on next deploy.\n';
|
||||
const lines = Object.entries(envVars).map(([k, v]) => `${k}=${v}`);
|
||||
await Bun.write(overrideEnvPath, header + lines.join('\n') + '\n');
|
||||
writeFileSync(overrideEnvPath, header + lines.join('\n') + '\n');
|
||||
args.push('--env-file', overrideEnvPath);
|
||||
}
|
||||
|
||||
@@ -1001,7 +1036,7 @@ async function executeLocalCompose(
|
||||
}
|
||||
break;
|
||||
case 'down':
|
||||
args.push('down');
|
||||
args.push('down', '--remove-orphans');
|
||||
if (removeVolumes) args.push('--volumes');
|
||||
break;
|
||||
case 'stop':
|
||||
@@ -1045,12 +1080,10 @@ async function executeLocalCompose(
|
||||
|
||||
try {
|
||||
console.log(`${logPrefix} Spawning docker compose process...`);
|
||||
const proc = Bun.spawn(args, {
|
||||
const proc = nodeSpawn(args[0], args.slice(1), {
|
||||
cwd: stackDir,
|
||||
env: spawnEnv,
|
||||
stdin: useStdin ? 'pipe' : 'inherit',
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe'
|
||||
stdio: [useStdin ? 'pipe' : 'inherit', 'pipe', 'pipe']
|
||||
});
|
||||
|
||||
// If using stdin (host path translation), write the modified compose content
|
||||
@@ -1077,12 +1110,7 @@ async function executeLocalCompose(
|
||||
}, COMPOSE_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const [stdout, stderr] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text()
|
||||
]);
|
||||
|
||||
const code = await proc.exited;
|
||||
const { exitCode: code, stdout, stderr } = await collectProcess(proc);
|
||||
|
||||
console.log(`${logPrefix} ----------------------------------------`);
|
||||
console.log(`${logPrefix} COMPOSE PROCESS COMPLETE`);
|
||||
@@ -1343,7 +1371,7 @@ async function executeComposeCommand(
|
||||
let hawserEnvVars = envVars;
|
||||
if (envPath && existsSync(envPath)) {
|
||||
try {
|
||||
const envFileContent = await Bun.file(envPath).text();
|
||||
const envFileContent = readFileSync(envPath, 'utf-8');
|
||||
const envFileVars = parseEnvFileContent(envFileContent, stackName);
|
||||
// Merge: envFileVars (lowest) < envVars (DB overrides)
|
||||
// secretVars are handled separately in executeComposeViaHawser
|
||||
@@ -1362,7 +1390,7 @@ async function executeComposeCommand(
|
||||
const overridePath = findComposeOverrideFile(composeDir, composeBaseName);
|
||||
if (overridePath) {
|
||||
try {
|
||||
const overrideContent = await Bun.file(overridePath).text();
|
||||
const overrideContent = readFileSync(overridePath, 'utf-8');
|
||||
hawserStackFiles = { ...(hawserStackFiles || {}), [basename(overridePath)]: overrideContent };
|
||||
console.log(`[Stack:${stackName}] Including override file for Hawser: ${basename(overridePath)}`);
|
||||
} catch (err) {
|
||||
@@ -1371,6 +1399,16 @@ async function executeComposeCommand(
|
||||
}
|
||||
}
|
||||
|
||||
// For git stacks: generate .env.dockhand with non-secret DB overrides
|
||||
// This mirrors executeLocalCompose behavior (lines 1017-1023).
|
||||
// envVars contains only the DB overrides (not merged repo .env values from hawserEnvVars).
|
||||
if (useOverrideFile && envVars && Object.keys(envVars).length > 0) {
|
||||
const header = '# Auto-generated by Dockhand. Do not edit - changes will be overwritten on next deploy.\n';
|
||||
const lines = Object.entries(envVars).map(([k, v]) => `${k}=${v}`);
|
||||
hawserStackFiles = { ...(hawserStackFiles || {}), '.env.dockhand': header + lines.join('\n') + '\n' };
|
||||
console.log(`[Stack:${stackName}] Including .env.dockhand override file for Hawser (${Object.keys(envVars).length} vars)`);
|
||||
}
|
||||
|
||||
return executeComposeViaHawser(
|
||||
operation,
|
||||
stackName,
|
||||
@@ -1766,8 +1804,9 @@ async function requireComposeFile(
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a stack using docker compose up
|
||||
* Falls back to individual container start for stacks without compose files
|
||||
* Start a stack using docker compose start (resumes stopped containers).
|
||||
* Falls back to docker compose up if containers don't exist (stack was removed/down).
|
||||
* Falls back to individual container start for stacks without compose files.
|
||||
*/
|
||||
export async function startStack(
|
||||
stackName: string,
|
||||
@@ -1780,9 +1819,17 @@ export async function startStack(
|
||||
return withContainerFallback(stackName, envId, 'start');
|
||||
}
|
||||
|
||||
const opts = { stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath };
|
||||
|
||||
// Check if containers exist for this stack. If they do, use 'start' to resume
|
||||
// them (preserves container IDs, avoids Traefik race conditions from recreation).
|
||||
// If no containers exist (stack was removed/down), use 'up' to create them.
|
||||
const containers = await getStackContainers(stackName, envId);
|
||||
const operation = containers.length > 0 ? 'start' : 'up';
|
||||
|
||||
return executeComposeCommand(
|
||||
'up',
|
||||
{ stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath },
|
||||
operation,
|
||||
opts,
|
||||
result.content!,
|
||||
result.nonSecretVars,
|
||||
result.secretVars
|
||||
@@ -2185,7 +2232,7 @@ export async function deployStack(options: DeployStackOptions): Promise<StackOpe
|
||||
}
|
||||
if (actualEnvPath && existsSync(actualEnvPath) && !stackFiles['.env']) {
|
||||
try {
|
||||
const envContent = await Bun.file(actualEnvPath).text();
|
||||
const envContent = readFileSync(actualEnvPath, 'utf-8');
|
||||
stackFiles['.env'] = envContent;
|
||||
console.log(`${logPrefix} Added .env to stackFiles for Hawser (${envContent.length} chars)`);
|
||||
} catch (err) {
|
||||
@@ -2415,7 +2462,7 @@ export async function writeStackEnvFile(
|
||||
.map(v => `${v.key.trim()}=${v.value}`)
|
||||
.join('\n') + '\n';
|
||||
|
||||
await Bun.write(envFilePath, rawContent);
|
||||
writeFileSync(envFilePath, rawContent);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2453,7 +2500,7 @@ export async function writeRawStackEnvFile(
|
||||
mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
await Bun.write(envFilePath, rawContent);
|
||||
writeFileSync(envFilePath, rawContent);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,687 +0,0 @@
|
||||
/**
|
||||
* Event Collection Subprocess
|
||||
*
|
||||
* Runs as a separate process via Bun.spawn to collect Docker container events
|
||||
* without blocking the main HTTP thread.
|
||||
*
|
||||
* Communication with main process via IPC (process.send).
|
||||
*/
|
||||
|
||||
import { getEnvironments, getEventCollectionMode, getEventPollInterval, type ContainerEventAction } from '../db';
|
||||
import { getDockerEvents } from '../docker';
|
||||
import type { MainProcessCommand } from '../subprocess-manager';
|
||||
|
||||
// Reconnection settings
|
||||
const RECONNECT_DELAY = 5000; // 5 seconds
|
||||
const MAX_RECONNECT_DELAY = 60000; // 1 minute max
|
||||
|
||||
// Track environment online status for notifications
|
||||
// Only send notifications on status CHANGES, not on every reconnect attempt
|
||||
const environmentOnlineStatus: Map<number, boolean> = new Map();
|
||||
|
||||
// Active collectors per environment (for streaming mode)
|
||||
const collectors: Map<number, { controller: AbortController; reconnectTimeout: ReturnType<typeof setTimeout> | null }> = new Map();
|
||||
|
||||
// Poll intervals per environment (for polling mode)
|
||||
const pollIntervals: Map<number, ReturnType<typeof setInterval>> = new Map();
|
||||
|
||||
// Last poll timestamp per environment (for polling mode)
|
||||
const lastPollTime: Map<number, number> = new Map();
|
||||
|
||||
// Recent event cache for deduplication (key: timeNano-containerId-action)
|
||||
const recentEvents: Map<string, number> = new Map();
|
||||
const DEDUP_WINDOW_MS = 5000; // 5 second window for deduplication
|
||||
const CACHE_CLEANUP_INTERVAL_MS = 5000; // Clean up cache every 5 seconds (match dedup window)
|
||||
const MAX_DEDUP_CACHE_SIZE = 500; // Hard limit to prevent unbounded growth
|
||||
|
||||
let cacheCleanupInterval: ReturnType<typeof setInterval> | null = null;
|
||||
let isShuttingDown = false;
|
||||
|
||||
// Track current settings to detect changes
|
||||
let currentPollInterval: number = 60000;
|
||||
let currentMode: 'stream' | 'poll' = 'stream';
|
||||
|
||||
// Actions we care about for container activity
|
||||
const CONTAINER_ACTIONS: ContainerEventAction[] = [
|
||||
'create',
|
||||
'start',
|
||||
'stop',
|
||||
'die',
|
||||
'kill',
|
||||
'restart',
|
||||
'pause',
|
||||
'unpause',
|
||||
'destroy',
|
||||
'rename',
|
||||
'update',
|
||||
'oom',
|
||||
'health_status'
|
||||
];
|
||||
|
||||
// Scanner image patterns to exclude from events
|
||||
const SCANNER_IMAGE_PATTERNS = [
|
||||
'anchore/grype',
|
||||
'aquasec/trivy',
|
||||
'ghcr.io/anchore/grype',
|
||||
'ghcr.io/aquasecurity/trivy'
|
||||
];
|
||||
|
||||
// Container name patterns to exclude from events
|
||||
const EXCLUDED_CONTAINER_PREFIXES = ['dockhand-browse-'];
|
||||
|
||||
/**
|
||||
* Send message to main process
|
||||
*/
|
||||
function send(message: any): void {
|
||||
if (process.send) {
|
||||
process.send(message);
|
||||
}
|
||||
}
|
||||
|
||||
function isScannerContainer(image: string | null | undefined): boolean {
|
||||
if (!image) return false;
|
||||
const lowerImage = image.toLowerCase();
|
||||
return SCANNER_IMAGE_PATTERNS.some((pattern) => lowerImage.includes(pattern.toLowerCase()));
|
||||
}
|
||||
|
||||
function isExcludedContainer(containerName: string | null | undefined): boolean {
|
||||
if (!containerName) return false;
|
||||
return EXCLUDED_CONTAINER_PREFIXES.some((prefix) => containerName.startsWith(prefix));
|
||||
}
|
||||
|
||||
/**
|
||||
* Update environment online status and notify main process on change
|
||||
*/
|
||||
function updateEnvironmentStatus(
|
||||
envId: number,
|
||||
envName: string,
|
||||
isOnline: boolean,
|
||||
errorMessage?: string
|
||||
) {
|
||||
const previousStatus = environmentOnlineStatus.get(envId);
|
||||
|
||||
// Only send notification on status CHANGE (not on first connection or repeated failures)
|
||||
if (previousStatus !== undefined && previousStatus !== isOnline) {
|
||||
send({
|
||||
type: 'env_status',
|
||||
envId,
|
||||
envName,
|
||||
online: isOnline,
|
||||
error: errorMessage
|
||||
});
|
||||
}
|
||||
|
||||
environmentOnlineStatus.set(envId, isOnline);
|
||||
}
|
||||
|
||||
interface DockerEvent {
|
||||
Type: string;
|
||||
Action: string;
|
||||
Actor: {
|
||||
ID: string;
|
||||
Attributes: Record<string, string>;
|
||||
};
|
||||
time: number;
|
||||
timeNano: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old entries from the deduplication cache
|
||||
* Also enforces max size limit with LRU eviction
|
||||
*/
|
||||
function cleanupRecentEvents() {
|
||||
const now = Date.now();
|
||||
// First pass: remove expired entries
|
||||
for (const [key, timestamp] of recentEvents.entries()) {
|
||||
if (now - timestamp > DEDUP_WINDOW_MS) {
|
||||
recentEvents.delete(key);
|
||||
}
|
||||
}
|
||||
// Second pass: enforce max size with LRU eviction if still too large
|
||||
if (recentEvents.size > MAX_DEDUP_CACHE_SIZE) {
|
||||
const entries = Array.from(recentEvents.entries())
|
||||
.sort((a, b) => a[1] - b[1]); // Sort by timestamp (oldest first)
|
||||
const toRemove = entries.slice(0, entries.length - MAX_DEDUP_CACHE_SIZE);
|
||||
for (const [key] of toRemove) {
|
||||
recentEvents.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a Docker event
|
||||
*/
|
||||
function processEvent(event: DockerEvent, envId: number) {
|
||||
// Only process container events
|
||||
if (event.Type !== 'container') return;
|
||||
|
||||
// Map Docker action to our action type
|
||||
// For health_status events, Docker sends "health_status: unhealthy" or "health_status: healthy"
|
||||
// We need to preserve the full string for notifications to distinguish healthy vs unhealthy
|
||||
const rawAction = event.Action;
|
||||
const baseAction = rawAction.split(':')[0] as ContainerEventAction;
|
||||
|
||||
// Skip actions we don't care about
|
||||
if (!CONTAINER_ACTIONS.includes(baseAction)) return;
|
||||
|
||||
// For notifications, preserve full action for health_status to enable proper mapping
|
||||
const action = rawAction.startsWith('health_status') ? rawAction : baseAction;
|
||||
|
||||
const containerId = event.Actor?.ID;
|
||||
const containerName = event.Actor?.Attributes?.name;
|
||||
const image = event.Actor?.Attributes?.image;
|
||||
|
||||
if (!containerId) return;
|
||||
|
||||
// Skip scanner containers (Trivy, Grype)
|
||||
if (isScannerContainer(image)) return;
|
||||
|
||||
// Skip internal Dockhand containers (volume browser helpers)
|
||||
if (isExcludedContainer(containerName)) return;
|
||||
|
||||
// Deduplicate events
|
||||
const dedupKey = `${envId}-${event.timeNano}-${containerId}-${action}`;
|
||||
if (recentEvents.has(dedupKey)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark as processed
|
||||
recentEvents.set(dedupKey, Date.now());
|
||||
|
||||
// Clean up if cache gets too large
|
||||
if (recentEvents.size > 200) {
|
||||
cleanupRecentEvents();
|
||||
}
|
||||
|
||||
// Convert Unix nanosecond timestamp to ISO string
|
||||
const timestamp = new Date(Math.floor(event.timeNano / 1000000)).toISOString();
|
||||
|
||||
// Prepare notification data
|
||||
// For health_status events, create a cleaner label
|
||||
const actionLabel = action.startsWith('health_status')
|
||||
? action.includes('unhealthy') ? 'Unhealthy' : 'Healthy'
|
||||
: action.charAt(0).toUpperCase() + action.slice(1);
|
||||
const containerLabel = containerName || containerId.substring(0, 12);
|
||||
const notificationType =
|
||||
action === 'die' || action === 'kill' || action === 'oom' || action.includes('unhealthy')
|
||||
? 'error'
|
||||
: action === 'stop'
|
||||
? 'warning'
|
||||
: action === 'start' || (action.includes('healthy') && !action.includes('unhealthy'))
|
||||
? 'success'
|
||||
: 'info';
|
||||
|
||||
// Send event to main process for DB save and SSE broadcast
|
||||
send({
|
||||
type: 'container_event',
|
||||
event: {
|
||||
environmentId: envId,
|
||||
containerId: containerId,
|
||||
containerName: containerName || null,
|
||||
image: image || null,
|
||||
action,
|
||||
actorAttributes: event.Actor?.Attributes || null,
|
||||
timestamp
|
||||
},
|
||||
notification: {
|
||||
action,
|
||||
title: `Container ${actionLabel}`,
|
||||
message: `Container "${containerLabel}" ${action}${image ? ` (${image})` : ''}`,
|
||||
notificationType,
|
||||
image
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll events for a specific environment (polling mode)
|
||||
*/
|
||||
async function pollEnvironmentEvents(envId: number, envName: string) {
|
||||
try {
|
||||
// Calculate 'since' timestamp (use last poll time, or start from 30s ago if first poll)
|
||||
const now = Math.floor(Date.now() / 1000); // Unix timestamp in seconds
|
||||
const since = lastPollTime.get(envId) || (now - 30); // Default to 30s ago on first poll
|
||||
|
||||
// Fetch events since last check until now
|
||||
// IMPORTANT: 'until' is required for polling mode, otherwise Docker keeps the connection open
|
||||
const eventStream = await getDockerEvents(
|
||||
{ type: ['container'] },
|
||||
envId,
|
||||
{ since: since.toString(), until: now.toString() }
|
||||
);
|
||||
|
||||
if (!eventStream) {
|
||||
console.error(`[EventSubprocess] Failed to fetch events for ${envName}`);
|
||||
updateEnvironmentStatus(envId, envName, false, 'Failed to fetch Docker events');
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark environment as online
|
||||
updateEnvironmentStatus(envId, envName, true);
|
||||
|
||||
// Read and process all events
|
||||
const reader = eventStream.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop() || '';
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.trim()) {
|
||||
try {
|
||||
const event = JSON.parse(line) as DockerEvent;
|
||||
processEvent(event, envId);
|
||||
} catch {
|
||||
// Ignore parse errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
// Cancel the stream first to ensure proper cleanup, then release lock
|
||||
await reader.cancel();
|
||||
reader.releaseLock();
|
||||
} catch {
|
||||
// Reader already released or stream closed
|
||||
}
|
||||
}
|
||||
|
||||
// Update last poll time
|
||||
lastPollTime.set(envId, now);
|
||||
|
||||
} catch (error: any) {
|
||||
if (!isShuttingDown) {
|
||||
console.error(`[EventSubprocess] Poll error for ${envName}:`, error.message);
|
||||
updateEnvironmentStatus(envId, envName, false, error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start collecting events for a specific environment
|
||||
*/
|
||||
async function startEnvironmentCollector(envId: number, envName: string) {
|
||||
// Stop existing collector if any
|
||||
stopEnvironmentCollector(envId);
|
||||
|
||||
const controller = new AbortController();
|
||||
const entry = { controller, reconnectTimeout: null as ReturnType<typeof setTimeout> | null };
|
||||
collectors.set(envId, entry);
|
||||
|
||||
let reconnectDelay = RECONNECT_DELAY;
|
||||
|
||||
const connect = async () => {
|
||||
if (controller.signal.aborted || isShuttingDown) return;
|
||||
|
||||
let reader: ReadableStreamDefaultReader<Uint8Array> | null = null;
|
||||
|
||||
try {
|
||||
console.log(
|
||||
`[EventSubprocess] Connecting to Docker events for ${envName} (env ${envId})...`
|
||||
);
|
||||
|
||||
const eventStream = await getDockerEvents({ type: ['container'] }, envId);
|
||||
|
||||
if (!eventStream) {
|
||||
console.error(`[EventSubprocess] Failed to get event stream for ${envName}`);
|
||||
updateEnvironmentStatus(envId, envName, false, 'Failed to connect to Docker');
|
||||
scheduleReconnect();
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset reconnect delay on successful connection
|
||||
reconnectDelay = RECONNECT_DELAY;
|
||||
console.log(`[EventSubprocess] Connected to Docker events for ${envName}`);
|
||||
|
||||
updateEnvironmentStatus(envId, envName, true);
|
||||
|
||||
reader = eventStream.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
|
||||
try {
|
||||
while (!controller.signal.aborted && !isShuttingDown) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop() || '';
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.trim()) {
|
||||
try {
|
||||
const event = JSON.parse(line) as DockerEvent;
|
||||
processEvent(event, envId);
|
||||
} catch {
|
||||
// Ignore parse errors for partial chunks
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (!controller.signal.aborted && !isShuttingDown) {
|
||||
if (error.name !== 'AbortError') {
|
||||
console.error(`[EventSubprocess] Stream error for ${envName}:`, error.message);
|
||||
updateEnvironmentStatus(envId, envName, false, error.message);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (reader) {
|
||||
try {
|
||||
// Cancel the stream first to ensure proper cleanup, then release lock
|
||||
await reader.cancel();
|
||||
reader.releaseLock();
|
||||
} catch {
|
||||
// Reader already released or stream closed - ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Connection closed, reconnect
|
||||
if (!controller.signal.aborted && !isShuttingDown) {
|
||||
scheduleReconnect();
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (reader) {
|
||||
try {
|
||||
// Cancel the stream first to ensure proper cleanup, then release lock
|
||||
await reader.cancel();
|
||||
reader.releaseLock();
|
||||
} catch {
|
||||
// Reader already released or stream closed - ignore
|
||||
}
|
||||
}
|
||||
|
||||
if (!controller.signal.aborted && !isShuttingDown && error.name !== 'AbortError') {
|
||||
console.error(`[EventSubprocess] Connection error for ${envName}:`, error.message);
|
||||
updateEnvironmentStatus(envId, envName, false, error.message);
|
||||
}
|
||||
|
||||
if (!controller.signal.aborted && !isShuttingDown) {
|
||||
scheduleReconnect();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const scheduleReconnect = () => {
|
||||
if (controller.signal.aborted || isShuttingDown) return;
|
||||
|
||||
console.log(`[EventSubprocess] Reconnecting to ${envName} in ${reconnectDelay / 1000}s...`);
|
||||
entry.reconnectTimeout = setTimeout(() => {
|
||||
entry.reconnectTimeout = null;
|
||||
if (!controller.signal.aborted && !isShuttingDown) {
|
||||
connect();
|
||||
}
|
||||
}, reconnectDelay);
|
||||
|
||||
// Exponential backoff
|
||||
reconnectDelay = Math.min(reconnectDelay * 2, MAX_RECONNECT_DELAY);
|
||||
};
|
||||
|
||||
// Start the connection
|
||||
connect();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start polling mode for a specific environment
|
||||
*/
|
||||
async function startEnvironmentPoller(envId: number, envName: string, interval: number) {
|
||||
// Stop existing poller if any
|
||||
stopEnvironmentPoller(envId);
|
||||
|
||||
console.log(`[EventSubprocess] Starting poller for ${envName} (every ${interval / 1000}s)`);
|
||||
|
||||
// Initial poll immediately
|
||||
await pollEnvironmentEvents(envId, envName);
|
||||
|
||||
// Set up interval for subsequent polls
|
||||
const intervalId = setInterval(async () => {
|
||||
if (!isShuttingDown) {
|
||||
await pollEnvironmentEvents(envId, envName);
|
||||
}
|
||||
}, interval);
|
||||
|
||||
pollIntervals.set(envId, intervalId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop polling for a specific environment
|
||||
*/
|
||||
function stopEnvironmentPoller(envId: number) {
|
||||
const intervalId = pollIntervals.get(envId);
|
||||
if (intervalId) {
|
||||
clearInterval(intervalId);
|
||||
pollIntervals.delete(envId);
|
||||
lastPollTime.delete(envId);
|
||||
environmentOnlineStatus.delete(envId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop collecting events for a specific environment (streaming mode)
|
||||
*/
|
||||
function stopEnvironmentCollector(envId: number) {
|
||||
const entry = collectors.get(envId);
|
||||
if (entry) {
|
||||
if (entry.reconnectTimeout !== null) {
|
||||
clearTimeout(entry.reconnectTimeout);
|
||||
}
|
||||
entry.controller.abort();
|
||||
collectors.delete(envId);
|
||||
environmentOnlineStatus.delete(envId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh collectors when environments change
|
||||
*/
|
||||
async function refreshEventCollectors() {
|
||||
if (isShuttingDown) return;
|
||||
|
||||
try {
|
||||
const environments = await getEnvironments();
|
||||
const mode = await getEventCollectionMode();
|
||||
const pollInterval = await getEventPollInterval();
|
||||
|
||||
// Detect if settings changed
|
||||
const modeChanged = mode !== currentMode;
|
||||
const intervalChanged = pollInterval !== currentPollInterval;
|
||||
|
||||
if (modeChanged) {
|
||||
console.log(`[EventSubprocess] Mode changed from ${currentMode} to ${mode}`);
|
||||
currentMode = mode;
|
||||
}
|
||||
if (intervalChanged) {
|
||||
console.log(`[EventSubprocess] Poll interval changed from ${currentPollInterval}ms to ${pollInterval}ms`);
|
||||
currentPollInterval = pollInterval;
|
||||
}
|
||||
|
||||
// Filter: only collect for environments with activity enabled AND not Hawser Edge
|
||||
const activeEnvIds = new Set(
|
||||
environments
|
||||
.filter((e) => e.collectActivity && e.connectionType !== 'hawser-edge')
|
||||
.map((e) => e.id)
|
||||
);
|
||||
|
||||
// Stop collectors for removed environments or those with collection disabled
|
||||
for (const envId of collectors.keys()) {
|
||||
if (!activeEnvIds.has(envId)) {
|
||||
console.log(`[EventSubprocess] Stopping stream collector for environment ${envId}`);
|
||||
stopEnvironmentCollector(envId);
|
||||
}
|
||||
}
|
||||
|
||||
// Stop pollers for removed environments or those with collection disabled
|
||||
// Also restart all pollers if interval changed
|
||||
for (const envId of pollIntervals.keys()) {
|
||||
if (!activeEnvIds.has(envId)) {
|
||||
console.log(`[EventSubprocess] Stopping poller for environment ${envId}`);
|
||||
stopEnvironmentPoller(envId);
|
||||
} else if (intervalChanged && mode === 'poll') {
|
||||
// Restart poller with new interval
|
||||
console.log(`[EventSubprocess] Restarting poller for environment ${envId} with new interval`);
|
||||
stopEnvironmentPoller(envId);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up stale map entries for deleted environments
|
||||
const allEnvIds = new Set(environments.map((e) => e.id));
|
||||
for (const envId of environmentOnlineStatus.keys()) {
|
||||
if (!allEnvIds.has(envId)) environmentOnlineStatus.delete(envId);
|
||||
}
|
||||
for (const envId of lastPollTime.keys()) {
|
||||
if (!allEnvIds.has(envId)) lastPollTime.delete(envId);
|
||||
}
|
||||
|
||||
// Start collectors based on mode
|
||||
for (const env of environments) {
|
||||
// Skip Hawser Edge (handled by main process)
|
||||
if (env.connectionType === 'hawser-edge') continue;
|
||||
|
||||
// Skip if activity collection is disabled
|
||||
if (!env.collectActivity) continue;
|
||||
|
||||
const hasStreamCollector = collectors.has(env.id);
|
||||
const hasPoller = pollIntervals.has(env.id);
|
||||
|
||||
if (mode === 'stream') {
|
||||
// Switch from polling to streaming if needed
|
||||
if (hasPoller) {
|
||||
console.log(`[EventSubprocess] Switching ${env.name} from poll to stream`);
|
||||
stopEnvironmentPoller(env.id);
|
||||
}
|
||||
// Start stream if not already running
|
||||
if (!hasStreamCollector) {
|
||||
startEnvironmentCollector(env.id, env.name);
|
||||
}
|
||||
} else if (mode === 'poll') {
|
||||
// Switch from streaming to polling if needed
|
||||
if (hasStreamCollector) {
|
||||
console.log(`[EventSubprocess] Switching ${env.name} from stream to poll`);
|
||||
stopEnvironmentCollector(env.id);
|
||||
}
|
||||
// Start poller if not already running (will also restart after interval change above)
|
||||
if (!hasPoller) {
|
||||
startEnvironmentPoller(env.id, env.name, pollInterval);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[EventSubprocess] Failed to refresh collectors: ${message}`);
|
||||
send({ type: 'error', message: `Failed to refresh collectors: ${message}` });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle commands from main process
|
||||
*/
|
||||
function handleCommand(command: MainProcessCommand): void {
|
||||
switch (command.type) {
|
||||
case 'refresh_environments':
|
||||
console.log('[EventSubprocess] Refreshing environments...');
|
||||
refreshEventCollectors();
|
||||
break;
|
||||
|
||||
case 'update_interval':
|
||||
// This is used by metrics subprocess, but we handle it here too for consistency
|
||||
// Event subprocess re-reads interval from DB on refresh
|
||||
console.log('[EventSubprocess] Interval update - refreshing collectors...');
|
||||
refreshEventCollectors();
|
||||
break;
|
||||
|
||||
case 'shutdown':
|
||||
console.log('[EventSubprocess] Shutdown requested');
|
||||
shutdown();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Graceful shutdown
|
||||
*/
|
||||
function shutdown(): void {
|
||||
isShuttingDown = true;
|
||||
|
||||
// Stop periodic cache cleanup
|
||||
if (cacheCleanupInterval) {
|
||||
clearInterval(cacheCleanupInterval);
|
||||
cacheCleanupInterval = null;
|
||||
}
|
||||
|
||||
// Stop all environment stream collectors
|
||||
for (const envId of collectors.keys()) {
|
||||
stopEnvironmentCollector(envId);
|
||||
}
|
||||
|
||||
// Stop all environment pollers
|
||||
for (const envId of pollIntervals.keys()) {
|
||||
stopEnvironmentPoller(envId);
|
||||
}
|
||||
|
||||
// Clear the deduplication cache
|
||||
recentEvents.clear();
|
||||
|
||||
console.log('[EventSubprocess] Stopped');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the event collector
|
||||
*/
|
||||
async function start(): Promise<void> {
|
||||
console.log('[EventSubprocess] Starting container event collection...');
|
||||
|
||||
// Initialize current settings from database
|
||||
try {
|
||||
currentMode = await getEventCollectionMode();
|
||||
currentPollInterval = await getEventPollInterval();
|
||||
console.log(`[EventSubprocess] Initial mode: ${currentMode}, poll interval: ${currentPollInterval}ms`);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[EventSubprocess] Failed to load settings, using defaults: ${message}`);
|
||||
}
|
||||
|
||||
// Start collectors for all environments
|
||||
await refreshEventCollectors();
|
||||
|
||||
// Start periodic cache cleanup
|
||||
cacheCleanupInterval = setInterval(cleanupRecentEvents, CACHE_CLEANUP_INTERVAL_MS);
|
||||
console.log('[EventSubprocess] Started deduplication cache cleanup (every 5s)');
|
||||
|
||||
// Start memory diagnostics logging (every 5 minutes)
|
||||
setInterval(() => {
|
||||
const mem = process.memoryUsage();
|
||||
console.log(
|
||||
`[EventSubprocess] Memory: heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB, ` +
|
||||
`rss=${Math.round(mem.rss / 1024 / 1024)}MB, ` +
|
||||
`dedup=${recentEvents.size}, collectors=${collectors.size}, pollers=${pollIntervals.size}`
|
||||
);
|
||||
}, 5 * 60 * 1000);
|
||||
|
||||
// Listen for commands from main process
|
||||
process.on('message', (message: MainProcessCommand) => {
|
||||
handleCommand(message);
|
||||
});
|
||||
|
||||
// Handle termination signals
|
||||
process.on('SIGTERM', shutdown);
|
||||
process.on('SIGINT', shutdown);
|
||||
|
||||
// Signal ready
|
||||
send({ type: 'ready' });
|
||||
|
||||
console.log('[EventSubprocess] Started successfully');
|
||||
}
|
||||
|
||||
// Start the subprocess
|
||||
start();
|
||||
@@ -1,498 +0,0 @@
|
||||
/**
|
||||
* Metrics Collection Subprocess
|
||||
*
|
||||
* Runs as a separate process via Bun.spawn to collect CPU/memory metrics
|
||||
* and check disk space without blocking the main HTTP thread.
|
||||
*
|
||||
* Communication with main process via IPC (process.send).
|
||||
*/
|
||||
|
||||
import { getEnvironments, getEnvSetting, getMetricsCollectionInterval } from '../db';
|
||||
import { listContainers, getContainerStats, getDockerInfo, getDiskUsage } from '../docker';
|
||||
import os from 'node:os';
|
||||
import type { MainProcessCommand } from '../subprocess-manager';
|
||||
|
||||
let COLLECT_INTERVAL = 30000; // 30 seconds (default, will be loaded from settings)
|
||||
const DISK_CHECK_INTERVAL = 300000; // 5 minutes
|
||||
const DEFAULT_DISK_THRESHOLD = 80; // 80% threshold for disk warnings
|
||||
const ENV_METRICS_TIMEOUT = 15000; // 15 seconds timeout per environment for metrics
|
||||
const ENV_DISK_TIMEOUT = 20000; // 20 seconds timeout per environment for disk checks
|
||||
|
||||
/**
|
||||
* Timeout wrapper - returns fallback if promise takes too long
|
||||
* IMPORTANT: Properly clears the timeout to prevent memory leaks
|
||||
*/
|
||||
function withTimeout<T>(promise: Promise<T>, ms: number, fallback: T): Promise<T> {
|
||||
let timeoutId: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
const timeoutPromise = new Promise<T>((resolve) => {
|
||||
timeoutId = setTimeout(() => resolve(fallback), ms);
|
||||
});
|
||||
|
||||
return Promise.race([promise, timeoutPromise]).finally(() => {
|
||||
if (timeoutId !== null) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Track last disk warning sent per environment to avoid spamming
|
||||
const lastDiskWarning: Map<number, number> = new Map();
|
||||
const DISK_WARNING_COOLDOWN = 3600000; // 1 hour between warnings
|
||||
|
||||
let collectInterval: ReturnType<typeof setInterval> | null = null;
|
||||
let diskCheckInterval: ReturnType<typeof setInterval> | null = null;
|
||||
let isShuttingDown = false;
|
||||
let collectionCycleCount = 0;
|
||||
const MEMORY_LOG_INTERVAL = 10; // Log memory every 10 cycles (~5 minutes at 30s interval)
|
||||
|
||||
/**
|
||||
* Send message to main process
|
||||
*/
|
||||
function send(message: any): void {
|
||||
if (process.send) {
|
||||
process.send(message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect metrics for a single environment
|
||||
*/
|
||||
async function collectEnvMetrics(env: { id: number; name: string; host?: string; socketPath?: string; collectMetrics?: boolean; connectionType?: string }) {
|
||||
try {
|
||||
// Skip environments where metrics collection is disabled
|
||||
if (env.collectMetrics === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip Hawser Edge environments (handled by main process)
|
||||
if (env.connectionType === 'hawser-edge') {
|
||||
return;
|
||||
}
|
||||
|
||||
// Get running containers
|
||||
const containers = await listContainers(false, env.id); // Only running
|
||||
let totalCpuPercent = 0;
|
||||
let totalContainerMemUsed = 0;
|
||||
|
||||
// Get stats for each running container
|
||||
const statsPromises = containers.map(async (container) => {
|
||||
try {
|
||||
const stats = (await getContainerStats(container.id, env.id)) as any;
|
||||
|
||||
// Calculate CPU percentage
|
||||
const cpuDelta =
|
||||
stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage;
|
||||
const systemDelta =
|
||||
stats.cpu_stats.system_cpu_usage - stats.precpu_stats.system_cpu_usage;
|
||||
const cpuCount = stats.cpu_stats.online_cpus || os.cpus().length;
|
||||
|
||||
let cpuPercent = 0;
|
||||
if (systemDelta > 0 && cpuDelta > 0) {
|
||||
cpuPercent = (cpuDelta / systemDelta) * cpuCount * 100;
|
||||
}
|
||||
|
||||
// Get container memory usage using the same formula as Docker CLI
|
||||
// Docker subtracts cache (inactive_file) from total usage
|
||||
// - cgroup v2: uses 'inactive_file'
|
||||
// - cgroup v1: uses 'total_inactive_file'
|
||||
const memUsage = stats.memory_stats?.usage || 0;
|
||||
const memStats = stats.memory_stats?.stats || {};
|
||||
const memCache = memStats.inactive_file ?? memStats.total_inactive_file ?? 0;
|
||||
const actualMemUsed = memCache > 0 && memCache < memUsage ? memUsage - memCache : memUsage;
|
||||
|
||||
return { cpuPercent, memUsage: actualMemUsed };
|
||||
} catch {
|
||||
return { cpuPercent: 0, memUsage: 0 };
|
||||
}
|
||||
});
|
||||
|
||||
const statsResults = await Promise.all(statsPromises);
|
||||
totalCpuPercent = statsResults.reduce((sum, r) => sum + r.cpuPercent, 0);
|
||||
totalContainerMemUsed = statsResults.reduce((sum, r) => sum + r.memUsage, 0);
|
||||
|
||||
// Get host memory info from Docker
|
||||
const info = (await getDockerInfo(env.id)) as any;
|
||||
const memTotal = info?.MemTotal || os.totalmem();
|
||||
|
||||
// Calculate memory: sum of all container memory vs host total
|
||||
const memUsed = totalContainerMemUsed;
|
||||
const memPercent = memTotal > 0 ? (memUsed / memTotal) * 100 : 0;
|
||||
|
||||
// Normalize CPU by number of cores from the Docker host
|
||||
const cpuCount = info?.NCPU || os.cpus().length;
|
||||
const normalizedCpu = totalCpuPercent / cpuCount;
|
||||
|
||||
// Validate values - skip if any are NaN, Infinity, or negative
|
||||
const finalCpu = Number.isFinite(normalizedCpu) && normalizedCpu >= 0 ? normalizedCpu : 0;
|
||||
const finalMemPercent = Number.isFinite(memPercent) && memPercent >= 0 ? memPercent : 0;
|
||||
const finalMemUsed = Number.isFinite(memUsed) && memUsed >= 0 ? memUsed : 0;
|
||||
const finalMemTotal = Number.isFinite(memTotal) && memTotal > 0 ? memTotal : 0;
|
||||
|
||||
// Only send if we have valid memory total (otherwise metrics are meaningless)
|
||||
if (finalMemTotal > 0) {
|
||||
send({
|
||||
type: 'metric',
|
||||
envId: env.id,
|
||||
cpu: finalCpu,
|
||||
memPercent: finalMemPercent,
|
||||
memUsed: finalMemUsed,
|
||||
memTotal: finalMemTotal
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
// Skip this environment if it fails (might be offline)
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.warn(`[MetricsSubprocess] Failed to collect metrics for ${env.name}: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect metrics for all environments
|
||||
*/
|
||||
async function collectMetrics() {
|
||||
if (isShuttingDown) return;
|
||||
|
||||
try {
|
||||
const environments = await getEnvironments();
|
||||
|
||||
// Filter enabled environments and collect metrics in parallel
|
||||
const enabledEnvs = environments.filter((env) => env.collectMetrics !== false);
|
||||
|
||||
// Process all environments in parallel with per-environment timeouts
|
||||
// Use Promise.allSettled so one slow/failed env doesn't block others
|
||||
const results = await Promise.allSettled(
|
||||
enabledEnvs.map((env) =>
|
||||
withTimeout(
|
||||
collectEnvMetrics(env).then(() => env.name),
|
||||
ENV_METRICS_TIMEOUT,
|
||||
null
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Log any environments that timed out
|
||||
results.forEach((result, index) => {
|
||||
if (result.status === 'fulfilled' && result.value === null) {
|
||||
console.warn(`[MetricsSubprocess] Environment "${enabledEnvs[index].name}" metrics timed out after ${ENV_METRICS_TIMEOUT}ms`);
|
||||
} else if (result.status === 'rejected') {
|
||||
const reason = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
||||
console.warn(`[MetricsSubprocess] Environment "${enabledEnvs[index].name}" metrics failed: ${reason}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Periodic memory logging for diagnostics
|
||||
collectionCycleCount++;
|
||||
if (collectionCycleCount % MEMORY_LOG_INTERVAL === 0) {
|
||||
const memUsage = process.memoryUsage();
|
||||
const heapMB = Math.round(memUsage.heapUsed / 1024 / 1024);
|
||||
const rssMB = Math.round(memUsage.rss / 1024 / 1024);
|
||||
console.log(`[MetricsSubprocess] Memory: heap=${heapMB}MB, rss=${rssMB}MB (cycle ${collectionCycleCount})`);
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[MetricsSubprocess] Metrics collection error: ${message}`);
|
||||
send({ type: 'error', message: `Metrics collection error: ${message}` });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse size string like "107.4GB" to bytes
|
||||
*/
|
||||
function parseSize(sizeStr: string): number {
|
||||
const units: Record<string, number> = {
|
||||
B: 1,
|
||||
KB: 1024,
|
||||
MB: 1024 * 1024,
|
||||
GB: 1024 * 1024 * 1024,
|
||||
TB: 1024 * 1024 * 1024 * 1024
|
||||
};
|
||||
|
||||
const match = sizeStr.match(/^([\d.]+)\s*([KMGT]?B)$/i);
|
||||
if (!match) return 0;
|
||||
|
||||
const value = parseFloat(match[1]);
|
||||
const unit = match[2].toUpperCase();
|
||||
return value * (units[unit] || 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes to human readable string
|
||||
*/
|
||||
function formatSize(bytes: number): string {
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let unitIndex = 0;
|
||||
let size = bytes;
|
||||
|
||||
while (size >= 1024 && unitIndex < units.length - 1) {
|
||||
size /= 1024;
|
||||
unitIndex++;
|
||||
}
|
||||
|
||||
return `${size.toFixed(1)} ${units[unitIndex]}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check disk space for a single environment
|
||||
*/
|
||||
async function checkEnvDiskSpace(env: { id: number; name: string; collectMetrics?: boolean; connectionType?: string }) {
|
||||
try {
|
||||
// Skip environments where metrics collection is disabled
|
||||
if (env.collectMetrics === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip Hawser Edge environments (handled by main process)
|
||||
if (env.connectionType === 'hawser-edge') {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if disk warnings are enabled for this environment
|
||||
const diskWarningEnabled = (await getEnvSetting('disk_warning_enabled', env.id)) ?? true;
|
||||
if (!diskWarningEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if we're in cooldown for this environment
|
||||
const lastWarningTime = lastDiskWarning.get(env.id);
|
||||
if (lastWarningTime && Date.now() - lastWarningTime < DISK_WARNING_COOLDOWN) {
|
||||
return; // Skip this environment, still in cooldown
|
||||
}
|
||||
|
||||
// Get Docker disk usage data
|
||||
const diskData = (await getDiskUsage(env.id)) as any;
|
||||
if (!diskData) return;
|
||||
|
||||
// Calculate total Docker disk usage using reduce for cleaner code
|
||||
let totalUsed = 0;
|
||||
if (diskData.Images) {
|
||||
totalUsed += diskData.Images.reduce((sum: number, img: any) => sum + (img.Size || 0), 0);
|
||||
}
|
||||
if (diskData.Containers) {
|
||||
totalUsed += diskData.Containers.reduce((sum: number, c: any) => sum + (c.SizeRw || 0), 0);
|
||||
}
|
||||
if (diskData.Volumes) {
|
||||
totalUsed += diskData.Volumes.reduce(
|
||||
(sum: number, v: any) => sum + (v.UsageData?.Size || 0),
|
||||
0
|
||||
);
|
||||
}
|
||||
if (diskData.BuildCache) {
|
||||
totalUsed += diskData.BuildCache.reduce((sum: number, bc: any) => sum + (bc.Size || 0), 0);
|
||||
}
|
||||
|
||||
// Get Docker root filesystem info from Docker info
|
||||
const info = (await getDockerInfo(env.id)) as any;
|
||||
const driverStatus = info?.DriverStatus;
|
||||
|
||||
// Try to find "Data Space Total" from driver status
|
||||
let dataSpaceTotal = 0;
|
||||
let diskPercentUsed = 0;
|
||||
|
||||
if (driverStatus) {
|
||||
for (const [key, value] of driverStatus) {
|
||||
if (key === 'Data Space Total' && typeof value === 'string') {
|
||||
dataSpaceTotal = parseSize(value);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine warning mode
|
||||
const diskWarningMode = (await getEnvSetting('disk_warning_mode', env.id)) ?? 'percentage';
|
||||
const GB = 1024 * 1024 * 1024;
|
||||
|
||||
if (diskWarningMode === 'absolute') {
|
||||
// Absolute mode: warn when usage exceeds GB threshold
|
||||
const thresholdGb = (await getEnvSetting('disk_warning_threshold_gb', env.id)) ?? 50;
|
||||
if (totalUsed > thresholdGb * GB) {
|
||||
send({
|
||||
type: 'disk_warning',
|
||||
envId: env.id,
|
||||
envName: env.name,
|
||||
message: `Environment "${env.name}" is using ${formatSize(totalUsed)} of Docker disk space (threshold: ${thresholdGb} GB)`
|
||||
});
|
||||
lastDiskWarning.set(env.id, Date.now());
|
||||
}
|
||||
} else {
|
||||
// Percentage mode: need total disk space
|
||||
if (dataSpaceTotal > 0) {
|
||||
diskPercentUsed = (totalUsed / dataSpaceTotal) * 100;
|
||||
} else {
|
||||
// Can't determine percentage without total space — skip
|
||||
return;
|
||||
}
|
||||
|
||||
const threshold =
|
||||
(await getEnvSetting('disk_warning_threshold', env.id)) || DEFAULT_DISK_THRESHOLD;
|
||||
if (diskPercentUsed >= threshold) {
|
||||
console.log(
|
||||
`[MetricsSubprocess] Docker disk usage for ${env.name}: ${diskPercentUsed.toFixed(1)}% (threshold: ${threshold}%)`
|
||||
);
|
||||
|
||||
send({
|
||||
type: 'disk_warning',
|
||||
envId: env.id,
|
||||
envName: env.name,
|
||||
message: `Environment "${env.name}" Docker disk usage is at ${diskPercentUsed.toFixed(1)}% (${formatSize(totalUsed)} used)`,
|
||||
diskPercent: diskPercentUsed
|
||||
});
|
||||
|
||||
lastDiskWarning.set(env.id, Date.now());
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Skip this environment if it fails
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.warn(`[MetricsSubprocess] Failed to check disk space for ${env.name}: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check disk space for all environments
|
||||
*/
|
||||
async function checkDiskSpace() {
|
||||
if (isShuttingDown) return;
|
||||
|
||||
try {
|
||||
const environments = await getEnvironments();
|
||||
|
||||
// Filter enabled environments and check disk space in parallel
|
||||
const enabledEnvs = environments.filter((env) => env.collectMetrics !== false);
|
||||
|
||||
// Process all environments in parallel with per-environment timeouts
|
||||
// Use Promise.allSettled so one slow/failed env doesn't block others
|
||||
const results = await Promise.allSettled(
|
||||
enabledEnvs.map((env) =>
|
||||
withTimeout(
|
||||
checkEnvDiskSpace(env).then(() => env.name),
|
||||
ENV_DISK_TIMEOUT,
|
||||
null
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Log any environments that timed out
|
||||
results.forEach((result, index) => {
|
||||
if (result.status === 'fulfilled' && result.value === null) {
|
||||
console.warn(`[MetricsSubprocess] Environment "${enabledEnvs[index].name}" disk check timed out after ${ENV_DISK_TIMEOUT}ms`);
|
||||
} else if (result.status === 'rejected') {
|
||||
const reason = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
||||
console.warn(`[MetricsSubprocess] Environment "${enabledEnvs[index].name}" disk check failed: ${reason}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Clean up stale lastDiskWarning entries for deleted environments
|
||||
const activeEnvIds = new Set(environments.map((e) => e.id));
|
||||
for (const envId of lastDiskWarning.keys()) {
|
||||
if (!activeEnvIds.has(envId)) lastDiskWarning.delete(envId);
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[MetricsSubprocess] Disk space check error: ${message}`);
|
||||
send({ type: 'error', message: `Disk space check error: ${message}` });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle commands from main process
|
||||
*/
|
||||
function handleCommand(command: MainProcessCommand): void {
|
||||
switch (command.type) {
|
||||
case 'refresh_environments':
|
||||
console.log('[MetricsSubprocess] Refreshing environments...');
|
||||
// The next collection cycle will pick up the new environments
|
||||
break;
|
||||
|
||||
case 'update_interval':
|
||||
console.log(`[MetricsSubprocess] Updating collection interval to ${command.intervalMs}ms`);
|
||||
COLLECT_INTERVAL = command.intervalMs;
|
||||
// Clear existing interval and restart with new timing
|
||||
if (collectInterval) {
|
||||
clearInterval(collectInterval);
|
||||
collectInterval = setInterval(collectMetrics, COLLECT_INTERVAL);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'shutdown':
|
||||
console.log('[MetricsSubprocess] Shutdown requested');
|
||||
shutdown();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Graceful shutdown
|
||||
*/
|
||||
function shutdown(): void {
|
||||
isShuttingDown = true;
|
||||
|
||||
if (collectInterval) {
|
||||
clearInterval(collectInterval);
|
||||
collectInterval = null;
|
||||
}
|
||||
if (diskCheckInterval) {
|
||||
clearInterval(diskCheckInterval);
|
||||
diskCheckInterval = null;
|
||||
}
|
||||
|
||||
lastDiskWarning.clear();
|
||||
console.log('[MetricsSubprocess] Stopped');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the metrics collector
|
||||
*/
|
||||
async function start(): Promise<void> {
|
||||
// Load interval from settings
|
||||
try {
|
||||
COLLECT_INTERVAL = await getMetricsCollectionInterval();
|
||||
console.log(`[MetricsSubprocess] Starting metrics collection (every ${COLLECT_INTERVAL / 1000}s)...`);
|
||||
} catch (error) {
|
||||
console.error('[MetricsSubprocess] Failed to load interval from settings, using default 30s');
|
||||
COLLECT_INTERVAL = 30000;
|
||||
}
|
||||
|
||||
// Initial collection
|
||||
collectMetrics();
|
||||
|
||||
// Schedule regular collection
|
||||
collectInterval = setInterval(collectMetrics, COLLECT_INTERVAL);
|
||||
|
||||
// Start disk space checking (every 5 minutes) - can be disabled for Synology NAS
|
||||
const skipDfCollection = process.env.SKIP_DF_COLLECTION === 'true' || process.env.SKIP_DF_COLLECTION === '1';
|
||||
if (!skipDfCollection) {
|
||||
console.log('[MetricsSubprocess] Starting disk space monitoring (every 5 minutes)');
|
||||
checkDiskSpace(); // Initial check
|
||||
diskCheckInterval = setInterval(checkDiskSpace, DISK_CHECK_INTERVAL);
|
||||
} else {
|
||||
console.log('[MetricsSubprocess] Disk space monitoring disabled (SKIP_DF_COLLECTION=true)');
|
||||
}
|
||||
|
||||
// Start memory diagnostics logging (every 5 minutes)
|
||||
setInterval(() => {
|
||||
const mem = process.memoryUsage();
|
||||
console.log(
|
||||
`[MetricsSubprocess] Memory: heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB, ` +
|
||||
`rss=${Math.round(mem.rss / 1024 / 1024)}MB`
|
||||
);
|
||||
}, 5 * 60 * 1000);
|
||||
|
||||
// Listen for commands from main process
|
||||
process.on('message', (message: MainProcessCommand) => {
|
||||
handleCommand(message);
|
||||
});
|
||||
|
||||
// Handle termination signals
|
||||
process.on('SIGTERM', shutdown);
|
||||
process.on('SIGINT', shutdown);
|
||||
|
||||
// Signal ready
|
||||
send({ type: 'ready' });
|
||||
|
||||
console.log('[MetricsSubprocess] Started successfully');
|
||||
}
|
||||
|
||||
// Start the subprocess
|
||||
start();
|
||||
Reference in new issue
Block a user