feat(deployment): add deployment history and rollback support
This commit is contained in:
KodekaTeam committed
1 parent
029b25b596
commit
4e9b173edb
36 files changed
+3663
-569
No files matched your search
+9
-8
@@ -5,6 +5,7 @@ import jwt from "@fastify/jwt";
|
||||
import rateLimit from "@fastify/rate-limit";
|
||||
import websocket from "@fastify/websocket";
|
||||
import prisma from "./lib/prisma";
|
||||
import { validateEncryptionConfiguration } from "./lib/crypto";
|
||||
|
||||
import { authRoutes } from "./routes/auth";
|
||||
import { serverRoutes } from "./routes/servers";
|
||||
@@ -109,10 +110,11 @@ async function ensureDatabaseConnection() {
|
||||
|
||||
async function start() {
|
||||
const jwtSecret = getJwtSecretOrThrow();
|
||||
validateEncryptionConfiguration();
|
||||
await ensureDatabaseConnection();
|
||||
setDefaultSecurityHeaders();
|
||||
|
||||
// ── Plugins ──────────────────────────────────────────
|
||||
// ── Plugins ──────────────────────────────────────────
|
||||
await app.register(cors, {
|
||||
origin: getCorsOrigins(),
|
||||
credentials: true,
|
||||
@@ -143,7 +145,7 @@ async function start() {
|
||||
|
||||
await app.register(websocket);
|
||||
|
||||
// ── Health check ──────────────────────────────────────
|
||||
// ── Health check ──────────────────────────────────────
|
||||
app.get("/health", { config: { rateLimit: false } }, async () => ({
|
||||
status: "ok",
|
||||
uptime: process.uptime(),
|
||||
@@ -151,7 +153,7 @@ async function start() {
|
||||
version: process.env.NEXT_PUBLIC_VERSION || "unknown",
|
||||
}));
|
||||
|
||||
// ── API Routes ────────────────────────────────────────
|
||||
// ── API Routes ────────────────────────────────────────
|
||||
const API_PREFIX = "/api/v1";
|
||||
|
||||
await app.register(authRoutes, { prefix: `${API_PREFIX}/auth` });
|
||||
@@ -183,7 +185,7 @@ async function start() {
|
||||
prefix: `${API_PREFIX}/notifications`,
|
||||
});
|
||||
|
||||
// ── Error handler ─────────────────────────────────────
|
||||
// ── Error handler ─────────────────────────────────────
|
||||
app.setErrorHandler((error: FastifyError, _req, reply) => {
|
||||
app.log.error(error);
|
||||
const code = error.statusCode || 500;
|
||||
@@ -194,14 +196,14 @@ async function start() {
|
||||
});
|
||||
});
|
||||
|
||||
// ── Start ─────────────────────────────────────────────
|
||||
// ── Start ─────────────────────────────────────────────
|
||||
try {
|
||||
await app.listen({ port: PORT, host: HOST });
|
||||
startS3StorageRetentionScheduler();
|
||||
console.log(
|
||||
`\n🚀 Doktainer Server Backend running on http://${HOST}:${PORT}`,
|
||||
`\n🚀 Doktainer Server Backend running on http://${HOST}:${PORT}`,
|
||||
);
|
||||
console.log(`📋 Health Check: http://localhost:${PORT}/health\n`);
|
||||
console.log(`📋 Health Check: http://localhost:${PORT}/health\n`);
|
||||
} catch (err) {
|
||||
const error = err as NodeJS.ErrnoException;
|
||||
if (error.code === "EADDRINUSE") {
|
||||
@@ -227,4 +229,3 @@ if (require.main === module) {
|
||||
}
|
||||
|
||||
export { app, start };
|
||||
|
||||
+62
-15
@@ -1,11 +1,56 @@
|
||||
import crypto from 'crypto';
|
||||
import crypto from "crypto";
|
||||
|
||||
const ALGORITHM = 'aes-256-gcm';
|
||||
const ALGORITHM = "aes-256-gcm";
|
||||
export const DEVELOPMENT_ENCRYPTION_KEY =
|
||||
"dev-32-char-encryption-key-here!";
|
||||
const REJECTED_PRODUCTION_KEYS = new Set([
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
"change-me",
|
||||
"changeme",
|
||||
"replace-me",
|
||||
"replace-with-a-secure-random-key",
|
||||
]);
|
||||
|
||||
export function getEncryptionKeyOrThrow(env = process.env): string {
|
||||
const configuredKey = env.ENCRYPTION_KEY?.trim();
|
||||
const isProduction = env.NODE_ENV === "production";
|
||||
|
||||
if (!configuredKey) {
|
||||
if (isProduction) {
|
||||
throw new Error(
|
||||
"ENCRYPTION_KEY must be configured before starting Doktainer in production",
|
||||
);
|
||||
}
|
||||
return DEVELOPMENT_ENCRYPTION_KEY;
|
||||
}
|
||||
|
||||
if (configuredKey.length < 32) {
|
||||
throw new Error("ENCRYPTION_KEY must be at least 32 characters long");
|
||||
}
|
||||
|
||||
if (
|
||||
isProduction &&
|
||||
(REJECTED_PRODUCTION_KEYS.has(configuredKey.toLowerCase()) ||
|
||||
/^dev[-_]/i.test(configuredKey) ||
|
||||
new Set(configuredKey).size < 8 ||
|
||||
/^(.)\1+$/.test(configuredKey))
|
||||
) {
|
||||
throw new Error(
|
||||
"ENCRYPTION_KEY must use a strong random value and must not use a development or placeholder value in production",
|
||||
);
|
||||
}
|
||||
|
||||
return configuredKey;
|
||||
}
|
||||
|
||||
export function validateEncryptionConfiguration(env = process.env): void {
|
||||
getEncryptionKeyOrThrow(env);
|
||||
}
|
||||
|
||||
function getKey(): Buffer {
|
||||
const key = process.env.ENCRYPTION_KEY || 'dev-32-char-encryption-key-here!';
|
||||
const key = getEncryptionKeyOrThrow();
|
||||
// Derive a 32-byte key using SHA-256
|
||||
return crypto.createHash('sha256').update(key).digest();
|
||||
return crypto.createHash("sha256").update(key).digest();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -17,31 +62,33 @@ export function encrypt(plaintext: string): string {
|
||||
const cipher = crypto.createCipheriv(ALGORITHM, getKey(), iv);
|
||||
|
||||
const encrypted = Buffer.concat([
|
||||
cipher.update(plaintext, 'utf8'),
|
||||
cipher.update(plaintext, "utf8"),
|
||||
cipher.final(),
|
||||
]);
|
||||
const authTag = cipher.getAuthTag();
|
||||
|
||||
return [
|
||||
iv.toString('base64'),
|
||||
authTag.toString('base64'),
|
||||
encrypted.toString('base64'),
|
||||
].join(':');
|
||||
iv.toString("base64"),
|
||||
authTag.toString("base64"),
|
||||
encrypted.toString("base64"),
|
||||
].join(":");
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrypt an encrypted string produced by `encrypt()`
|
||||
*/
|
||||
export function decrypt(encoded: string): string {
|
||||
const [ivB64, tagB64, dataB64] = encoded.split(':');
|
||||
if (!ivB64 || !tagB64 || !dataB64) throw new Error('Invalid encrypted format');
|
||||
const [ivB64, tagB64, dataB64] = encoded.split(":");
|
||||
if (!ivB64 || !tagB64 || !dataB64) throw new Error("Invalid encrypted format");
|
||||
|
||||
const iv = Buffer.from(ivB64, 'base64');
|
||||
const authTag = Buffer.from(tagB64, 'base64');
|
||||
const data = Buffer.from(dataB64, 'base64');
|
||||
const iv = Buffer.from(ivB64, "base64");
|
||||
const authTag = Buffer.from(tagB64, "base64");
|
||||
const data = Buffer.from(dataB64, "base64");
|
||||
|
||||
const decipher = crypto.createDecipheriv(ALGORITHM, getKey(), iv);
|
||||
decipher.setAuthTag(authTag);
|
||||
|
||||
return Buffer.concat([decipher.update(data), decipher.final()]).toString('utf8');
|
||||
return Buffer.concat([decipher.update(data), decipher.final()]).toString(
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
+143
-58
@@ -7,6 +7,11 @@ import { auditLog } from "../services/audit.service";
|
||||
import { dispatchRuntimeNotification } from "../services/notification.service";
|
||||
import * as ssh from "../services/ssh.service";
|
||||
import { rebuildAppInstall } from "../services/app-install-rebuild.service";
|
||||
import {
|
||||
createDeployment,
|
||||
updateDeployment,
|
||||
} from "../services/deployment.service";
|
||||
import { sanitizeDeploymentError } from "../services/deployment-error.service";
|
||||
import {
|
||||
APP_TEMPLATES,
|
||||
AppTemplate,
|
||||
@@ -98,48 +103,6 @@ type DockerInspectRuntime = {
|
||||
}>;
|
||||
};
|
||||
|
||||
type DockerPortBindings = Record<
|
||||
string,
|
||||
Array<{ HostIp?: string; HostPort?: string }> | null
|
||||
>;
|
||||
|
||||
function formatEnvLines(env?: string[] | null): string {
|
||||
return (env ?? []).filter(Boolean).join("\n");
|
||||
}
|
||||
|
||||
function formatMountBindings(mounts?: DockerInspectRuntime["Mounts"]): string {
|
||||
return (mounts ?? [])
|
||||
.map((mount) => {
|
||||
if (!mount.Source || !mount.Destination) return "";
|
||||
return `${mount.Source}:${mount.Destination}${mount.RW === false ? ":ro" : ""}`;
|
||||
})
|
||||
.filter(Boolean)
|
||||
.join(",");
|
||||
}
|
||||
|
||||
function formatPortBindings(bindings?: DockerPortBindings): string {
|
||||
if (!bindings) return "";
|
||||
|
||||
return Object.entries(bindings)
|
||||
.flatMap(([containerPortSpec, hostBindings]) => {
|
||||
if (!hostBindings || hostBindings.length === 0) return [];
|
||||
const [containerPort, protocol = "tcp"] = containerPortSpec.split("/");
|
||||
|
||||
return hostBindings
|
||||
.map((binding) => {
|
||||
const hostPort = binding.HostPort?.trim();
|
||||
if (!hostPort) return "";
|
||||
return `${hostPort}:${containerPort}${protocol !== "tcp" ? `/${protocol}` : ""}`;
|
||||
})
|
||||
.filter(Boolean);
|
||||
})
|
||||
.join(",");
|
||||
}
|
||||
|
||||
function formatCommandParts(cmd?: string[] | null): string {
|
||||
return (cmd ?? []).filter(Boolean).join(" ").trim();
|
||||
}
|
||||
|
||||
function parseCsvBindings(value: string): string[] {
|
||||
return value
|
||||
.split(",")
|
||||
@@ -1045,16 +1008,88 @@ export async function appsRoutes(app: FastifyInstance) {
|
||||
});
|
||||
}
|
||||
|
||||
const install = await prisma.appInstall.create({
|
||||
data: {
|
||||
appId: finalAppId,
|
||||
appName: finalAppName,
|
||||
serverId: server.id,
|
||||
containerName,
|
||||
port: finalPorts || null,
|
||||
status: "INSTALLING",
|
||||
const { install, provisionalContainer } = await prisma.$transaction(
|
||||
async (tx) => {
|
||||
const createdInstall = await tx.appInstall.create({
|
||||
data: {
|
||||
appId: finalAppId,
|
||||
appName: finalAppName,
|
||||
serverId: server.id,
|
||||
containerName,
|
||||
port: finalPorts || null,
|
||||
status: "INSTALLING",
|
||||
},
|
||||
});
|
||||
const createdContainer = await tx.container.create({
|
||||
data: {
|
||||
name: containerName,
|
||||
image: finalImage,
|
||||
serverId: server.id,
|
||||
environmentId: selectedEnvironmentId,
|
||||
status: "STARTING",
|
||||
sourceType: "APP_INSTALLER",
|
||||
deployMode: "IMAGE",
|
||||
ports: JSON.parse(JSON.stringify(parseCsvBindings(finalPorts))),
|
||||
envVars: JSON.parse(
|
||||
JSON.stringify(parseEnvironmentVariables(finalEnv)),
|
||||
),
|
||||
volumes: JSON.parse(
|
||||
JSON.stringify(parseCsvBindings(finalVolumes)),
|
||||
),
|
||||
restartPolicy: finalRestartPolicy,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
install: createdInstall,
|
||||
provisionalContainer: createdContainer,
|
||||
};
|
||||
},
|
||||
});
|
||||
);
|
||||
const deploymentStartedAt = new Date();
|
||||
let deployment;
|
||||
try {
|
||||
deployment = await createDeployment({
|
||||
containerId: provisionalContainer.id,
|
||||
organizationId: req.organizationId!,
|
||||
serverId: server.id,
|
||||
userId: req.userId,
|
||||
status: "RUNNING",
|
||||
trigger: "APP_INSTALLER",
|
||||
version: finalImage,
|
||||
image: finalImage,
|
||||
configSnapshot: {
|
||||
sourceType: "APP_INSTALLER",
|
||||
deployMode: "IMAGE",
|
||||
image: finalImage,
|
||||
ports: finalPorts,
|
||||
env: finalEnv,
|
||||
volumes: finalVolumes,
|
||||
network: finalNetwork,
|
||||
restartPolicy: finalRestartPolicy,
|
||||
command: finalCommand,
|
||||
appId: finalAppId,
|
||||
appName: finalAppName,
|
||||
presetId: body.data.presetId ?? null,
|
||||
},
|
||||
startedAt: deploymentStartedAt,
|
||||
});
|
||||
} catch (error) {
|
||||
const message = sanitizeDeploymentError(error, {
|
||||
fallback: "Failed to create deployment history",
|
||||
});
|
||||
await prisma.$transaction([
|
||||
prisma.container.update({
|
||||
where: { id: provisionalContainer.id },
|
||||
data: { status: "ERROR" },
|
||||
}),
|
||||
prisma.appInstall.update({
|
||||
where: { id: install.id },
|
||||
data: { status: "FAILED", error: message },
|
||||
}),
|
||||
]);
|
||||
return reply.status(500).send({ success: false, error: message });
|
||||
}
|
||||
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
@@ -1069,6 +1104,13 @@ export async function appsRoutes(app: FastifyInstance) {
|
||||
command: finalCommand,
|
||||
mountValidation: resolveAppMountValidation(finalAppId),
|
||||
});
|
||||
await prisma.container.update({
|
||||
where: { id: provisionalContainer.id },
|
||||
data: {
|
||||
dockerId: dockerId.trim().slice(0, 12) || null,
|
||||
status: "STARTING",
|
||||
},
|
||||
});
|
||||
|
||||
await upsertAppInstallerContainer({
|
||||
serverId: server.id,
|
||||
@@ -1082,6 +1124,32 @@ export async function appsRoutes(app: FastifyInstance) {
|
||||
restartPolicy: finalRestartPolicy,
|
||||
});
|
||||
|
||||
const deployedContainer = await prisma.container.findUnique({
|
||||
where: { id: provisionalContainer.id },
|
||||
select: { id: true, image: true },
|
||||
});
|
||||
if (deployedContainer) {
|
||||
await updateDeployment(deployment.id, {
|
||||
status: "SUCCESS",
|
||||
image: deployedContainer.image || finalImage,
|
||||
configSnapshot: {
|
||||
sourceType: "APP_INSTALLER",
|
||||
deployMode: "IMAGE",
|
||||
image: deployedContainer.image || finalImage,
|
||||
ports: finalPorts,
|
||||
env: finalEnv,
|
||||
volumes: finalVolumes,
|
||||
network: finalNetwork,
|
||||
restartPolicy: finalRestartPolicy,
|
||||
command: finalCommand,
|
||||
appId: finalAppId,
|
||||
appName: finalAppName,
|
||||
presetId: body.data.presetId ?? null,
|
||||
},
|
||||
completedAt: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
await prisma.appInstall.update({
|
||||
where: { id: install.id },
|
||||
data: { status: "RUNNING" },
|
||||
@@ -1125,11 +1193,28 @@ export async function appsRoutes(app: FastifyInstance) {
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (err: any) {
|
||||
await prisma.appInstall.update({
|
||||
where: { id: install.id },
|
||||
data: { status: "FAILED", error: err.message },
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = sanitizeDeploymentError(err, {
|
||||
fallback: "App installation failed",
|
||||
});
|
||||
await prisma.$transaction([
|
||||
prisma.appInstall.update({
|
||||
where: { id: install.id },
|
||||
data: { status: "FAILED", error: errorMessage },
|
||||
}),
|
||||
prisma.container.update({
|
||||
where: { id: provisionalContainer.id },
|
||||
data: { status: "ERROR" },
|
||||
}),
|
||||
prisma.deployment.update({
|
||||
where: { id: deployment.id },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
error: errorMessage,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
await auditLog({
|
||||
userId: req.userId,
|
||||
@@ -1138,7 +1223,7 @@ export async function appsRoutes(app: FastifyInstance) {
|
||||
action: "APP_INSTALL_FAILED",
|
||||
category: "SYSTEM",
|
||||
level: "ERROR",
|
||||
message: `App "${finalAppName}" install failed on "${server.name}": ${err.message}`,
|
||||
message: `App "${finalAppName}" install failed on "${server.name}": ${errorMessage}`,
|
||||
meta: {
|
||||
mode: body.data.mode,
|
||||
image: finalImage,
|
||||
@@ -1151,7 +1236,7 @@ export async function appsRoutes(app: FastifyInstance) {
|
||||
organizationId: req.organizationId,
|
||||
action: "app_build_error",
|
||||
title: `App deploy failed: ${finalAppName}`,
|
||||
message: `App ${finalAppName} failed to deploy on ${server.name}. ${err.message}`,
|
||||
message: `App ${finalAppName} failed to deploy on ${server.name}. ${errorMessage}`,
|
||||
serverId: server.id,
|
||||
resourceType: "app_install",
|
||||
resourceId: install.id,
|
||||
@@ -1164,7 +1249,7 @@ export async function appsRoutes(app: FastifyInstance) {
|
||||
presetId: body.data.presetId ?? null,
|
||||
serverId: server.id,
|
||||
serverName: server.name,
|
||||
error: err.message,
|
||||
error: errorMessage,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
+439
-42
@@ -25,6 +25,19 @@ import {
|
||||
import * as ssh from "../services/ssh.service";
|
||||
import { withCommandLogSink } from "../services/ssh.service";
|
||||
import { rebuildAppInstall } from "../services/app-install-rebuild.service";
|
||||
import {
|
||||
createDeployment,
|
||||
getDeployment,
|
||||
listDeployments,
|
||||
updateDeployment,
|
||||
} from "../services/deployment.service";
|
||||
import { rollbackContainerToDeployment } from "../services/deployment-rollback.service";
|
||||
import {
|
||||
acquireDeploymentLock,
|
||||
releaseDeploymentLock,
|
||||
startDeploymentLockHeartbeat,
|
||||
} from "../services/deployment-lock.service";
|
||||
import { sanitizeDeploymentError } from "../services/deployment-error.service";
|
||||
|
||||
const DeploySourceTypeSchema = z.enum([
|
||||
"APP_INSTALLER",
|
||||
@@ -707,8 +720,7 @@ export function resolveContainerPathToHostMountPath(
|
||||
const matchedMount = mounts
|
||||
.filter(
|
||||
(mount) =>
|
||||
isUsableMountPath(mount.Source) &&
|
||||
isUsableMountPath(mount.Destination),
|
||||
isUsableMountPath(mount.Source) && isUsableMountPath(mount.Destination),
|
||||
)
|
||||
.map((mount) => ({
|
||||
source: normalizeRemotePath(mount.Source!),
|
||||
@@ -732,7 +744,10 @@ export function resolveContainerPathToHostMountPath(
|
||||
return pathPosix.normalize(pathPosix.join(matchedMount.source, relativePath));
|
||||
}
|
||||
|
||||
function getWritableMountRootsForPath(targetPath: string, mounts: DockerInspectMount[]) {
|
||||
function getWritableMountRootsForPath(
|
||||
targetPath: string,
|
||||
mounts: DockerInspectMount[],
|
||||
) {
|
||||
const normalizedTargetPath = normalizeRemotePath(targetPath);
|
||||
|
||||
return mounts
|
||||
@@ -944,6 +959,7 @@ function resolveGitBuildType(input: z.infer<typeof DeploySchema>) {
|
||||
}
|
||||
|
||||
type DockerInspectRuntime = {
|
||||
Image?: string;
|
||||
Config?: {
|
||||
Image?: string;
|
||||
Env?: string[];
|
||||
@@ -1564,7 +1580,7 @@ function parseInjectedResponsePayload(response: {
|
||||
|
||||
function runInjectedContainerJob(input: {
|
||||
app: FastifyInstance;
|
||||
job: ReturnType<typeof createProcessJob>;
|
||||
job: Awaited<ReturnType<typeof createProcessJob>>;
|
||||
method: "POST";
|
||||
url: string;
|
||||
headers: Record<string, unknown>;
|
||||
@@ -1574,19 +1590,22 @@ function runInjectedContainerJob(input: {
|
||||
void Promise.resolve().then(async () => {
|
||||
const abortController = new AbortController();
|
||||
if (isProcessJobCancelling(input.job)) {
|
||||
updateProcessJob(input.job, {
|
||||
await updateProcessJob(input.job, {
|
||||
status: "cancelled",
|
||||
error: input.job.cancelReason ?? "Job cancelled",
|
||||
});
|
||||
appendProcessJobLog(input.job, "[job] Cancelled");
|
||||
await appendProcessJobLog(input.job, "[job] Cancelled");
|
||||
return;
|
||||
}
|
||||
|
||||
updateProcessJob(input.job, {
|
||||
await updateProcessJob(input.job, {
|
||||
status: "running",
|
||||
cancel: (reason) => abortController.abort(reason),
|
||||
});
|
||||
appendProcessJobLog(input.job, `[job] Started ${input.job.type}`);
|
||||
await appendProcessJobLog(
|
||||
input.job,
|
||||
`[job] Started ${input.job.type}`,
|
||||
);
|
||||
|
||||
try {
|
||||
const response = await withCommandLogSink(
|
||||
@@ -1601,7 +1620,10 @@ function runInjectedContainerJob(input: {
|
||||
await input.app.inject({
|
||||
method: input.method,
|
||||
url: input.url,
|
||||
headers: buildInternalJobHeaders(input.headers),
|
||||
headers: {
|
||||
...buildInternalJobHeaders(input.headers),
|
||||
"x-process-job-sync": "true",
|
||||
},
|
||||
payload: input.payload,
|
||||
}),
|
||||
{
|
||||
@@ -1610,12 +1632,12 @@ function runInjectedContainerJob(input: {
|
||||
},
|
||||
);
|
||||
|
||||
if (abortController.signal.aborted || isProcessJobCancelling(input.job)) {
|
||||
updateProcessJob(input.job, {
|
||||
if (isProcessJobCancelling(input.job)) {
|
||||
await updateProcessJob(input.job, {
|
||||
status: "cancelled",
|
||||
error: input.job.cancelReason ?? "Job cancelled",
|
||||
});
|
||||
appendProcessJobLog(input.job, "[job] Cancelled");
|
||||
await appendProcessJobLog(input.job, "[job] Cancelled");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1628,29 +1650,56 @@ function runInjectedContainerJob(input: {
|
||||
throw new Error(error || `Job failed with HTTP ${response.statusCode}`);
|
||||
}
|
||||
|
||||
updateProcessJob(input.job, { status: "success", result: payload });
|
||||
appendProcessJobLog(input.job, "[job] Completed successfully");
|
||||
await updateProcessJob(input.job, {
|
||||
status: "success",
|
||||
result: payload,
|
||||
});
|
||||
await appendProcessJobLog(input.job, "[job] Completed successfully");
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Job failed";
|
||||
if (
|
||||
abortController.signal.aborted ||
|
||||
isProcessJobCancelling(input.job) ||
|
||||
message === "Command cancelled"
|
||||
) {
|
||||
updateProcessJob(input.job, {
|
||||
await updateProcessJob(input.job, {
|
||||
status: "cancelled",
|
||||
error: input.job.cancelReason ?? "Job cancelled",
|
||||
});
|
||||
appendProcessJobLog(input.job, "[job] Cancelled");
|
||||
await appendProcessJobLog(input.job, "[job] Cancelled");
|
||||
return;
|
||||
}
|
||||
|
||||
updateProcessJob(input.job, { status: "error", error: message });
|
||||
appendProcessJobLog(input.job, `[job] Failed: ${message}`);
|
||||
await updateProcessJob(input.job, {
|
||||
status: "error",
|
||||
error: message,
|
||||
});
|
||||
await appendProcessJobLog(input.job, `[job] Failed: ${message}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function buildDeploymentSnapshot(
|
||||
body: z.infer<typeof DeploySchema>,
|
||||
runtimeImage?: string,
|
||||
) {
|
||||
return {
|
||||
sourceType: body.sourceType,
|
||||
deployMode: body.deployMode,
|
||||
buildType: body.buildType,
|
||||
image: runtimeImage || body.image,
|
||||
ports: body.ports,
|
||||
env: body.env,
|
||||
volumes: body.volumes,
|
||||
restartPolicy: body.restartPolicy,
|
||||
repoUrl: body.repoUrl,
|
||||
repoBranch: body.repoBranch,
|
||||
buildPath: body.buildPath,
|
||||
composeFilePath: body.composeFilePath,
|
||||
dockerfilePath: body.dockerfilePath,
|
||||
dockerContextPath: body.dockerContextPath,
|
||||
imageTag: body.imageTag,
|
||||
};
|
||||
}
|
||||
|
||||
export async function containerRoutes(app: FastifyInstance) {
|
||||
const containerReadAccess = [
|
||||
authenticate,
|
||||
@@ -1662,6 +1711,143 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
];
|
||||
|
||||
// GET /containers — all containers from DB
|
||||
const recordDeployment = async (input: {
|
||||
container: { id: string; serverId: string; image: string };
|
||||
body: z.infer<typeof DeploySchema>;
|
||||
trigger: "MANUAL" | "GIT_WEBHOOK" | "APP_INSTALLER";
|
||||
userId?: string;
|
||||
organizationId: string;
|
||||
}) => {
|
||||
const body = input.body;
|
||||
return createDeployment({
|
||||
containerId: input.container.id,
|
||||
organizationId: input.organizationId,
|
||||
serverId: input.container.serverId,
|
||||
userId: input.userId,
|
||||
status: "SUCCESS",
|
||||
trigger: input.trigger,
|
||||
version: body.imageTag || body.repoBranch || input.container.image,
|
||||
branch: body.repoBranch || null,
|
||||
image: input.container.image,
|
||||
configSnapshot: buildDeploymentSnapshot(body, input.container.image),
|
||||
startedAt: new Date(),
|
||||
completedAt: new Date(),
|
||||
});
|
||||
};
|
||||
|
||||
const recordFailedDeployment = async (input: {
|
||||
container: { id: string; serverId: string; image: string };
|
||||
body: z.infer<typeof DeploySchema>;
|
||||
trigger: "MANUAL" | "GIT_WEBHOOK" | "APP_INSTALLER";
|
||||
userId?: string;
|
||||
organizationId: string;
|
||||
error: string;
|
||||
}) =>
|
||||
createDeployment({
|
||||
containerId: input.container.id,
|
||||
organizationId: input.organizationId,
|
||||
serverId: input.container.serverId,
|
||||
userId: input.userId,
|
||||
status: "FAILED",
|
||||
trigger: input.trigger,
|
||||
version: input.body.imageTag || input.body.repoBranch || input.container.image,
|
||||
branch: input.body.repoBranch || null,
|
||||
image: input.container.image,
|
||||
configSnapshot: buildDeploymentSnapshot(input.body, input.container.image),
|
||||
error: input.error,
|
||||
startedAt: new Date(),
|
||||
completedAt: new Date(),
|
||||
});
|
||||
|
||||
app.get(
|
||||
"/:id/deployments",
|
||||
{ preHandler: containerReadAccess },
|
||||
async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
const query = req.query as { page?: string; pageSize?: string };
|
||||
const page = Math.max(1, Number.parseInt(query.page ?? "1", 10) || 1);
|
||||
const pageSize = Math.min(
|
||||
50,
|
||||
Math.max(1, Number.parseInt(query.pageSize ?? "20", 10) || 20),
|
||||
);
|
||||
const container = await prisma.container.findFirst({
|
||||
where: { id, server: { organizationId: req.organizationId! } },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!container) {
|
||||
return reply
|
||||
.status(404)
|
||||
.send({ success: false, error: "Container not found" });
|
||||
}
|
||||
|
||||
const data = await listDeployments({
|
||||
containerId: id,
|
||||
organizationId: req.organizationId!,
|
||||
page,
|
||||
pageSize,
|
||||
});
|
||||
return reply.send({ success: true, data });
|
||||
},
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/:id/deployments/:deploymentId",
|
||||
{ preHandler: containerReadAccess },
|
||||
async (req, reply) => {
|
||||
const { id, deploymentId } = req.params as {
|
||||
id: string;
|
||||
deploymentId: string;
|
||||
};
|
||||
const deployment = await getDeployment({
|
||||
containerId: id,
|
||||
deploymentId,
|
||||
organizationId: req.organizationId!,
|
||||
});
|
||||
if (!deployment) {
|
||||
return reply
|
||||
.status(404)
|
||||
.send({ success: false, error: "Deployment not found" });
|
||||
}
|
||||
return reply.send({ success: true, data: deployment });
|
||||
},
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/:id/deployments/:deploymentId/rollback",
|
||||
{ preHandler: containerWriteAccess },
|
||||
async (req, reply) => {
|
||||
const { id, deploymentId } = req.params as {
|
||||
id: string;
|
||||
deploymentId: string;
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await rollbackContainerToDeployment({
|
||||
containerId: id,
|
||||
deploymentId,
|
||||
organizationId: req.organizationId!,
|
||||
userId: req.userId,
|
||||
});
|
||||
return reply.send({
|
||||
success: true,
|
||||
data: result.updated,
|
||||
meta: {
|
||||
rollbackDeploymentId: result.deploymentId,
|
||||
targetDeploymentId: result.targetDeploymentId,
|
||||
},
|
||||
message: "Container rolled back successfully",
|
||||
});
|
||||
} catch (error) {
|
||||
return reply.status(400).send({
|
||||
success: false,
|
||||
error: sanitizeDeploymentError(error, {
|
||||
fallback: "Rollback failed",
|
||||
}),
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/jobs/:jobId",
|
||||
{ preHandler: containerReadAccess },
|
||||
@@ -1700,7 +1886,7 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
.send({ success: false, error: "Job not found" });
|
||||
}
|
||||
|
||||
const cancelled = cancelProcessJob(job);
|
||||
const cancelled = await cancelProcessJob(job);
|
||||
|
||||
return reply.send({
|
||||
success: true,
|
||||
@@ -1792,7 +1978,7 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
.send({ success: false, error: validationError });
|
||||
}
|
||||
|
||||
const job = createProcessJob({
|
||||
const job = await createProcessJob({
|
||||
type: "container_deploy",
|
||||
userId: req.userId,
|
||||
organizationId: req.organizationId,
|
||||
@@ -2196,10 +2382,10 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
container.dockerId || container.name,
|
||||
);
|
||||
canWriteProjectEnv =
|
||||
getWritableMountRootsForPath(
|
||||
normalizedEnvPath,
|
||||
[...getInspectMounts(inspect), ...storedMounts],
|
||||
).length > 0;
|
||||
getWritableMountRootsForPath(normalizedEnvPath, [
|
||||
...getInspectMounts(inspect),
|
||||
...storedMounts,
|
||||
]).length > 0;
|
||||
} catch {
|
||||
canWriteProjectEnv = false;
|
||||
}
|
||||
@@ -3082,7 +3268,6 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
volumes,
|
||||
network: selectedNetworkName,
|
||||
});
|
||||
|
||||
const updatedContainer = await prisma.container.update({
|
||||
where: { id: container.id },
|
||||
data: {
|
||||
@@ -3092,6 +3277,14 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
include: { server: { select: { name: true, ip: true } } },
|
||||
});
|
||||
|
||||
await recordDeployment({
|
||||
container: updatedContainer,
|
||||
body: body.data,
|
||||
trigger: "MANUAL",
|
||||
userId: req.userId,
|
||||
organizationId: req.organizationId!,
|
||||
});
|
||||
|
||||
await auditLog({
|
||||
userId: req.userId,
|
||||
serverId,
|
||||
@@ -3107,14 +3300,22 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
message: "Deployment completed successfully",
|
||||
});
|
||||
} catch (err) {
|
||||
const errorMessage = redactSecret(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
accessToken,
|
||||
);
|
||||
const errorMessage = sanitizeDeploymentError(err, {
|
||||
fallback: "Container deployment failed",
|
||||
secrets: [accessToken],
|
||||
});
|
||||
await prisma.container.update({
|
||||
where: { id: container.id },
|
||||
data: { status: "ERROR" },
|
||||
});
|
||||
await recordFailedDeployment({
|
||||
container,
|
||||
body: body.data,
|
||||
trigger: "MANUAL",
|
||||
userId: req.userId,
|
||||
organizationId: req.organizationId!,
|
||||
error: errorMessage,
|
||||
}).catch(() => undefined);
|
||||
await auditLog({
|
||||
userId: req.userId,
|
||||
serverId,
|
||||
@@ -3150,6 +3351,17 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
dockerId: dockerId.trim().slice(0, 12),
|
||||
},
|
||||
});
|
||||
await recordDeployment({
|
||||
container: {
|
||||
id: container.id,
|
||||
serverId: container.serverId,
|
||||
image: container.image,
|
||||
},
|
||||
body: body.data,
|
||||
trigger: "MANUAL",
|
||||
userId: req.userId,
|
||||
organizationId: req.organizationId!,
|
||||
});
|
||||
await auditLog({
|
||||
userId: req.userId,
|
||||
serverId,
|
||||
@@ -3160,14 +3372,22 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
});
|
||||
})
|
||||
.catch(async (err) => {
|
||||
const errorMessage = redactSecret(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
accessToken,
|
||||
);
|
||||
const errorMessage = sanitizeDeploymentError(err, {
|
||||
fallback: "Container deployment failed",
|
||||
secrets: [accessToken],
|
||||
});
|
||||
await prisma.container.update({
|
||||
where: { id: container.id },
|
||||
data: { status: "ERROR" },
|
||||
});
|
||||
await recordFailedDeployment({
|
||||
container,
|
||||
body: body.data,
|
||||
trigger: "MANUAL",
|
||||
userId: req.userId,
|
||||
organizationId: req.organizationId!,
|
||||
error: errorMessage,
|
||||
}).catch(() => undefined);
|
||||
await auditLog({
|
||||
userId: req.userId,
|
||||
serverId,
|
||||
@@ -3290,6 +3510,23 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
include: { server: { select: { name: true, ip: true } } },
|
||||
});
|
||||
|
||||
await Promise.all(
|
||||
responseContainers.map((matchedContainer) =>
|
||||
recordDeployment({
|
||||
container: matchedContainer,
|
||||
body: body.data,
|
||||
trigger:
|
||||
body.data.sourceType === "APP_INSTALLER"
|
||||
? "APP_INSTALLER"
|
||||
: "MANUAL",
|
||||
userId: req.userId,
|
||||
organizationId: req.organizationId!,
|
||||
}),
|
||||
),
|
||||
).catch((error) => {
|
||||
app.log.error({ error }, "Failed to persist deployment history");
|
||||
});
|
||||
|
||||
await auditLog({
|
||||
userId: req.userId,
|
||||
serverId,
|
||||
@@ -3315,11 +3552,32 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
: "Deployment completed successfully",
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage = redactSecret(
|
||||
const errorMessage = sanitizeDeploymentError(
|
||||
ssh.formatDeploymentErrorMessage(error),
|
||||
accessToken,
|
||||
{
|
||||
fallback: "Container deployment failed",
|
||||
secrets: [accessToken],
|
||||
},
|
||||
);
|
||||
|
||||
const failedContainer = await prisma.container.findFirst({
|
||||
where: {
|
||||
serverId,
|
||||
name: { equals: rest.name, mode: "insensitive" },
|
||||
},
|
||||
select: { id: true, serverId: true, image: true },
|
||||
});
|
||||
if (failedContainer) {
|
||||
await recordFailedDeployment({
|
||||
container: failedContainer,
|
||||
body: body.data,
|
||||
trigger: body.data.sourceType === "APP_INSTALLER" ? "APP_INSTALLER" : "MANUAL",
|
||||
userId: req.userId,
|
||||
organizationId: req.organizationId!,
|
||||
error: errorMessage,
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
|
||||
await auditLog({
|
||||
userId: req.userId,
|
||||
serverId,
|
||||
@@ -3501,7 +3759,7 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
.send({ success: false, error: "Container not found" });
|
||||
}
|
||||
|
||||
const job = createProcessJob({
|
||||
const job = await createProcessJob({
|
||||
type: "container_rebuild",
|
||||
userId: req.userId,
|
||||
organizationId: req.organizationId,
|
||||
@@ -3545,6 +3803,13 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
.send({ success: false, error: "Container not found" });
|
||||
}
|
||||
|
||||
let gitDeploymentId: string | null = null;
|
||||
let gitDeploymentLock: { token: string } | null = null;
|
||||
let gitDeploymentHeartbeat: ReturnType<
|
||||
typeof startDeploymentLockHeartbeat
|
||||
> | null = null;
|
||||
let gitAccessToken: string | undefined;
|
||||
|
||||
try {
|
||||
if (container.sourceType === "APP_INSTALLER") {
|
||||
const install = await prisma.appInstall.findFirst({
|
||||
@@ -3626,6 +3891,50 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
const accessToken = container.deploymentSource.accessTokenEnc
|
||||
? decrypt(container.deploymentSource.accessTokenEnc)
|
||||
: undefined;
|
||||
gitAccessToken = accessToken;
|
||||
const gitDeploymentSnapshot = {
|
||||
sourceType: container.sourceType,
|
||||
deployMode: container.deployMode,
|
||||
buildType,
|
||||
image: container.image,
|
||||
ports: runtimeConfig.ports,
|
||||
env: runtimeConfig.env,
|
||||
volumes: runtimeConfig.volumes,
|
||||
network: runtimeConfig.network,
|
||||
restartPolicy: runtimeConfig.restartPolicy,
|
||||
repoUrl: container.deploymentSource.repoUrl,
|
||||
repoBranch: container.deploymentSource.repoBranch,
|
||||
buildPath: container.deploymentSource.buildPath,
|
||||
composeFilePath: container.deploymentSource.composeFilePath,
|
||||
dockerfilePath: container.deploymentSource.dockerfilePath,
|
||||
dockerContextPath:
|
||||
container.deploymentSource.dockerContextPath,
|
||||
imageTag: container.deploymentSource.imageTag,
|
||||
deploymentPath: container.deploymentSource.deploymentPath,
|
||||
};
|
||||
|
||||
gitDeploymentLock = await acquireDeploymentLock({
|
||||
containerId: container.id,
|
||||
});
|
||||
gitDeploymentHeartbeat = startDeploymentLockHeartbeat({
|
||||
containerId: container.id,
|
||||
token: gitDeploymentLock.token,
|
||||
});
|
||||
const runningDeployment = await createDeployment({
|
||||
containerId: container.id,
|
||||
organizationId: req.organizationId!,
|
||||
serverId: container.serverId,
|
||||
userId: req.userId,
|
||||
status: "RUNNING",
|
||||
trigger: "REBUILD",
|
||||
version:
|
||||
container.deploymentSource.repoBranch?.trim() || container.image,
|
||||
branch: container.deploymentSource.repoBranch?.trim() || null,
|
||||
image: container.image,
|
||||
configSnapshot: gitDeploymentSnapshot,
|
||||
startedAt: new Date(),
|
||||
});
|
||||
gitDeploymentId = runningDeployment.id;
|
||||
|
||||
if (buildType !== "COMPOSE") {
|
||||
for (const candidate of [container.dockerId, container.name]) {
|
||||
@@ -3650,7 +3959,9 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
}
|
||||
}
|
||||
|
||||
await ssh.deployContainerFromGitSource(container.server, {
|
||||
const gitDeploymentResult = await ssh.deployContainerFromGitSource(
|
||||
container.server,
|
||||
{
|
||||
projectName: deploymentProjectName,
|
||||
repoUrl: container.deploymentSource.repoUrl,
|
||||
branch: toOptionalValue(container.deploymentSource.repoBranch),
|
||||
@@ -3686,7 +3997,9 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
deploymentPath: toOptionalValue(
|
||||
container.deploymentSource.deploymentPath,
|
||||
),
|
||||
});
|
||||
},
|
||||
);
|
||||
gitDeploymentHeartbeat.assertOwned();
|
||||
|
||||
await syncContainersForServers(
|
||||
[container.server],
|
||||
@@ -3740,6 +4053,46 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
(candidate) =>
|
||||
candidate.name.toLowerCase() === container.name.toLowerCase(),
|
||||
) ?? responseContainers[0];
|
||||
if (!updatedContainer) {
|
||||
throw new Error(
|
||||
"Redeploy from Git finished but no primary container could be resolved",
|
||||
);
|
||||
}
|
||||
|
||||
gitDeploymentHeartbeat.assertOwned();
|
||||
|
||||
let imageDigest: string | null = null;
|
||||
try {
|
||||
const inspect = (await ssh.dockerInspect(
|
||||
container.server,
|
||||
updatedContainer.dockerId || updatedContainer.name,
|
||||
)) as DockerInspectRuntime;
|
||||
imageDigest =
|
||||
typeof inspect.Image === "string" && inspect.Image.trim()
|
||||
? inspect.Image.trim()
|
||||
: null;
|
||||
} catch {
|
||||
// The canonical runtime image is still persisted even when Docker
|
||||
// does not return an image ID after the successful health gate.
|
||||
}
|
||||
|
||||
await updateDeployment(runningDeployment.id, {
|
||||
status: "SUCCESS",
|
||||
completedAt: new Date(),
|
||||
image: updatedContainer.image,
|
||||
imageDigest,
|
||||
commitSha: gitDeploymentResult.commitSha,
|
||||
branch: container.deploymentSource.repoBranch?.trim() || null,
|
||||
configSnapshot: {
|
||||
...gitDeploymentSnapshot,
|
||||
image: updatedContainer.image,
|
||||
imageDigest,
|
||||
commitSha: gitDeploymentResult.commitSha,
|
||||
matchedContainerIds: responseContainers.map(
|
||||
(candidate) => candidate.id,
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
await auditLog({
|
||||
userId: req.userId,
|
||||
@@ -3748,6 +4101,16 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
category: "CONTAINER",
|
||||
level: "SUCCESS",
|
||||
message: `Container "${container.name}" redeployed from Git on ${container.server.name}`,
|
||||
meta: {
|
||||
deploymentId: runningDeployment.id,
|
||||
commitSha: gitDeploymentResult.commitSha,
|
||||
branch: container.deploymentSource.repoBranch,
|
||||
image: updatedContainer.image,
|
||||
imageDigest,
|
||||
matchedContainerIds: responseContainers.map(
|
||||
(candidate) => candidate.id,
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
return reply.send({
|
||||
@@ -3759,11 +4122,45 @@ export async function containerRoutes(app: FastifyInstance) {
|
||||
: "Container redeployed successfully from Git",
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = gitDeploymentId
|
||||
? sanitizeDeploymentError(ssh.formatDeploymentErrorMessage(err), {
|
||||
fallback: "Git redeploy failed",
|
||||
secrets: [gitAccessToken],
|
||||
})
|
||||
: err instanceof Error
|
||||
? err.message
|
||||
: "Failed to rebuild container";
|
||||
|
||||
if (gitDeploymentId) {
|
||||
await updateDeployment(gitDeploymentId, {
|
||||
status: "FAILED",
|
||||
error: errorMessage,
|
||||
completedAt: new Date(),
|
||||
}).catch(() => undefined);
|
||||
await auditLog({
|
||||
userId: req.userId,
|
||||
organizationId: req.organizationId,
|
||||
serverId: container.serverId,
|
||||
action: "CONTAINER_REDEPLOY_GIT",
|
||||
category: "CONTAINER",
|
||||
level: "ERROR",
|
||||
message: `Git redeploy failed for container "${container.name}": ${errorMessage}`,
|
||||
meta: { deploymentId: gitDeploymentId },
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
|
||||
return reply.status(500).send({
|
||||
success: false,
|
||||
error:
|
||||
err instanceof Error ? err.message : "Failed to rebuild container",
|
||||
error: errorMessage,
|
||||
});
|
||||
} finally {
|
||||
gitDeploymentHeartbeat?.stop();
|
||||
if (gitDeploymentLock) {
|
||||
await releaseDeploymentLock({
|
||||
containerId: container.id,
|
||||
token: gitDeploymentLock.token,
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
function getErrorText(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error ?? "");
|
||||
}
|
||||
|
||||
export function formatServerActionError(
|
||||
error: unknown,
|
||||
fallbackMessage: string,
|
||||
) {
|
||||
const message = getErrorText(error);
|
||||
|
||||
const portMatch = message.match(/(?:0\.0\.0\.0|:::):(?:80|443).*?(?:address already in use|already in use)/i);
|
||||
if (portMatch || /address already in use/i.test(message)) {
|
||||
return "The service could not start because port 80 or 443 is already in use by another host process or Docker container.";
|
||||
}
|
||||
|
||||
if (/syntax error|unexpected token|command not found|bad substitution/i.test(message)) {
|
||||
return "The remote host rejected the generated shell command before it could finish. Refresh the server snapshot and retry the action.";
|
||||
}
|
||||
|
||||
return fallbackMessage;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,13 @@ import { auditLog } from "./audit.service";
|
||||
import { dispatchRuntimeNotification } from "./notification.service";
|
||||
import * as ssh from "./ssh.service";
|
||||
import { APP_TEMPLATES } from "../config/app-templates";
|
||||
import { createDeployment, updateDeployment } from "./deployment.service";
|
||||
import {
|
||||
acquireDeploymentLock,
|
||||
DeploymentLockConflictError,
|
||||
releaseDeploymentLock,
|
||||
startDeploymentLockHeartbeat,
|
||||
} from "./deployment-lock.service";
|
||||
|
||||
type DockerInspectRuntime = {
|
||||
Config?: {
|
||||
@@ -176,15 +183,51 @@ export async function rebuildAppInstall(options: {
|
||||
throw new Error("Unable to determine which image should be rebuilt");
|
||||
}
|
||||
|
||||
await prisma.appInstall.update({
|
||||
where: { id: install.id },
|
||||
data: { status: "INSTALLING", error: null },
|
||||
});
|
||||
|
||||
let pulledLatestImage = false;
|
||||
let usedImageCache = false;
|
||||
let deploymentId: string | null = null;
|
||||
let deploymentLockToken: string | null = null;
|
||||
let deploymentLockContainerId: string | null = null;
|
||||
let deploymentLockHeartbeat: ReturnType<
|
||||
typeof startDeploymentLockHeartbeat
|
||||
> | null = null;
|
||||
|
||||
try {
|
||||
const currentContainer = await prisma.container.findFirst({
|
||||
where: {
|
||||
serverId: install.serverId,
|
||||
name: { equals: install.containerName, mode: "insensitive" },
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
if (currentContainer) {
|
||||
const lock = await acquireDeploymentLock({ containerId: currentContainer.id });
|
||||
deploymentLockToken = lock.token;
|
||||
deploymentLockContainerId = currentContainer.id;
|
||||
deploymentLockHeartbeat = startDeploymentLockHeartbeat({
|
||||
containerId: currentContainer.id,
|
||||
token: lock.token,
|
||||
});
|
||||
const deployment = await createDeployment({
|
||||
containerId: currentContainer.id,
|
||||
organizationId: options.organizationId ?? install.server.organizationId,
|
||||
serverId: install.serverId,
|
||||
userId: options.userId,
|
||||
status: "RUNNING",
|
||||
trigger: "REBUILD",
|
||||
version: runtimeConfig.image,
|
||||
image: runtimeConfig.image,
|
||||
configSnapshot: runtimeConfig,
|
||||
startedAt: new Date(),
|
||||
});
|
||||
deploymentId = deployment.id;
|
||||
}
|
||||
|
||||
await prisma.appInstall.update({
|
||||
where: { id: install.id },
|
||||
data: { status: "INSTALLING", error: null },
|
||||
});
|
||||
|
||||
try {
|
||||
await ssh.dockerPullImage(install.server, runtimeConfig.image);
|
||||
pulledLatestImage = true;
|
||||
@@ -218,6 +261,8 @@ export async function rebuildAppInstall(options: {
|
||||
|
||||
const normalizedDockerId = dockerId.trim().slice(0, 12);
|
||||
|
||||
deploymentLockHeartbeat?.assertOwned();
|
||||
|
||||
await prisma.container.updateMany({
|
||||
where: {
|
||||
serverId: install.serverId,
|
||||
@@ -238,6 +283,14 @@ export async function rebuildAppInstall(options: {
|
||||
},
|
||||
});
|
||||
|
||||
if (deploymentId) {
|
||||
await updateDeployment(deploymentId, {
|
||||
status: "SUCCESS",
|
||||
completedAt: new Date(),
|
||||
image: runtimeConfig.image,
|
||||
});
|
||||
}
|
||||
|
||||
await auditLog({
|
||||
userId: options.userId,
|
||||
serverId: install.serverId,
|
||||
@@ -283,6 +336,9 @@ export async function rebuildAppInstall(options: {
|
||||
usedImageCache,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof DeploymentLockConflictError) {
|
||||
throw error;
|
||||
}
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Failed to rebuild app";
|
||||
|
||||
@@ -299,6 +355,15 @@ export async function rebuildAppInstall(options: {
|
||||
data: { status: "ERROR" },
|
||||
});
|
||||
|
||||
if (deploymentId) {
|
||||
await updateDeployment(deploymentId, {
|
||||
status: "FAILED",
|
||||
error: message,
|
||||
completedAt: new Date(),
|
||||
image: runtimeConfig.image,
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
|
||||
await auditLog({
|
||||
userId: options.userId,
|
||||
serverId: install.serverId,
|
||||
@@ -335,5 +400,13 @@ export async function rebuildAppInstall(options: {
|
||||
}
|
||||
|
||||
throw error instanceof Error ? error : new Error(message);
|
||||
} finally {
|
||||
deploymentLockHeartbeat?.stop();
|
||||
if (deploymentLockToken && deploymentLockContainerId) {
|
||||
await releaseDeploymentLock({
|
||||
containerId: deploymentLockContainerId,
|
||||
token: deploymentLockToken,
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import type { Server } from "@prisma/client";
|
||||
import * as ssh from "./ssh.service";
|
||||
import { sanitizeDeploymentError } from "./deployment-error.service";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_INTERVAL_MS = 2_000;
|
||||
|
||||
type DockerInspectHealth = {
|
||||
State?: {
|
||||
Status?: string;
|
||||
Health?: { Status?: string };
|
||||
};
|
||||
};
|
||||
|
||||
export type ContainerHealthResult = {
|
||||
healthy: boolean;
|
||||
status:
|
||||
| "healthy"
|
||||
| "unhealthy"
|
||||
| "starting"
|
||||
| "not_running"
|
||||
| "unknown";
|
||||
reason: string;
|
||||
attempts: number;
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
export function evaluateDockerHealth(
|
||||
inspect: DockerInspectHealth | null | undefined,
|
||||
): Omit<ContainerHealthResult, "attempts" | "durationMs"> {
|
||||
const state = inspect?.State?.Status?.toLowerCase();
|
||||
if (state === "created" || state === "restarting") {
|
||||
return {
|
||||
healthy: false,
|
||||
status: "starting",
|
||||
reason: `Container runtime state is ${state}`,
|
||||
};
|
||||
}
|
||||
if (state !== "running") {
|
||||
return {
|
||||
healthy: false,
|
||||
status: "not_running",
|
||||
reason: `Container runtime state is ${state || "unknown"}`,
|
||||
};
|
||||
}
|
||||
|
||||
const health = inspect?.State?.Health?.Status?.toLowerCase();
|
||||
if (health === "unhealthy") {
|
||||
return {
|
||||
healthy: false,
|
||||
status: "unhealthy",
|
||||
reason: "Docker health check reported unhealthy",
|
||||
};
|
||||
}
|
||||
if (health === "starting") {
|
||||
return {
|
||||
healthy: false,
|
||||
status: "starting",
|
||||
reason: "Docker health check is still starting",
|
||||
};
|
||||
}
|
||||
if (health === "healthy") {
|
||||
return {
|
||||
healthy: true,
|
||||
status: "healthy",
|
||||
reason: "Docker health check reported healthy",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
healthy: true,
|
||||
status: "healthy",
|
||||
reason: "Container is running and the image has no Docker health check",
|
||||
};
|
||||
}
|
||||
|
||||
export async function waitForDockerHealth(input: {
|
||||
server: Server;
|
||||
containerRef: string;
|
||||
timeoutMs?: number;
|
||||
intervalMs?: number;
|
||||
}): Promise<ContainerHealthResult> {
|
||||
const startedAt = Date.now();
|
||||
const timeoutMs = Math.max(1_000, input.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
const intervalMs = Math.max(250, input.intervalMs ?? DEFAULT_INTERVAL_MS);
|
||||
let attempts = 0;
|
||||
let lastReason = "Docker health check did not complete";
|
||||
|
||||
while (Date.now() - startedAt <= timeoutMs) {
|
||||
attempts += 1;
|
||||
try {
|
||||
const result = evaluateDockerHealth(
|
||||
(await ssh.dockerInspect(
|
||||
input.server,
|
||||
input.containerRef,
|
||||
)) as DockerInspectHealth,
|
||||
);
|
||||
lastReason = result.reason;
|
||||
if (result.healthy) {
|
||||
return { ...result, attempts, durationMs: Date.now() - startedAt };
|
||||
}
|
||||
if (
|
||||
result.status === "unhealthy" ||
|
||||
result.status === "not_running"
|
||||
) {
|
||||
return { ...result, attempts, durationMs: Date.now() - startedAt };
|
||||
}
|
||||
} catch (error) {
|
||||
lastReason = sanitizeDeploymentError(error, {
|
||||
fallback: "Docker health inspection failed",
|
||||
});
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||
}
|
||||
|
||||
return {
|
||||
healthy: false,
|
||||
status: "unknown",
|
||||
reason: `Docker health check timed out after ${timeoutMs}ms: ${lastReason}`,
|
||||
attempts,
|
||||
durationMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
const SENSITIVE_ASSIGNMENT_PATTERN =
|
||||
/\b(password|passwd|secret|token|api[-_]?key|private[-_]?key|credential|authorization)\s*[:=]\s*(?:"[^"]*"|'[^']*'|[^\s,;]+)/gi;
|
||||
const URL_CREDENTIAL_PATTERN =
|
||||
/([a-z][a-z0-9+.-]*:\/\/)([^/\s:@]+):([^@\s/]+)@/gi;
|
||||
const SENSITIVE_QUERY_PATTERN =
|
||||
/([?&](?:access_token|token|api_key|key|secret)=)[^&\s]+/gi;
|
||||
const PRIVATE_KEY_PATTERN =
|
||||
/-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/gi;
|
||||
|
||||
function errorText(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error ?? "");
|
||||
}
|
||||
|
||||
function redactKnownSecrets(message: string, secrets: Array<string | null | undefined>) {
|
||||
return secrets.reduce<string>((current, secret) => {
|
||||
const normalized = secret?.trim();
|
||||
return normalized ? current.split(normalized).join("[REDACTED]") : current;
|
||||
}, message);
|
||||
}
|
||||
|
||||
export function redactDeploymentErrorDetails(
|
||||
error: unknown,
|
||||
secrets: Array<string | null | undefined> = [],
|
||||
) {
|
||||
return redactKnownSecrets(errorText(error), secrets)
|
||||
.replace(PRIVATE_KEY_PATTERN, "[REDACTED PRIVATE KEY]")
|
||||
.replace(
|
||||
/\bauthorization\s*:\s*(?:Bearer|Basic)\s+\S+/gi,
|
||||
"authorization: [REDACTED]",
|
||||
)
|
||||
.replace(
|
||||
/\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi,
|
||||
"[REDACTED AUTHORIZATION]",
|
||||
)
|
||||
.replace(URL_CREDENTIAL_PATTERN, "$1[REDACTED]@")
|
||||
.replace(SENSITIVE_QUERY_PATTERN, "$1[REDACTED]")
|
||||
.replace(SENSITIVE_ASSIGNMENT_PATTERN, "$1=[REDACTED]")
|
||||
.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function sanitizeDeploymentError(
|
||||
error: unknown,
|
||||
options: {
|
||||
fallback?: string;
|
||||
secrets?: Array<string | null | undefined>;
|
||||
} = {},
|
||||
) {
|
||||
const fallback = options.fallback || "Deployment operation failed";
|
||||
const message = redactDeploymentErrorDetails(error, options.secrets);
|
||||
if (!message) return fallback;
|
||||
|
||||
if (/address already in use|port is already allocated|bind.*failed/i.test(message)) {
|
||||
return "The deployment could not start because a required host port is already in use.";
|
||||
}
|
||||
|
||||
if (/no such image|pull access denied|manifest unknown/i.test(message)) {
|
||||
return "The deployment image could not be resolved or pulled from its registry.";
|
||||
}
|
||||
|
||||
if (/no such container|container .* not found/i.test(message)) {
|
||||
return "The target container could not be found on the selected server.";
|
||||
}
|
||||
|
||||
if (
|
||||
/command failed|docker (?:run|pull|inspect|rename|stop|rm)|bash:|sh:|stderr|stdout|prisma|stack|ssh/i.test(
|
||||
message,
|
||||
)
|
||||
) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
return message.slice(0, 800);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { Prisma } from "@prisma/client";
|
||||
import prisma from "../lib/prisma";
|
||||
|
||||
export const DEFAULT_DEPLOYMENT_LOCK_TTL_MS = 15 * 60 * 1000;
|
||||
export const DEFAULT_DEPLOYMENT_LOCK_HEARTBEAT_MS = 60 * 1000;
|
||||
|
||||
export class DeploymentLockConflictError extends Error {
|
||||
constructor() {
|
||||
super("Another deployment operation is already running for this container");
|
||||
this.name = "DeploymentLockConflictError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function acquireDeploymentLock(input: {
|
||||
containerId: string;
|
||||
ttlMs?: number;
|
||||
}) {
|
||||
const token = randomUUID();
|
||||
const expiresAt = new Date(
|
||||
Date.now() +
|
||||
Math.max(30_000, input.ttlMs ?? DEFAULT_DEPLOYMENT_LOCK_TTL_MS),
|
||||
);
|
||||
|
||||
try {
|
||||
return await prisma.$transaction(async (tx) => {
|
||||
const reclaimed = await tx.deploymentLock.updateMany({
|
||||
where: { containerId: input.containerId, expiresAt: { lte: new Date() } },
|
||||
data: { token, expiresAt },
|
||||
});
|
||||
|
||||
if (reclaimed.count > 0) return { token, expiresAt };
|
||||
|
||||
await tx.deploymentLock.create({
|
||||
data: { containerId: input.containerId, token, expiresAt },
|
||||
});
|
||||
return { token, expiresAt };
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") {
|
||||
throw new DeploymentLockConflictError();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function releaseDeploymentLock(input: {
|
||||
containerId: string;
|
||||
token: string;
|
||||
}) {
|
||||
await prisma.deploymentLock.deleteMany({
|
||||
where: { containerId: input.containerId, token: input.token },
|
||||
});
|
||||
}
|
||||
|
||||
export async function renewDeploymentLock(input: {
|
||||
containerId: string;
|
||||
token: string;
|
||||
ttlMs?: number;
|
||||
}) {
|
||||
const result = await prisma.deploymentLock.updateMany({
|
||||
where: { containerId: input.containerId, token: input.token, expiresAt: { gt: new Date() } },
|
||||
data: {
|
||||
expiresAt: new Date(
|
||||
Date.now() +
|
||||
Math.max(30_000, input.ttlMs ?? DEFAULT_DEPLOYMENT_LOCK_TTL_MS),
|
||||
),
|
||||
},
|
||||
});
|
||||
if (result.count !== 1) throw new DeploymentLockConflictError();
|
||||
}
|
||||
|
||||
export function startDeploymentLockHeartbeat(
|
||||
input: {
|
||||
containerId: string;
|
||||
token: string;
|
||||
ttlMs?: number;
|
||||
intervalMs?: number;
|
||||
},
|
||||
renew = renewDeploymentLock,
|
||||
) {
|
||||
const ttlMs = Math.max(
|
||||
30_000,
|
||||
input.ttlMs ?? DEFAULT_DEPLOYMENT_LOCK_TTL_MS,
|
||||
);
|
||||
const intervalMs = Math.min(
|
||||
Math.max(10, input.intervalMs ?? DEFAULT_DEPLOYMENT_LOCK_HEARTBEAT_MS),
|
||||
Math.max(10, Math.floor(ttlMs / 3)),
|
||||
);
|
||||
let stopped = false;
|
||||
let renewing = false;
|
||||
let ownershipError: Error | null = null;
|
||||
|
||||
const timer = setInterval(() => {
|
||||
if (stopped || renewing || ownershipError) return;
|
||||
renewing = true;
|
||||
void renew({
|
||||
containerId: input.containerId,
|
||||
token: input.token,
|
||||
ttlMs,
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
ownershipError =
|
||||
error instanceof Error
|
||||
? error
|
||||
: new DeploymentLockConflictError();
|
||||
})
|
||||
.finally(() => {
|
||||
renewing = false;
|
||||
});
|
||||
}, intervalMs);
|
||||
timer.unref?.();
|
||||
|
||||
return {
|
||||
assertOwned() {
|
||||
if (ownershipError) {
|
||||
throw new DeploymentLockConflictError();
|
||||
}
|
||||
},
|
||||
hasLostOwnership() {
|
||||
return ownershipError !== null;
|
||||
},
|
||||
stop() {
|
||||
stopped = true;
|
||||
clearInterval(timer);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,714 @@
|
||||
import prisma from "../lib/prisma";
|
||||
import { auditLog } from "./audit.service";
|
||||
import {
|
||||
createDeployment,
|
||||
getRollbackSnapshot,
|
||||
updateDeployment,
|
||||
type DeploymentSnapshot,
|
||||
} from "./deployment.service";
|
||||
import * as ssh from "./ssh.service";
|
||||
import { waitForDockerHealth } from "./container-health.service";
|
||||
import { resolveDeploymentStrategy } from "./deployment-strategy";
|
||||
import {
|
||||
acquireDeploymentLock,
|
||||
releaseDeploymentLock,
|
||||
startDeploymentLockHeartbeat,
|
||||
} from "./deployment-lock.service";
|
||||
import { sanitizeDeploymentError } from "./deployment-error.service";
|
||||
|
||||
type RollbackRuntime = {
|
||||
image: string;
|
||||
ports: string;
|
||||
env: string;
|
||||
volumes: string;
|
||||
network: string;
|
||||
restartPolicy: string;
|
||||
entrypoint?: string;
|
||||
commandArgs?: string[];
|
||||
command: string;
|
||||
};
|
||||
|
||||
type DockerInspectRuntime = {
|
||||
Config?: {
|
||||
Image?: string;
|
||||
Env?: string[];
|
||||
Entrypoint?: string[] | string | null;
|
||||
Cmd?: string[] | null;
|
||||
};
|
||||
HostConfig?: {
|
||||
RestartPolicy?: { Name?: string | null };
|
||||
PortBindings?: Record<
|
||||
string,
|
||||
Array<{ HostPort?: string }> | null
|
||||
>;
|
||||
NetworkMode?: string | null;
|
||||
};
|
||||
Mounts?: Array<{
|
||||
Source?: string;
|
||||
Destination?: string;
|
||||
RW?: boolean;
|
||||
}>;
|
||||
};
|
||||
|
||||
type DockerPortBindings = NonNullable<
|
||||
NonNullable<DockerInspectRuntime["HostConfig"]>["PortBindings"]
|
||||
>;
|
||||
|
||||
type RollbackServer = Parameters<typeof ssh.runContainer>[0];
|
||||
|
||||
type RollbackRuntimeDependencies = {
|
||||
runContainer: typeof ssh.runContainer;
|
||||
stopAndRemoveContainer: typeof stopAndRemoveContainer;
|
||||
dockerRename: typeof ssh.dockerRename;
|
||||
waitForDockerHealth: typeof waitForDockerHealth;
|
||||
};
|
||||
|
||||
const rollbackRuntimeDependencies: RollbackRuntimeDependencies = {
|
||||
runContainer: ssh.runContainer,
|
||||
stopAndRemoveContainer,
|
||||
dockerRename: ssh.dockerRename,
|
||||
waitForDockerHealth,
|
||||
};
|
||||
|
||||
function snapshotString(snapshot: DeploymentSnapshot, key: string, fallback = "") {
|
||||
const value = snapshot[key];
|
||||
return typeof value === "string" ? value : fallback;
|
||||
}
|
||||
|
||||
export function normalizeRollbackPortMappings(value: unknown) {
|
||||
const entries = Array.isArray(value)
|
||||
? value.filter((item): item is string => typeof item === "string")
|
||||
: typeof value === "string"
|
||||
? value.split(",")
|
||||
: [];
|
||||
const normalized = new Map<string, string>();
|
||||
|
||||
for (const rawEntry of entries) {
|
||||
const entry = rawEntry.trim();
|
||||
if (!entry) continue;
|
||||
|
||||
if (!entry.includes("->")) {
|
||||
normalized.set(entry, entry);
|
||||
continue;
|
||||
}
|
||||
|
||||
const [publishedRaw, containerRaw] = entry.split("->", 2);
|
||||
const published = publishedRaw?.trim();
|
||||
const containerSpec = containerRaw?.trim();
|
||||
if (!published || !containerSpec) continue;
|
||||
|
||||
const [containerPort, protocol = "tcp"] = containerSpec.split("/");
|
||||
const hostPort = published.match(/(?:^|:)(\d+)$/)?.[1];
|
||||
if (!hostPort || !/^\d+$/.test(containerPort ?? "")) continue;
|
||||
|
||||
const hostIpMatch = published.match(
|
||||
/^((?:\d{1,3}\.){3}\d{1,3}):\d+$/,
|
||||
);
|
||||
const hostIp =
|
||||
hostIpMatch?.[1] && hostIpMatch[1] !== "0.0.0.0"
|
||||
? `${hostIpMatch[1]}:`
|
||||
: "";
|
||||
const mapping = `${hostIp}${hostPort}:${containerPort}${
|
||||
protocol !== "tcp" ? `/${protocol}` : ""
|
||||
}`;
|
||||
normalized.set(`${hostPort}:${containerPort}/${protocol}`, mapping);
|
||||
}
|
||||
|
||||
return [...normalized.values()].join(",");
|
||||
}
|
||||
|
||||
function storedList(value: unknown) {
|
||||
return Array.isArray(value)
|
||||
? value.filter((item): item is string => typeof item === "string")
|
||||
: [];
|
||||
}
|
||||
|
||||
function formatPortBindings(bindings?: DockerPortBindings) {
|
||||
if (!bindings) return "";
|
||||
|
||||
return Object.entries(bindings)
|
||||
.flatMap(([containerPortSpec, hostBindings]) => {
|
||||
if (!hostBindings?.length) return [];
|
||||
const [containerPort, protocol = "tcp"] = containerPortSpec.split("/");
|
||||
return hostBindings.flatMap((binding) => {
|
||||
const hostPort = binding.HostPort?.trim();
|
||||
if (!hostPort) return [];
|
||||
return `${hostPort}:${containerPort}${protocol !== "tcp" ? `/${protocol}` : ""}`;
|
||||
});
|
||||
})
|
||||
.join(",");
|
||||
}
|
||||
|
||||
function formatMountBindings(mounts?: DockerInspectRuntime["Mounts"]) {
|
||||
return (mounts ?? [])
|
||||
.flatMap((mount) => {
|
||||
if (!mount.Source || !mount.Destination) return [];
|
||||
return `${mount.Source}:${mount.Destination}${mount.RW === false ? ":ro" : ""}`;
|
||||
})
|
||||
.join(",");
|
||||
}
|
||||
|
||||
function formatEntrypoint(value?: string[] | string | null) {
|
||||
if (typeof value === "string") return value.trim() || undefined;
|
||||
return value?.[0]?.trim() || undefined;
|
||||
}
|
||||
|
||||
function snapshotRuntime(
|
||||
snapshot: DeploymentSnapshot,
|
||||
fallbackImage?: string | null,
|
||||
): RollbackRuntime {
|
||||
const image = (snapshotString(snapshot, "image") || fallbackImage || "").trim();
|
||||
if (!image) throw new Error("Rollback artifact does not contain an image");
|
||||
|
||||
return {
|
||||
image,
|
||||
ports: normalizeRollbackPortMappings(snapshot.ports),
|
||||
env: snapshotString(snapshot, "env"),
|
||||
volumes: snapshotString(snapshot, "volumes"),
|
||||
network: snapshotString(snapshot, "network", "bridge"),
|
||||
restartPolicy: snapshotString(snapshot, "restartPolicy", "unless-stopped"),
|
||||
entrypoint: snapshotString(snapshot, "entrypoint") || undefined,
|
||||
commandArgs: storedList(snapshot.commandArgs),
|
||||
command: snapshotString(snapshot, "command"),
|
||||
};
|
||||
}
|
||||
|
||||
type RollbackImageDependencies = {
|
||||
dockerInspect: typeof ssh.dockerInspect;
|
||||
dockerPullImage: typeof ssh.dockerPullImage;
|
||||
};
|
||||
|
||||
const rollbackImageDependencies: RollbackImageDependencies = {
|
||||
dockerInspect: ssh.dockerInspect,
|
||||
dockerPullImage: ssh.dockerPullImage,
|
||||
};
|
||||
|
||||
async function pullOrUseLocalImage(
|
||||
server: RollbackServer,
|
||||
image: string,
|
||||
dependencies: RollbackImageDependencies,
|
||||
) {
|
||||
try {
|
||||
await dependencies.dockerPullImage(server, image);
|
||||
return image;
|
||||
} catch (pullError) {
|
||||
try {
|
||||
await dependencies.dockerInspect(server, image);
|
||||
return image;
|
||||
} catch {
|
||||
throw pullError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveRollbackImageReference(
|
||||
input: {
|
||||
server: RollbackServer;
|
||||
image: string;
|
||||
imageDigest?: string | null;
|
||||
},
|
||||
dependencies: RollbackImageDependencies = rollbackImageDependencies,
|
||||
) {
|
||||
const image = input.image.trim();
|
||||
const digest = input.imageDigest?.trim() ?? "";
|
||||
|
||||
// Existing records store Docker's local image ID (`sha256:...`) in
|
||||
// imageDigest. It is not a registry manifest digest and must never be
|
||||
// appended to an image name as `image@sha256:...`.
|
||||
if (/^sha256:[a-f0-9]{64}$/i.test(digest)) {
|
||||
try {
|
||||
await dependencies.dockerInspect(input.server, digest);
|
||||
return digest;
|
||||
} catch {
|
||||
// The local immutable artifact may have been pruned. Fall back to the
|
||||
// stored image reference so legacy records remain usable when possible.
|
||||
}
|
||||
}
|
||||
|
||||
if (digest.includes("@sha256:")) {
|
||||
try {
|
||||
return await pullOrUseLocalImage(input.server, digest, dependencies);
|
||||
} catch {
|
||||
// Fall back to the image reference below for older/incomplete records.
|
||||
}
|
||||
}
|
||||
|
||||
return pullOrUseLocalImage(input.server, image, dependencies);
|
||||
}
|
||||
|
||||
async function resolvePreviousRuntime(input: {
|
||||
container: {
|
||||
name: string;
|
||||
dockerId: string | null;
|
||||
image: string;
|
||||
ports: unknown;
|
||||
envVars: unknown;
|
||||
volumes: unknown;
|
||||
restartPolicy: string;
|
||||
server: RollbackServer;
|
||||
};
|
||||
}): Promise<RollbackRuntime> {
|
||||
try {
|
||||
const inspect = (await ssh.dockerInspect(
|
||||
input.container.server,
|
||||
input.container.dockerId || input.container.name,
|
||||
)) as DockerInspectRuntime;
|
||||
|
||||
return {
|
||||
image: inspect.Config?.Image?.trim() || input.container.image,
|
||||
ports: formatPortBindings(inspect.HostConfig?.PortBindings),
|
||||
env: (inspect.Config?.Env ?? []).filter(Boolean).join("\n"),
|
||||
volumes: formatMountBindings(inspect.Mounts),
|
||||
network: inspect.HostConfig?.NetworkMode?.trim() || "bridge",
|
||||
restartPolicy:
|
||||
inspect.HostConfig?.RestartPolicy?.Name?.trim() ||
|
||||
input.container.restartPolicy ||
|
||||
"unless-stopped",
|
||||
entrypoint: formatEntrypoint(inspect.Config?.Entrypoint),
|
||||
commandArgs: inspect.Config?.Cmd?.filter(
|
||||
(argument): argument is string => typeof argument === "string",
|
||||
),
|
||||
command: "",
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
image: input.container.image,
|
||||
ports: storedList(input.container.ports).join(","),
|
||||
env: storedList(input.container.envVars).join("\n"),
|
||||
volumes: storedList(input.container.volumes).join(","),
|
||||
network: "bridge",
|
||||
restartPolicy: input.container.restartPolicy || "unless-stopped",
|
||||
command: "",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function stopAndRemoveContainer(
|
||||
server: Parameters<typeof ssh.dockerAction>[0],
|
||||
candidates: string[],
|
||||
) {
|
||||
for (const candidate of candidates.filter(Boolean)) {
|
||||
try {
|
||||
await ssh.dockerAction(server, candidate, "stop");
|
||||
} catch {
|
||||
// The runtime may already be stopped; removal below is still attempted.
|
||||
}
|
||||
|
||||
try {
|
||||
await ssh.dockerAction(server, candidate, "rm");
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!/not found|no such container/i.test(String(error))) throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForRuntimeHealth(input: {
|
||||
server: RollbackServer;
|
||||
containerRef: string;
|
||||
runtime: RollbackRuntime;
|
||||
label: string;
|
||||
dependencies: RollbackRuntimeDependencies;
|
||||
}) {
|
||||
const health = await input.dependencies.waitForDockerHealth({
|
||||
server: input.server,
|
||||
containerRef: input.containerRef,
|
||||
});
|
||||
if (!health.healthy) {
|
||||
throw new Error(`${input.label} runtime check failed: ${health.reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateRuntimeBeforeMutation(input: {
|
||||
name: string;
|
||||
runtime: RollbackRuntime;
|
||||
ports: string;
|
||||
}) {
|
||||
ssh.buildDockerRunCommand({
|
||||
name: input.name,
|
||||
image: input.runtime.image,
|
||||
ports: input.ports,
|
||||
env: input.runtime.env,
|
||||
volumes: input.runtime.volumes,
|
||||
network: input.runtime.network,
|
||||
restartPolicy: input.runtime.restartPolicy,
|
||||
entrypoint: input.runtime.entrypoint,
|
||||
commandArgs: input.runtime.commandArgs,
|
||||
command: input.runtime.command,
|
||||
});
|
||||
}
|
||||
|
||||
export class RollbackRuntimeError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly recoveryDockerId: string | null,
|
||||
readonly recoveryMode: "UNCHANGED" | "RECREATED" | "FAILED",
|
||||
readonly recoveryError: string | null = null,
|
||||
) {
|
||||
super(
|
||||
recoveryError
|
||||
? `${message} Previous runtime recovery failed: ${recoveryError}`
|
||||
: message,
|
||||
);
|
||||
this.name = "RollbackRuntimeError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function replaceRuntimeForRollback<TResult = undefined>(
|
||||
input: {
|
||||
server: RollbackServer;
|
||||
containerName: string;
|
||||
currentContainerRefs: string[];
|
||||
temporaryName: string;
|
||||
strategy: ReturnType<typeof resolveDeploymentStrategy>;
|
||||
targetRuntime: RollbackRuntime;
|
||||
previousRuntime: RollbackRuntime;
|
||||
finalize?: (result: {
|
||||
dockerId: string;
|
||||
runtimeRef: string;
|
||||
}) => Promise<TResult>;
|
||||
},
|
||||
dependencies: RollbackRuntimeDependencies = rollbackRuntimeDependencies,
|
||||
) {
|
||||
const targetName =
|
||||
input.strategy === "ATOMIC_RENAME"
|
||||
? input.temporaryName
|
||||
: input.containerName;
|
||||
const targetPorts =
|
||||
input.strategy === "ATOMIC_RENAME" ? "" : input.targetRuntime.ports;
|
||||
|
||||
// Both the target and recovery commands must be valid before the active
|
||||
// runtime is touched. This prevents a deterministic validation error from
|
||||
// causing avoidable downtime.
|
||||
validateRuntimeBeforeMutation({
|
||||
name: targetName,
|
||||
runtime: input.targetRuntime,
|
||||
ports: targetPorts,
|
||||
});
|
||||
validateRuntimeBeforeMutation({
|
||||
name: input.containerName,
|
||||
runtime: input.previousRuntime,
|
||||
ports: input.previousRuntime.ports,
|
||||
});
|
||||
|
||||
let previousRuntimeTouched = false;
|
||||
let candidateRef: string | null = null;
|
||||
|
||||
try {
|
||||
if (input.strategy === "RECREATE_WITH_RECOVERY") {
|
||||
previousRuntimeTouched = true;
|
||||
await dependencies.stopAndRemoveContainer(
|
||||
input.server,
|
||||
input.currentContainerRefs,
|
||||
);
|
||||
}
|
||||
|
||||
const dockerId = await dependencies.runContainer(input.server, {
|
||||
name: targetName,
|
||||
image: input.targetRuntime.image,
|
||||
ports: targetPorts,
|
||||
env: input.targetRuntime.env,
|
||||
volumes: input.targetRuntime.volumes,
|
||||
network: input.targetRuntime.network,
|
||||
restartPolicy: input.targetRuntime.restartPolicy,
|
||||
entrypoint: input.targetRuntime.entrypoint,
|
||||
commandArgs: input.targetRuntime.commandArgs,
|
||||
command: input.targetRuntime.command,
|
||||
});
|
||||
candidateRef = dockerId.trim() || targetName;
|
||||
|
||||
await waitForRuntimeHealth({
|
||||
server: input.server,
|
||||
containerRef: candidateRef,
|
||||
runtime: input.targetRuntime,
|
||||
label: "Rollback",
|
||||
dependencies,
|
||||
});
|
||||
|
||||
if (input.strategy === "ATOMIC_RENAME") {
|
||||
previousRuntimeTouched = true;
|
||||
await dependencies.stopAndRemoveContainer(
|
||||
input.server,
|
||||
input.currentContainerRefs,
|
||||
);
|
||||
await dependencies.dockerRename(
|
||||
input.server,
|
||||
input.temporaryName,
|
||||
input.containerName,
|
||||
);
|
||||
}
|
||||
|
||||
const result = input.finalize
|
||||
? await input.finalize({
|
||||
dockerId: dockerId.trim(),
|
||||
runtimeRef: candidateRef,
|
||||
})
|
||||
: (undefined as TResult);
|
||||
|
||||
return { dockerId: dockerId.trim(), runtimeRef: candidateRef, result };
|
||||
} catch (error) {
|
||||
const message = sanitizeDeploymentError(error, {
|
||||
fallback: "Rollback runtime replacement failed",
|
||||
});
|
||||
|
||||
if (!previousRuntimeTouched) {
|
||||
try {
|
||||
await dependencies.stopAndRemoveContainer(input.server, [
|
||||
candidateRef ?? "",
|
||||
input.temporaryName,
|
||||
]);
|
||||
const previousRef =
|
||||
input.currentContainerRefs.find((value) => value.trim()) ??
|
||||
input.containerName;
|
||||
await waitForRuntimeHealth({
|
||||
server: input.server,
|
||||
containerRef: previousRef,
|
||||
runtime: input.previousRuntime,
|
||||
label: "Previous runtime",
|
||||
dependencies,
|
||||
});
|
||||
throw new RollbackRuntimeError(message, previousRef, "UNCHANGED");
|
||||
} catch (recoveryError) {
|
||||
if (recoveryError instanceof RollbackRuntimeError) throw recoveryError;
|
||||
throw new RollbackRuntimeError(
|
||||
message,
|
||||
null,
|
||||
"FAILED",
|
||||
recoveryError instanceof Error
|
||||
? sanitizeDeploymentError(recoveryError, {
|
||||
fallback: "Previous runtime recovery failed",
|
||||
})
|
||||
: "Previous runtime recovery failed",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await dependencies.stopAndRemoveContainer(input.server, [
|
||||
candidateRef ?? "",
|
||||
input.temporaryName,
|
||||
input.containerName,
|
||||
]);
|
||||
const recoveryDockerId = await dependencies.runContainer(input.server, {
|
||||
name: input.containerName,
|
||||
image: input.previousRuntime.image,
|
||||
ports: input.previousRuntime.ports,
|
||||
env: input.previousRuntime.env,
|
||||
volumes: input.previousRuntime.volumes,
|
||||
network: input.previousRuntime.network,
|
||||
restartPolicy: input.previousRuntime.restartPolicy,
|
||||
entrypoint: input.previousRuntime.entrypoint,
|
||||
commandArgs: input.previousRuntime.commandArgs,
|
||||
command: input.previousRuntime.command,
|
||||
});
|
||||
const recoveryRef = recoveryDockerId.trim() || input.containerName;
|
||||
await waitForRuntimeHealth({
|
||||
server: input.server,
|
||||
containerRef: recoveryRef,
|
||||
runtime: input.previousRuntime,
|
||||
label: "Previous runtime",
|
||||
dependencies,
|
||||
});
|
||||
throw new RollbackRuntimeError(
|
||||
message,
|
||||
recoveryDockerId.trim(),
|
||||
"RECREATED",
|
||||
);
|
||||
} catch (recoveryError) {
|
||||
if (recoveryError instanceof RollbackRuntimeError) throw recoveryError;
|
||||
throw new RollbackRuntimeError(
|
||||
message,
|
||||
null,
|
||||
"FAILED",
|
||||
recoveryError instanceof Error
|
||||
? sanitizeDeploymentError(recoveryError, {
|
||||
fallback: "Previous runtime recovery failed",
|
||||
})
|
||||
: "Previous runtime recovery failed",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function rollbackContainerToDeployment(input: {
|
||||
containerId: string;
|
||||
deploymentId: string;
|
||||
organizationId: string;
|
||||
userId?: string;
|
||||
}) {
|
||||
const container = await prisma.container.findFirst({
|
||||
where: {
|
||||
id: input.containerId,
|
||||
server: { organizationId: input.organizationId },
|
||||
},
|
||||
include: { server: true },
|
||||
});
|
||||
if (!container) throw new Error("Container not found");
|
||||
|
||||
const target = await getRollbackSnapshot(input);
|
||||
if (!target) {
|
||||
throw new Error("Only a successful deployment in this container can be rolled back");
|
||||
}
|
||||
if (target.serverId !== container.serverId) {
|
||||
throw new Error("Rollback target belongs to a different server");
|
||||
}
|
||||
|
||||
const runningDeployment = await prisma.deployment.findFirst({
|
||||
where: {
|
||||
containerId: container.id,
|
||||
organizationId: input.organizationId,
|
||||
status: "RUNNING",
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
if (runningDeployment) {
|
||||
throw new Error("Another deployment operation is already running for this container");
|
||||
}
|
||||
|
||||
const runtime = snapshotRuntime(target.snapshot, target.image);
|
||||
const strategy = resolveDeploymentStrategy(runtime.ports);
|
||||
const lock = await acquireDeploymentLock({ containerId: container.id });
|
||||
const lockHeartbeat = startDeploymentLockHeartbeat({
|
||||
containerId: container.id,
|
||||
token: lock.token,
|
||||
});
|
||||
const rollback = await createDeployment({
|
||||
containerId: container.id,
|
||||
organizationId: input.organizationId,
|
||||
serverId: container.serverId,
|
||||
userId: input.userId,
|
||||
status: "RUNNING",
|
||||
trigger: "ROLLBACK",
|
||||
version: target.version ?? runtime.image,
|
||||
image: runtime.image,
|
||||
imageDigest: target.imageDigest,
|
||||
configSnapshot: { ...target.snapshot, deploymentStrategy: strategy },
|
||||
startedAt: new Date(),
|
||||
}).catch(async (error) => {
|
||||
lockHeartbeat.stop();
|
||||
await releaseDeploymentLock({ containerId: container.id, token: lock.token });
|
||||
throw error;
|
||||
});
|
||||
|
||||
const previousRuntime = await resolvePreviousRuntime({
|
||||
container,
|
||||
});
|
||||
|
||||
try {
|
||||
const image = await resolveRollbackImageReference({
|
||||
server: container.server,
|
||||
image: runtime.image,
|
||||
imageDigest: target.imageDigest,
|
||||
});
|
||||
runtime.image = image;
|
||||
|
||||
const atomicName = `${container.name}-rollback-${rollback.id.slice(-8)}`.slice(0, 128);
|
||||
const replacement = await replaceRuntimeForRollback({
|
||||
server: container.server,
|
||||
containerName: container.name,
|
||||
currentContainerRefs: [container.dockerId ?? "", container.name],
|
||||
temporaryName: atomicName,
|
||||
strategy,
|
||||
targetRuntime: runtime,
|
||||
previousRuntime,
|
||||
finalize: async ({ dockerId }) => {
|
||||
lockHeartbeat.assertOwned();
|
||||
const updated = await prisma.container.update({
|
||||
where: { id: container.id },
|
||||
data: {
|
||||
image,
|
||||
status: "RUNNING",
|
||||
dockerId: dockerId.trim().slice(0, 12) || null,
|
||||
},
|
||||
include: { server: { select: { name: true, ip: true } } },
|
||||
});
|
||||
|
||||
await updateDeployment(rollback.id, {
|
||||
status: "SUCCESS",
|
||||
completedAt: new Date(),
|
||||
image,
|
||||
imageDigest: target.imageDigest,
|
||||
});
|
||||
await auditLog({
|
||||
userId: input.userId,
|
||||
organizationId: input.organizationId,
|
||||
serverId: container.serverId,
|
||||
action: "CONTAINER_ROLLBACK",
|
||||
category: "CONTAINER",
|
||||
level: "SUCCESS",
|
||||
message: `Container "${container.name}" rolled back to deployment ${target.id}`,
|
||||
meta: { deploymentId: rollback.id, targetDeploymentId: target.id, image },
|
||||
});
|
||||
|
||||
return updated;
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
updated: replacement.result,
|
||||
deploymentId: rollback.id,
|
||||
targetDeploymentId: target.id,
|
||||
};
|
||||
} catch (error) {
|
||||
const message = sanitizeDeploymentError(error, {
|
||||
fallback: "Rollback failed",
|
||||
});
|
||||
const runtimeError =
|
||||
error instanceof RollbackRuntimeError ? error : null;
|
||||
if (runtimeError?.recoveryMode !== "FAILED") {
|
||||
await prisma.container.update({
|
||||
where: { id: container.id },
|
||||
data: {
|
||||
status: "RUNNING",
|
||||
image: previousRuntime.image,
|
||||
dockerId:
|
||||
runtimeError?.recoveryDockerId?.trim().slice(0, 12) ||
|
||||
container.dockerId,
|
||||
},
|
||||
});
|
||||
await auditLog({
|
||||
userId: input.userId,
|
||||
organizationId: input.organizationId,
|
||||
serverId: container.serverId,
|
||||
action: "CONTAINER_ROLLBACK_RECOVERY",
|
||||
category: "CONTAINER",
|
||||
level: "WARNING",
|
||||
message: `Previous runtime for container "${container.name}" was recovered after rollback failure`,
|
||||
meta: {
|
||||
deploymentId: rollback.id,
|
||||
recovery:
|
||||
runtimeError?.recoveryMode === "UNCHANGED"
|
||||
? "UNCHANGED_HEALTHY"
|
||||
: "RECREATED_HEALTHY",
|
||||
},
|
||||
});
|
||||
} else {
|
||||
await prisma.container.update({
|
||||
where: { id: container.id },
|
||||
data: { status: "ERROR" },
|
||||
});
|
||||
await auditLog({
|
||||
userId: input.userId,
|
||||
organizationId: input.organizationId,
|
||||
serverId: container.serverId,
|
||||
action: "CONTAINER_ROLLBACK_RECOVERY",
|
||||
category: "CONTAINER",
|
||||
level: "ERROR",
|
||||
message: `Previous runtime recovery failed for container "${container.name}"`,
|
||||
meta: { deploymentId: rollback.id, recovery: "FAILED" },
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
|
||||
await updateDeployment(rollback.id, {
|
||||
status: "FAILED",
|
||||
error: message,
|
||||
completedAt: new Date(),
|
||||
}).catch(() => undefined);
|
||||
throw new Error(message);
|
||||
} finally {
|
||||
lockHeartbeat.stop();
|
||||
await releaseDeploymentLock({ containerId: container.id, token: lock.token });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
export type DeploymentStrategy = "ATOMIC_RENAME" | "RECREATE_WITH_RECOVERY";
|
||||
|
||||
export function resolveDeploymentStrategy(ports: string): DeploymentStrategy {
|
||||
return ports.trim() ? "RECREATE_WITH_RECOVERY" : "ATOMIC_RENAME";
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
import {
|
||||
DeploymentStatus,
|
||||
DeploymentTrigger,
|
||||
Prisma,
|
||||
} from "@prisma/client";
|
||||
import prisma from "../lib/prisma";
|
||||
import { decrypt, encrypt } from "../lib/crypto";
|
||||
|
||||
const SENSITIVE_KEY_PATTERN = /(pass(word)?|secret|token|api[-_]?key|private[-_]?key|credential)/i;
|
||||
|
||||
function redactEnvironmentLine(line: string) {
|
||||
const separator = line.indexOf("=");
|
||||
if (separator < 1) return line;
|
||||
|
||||
const key = line.slice(0, separator).trim();
|
||||
return SENSITIVE_KEY_PATTERN.test(key)
|
||||
? `${key}=<redacted>`
|
||||
: line;
|
||||
}
|
||||
|
||||
function sanitizeSnapshotValue(value: unknown, key?: string): unknown {
|
||||
if (key && SENSITIVE_KEY_PATTERN.test(key)) return "<redacted>";
|
||||
|
||||
if (typeof value === "string") {
|
||||
if (key?.toLowerCase().includes("env")) {
|
||||
return value
|
||||
.split(/\r?\n/)
|
||||
.map(redactEnvironmentLine)
|
||||
.join("\n");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => sanitizeSnapshotValue(item));
|
||||
}
|
||||
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([entryKey, entryValue]) => [
|
||||
entryKey,
|
||||
sanitizeSnapshotValue(entryValue, entryKey),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
export type DeploymentSnapshot = Record<string, unknown>;
|
||||
|
||||
export type CreateDeploymentInput = {
|
||||
containerId: string;
|
||||
organizationId: string;
|
||||
serverId: string;
|
||||
userId?: string;
|
||||
status: DeploymentStatus;
|
||||
trigger: DeploymentTrigger;
|
||||
version?: string | null;
|
||||
commitSha?: string | null;
|
||||
branch?: string | null;
|
||||
image?: string | null;
|
||||
imageDigest?: string | null;
|
||||
configSnapshot: DeploymentSnapshot;
|
||||
error?: string | null;
|
||||
startedAt?: Date | null;
|
||||
completedAt?: Date | null;
|
||||
};
|
||||
|
||||
export function sanitizeDeploymentSnapshot(
|
||||
snapshot: DeploymentSnapshot,
|
||||
): Prisma.InputJsonValue {
|
||||
return sanitizeSnapshotValue(snapshot) as Prisma.InputJsonValue;
|
||||
}
|
||||
|
||||
export async function createDeployment(input: CreateDeploymentInput) {
|
||||
return prisma.deployment.create({
|
||||
data: {
|
||||
containerId: input.containerId,
|
||||
organizationId: input.organizationId,
|
||||
serverId: input.serverId,
|
||||
userId: input.userId ?? null,
|
||||
status: input.status,
|
||||
trigger: input.trigger,
|
||||
version: input.version ?? null,
|
||||
commitSha: input.commitSha ?? null,
|
||||
branch: input.branch ?? null,
|
||||
image: input.image ?? null,
|
||||
imageDigest: input.imageDigest ?? null,
|
||||
configSnapshot: sanitizeDeploymentSnapshot(input.configSnapshot),
|
||||
rollbackSnapshotEnc: encrypt(JSON.stringify(input.configSnapshot)),
|
||||
error: input.error ?? null,
|
||||
startedAt: input.startedAt ?? null,
|
||||
completedAt: input.completedAt ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function getRollbackSnapshot(input: {
|
||||
containerId: string;
|
||||
deploymentId: string;
|
||||
organizationId: string;
|
||||
}) {
|
||||
const deployment = await prisma.deployment.findFirst({
|
||||
where: {
|
||||
id: input.deploymentId,
|
||||
containerId: input.containerId,
|
||||
organizationId: input.organizationId,
|
||||
status: "SUCCESS",
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
containerId: true,
|
||||
serverId: true,
|
||||
version: true,
|
||||
image: true,
|
||||
imageDigest: true,
|
||||
configSnapshot: true,
|
||||
rollbackSnapshotEnc: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) return null;
|
||||
if (!deployment.rollbackSnapshotEnc) {
|
||||
throw new Error(
|
||||
"This deployment has no encrypted rollback artifact. It cannot be restored safely.",
|
||||
);
|
||||
}
|
||||
|
||||
let snapshot: DeploymentSnapshot;
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(decrypt(deployment.rollbackSnapshotEnc));
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
throw new Error("Invalid rollback snapshot");
|
||||
}
|
||||
snapshot = parsed as DeploymentSnapshot;
|
||||
} catch {
|
||||
throw new Error("Rollback artifact is invalid or cannot be decrypted");
|
||||
}
|
||||
|
||||
return { ...deployment, snapshot };
|
||||
}
|
||||
|
||||
export async function updateDeployment(
|
||||
id: string,
|
||||
patch: {
|
||||
status: DeploymentStatus;
|
||||
error?: string | null;
|
||||
completedAt?: Date | null;
|
||||
image?: string | null;
|
||||
imageDigest?: string | null;
|
||||
commitSha?: string | null;
|
||||
branch?: string | null;
|
||||
configSnapshot?: DeploymentSnapshot;
|
||||
},
|
||||
) {
|
||||
return prisma.deployment.update({
|
||||
where: { id },
|
||||
data: {
|
||||
status: patch.status,
|
||||
error: patch.error ?? null,
|
||||
completedAt: patch.completedAt ?? null,
|
||||
...(patch.image === undefined ? {} : { image: patch.image }),
|
||||
...(patch.imageDigest === undefined
|
||||
? {}
|
||||
: { imageDigest: patch.imageDigest }),
|
||||
...(patch.commitSha === undefined ? {} : { commitSha: patch.commitSha }),
|
||||
...(patch.branch === undefined ? {} : { branch: patch.branch }),
|
||||
...(patch.configSnapshot === undefined
|
||||
? {}
|
||||
: {
|
||||
configSnapshot: sanitizeDeploymentSnapshot(patch.configSnapshot),
|
||||
rollbackSnapshotEnc: encrypt(
|
||||
JSON.stringify(patch.configSnapshot),
|
||||
),
|
||||
}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function listDeployments(input: {
|
||||
containerId: string;
|
||||
organizationId: string;
|
||||
page: number;
|
||||
pageSize: number;
|
||||
}) {
|
||||
const skip = (input.page - 1) * input.pageSize;
|
||||
const where = {
|
||||
containerId: input.containerId,
|
||||
organizationId: input.organizationId,
|
||||
};
|
||||
|
||||
const [items, total] = await prisma.$transaction([
|
||||
prisma.deployment.findMany({
|
||||
where,
|
||||
orderBy: { createdAt: "desc" },
|
||||
skip,
|
||||
take: input.pageSize,
|
||||
select: {
|
||||
id: true,
|
||||
containerId: true,
|
||||
status: true,
|
||||
trigger: true,
|
||||
version: true,
|
||||
commitSha: true,
|
||||
branch: true,
|
||||
image: true,
|
||||
imageDigest: true,
|
||||
error: true,
|
||||
startedAt: true,
|
||||
completedAt: true,
|
||||
createdAt: true,
|
||||
user: { select: { id: true, name: true } },
|
||||
},
|
||||
}),
|
||||
prisma.deployment.count({ where }),
|
||||
]);
|
||||
|
||||
return { items, total, page: input.page, pageSize: input.pageSize };
|
||||
}
|
||||
|
||||
export async function getDeployment(input: {
|
||||
containerId: string;
|
||||
deploymentId: string;
|
||||
organizationId: string;
|
||||
}) {
|
||||
return prisma.deployment.findFirst({
|
||||
where: {
|
||||
id: input.deploymentId,
|
||||
containerId: input.containerId,
|
||||
organizationId: input.organizationId,
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
containerId: true,
|
||||
serverId: true,
|
||||
status: true,
|
||||
trigger: true,
|
||||
version: true,
|
||||
commitSha: true,
|
||||
branch: true,
|
||||
image: true,
|
||||
imageDigest: true,
|
||||
configSnapshot: true,
|
||||
error: true,
|
||||
startedAt: true,
|
||||
completedAt: true,
|
||||
createdAt: true,
|
||||
user: { select: { id: true, name: true } },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import yaml from "js-yaml";
|
||||
import { randomBytes } from "crypto";
|
||||
import { posix as pathPosix } from "path";
|
||||
import { Server } from "@prisma/client";
|
||||
import { TextDecoder } from "util";
|
||||
import { Server } from "@prisma/client";
|
||||
import { execStrict } from "./commands";
|
||||
import { execDocker, execDockerStrict } from "./internal/docker";
|
||||
import { privilegedCommand } from "./internal/privilege";
|
||||
@@ -1140,6 +1140,18 @@ export async function listContainerFiles(
|
||||
};
|
||||
}
|
||||
|
||||
export async function dockerRename(
|
||||
server: Server,
|
||||
containerId: string,
|
||||
newName: string,
|
||||
): Promise<void> {
|
||||
await execDockerStrict(
|
||||
server,
|
||||
`docker rename ${escapeShellArg(containerId)} ${escapeShellArg(newName)}`,
|
||||
shortDockerCommandTimeout(DOCKER_ACTION_TIMEOUT_MS),
|
||||
);
|
||||
}
|
||||
|
||||
export async function getContainerMainProcessWorkingDirectory(
|
||||
server: Server,
|
||||
containerId: string,
|
||||
@@ -2518,6 +2530,7 @@ export async function deployContainerFromGitSource(
|
||||
dockerId?: string;
|
||||
imageTag?: string;
|
||||
composeFilePath?: string;
|
||||
commitSha: string;
|
||||
}> {
|
||||
const projectName = sanitizeProjectName(opts.projectName);
|
||||
const deploymentPath =
|
||||
@@ -2572,6 +2585,24 @@ export async function deployContainerFromGitSource(
|
||||
throw new Error(formatDeploymentErrorMessage(error));
|
||||
}
|
||||
|
||||
let commitSha = "";
|
||||
try {
|
||||
commitSha = (
|
||||
await execStrict(
|
||||
server,
|
||||
privilegedCommand(
|
||||
server,
|
||||
`git -C ${escapeShellArg(deploymentPath)} rev-parse HEAD`,
|
||||
),
|
||||
shortDockerCommandTimeout(DOCKER_INSPECT_TIMEOUT_MS),
|
||||
)
|
||||
).trim();
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Repository was cloned but its commit SHA could not be resolved: ${formatDeploymentErrorMessage(error)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (buildType === "COMPOSE") {
|
||||
try {
|
||||
await writeComposeEnvFiles({
|
||||
@@ -2608,6 +2639,7 @@ export async function deployContainerFromGitSource(
|
||||
return {
|
||||
deploymentPath,
|
||||
composeFilePath,
|
||||
commitSha,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2708,6 +2740,7 @@ export async function deployContainerFromGitSource(
|
||||
deploymentPath,
|
||||
dockerId,
|
||||
imageTag,
|
||||
commitSha,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2813,6 +2846,7 @@ export async function deployContainerFromGitSource(
|
||||
deploymentPath,
|
||||
dockerId,
|
||||
imageTag,
|
||||
commitSha,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2851,6 +2885,7 @@ export async function deployContainerFromGitSource(
|
||||
deploymentPath,
|
||||
dockerId,
|
||||
imageTag,
|
||||
commitSha,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2916,5 +2951,6 @@ export async function deployContainerFromGitSource(
|
||||
deploymentPath,
|
||||
dockerId,
|
||||
imageTag,
|
||||
commitSha,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user