feat(deployment): add deployment history and rollback support

This commit is contained in:
KodekaTeam committed 2026-07-17 00:27:07 +07:00
1 parent 029b25b596
commit 4e9b173edb
36 files changed
+3663 -569

No files matched your search

+31
View File
@@ -0,0 +1,31 @@
import assert from "node:assert/strict";
import test from "node:test";
import { evaluateDockerHealth } from "../../src/server/services/container-health.service";
test("docker health gate accepts a running image without a healthcheck", () => {
const result = evaluateDockerHealth({ State: { Status: "running" } });
assert.equal(result.healthy, true);
assert.equal(result.status, "healthy");
});
test("docker health gate rejects unhealthy containers", () => {
const result = evaluateDockerHealth({
State: { Status: "running", Health: { Status: "unhealthy" } },
});
assert.equal(result.healthy, false);
assert.equal(result.status, "unhealthy");
});
test("docker health gate rejects containers that are not running", () => {
const result = evaluateDockerHealth({ State: { Status: "exited" } });
assert.equal(result.healthy, false);
assert.equal(result.status, "not_running");
});
test("docker runtime verification keeps polling while a container is restarting", () => {
const result = evaluateDockerHealth({ State: { Status: "restarting" } });
assert.equal(result.healthy, false);
assert.equal(result.status, "starting");
});
@@ -0,0 +1,42 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
DeploymentLockConflictError,
startDeploymentLockHeartbeat,
} from "../../src/server/services/deployment-lock.service";
test("deployment lock heartbeat renews with the owner token and detects ownership loss", async () => {
const renewals: Array<{ containerId: string; token: string }> = [];
let shouldFail = false;
const heartbeat = startDeploymentLockHeartbeat(
{
containerId: "container-1",
token: "owner-token",
ttlMs: 30_000,
intervalMs: 10,
},
async (input) => {
renewals.push({
containerId: input.containerId,
token: input.token,
});
if (shouldFail) throw new DeploymentLockConflictError();
},
);
await new Promise((resolve) => setTimeout(resolve, 25));
assert.ok(renewals.length >= 1);
assert.deepEqual(renewals[0], {
containerId: "container-1",
token: "owner-token",
});
heartbeat.assertOwned();
shouldFail = true;
await new Promise((resolve) => setTimeout(resolve, 25));
assert.equal(heartbeat.hasLostOwnership(), true);
assert.throws(() => heartbeat.assertOwned(), DeploymentLockConflictError);
heartbeat.stop();
});
+22
View File
@@ -0,0 +1,22 @@
import assert from "node:assert/strict";
import test from "node:test";
import { sanitizeDeploymentSnapshot } from "../../src/server/services/deployment.service";
test("deployment snapshots redact sensitive configuration values", () => {
const snapshot = sanitizeDeploymentSnapshot({
env: "APP_ENV=production\nDATABASE_PASSWORD=super-secret\nPUBLIC_URL=https://example.test",
accessToken: "provider-token",
nested: { apiKey: "another-secret" },
image: "nginx:latest",
}) as Record<string, unknown>;
assert.equal(
snapshot.env,
"APP_ENV=production\nDATABASE_PASSWORD=<redacted>\nPUBLIC_URL=https://example.test",
);
assert.equal(snapshot.accessToken, "<redacted>");
assert.deepEqual(snapshot.nested, { apiKey: "<redacted>" });
assert.equal(snapshot.image, "nginx:latest");
});
@@ -0,0 +1,85 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
const CONTAINER_ROUTES = "src/server/routes/containers.ts";
const APP_ROUTES = "src/server/routes/apps.ts";
const DOCKER_CONTAINERS =
"src/server/services/ssh-services/docker-containers.ts";
function readSource(path: string) {
return readFileSync(path, "utf8");
}
function sourceBlock(source: string, startPattern: string, endPattern: string) {
const start = source.indexOf(startPattern);
assert.notEqual(start, -1, `Missing source block start: ${startPattern}`);
const end = source.indexOf(endPattern, start);
assert.notEqual(end, -1, `Missing source block end: ${endPattern}`);
return source.slice(start, end);
}
test("Git rebuild follows the deployment history lifecycle", () => {
const source = readSource(CONTAINER_ROUTES);
const rebuildRoute = sourceBlock(
source,
'"/:id/rebuild"',
"GET /containers/:id/logs",
);
const acquireIndex = rebuildRoute.indexOf("acquireDeploymentLock");
const runningIndex = rebuildRoute.indexOf('status: "RUNNING"');
const mutationIndex = rebuildRoute.indexOf("deployContainerFromGitSource");
const successIndex = rebuildRoute.indexOf('status: "SUCCESS"', mutationIndex);
assert.ok(acquireIndex >= 0);
assert.ok(runningIndex > acquireIndex);
assert.ok(mutationIndex > runningIndex);
assert.ok(successIndex > mutationIndex);
assert.doesNotMatch(rebuildRoute, /waitForDeploymentHealth/);
assert.match(rebuildRoute, /status: "FAILED"/);
assert.match(rebuildRoute, /releaseDeploymentLock/);
assert.match(rebuildRoute, /commitSha: gitDeploymentResult\.commitSha/);
assert.match(rebuildRoute, /imageDigest/);
});
test("Fresh App Installer creates a provisional parent and RUNNING deployment before Docker mutation", () => {
const source = readSource(APP_ROUTES);
const installRoute = sourceBlock(
source,
'app.post("/install"',
'"/installs/:id/action"',
);
const provisionalIndex = installRoute.indexOf(
"const createdContainer = await tx.container.create",
);
const runningIndex = installRoute.indexOf('status: "RUNNING"');
const mutationIndex = installRoute.indexOf(
"const dockerId = await ssh.runContainer",
);
assert.ok(provisionalIndex >= 0);
assert.ok(runningIndex > provisionalIndex);
assert.ok(mutationIndex > runningIndex);
assert.match(installRoute, /where: \{ id: deployment\.id \}/);
assert.match(installRoute, /status: "FAILED"/);
assert.match(installRoute, /where: \{ id: provisionalContainer\.id \}/);
});
test("Git deployment resolves and returns the cloned commit SHA", () => {
const source = readSource(DOCKER_CONTAINERS);
const start = source.indexOf(
"export async function deployContainerFromGitSource",
);
assert.notEqual(start, -1);
const gitDeployFunction = source.slice(start);
assert.match(gitDeployFunction, /git -C .* rev-parse HEAD/);
assert.match(gitDeployFunction, /commitSha: string/);
assert.match(gitDeployFunction, /commitSha,/);
});
@@ -0,0 +1,42 @@
import assert from "node:assert/strict";
import { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import test from "node:test";
import { sanitizeDeploymentSnapshot } from "../../src/server/services/deployment.service";
test("deployment snapshot keeps rollback-safe runtime fields sanitized", () => {
const snapshot = sanitizeDeploymentSnapshot({
image: "example/app@sha256:abc",
ports: "8080:80",
env: "APP_ENV=production\nAPI_TOKEN=secret",
volumes: "/data:/data",
}) as Record<string, unknown>;
assert.equal(snapshot.image, "example/app@sha256:abc");
assert.equal(snapshot.env, "APP_ENV=production\nAPI_TOKEN=<redacted>");
});
test("deployment migrations include the history and rollback schema", () => {
const migrationsPath = join(process.cwd(), "prisma", "migrations");
const migrations = readdirSync(migrationsPath);
assert.ok(migrations.some((name) => name.includes("add_deployment_history")));
assert.ok(migrations.some((name) => name.includes("add_deployment_rollback_snapshot")));
assert.ok(migrations.some((name) => name.includes("add_deployment_locks")));
assert.equal(
migrations.some((name) =>
/deployment_attempt|process_job|service_lease|deployment_health_policy/.test(
name,
),
),
false,
);
const schema = readFileSync(join(process.cwd(), "prisma", "schema.prisma"), "utf8");
assert.match(schema, /model Deployment\s*\{/);
assert.match(schema, /rollbackSnapshotEnc\s+String\?/);
assert.doesNotMatch(schema, /model DeploymentAttempt\s*\{/);
assert.doesNotMatch(schema, /model ProcessJob\s*\{/);
assert.doesNotMatch(schema, /model ServiceLease\s*\{/);
});
@@ -0,0 +1,275 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
normalizeRollbackPortMappings,
replaceRuntimeForRollback,
resolveRollbackImageReference,
RollbackRuntimeError,
} from "../../src/server/services/deployment-rollback.service";
type RollbackInput = Parameters<typeof replaceRuntimeForRollback>[0];
type RollbackDependencies = NonNullable<
Parameters<typeof replaceRuntimeForRollback>[1]
>;
const server = {} as RollbackInput["server"];
test("normalizes Docker display port mappings for rollback", () => {
assert.equal(
normalizeRollbackPortMappings(
"0.0.0.0:8080->80/tcp, [::]:8080->80/tcp, 127.0.0.1:8443->443/tcp",
),
"8080:80,127.0.0.1:8443:443",
);
assert.equal(
normalizeRollbackPortMappings(["8081:81", "9000->90/udp"]),
"8081:81,9000:90/udp",
);
});
test("uses a locally available Docker image ID without creating an invalid registry reference", async () => {
const inspected: string[] = [];
const pulled: string[] = [];
const image = await resolveRollbackImageReference(
{
server,
image: "example/app:latest",
imageDigest: `sha256:${"a".repeat(64)}`,
},
{
dockerInspect: async (_server, imageRef) => {
inspected.push(imageRef);
return {};
},
dockerPullImage: async (_server, imageRef) => {
pulled.push(imageRef);
},
},
);
assert.equal(image, `sha256:${"a".repeat(64)}`);
assert.deepEqual(inspected, [`sha256:${"a".repeat(64)}`]);
assert.deepEqual(pulled, []);
});
test("falls back to the stored image when a historical local image ID was pruned", async () => {
const pulled: string[] = [];
const image = await resolveRollbackImageReference(
{
server,
image: "example/app:stable",
imageDigest: `sha256:${"b".repeat(64)}`,
},
{
dockerInspect: async () => {
throw new Error("image not found");
},
dockerPullImage: async (_server, imageRef) => {
pulled.push(imageRef);
},
},
);
assert.equal(image, "example/app:stable");
assert.deepEqual(pulled, ["example/app:stable"]);
});
function runtime(overrides: Partial<RollbackInput["targetRuntime"]> = {}) {
return {
image: "example/app:target",
ports: "",
env: "APP_ENV=production",
volumes: "/data:/data",
network: "bridge",
restartPolicy: "unless-stopped",
command: "",
...overrides,
};
}
function dependencies(
overrides: Partial<RollbackDependencies> = {},
): RollbackDependencies {
return {
runContainer: async () => "new-container-id",
stopAndRemoveContainer: async () => undefined,
dockerRename: async () => undefined,
waitForDockerHealth: async () => ({
healthy: true,
reason: "running",
status: "healthy",
attempts: 1,
durationMs: 0,
}),
...overrides,
};
}
test("published-port rollback removes the active runtime before claiming its name and host port", async () => {
const events: string[] = [];
await replaceRuntimeForRollback(
{
server,
containerName: "app",
currentContainerRefs: ["old-container-id", "app"],
temporaryName: "app-rollback",
strategy: "RECREATE_WITH_RECOVERY",
targetRuntime: runtime({ ports: "8080:80" }),
previousRuntime: runtime({
image: "example/app:previous",
ports: "8080:80",
}),
},
dependencies({
stopAndRemoveContainer: async (_server, refs) => {
events.push(`remove:${refs.join(",")}`);
},
runContainer: async (_server, options) => {
events.push(`run:${options.name}:${options.ports}`);
return "new-container-id";
},
waitForDockerHealth: async () => {
events.push("runtime-check");
return {
healthy: true,
reason: "running",
status: "healthy",
attempts: 1,
durationMs: 0,
};
},
}),
);
assert.deepEqual(events, [
"remove:old-container-id,app",
"run:app:8080:80",
"runtime-check",
]);
});
test("failed published-port rollback recreates and verifies the previous runtime", async () => {
const events: string[] = [];
let runCount = 0;
await assert.rejects(
replaceRuntimeForRollback(
{
server,
containerName: "app",
currentContainerRefs: ["old-container-id", "app"],
temporaryName: "app-rollback",
strategy: "RECREATE_WITH_RECOVERY",
targetRuntime: runtime({ ports: "8080:80" }),
previousRuntime: runtime({
image: "example/app:previous",
ports: "8080:80",
}),
},
dependencies({
stopAndRemoveContainer: async (_server, refs) => {
events.push(`remove:${refs.join(",")}`);
},
runContainer: async (_server, options) => {
runCount += 1;
events.push(`run:${options.image}`);
return runCount === 1 ? "failed-target-id" : "recovered-container-id";
},
waitForDockerHealth: async ({ containerRef }) => {
events.push(`runtime-check:${containerRef}`);
if (containerRef === "failed-target-id") {
return {
healthy: false,
reason: "not running",
status: "not_running",
attempts: 1,
durationMs: 0,
};
}
return {
healthy: true,
reason: "running",
status: "healthy",
attempts: 1,
durationMs: 0,
};
},
}),
),
(error: unknown) => {
assert.ok(error instanceof RollbackRuntimeError);
assert.equal(error.recoveryMode, "RECREATED");
assert.equal(error.recoveryDockerId, "recovered-container-id");
return true;
},
);
assert.deepEqual(events, [
"remove:old-container-id,app",
"run:example/app:target",
"runtime-check:failed-target-id",
"remove:failed-target-id,app-rollback,app",
"run:example/app:previous",
"runtime-check:recovered-container-id",
]);
});
test("failed atomic candidate leaves the active runtime in place", async () => {
const removedRefs: string[][] = [];
let runCount = 0;
await assert.rejects(
replaceRuntimeForRollback(
{
server,
containerName: "app",
currentContainerRefs: ["old-container-id", "app"],
temporaryName: "app-rollback",
strategy: "ATOMIC_RENAME",
targetRuntime: runtime(),
previousRuntime: runtime({ image: "example/app:previous" }),
},
dependencies({
stopAndRemoveContainer: async (_server, refs) => {
removedRefs.push(refs);
},
runContainer: async () => {
runCount += 1;
return "failed-target-id";
},
waitForDockerHealth: async ({ containerRef }) => {
if (containerRef === "failed-target-id") {
return {
healthy: false,
reason: "not running",
status: "not_running",
attempts: 1,
durationMs: 0,
};
}
return {
healthy: true,
reason: "running",
status: "healthy",
attempts: 1,
durationMs: 0,
};
},
}),
),
(error: unknown) => {
assert.ok(error instanceof RollbackRuntimeError);
assert.equal(error.recoveryMode, "UNCHANGED");
assert.equal(error.recoveryDockerId, "old-container-id");
return true;
},
);
assert.equal(runCount, 1);
assert.deepEqual(removedRefs, [["failed-target-id", "app-rollback"]]);
});
+141
View File
@@ -0,0 +1,141 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
import {
DEVELOPMENT_ENCRYPTION_KEY,
getEncryptionKeyOrThrow,
} from "../../src/server/lib/crypto";
import {
redactDeploymentErrorDetails,
sanitizeDeploymentError,
} from "../../src/server/services/deployment-error.service";
test("production rejects a missing, short, development, or placeholder encryption key", () => {
assert.throws(
() => getEncryptionKeyOrThrow({ NODE_ENV: "production" }),
/must be configured/,
);
assert.throws(
() =>
getEncryptionKeyOrThrow({
NODE_ENV: "production",
ENCRYPTION_KEY: "short",
}),
/at least 32 characters/,
);
assert.throws(
() =>
getEncryptionKeyOrThrow({
NODE_ENV: "production",
ENCRYPTION_KEY: DEVELOPMENT_ENCRYPTION_KEY,
}),
/strong random value/,
);
assert.throws(
() =>
getEncryptionKeyOrThrow({
NODE_ENV: "production",
ENCRYPTION_KEY:
"dev-this-key-is-long-enough-but-still-not-production",
}),
/strong random value/,
);
assert.throws(
() =>
getEncryptionKeyOrThrow({
NODE_ENV: "production",
ENCRYPTION_KEY: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}),
/strong random value/,
);
});
test("development keeps the legacy fallback while production accepts a strong key", () => {
assert.equal(
getEncryptionKeyOrThrow({ NODE_ENV: "development" }),
DEVELOPMENT_ENCRYPTION_KEY,
);
assert.equal(
getEncryptionKeyOrThrow({
NODE_ENV: "production",
ENCRYPTION_KEY:
"47c2ecffbb4a488eb9a8d1939b54810f1db621bccf1bbf87c87a07de1e6e42d1",
}),
"47c2ecffbb4a488eb9a8d1939b54810f1db621bccf1bbf87c87a07de1e6e42d1",
);
});
test("backend startup validates encryption configuration before connecting to the database", () => {
const source = readFileSync("src/server/index.ts", "utf8");
const validationIndex = source.indexOf("validateEncryptionConfiguration();");
const databaseIndex = source.indexOf("await ensureDatabaseConnection();");
assert.ok(validationIndex >= 0);
assert.ok(databaseIndex > validationIndex);
});
test("deployment error redaction removes credentials, authorization, query secrets, and private keys", () => {
const message = redactDeploymentErrorDetails(
new Error(
[
"Command failed with password=super-secret",
"Authorization: Bearer abc.def.ghi",
"https://deploy-user:deploy-pass@example.test/repo.git?access_token=query-token",
"-----BEGIN PRIVATE KEY-----",
"private-material",
"-----END PRIVATE KEY-----",
].join("\n"),
),
["super-secret"],
);
assert.doesNotMatch(
message,
/super-secret|abc\.def\.ghi|deploy-user|deploy-pass|query-token|private-material/,
);
assert.match(message, /REDACTED/);
});
test("deployment errors normalize raw command details", () => {
assert.equal(
sanitizeDeploymentError(
new Error(
"Command failed: docker run -e API_TOKEN=secret example/app; stderr: access denied",
),
{ fallback: "Git redeploy failed" },
),
"Git redeploy failed",
);
});
test("deployment and rollback boundaries use the centralized sanitizer before persistence and API responses", () => {
const rollbackService = readFileSync(
"src/server/services/deployment-rollback.service.ts",
"utf8",
);
const containerRoutes = readFileSync(
"src/server/routes/containers.ts",
"utf8",
);
const containerHealth = readFileSync(
"src/server/services/container-health.service.ts",
"utf8",
);
assert.match(
rollbackService,
/const message = sanitizeDeploymentError\(error,[\s\S]*?updateDeployment\(rollback\.id,[\s\S]*?error: message/,
);
assert.match(
containerRoutes,
/"\/:id\/deployments\/:deploymentId\/rollback"[\s\S]*?error: sanitizeDeploymentError\(error/,
);
assert.match(
containerRoutes,
/sanitizeDeploymentError\(ssh\.formatDeploymentErrorMessage\(err\)/,
);
assert.match(containerHealth, /lastReason = sanitizeDeploymentError\(error/);
});
+16
View File
@@ -0,0 +1,16 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { resolveDeploymentStrategy } from "../../src/server/services/deployment-strategy";
describe("deployment strategy", () => {
it("uses atomic rename when no published ports are configured", () => {
assert.equal(resolveDeploymentStrategy(""), "ATOMIC_RENAME");
assert.equal(resolveDeploymentStrategy(" "), "ATOMIC_RENAME");
});
it("uses recreate with recovery when published ports are configured", () => {
assert.equal(resolveDeploymentStrategy("8080:80"), "RECREATE_WITH_RECOVERY");
});
});