From 4e9b173edb2a78077f218ba1f4613b4bc9957690 Mon Sep 17 00:00:00 2001 From: KodekaTeam Date: Fri, 17 Jul 2026 00:27:07 +0700 Subject: [PATCH] feat(deployment): add deployment history and rollback support --- .env.docker | 5 +- .env.example | 3 +- SECURITY.md | 31 + .../migration.sql | 38 + .../migration.sql | 15 + .../migration.sql | 2 + prisma/schema.prisma | 67 +- src/app/globals.css | 6 + .../deployments/DeploymentDetailsModal.tsx | 272 +++++++ .../deployments/DeploymentSummaryCard.tsx | 16 +- .../deployments/DeploymentsTabPanel.tsx | 464 +++++------- .../[containerId]/data/deployments-data.ts | 177 ++--- .../containers/[containerId]/page.tsx | 136 +++- .../[containerId]/types/app-detail-types.ts | 25 +- src/lib/api.ts | 94 ++- src/server/index.ts | 17 +- src/server/lib/crypto.ts | 77 +- src/server/routes/apps.ts | 201 +++-- src/server/routes/containers.ts | 481 ++++++++++-- src/server/routes/server-action-errors.ts | 23 + .../services/app-install-rebuild.service.ts | 83 +- .../services/container-health.service.ts | 126 ++++ .../services/deployment-error.service.ts | 76 ++ .../services/deployment-lock.service.ts | 130 ++++ .../services/deployment-rollback.service.ts | 714 ++++++++++++++++++ src/server/services/deployment-strategy.ts | 7 + src/server/services/deployment.service.ts | 254 +++++++ .../ssh-services/docker-containers.ts | 38 +- tests/backend/container-health.test.ts | 31 + tests/backend/deployment-concurrency.test.ts | 42 ++ tests/backend/deployment-history.test.ts | 22 + .../deployment-lifecycle-regressions.test.ts | 85 +++ tests/backend/deployment-reliability.test.ts | 42 ++ .../deployment-rollback-runtime.test.ts | 275 +++++++ tests/backend/deployment-security.test.ts | 141 ++++ tests/backend/deployment-strategy.test.ts | 16 + 36 files changed, 3663 insertions(+), 569 deletions(-) create mode 100644 prisma/migrations/20260715_add_deployment_history/migration.sql create mode 100644 prisma/migrations/20260715_add_deployment_locks/migration.sql create mode 100644 prisma/migrations/20260715_add_deployment_rollback_snapshot/migration.sql create mode 100644 src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentDetailsModal.tsx create mode 100644 src/server/routes/server-action-errors.ts create mode 100644 src/server/services/container-health.service.ts create mode 100644 src/server/services/deployment-error.service.ts create mode 100644 src/server/services/deployment-lock.service.ts create mode 100644 src/server/services/deployment-rollback.service.ts create mode 100644 src/server/services/deployment-strategy.ts create mode 100644 src/server/services/deployment.service.ts create mode 100644 tests/backend/container-health.test.ts create mode 100644 tests/backend/deployment-concurrency.test.ts create mode 100644 tests/backend/deployment-history.test.ts create mode 100644 tests/backend/deployment-lifecycle-regressions.test.ts create mode 100644 tests/backend/deployment-reliability.test.ts create mode 100644 tests/backend/deployment-rollback-runtime.test.ts create mode 100644 tests/backend/deployment-security.test.ts create mode 100644 tests/backend/deployment-strategy.test.ts diff --git a/.env.docker b/.env.docker index 09e7ee2..dc587ea 100644 --- a/.env.docker +++ b/.env.docker @@ -35,7 +35,10 @@ PORT=4000 JWT_SECRET=dev-super-secret-jwt-key-change-in-production-32chars JWT_EXPIRES_IN=7d -# Encryption key for SSH private keys (AES-256) +# Required in production. Generate once, store securely, and keep it stable: +# openssl rand -hex 32 +# Changing this value without re-encrypting stored secrets makes existing +# credentials and deployment rollback snapshots unreadable. ENCRYPTION_KEY=dev-32-char-encryption-key-here! # /* -------------------------- Public URL / callbacks ------------------------ */ diff --git a/.env.example b/.env.example index 329c34c..894fe6a 100644 --- a/.env.example +++ b/.env.example @@ -45,7 +45,8 @@ RATE_LIMIT_TIME_WINDOW=1 minute # Optional internal backend URL. # INTERNAL_API_URL=http://127.0.0.1:4000 -# Encryption key for SSH private keys (AES-256) +# Development-only example. Production rejects this value. +# Generate a production key with: openssl rand -hex 32 ENCRYPTION_KEY=dev-32-char-encryption-key-here! # /* -------------------------- Optional integrations ------------------------- */ diff --git a/SECURITY.md b/SECURITY.md index 077e298..1f0897d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -69,6 +69,37 @@ Security fixes may be released as maintenance releases: or, when necessary, included in the next milestone release. +## Production Encryption Key + +Doktainer uses `ENCRYPTION_KEY` to encrypt SSH credentials, integration secrets, +and deployment rollback snapshots at rest. + +- Production startup requires a non-placeholder key of at least 32 characters. +- Generate a key with a cryptographically secure generator, for example + `openssl rand -hex 32`. +- Store the key in a secret manager or protected runtime environment file. Do + not commit the production value to Git. +- Keep the key stable across application restarts and deployments. Losing it + makes existing encrypted records unreadable. + +### Rotation procedure + +Do not replace `ENCRYPTION_KEY` directly while encrypted records still use the +old key. A safe rotation requires: + +1. Back up the database and verify the backup can be restored. +2. Stop writes that create or update encrypted records. +3. Decrypt every encrypted record with the old key and re-encrypt it with the + new key, including deployment `rollbackSnapshotEnc` values. +4. Verify representative SSH credentials, integrations, 2FA secrets, storage + destinations, and rollback snapshots before switching traffic. +5. Deploy the new key, restart all backend instances, and remove the old key + only after verification succeeds. + +The current encrypted payload format does not contain a key identifier. +Operators should therefore treat key rotation as a controlled maintenance +operation until versioned key-ring support is implemented. + ## Acknowledgements We appreciate responsible security researchers and community members who help improve the security of Doktainer. diff --git a/prisma/migrations/20260715_add_deployment_history/migration.sql b/prisma/migrations/20260715_add_deployment_history/migration.sql new file mode 100644 index 0000000..a05754b --- /dev/null +++ b/prisma/migrations/20260715_add_deployment_history/migration.sql @@ -0,0 +1,38 @@ +CREATE TYPE "DeploymentStatus" AS ENUM ('QUEUED', 'RUNNING', 'SUCCESS', 'FAILED', 'CANCELLED', 'ROLLED_BACK'); + +CREATE TYPE "DeploymentTrigger" AS ENUM ('MANUAL', 'GIT_WEBHOOK', 'REBUILD', 'ROLLBACK', 'APP_INSTALLER'); + +CREATE TABLE "deployments" ( + "id" TEXT NOT NULL, + "containerId" TEXT NOT NULL, + "organizationId" TEXT NOT NULL, + "serverId" TEXT NOT NULL, + "userId" TEXT, + "status" "DeploymentStatus" NOT NULL, + "trigger" "DeploymentTrigger" NOT NULL, + "version" TEXT, + "commitSha" TEXT, + "branch" TEXT, + "image" TEXT, + "imageDigest" TEXT, + "configSnapshot" JSONB NOT NULL, + "error" TEXT, + "startedAt" TIMESTAMP(3), + "completedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "deployments_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX "deployments_containerId_createdAt_idx" ON "deployments"("containerId", "createdAt"); +CREATE INDEX "deployments_organizationId_createdAt_idx" ON "deployments"("organizationId", "createdAt"); +CREATE INDEX "deployments_serverId_createdAt_idx" ON "deployments"("serverId", "createdAt"); + +ALTER TABLE "deployments" ADD CONSTRAINT "deployments_containerId_fkey" + FOREIGN KEY ("containerId") REFERENCES "containers"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "deployments" ADD CONSTRAINT "deployments_organizationId_fkey" + FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "deployments" ADD CONSTRAINT "deployments_serverId_fkey" + FOREIGN KEY ("serverId") REFERENCES "servers"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "deployments" ADD CONSTRAINT "deployments_userId_fkey" + FOREIGN KEY ("userId") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/prisma/migrations/20260715_add_deployment_locks/migration.sql b/prisma/migrations/20260715_add_deployment_locks/migration.sql new file mode 100644 index 0000000..374990c --- /dev/null +++ b/prisma/migrations/20260715_add_deployment_locks/migration.sql @@ -0,0 +1,15 @@ +CREATE TABLE "deployment_locks" ( + "containerId" TEXT NOT NULL, + "token" TEXT NOT NULL, + "expiresAt" TIMESTAMP(3) NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "deployment_locks_pkey" PRIMARY KEY ("containerId") +); + +CREATE UNIQUE INDEX "deployment_locks_token_key" ON "deployment_locks"("token"); +CREATE INDEX "deployment_locks_expiresAt_idx" ON "deployment_locks"("expiresAt"); + +ALTER TABLE "deployment_locks" ADD CONSTRAINT "deployment_locks_containerId_fkey" + FOREIGN KEY ("containerId") REFERENCES "containers"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/migrations/20260715_add_deployment_rollback_snapshot/migration.sql b/prisma/migrations/20260715_add_deployment_rollback_snapshot/migration.sql new file mode 100644 index 0000000..a252d7d --- /dev/null +++ b/prisma/migrations/20260715_add_deployment_rollback_snapshot/migration.sql @@ -0,0 +1,2 @@ +ALTER TABLE "deployments" +ADD COLUMN "rollbackSnapshotEnc" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 00efb60..bd5651a 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -31,6 +31,7 @@ model User { createdOrganizations Organization[] @relation("OrganizationCreator") apiKeys ApiKey[] auditLogs AuditLog[] + deployments Deployment[] settings UserSettings? notificationProviders UserNotificationProvider[] integrations UserIntegration[] @@ -61,6 +62,7 @@ model Organization { apiKeys ApiKey[] invitations UserInvitation[] auditLogs AuditLog[] + deployments Deployment[] @@index([createdById]) @@map("organizations") @@ -397,6 +399,7 @@ model Server { storageDestinations UserStorageDestination[] domains Domain[] auditLogs AuditLog[] + deployments Deployment[] metrics ServerMetric[] appInstalls AppInstall[] backups Backup[] @@ -456,6 +459,8 @@ model Container { environment Environment? @relation(fields: [environmentId], references: [id], onDelete: SetNull) domains Domain[] deploymentSource ContainerDeploymentSource? + deployments Deployment[] + deploymentLock DeploymentLock? ports Json @default("[]") // [{host: 80, container: 80, protocol: "tcp"}] envVars Json @default("[]") // [{key: "NODE_ENV", value: "production"}] volumes Json @default("[]") @@ -501,6 +506,67 @@ model ContainerDeploymentSource { @@map("container_deployment_sources") } +model Deployment { + id String @id @default(cuid()) + containerId String + organizationId String + serverId String + userId String? + status DeploymentStatus + trigger DeploymentTrigger + version String? + commitSha String? + branch String? + image String? + imageDigest String? + configSnapshot Json + rollbackSnapshotEnc String? + error String? + startedAt DateTime? + completedAt DateTime? + createdAt DateTime @default(now()) + + container Container @relation(fields: [containerId], references: [id], onDelete: Cascade) + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + server Server @relation(fields: [serverId], references: [id], onDelete: Cascade) + user User? @relation(fields: [userId], references: [id], onDelete: SetNull) + + @@index([containerId, createdAt]) + @@index([organizationId, createdAt]) + @@index([serverId, createdAt]) + @@map("deployments") +} + +model DeploymentLock { + containerId String @id + token String @unique + expiresAt DateTime + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + container Container @relation(fields: [containerId], references: [id], onDelete: Cascade) + + @@index([expiresAt]) + @@map("deployment_locks") +} + +enum DeploymentStatus { + QUEUED + RUNNING + SUCCESS + FAILED + CANCELLED + ROLLED_BACK +} + +enum DeploymentTrigger { + MANUAL + GIT_WEBHOOK + REBUILD + ROLLBACK + APP_INSTALLER +} + model Network { id String @id @default(cuid()) name String @@ -781,4 +847,3 @@ enum BackupStatus { COMPLETED FAILED } - diff --git a/src/app/globals.css b/src/app/globals.css index eb144b1..5d3e68d 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -384,6 +384,9 @@ body { flex: 1 1 240px; } +.deployment-filter-toolbar { display: flex; align-items: flex-end; justify-content: flex-end; gap: 12px; flex-wrap: wrap; } +.deployment-filter-control { display: grid; gap: 4px; min-width: 160px; flex: 0 1 190px; } + @media (max-width: 768px) { .ui-responsive-toolbar, .ui-tab-shell { @@ -416,6 +419,9 @@ body { .ui-tab-scroll { width: 100%; } + + .deployment-filter-toolbar { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); align-items: end; justify-content: stretch; } + .deployment-filter-control { min-width: 0; width: 100%; flex: none; } } /* Gradient text */ diff --git a/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentDetailsModal.tsx b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentDetailsModal.tsx new file mode 100644 index 0000000..f4510cb --- /dev/null +++ b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentDetailsModal.tsx @@ -0,0 +1,272 @@ +"use client"; + +import { AlertTriangle, History, X } from "lucide-react"; +import type { DeploymentRecord } from "@/lib/api"; + +interface DeploymentDetailsModalProps { + deployment: DeploymentRecord | null; + loading: boolean; + error: string | null; + onClose: () => void; +} + +function formatDate(value: string | null) { + if (!value) return "-"; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? "-" : date.toLocaleString(); +} + +function getStatusClass(status?: DeploymentRecord["status"]) { + if (status === "SUCCESS") return "badge-online"; + if (status === "FAILED") return "badge-danger"; + return "badge-warning"; +} + +export default function DeploymentDetailsModal({ + deployment, + loading, + error, + onClose, +}: DeploymentDetailsModalProps) { + if (!deployment && !loading && !error) return null; + + const metadata = deployment + ? [ + ["Status", deployment.status], + ["Trigger", deployment.trigger], + ["Commit", deployment.commitSha ?? "-"], + ["Branch", deployment.branch ?? "-"], + ["Started", formatDate(deployment.startedAt)], + ["Completed", formatDate(deployment.completedAt)], + ["Actor", deployment.user?.name ?? "System"], + ["Image digest", deployment.imageDigest ?? "-"], + ] + : []; + + return ( +
+
event.stopPropagation()} + > + + +
+
+ + + +
+

+ Deployment details +

+

+ {deployment?.version ?? "Loading deployment..."} +

+
+
+ + {loading ? ( +
+ Loading deployment details... +
+ ) : error ? ( +
+ + {error} +
+ ) : deployment ? ( + <> +
+ {metadata.map(([label, value]) => ( +
+

+ {label} +

+ {label === "Status" ? ( + + {value} + + ) : ( +

+ {value} +

+ )} +
+ ))} +
+ + {deployment.error ? ( +
+ {deployment.error} +
+ ) : null} + +
+

+ Sanitized configuration snapshot +

+ 
+                  {JSON.stringify(deployment.configSnapshot ?? {}, null, 2)}
+
+
+ + ) : null} +
+
+
+ ); +} + + diff --git a/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentSummaryCard.tsx b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentSummaryCard.tsx index b0ad2c2..2ed4a2e 100644 --- a/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentSummaryCard.tsx +++ b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentSummaryCard.tsx @@ -20,6 +20,7 @@ export default function DeploymentSummaryCard({ className="card" style={{ padding: 14, + minWidth: 0, minHeight: 104, display: "flex", flexDirection: "column", @@ -42,11 +43,24 @@ export default function DeploymentSummaryCard({ color: "var(--text-primary)", fontSize: 20, lineHeight: 1.1, + minWidth: 0, + overflowWrap: "anywhere", + wordBreak: "break-word", }} + title={item.value} > {item.value} - + {item.subvalue} diff --git a/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentsTabPanel.tsx b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentsTabPanel.tsx index 4e022b3..be0599e 100644 --- a/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentsTabPanel.tsx +++ b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/components/deployments/DeploymentsTabPanel.tsx @@ -2,19 +2,10 @@ import TablePagination from "@/components/TablePagination"; import { useTablePagination } from "@/lib/use-table-pagination"; -import { - CheckCircle2, - Circle, - Clock3, - ExternalLink, - GitBranch, - GitCommit, - Rocket, - XCircle, -} from "lucide-react"; +import { useMemo, useState } from "react"; +import { Eye, GitBranch, GitCommit, Loader2, RotateCcw } from "lucide-react"; import type { DeploymentHistoryItem, - DeploymentPipelineStep, DeploymentTabData, } from "../../types/app-detail-types"; import PanelShell from "../overview/PanelShell"; @@ -22,6 +13,9 @@ import DeploymentSummaryCard from "./DeploymentSummaryCard"; interface DeploymentsTabPanelProps { deployments: DeploymentTabData; + onRollback: (deploymentId: string) => void; + onViewDetails: (deploymentId: string) => void; + rollingBackId?: string | null; } const statusClass: Record = { @@ -31,27 +25,29 @@ const statusClass: Record = { "Rolled Back": "badge-warning", }; -function getStepIcon(status: DeploymentPipelineStep["status"]) { - if (status === "success") return ; - if (status === "failed") return ; - if (status === "running") return ; - return ; -} - -function getStepColor(status: DeploymentPipelineStep["status"]) { - if (status === "success") return "var(--accent-green)"; - if (status === "failed") return "var(--text-danger)"; - if (status === "running") return "var(--accent-blue)"; - return "var(--text-muted)"; -} - export default function DeploymentsTabPanel({ deployments, + onRollback, + onViewDetails, + rollingBackId = null, }: DeploymentsTabPanelProps) { + const [statusFilter, setStatusFilter] = useState< + "ALL" | DeploymentHistoryItem["status"] + >("ALL"); + const [triggerFilter, setTriggerFilter] = useState("ALL"); + const filteredHistory = useMemo( + () => + deployments.history.filter( + (item) => + (statusFilter === "ALL" || item.status === statusFilter) && + (triggerFilter === "ALL" || item.trigger === triggerFilter), + ), + [deployments.history, statusFilter, triggerFilter], + ); const pagination = useTablePagination({ - items: deployments.history, + items: filteredHistory, pageSize: 5, - resetKey: deployments.history.map((item) => item.id).join("|"), + resetKey: `${statusFilter}:${triggerFilter}:${filteredHistory.map((item) => item.id).join("|")}`, }); return ( @@ -69,184 +65,120 @@ export default function DeploymentsTabPanel({ ))} -
- -
- - - -
-
-

- {deployments.latest.version} -

- - {deployments.latest.status} - -
-

- Deployed by {deployments.latest.deployedBy} from{" "} - {deployments.latest.source}. This panel reflects the current - container deployment record and runtime inspection state. -

-

- {deployments.latest.image} -

-
-
- - - -
- {deployments.artifacts.map((artifact) => ( -
-

- {artifact.label} -

-

- {artifact.value} -

-

- {artifact.meta} -

-
- ))} -
- -
- -
- +
-
- - + + + + +
+
+ + + + + + + + + + + + + + {pagination.paginatedItems.length === 0 ? ( - - - - - - - + - - - {pagination.paginatedItems.map((deployment) => ( + ) : ( + pagination.paginatedItems.map((deployment) => ( - + - ))} - -
VersionStatusTriggerCommitBranchDurationDeployed AtAction
VersionStatusTriggerCommitBranchDurationDeployed At + {deployments.history.length === 0 + ? "No deployment history yet." + : "No deployments match the selected filters."} +
+ {deployment.version} @@ -267,83 +199,61 @@ export default function DeploymentsTabPanel({ {deployment.duration} {deployment.deployedAt} +
+ + {deployment.canRollback ? ( + + ) : null} +
+
-
- - - - -
- {deployments.pipeline.map((step) => ( -
- - {getStepIcon(step.status)} - -
-

- {step.label} -

-

- {step.description} -

-
- - {step.duration} - -
- ))} -
- -
+ )) + )} + + +
+ + ); } diff --git a/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/data/deployments-data.ts b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/data/deployments-data.ts index 03d197a..6ca4ae5 100644 --- a/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/data/deployments-data.ts +++ b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/data/deployments-data.ts @@ -1,4 +1,8 @@ -import type { Container, ContainerDetails } from "@/lib/api"; +import type { + Container, + ContainerDetails, + DeploymentRecord, +} from "@/lib/api"; import type { DeploymentHistoryItem, DeploymentTabData, @@ -21,70 +25,71 @@ function getDeploymentStatus( return "Rolled Back"; } -function getDuration(detail: ContainerDetails | null) { - const startedAt = detail?.inspect.State; - if ( - typeof startedAt === "object" && - startedAt !== null && - "StartedAt" in startedAt && - typeof startedAt.StartedAt === "string" - ) { - const startedDate = new Date(startedAt.StartedAt); - if (!Number.isNaN(startedDate.getTime())) return "Runtime active"; - } - - return "-"; -} - -function getCommit(container: Container) { - if (container.sourceType === "GIT_CLONE") return "Git source"; - if (container.sourceType === "GIT_PROVIDER") return "Provider"; - return "-"; -} - -function getBranch(container: Container) { - if ( - container.sourceType === "GIT_CLONE" || - container.sourceType === "GIT_PROVIDER" - ) { - return "Configured source"; - } - - return "-"; -} - function getImageTag(image: string) { - if (!image.trim()) return "-"; - if (image.includes("@")) return "digest"; + const normalized = image.trim(); + if (!normalized) return "-"; - const lastSegment = image.split("/").at(-1) ?? image; + const digest = normalized.includes("@") + ? normalized.split("@").at(-1) + : normalized; + const digestValue = digest ?? ""; + if (/^sha256:[a-f0-9]{64}$/i.test(digestValue)) { + return `sha256:${digestValue.slice(7, 19)}…`; + } + + const lastSegment = normalized.split("/").at(-1) ?? normalized; const tagIndex = lastSegment.lastIndexOf(":"); return tagIndex >= 0 ? lastSegment.slice(tagIndex + 1) || "latest" : "latest"; } +function formatDeploymentDuration(item: DeploymentRecord) { + if (!item.startedAt || !item.completedAt) return "-"; + const durationMs = new Date(item.completedAt).getTime() - new Date(item.startedAt).getTime(); + if (!Number.isFinite(durationMs) || durationMs < 0) return "-"; + return `${Math.floor(durationMs / 60000)}m ${Math.floor((durationMs % 60000) / 1000)}s`; +} + +function formatDeploymentStatus( + status: DeploymentRecord["status"], +): DeploymentHistoryItem["status"] { + if (status === "SUCCESS") return "Success"; + if (status === "FAILED") return "Failed"; + if (status === "RUNNING" || status === "QUEUED") return "Running"; + return status === "ROLLED_BACK" ? "Rolled Back" : "Failed"; +} + +function formatDeploymentTrigger(trigger: DeploymentRecord["trigger"]) { + return { + MANUAL: "Manual deploy", + GIT_WEBHOOK: "Git webhook", + REBUILD: "Rebuild", + ROLLBACK: "Rollback", + APP_INSTALLER: "App installer", + }[trigger]; +} + +function mapDeploymentHistory(items: DeploymentRecord[]) { + return items.map((item) => ({ + id: item.id, + version: item.version || item.image || item.id, + status: formatDeploymentStatus(item.status), + trigger: formatDeploymentTrigger(item.trigger), + commit: item.commitSha || "-", + branch: item.branch || "-", + duration: formatDeploymentDuration(item), + deployedAt: formatDate(item.createdAt), + canRollback: item.status === "SUCCESS" && Boolean(item.image), + })); +} + export function createDeploymentsData( container: Container, detail: ContainerDetails | null, + deploymentRecords: DeploymentRecord[] = [], ): DeploymentTabData { const status = getDeploymentStatus(container); - const history: DeploymentHistoryItem[] = [ - { - id: container.id, - version: container.name, - status, - trigger: - container.sourceType === "APP_INSTALLER" - ? "App installer" - : container.sourceType === "MANUAL" - ? "Manual deploy" - : "Git deploy", - commit: getCommit(container), - branch: getBranch(container), - duration: getDuration(detail), - deployedAt: formatDate(container.createdAt), - }, - ]; + const history = mapDeploymentHistory(deploymentRecords); const latest = history[0]; const runtimeLoaded = detail !== null; const isSuccessful = status === "Success"; @@ -93,8 +98,8 @@ export function createDeploymentsData( summaries: [ { label: "Latest Deployment", - value: latest.version, - subvalue: `${latest.status} - ${latest.deployedAt}`, + value: latest?.version ?? "No deployments", + subvalue: latest ? `${latest.status} - ${latest.deployedAt}` : "No history yet", tone: isSuccessful ? "green" : "amber", }, { @@ -116,68 +121,6 @@ export function createDeploymentsData( tone: runtimeLoaded ? "blue" : "amber", }, ], - latest: { - version: latest.version, - status: latest.status, - source: container.sourceType ?? "MANUAL", - image: container.image, - deployedBy: "-", - }, history, - pipeline: [ - { - id: "source", - label: "Source metadata", - description: `${container.sourceType ?? "MANUAL"} deployment metadata loaded from the container record.`, - status: "success", - duration: "-", - }, - { - id: "image", - label: "Image resolved", - description: `Runtime image resolved as ${container.image}.`, - status: container.image ? "success" : "failed", - duration: "-", - }, - { - id: "runtime", - label: "Runtime inspected", - description: runtimeLoaded - ? "Docker inspect and runtime statistics are available." - : "Runtime inspect is unavailable, usually because the container is stopped or Docker returned an error.", - status: runtimeLoaded ? "success" : "pending", - duration: "-", - }, - { - id: "health", - label: "Container status", - description: `Current container status is ${container.status}.`, - status: isSuccessful ? "success" : status === "Failed" ? "failed" : "pending", - duration: "-", - }, - ], - artifacts: [ - { - label: "Image", - value: container.image, - meta: "Runtime image", - }, - { - label: "Deploy Mode", - value: container.deployMode ?? "IMAGE", - meta: "Deployment strategy", - }, - { - label: "Restart Policy", - value: container.restartPolicy || "unless-stopped", - meta: "Docker policy", - }, - { - label: "Server", - value: detail?.server.name ?? container.server?.name ?? container.serverId, - meta: detail?.server.ip ?? container.server?.ip ?? "Target host", - }, - ], - hasHistoricalData: false, }; } diff --git a/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/page.tsx b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/page.tsx index bbc0b7c..8702e4d 100644 --- a/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/page.tsx +++ b/src/app/projects/[projectId]/environments/[environmentId]/containers/[containerId]/page.tsx @@ -18,6 +18,7 @@ import { type ContainerProcess, type ContainerProjectEnvFile, type ContainerRuntimeStats, + type DeploymentRecord, type Domain, type ProjectEnvironmentRecord, type ProjectRecord, @@ -25,6 +26,7 @@ import { import ContainerFileManagerModal from "@/components/containers/modals/ContainerFileManagerModal"; import AdvancedTabPanel from "./components/advanced/AdvancedTabPanel"; import DeploymentsTabPanel from "./components/deployments/DeploymentsTabPanel"; +import DeploymentDetailsModal from "./components/deployments/DeploymentDetailsModal"; import EnvironmentTabPanel from "./components/environment/EnvironmentTabPanel"; import AppDetailHeader from "./components/header/AppDetailHeader"; import LogsTabPanel from "./components/logs/LogsTabPanel"; @@ -467,6 +469,16 @@ export default function AppContainerDetailPage() { ); const [loading, setLoading] = useState(true); const [acting, setActing] = useState(false); + const [rollingBackDeploymentId, setRollingBackDeploymentId] = useState< + string | null + >(null); + const [selectedDeployment, setSelectedDeployment] = + useState(null); + const [deploymentDetailLoading, setDeploymentDetailLoading] = + useState(false); + const [deploymentDetailError, setDeploymentDetailError] = useState< + string | null + >(null); const [activeAction, setActiveAction] = useState( null, ); @@ -491,6 +503,7 @@ export default function AppContainerDetailPage() { const runtimeMetricsInFlightRef = useRef(false); const runtimeDetailRef = useRef(null); const environmentRecordRef = useRef(null); + const deploymentHistoryRef = useRef([]); const projectEnvRef = useRef(null); const domainsLoadedRef = useRef(false); const domainsLoadingRef = useRef(false); @@ -518,6 +531,7 @@ export default function AppContainerDetailPage() { runtimeDetailLoadedRef.current = false; projectEnvLoadedRef.current = false; logsLoadedRef.current = false; + deploymentHistoryRef.current = []; try { const [projectResponse, containerResponse, metricsResult] = @@ -614,6 +628,105 @@ export default function AppContainerDetailPage() { containerRecordRef.current = containerRecord; }, [containerRecord]); + const hydrateDeployments = useCallback(async () => { + const container = containerRecordRef.current; + if (!container) return; + + try { + const response = await containersApi.deployments(params.containerId, { + page: 1, + pageSize: 50, + }); + deploymentHistoryRef.current = response.data.items; + setAppDetail((current) => + current + ? { + ...current, + deployments: createDeploymentsData( + container, + runtimeDetailRef.current, + response.data.items, + ), + } + : current, + ); + } catch { + // The detail page remains usable if deployment history is unavailable. + } + }, [params.containerId]); + + const runRollback = useCallback( + async (deploymentId: string) => { + if (acting || rollingBackDeploymentId) return; + + setRollingBackDeploymentId(deploymentId); + setActionError(""); + try { + await containersApi.rollbackDeployment(params.containerId, deploymentId); + await load(); + setActiveTab("deployments"); + } catch (rollbackError) { + setActionError( + rollbackError instanceof Error + ? rollbackError.message + : "Rollback failed", + ); + } finally { + setRollingBackDeploymentId(null); + } + }, + [acting, load, params.containerId, rollingBackDeploymentId], + ); + + const requestRollback = useCallback( + (deploymentId: string) => { + const deployment = appDetail?.deployments.history.find( + (item) => item.id === deploymentId, + ); + if (!deployment || !deployment.canRollback) return; + + setConfirmDialog({ + title: "Rollback Deployment", + description: `Rollback container "${appDetail?.name ?? "this container"}" to ${deployment.version}?`, + confirmLabel: "Rollback Deployment", + tone: "warning", + note: "The current container will be replaced using the selected deployment artifact. Doktainer will attempt to recover the current runtime if rollback fails.", + onConfirm: () => { + void runRollback(deploymentId); + }, + }); + }, + [appDetail, runRollback], + ); + + const openDeploymentDetails = useCallback( + async (deploymentId: string) => { + setSelectedDeployment(null); + setDeploymentDetailError(null); + setDeploymentDetailLoading(true); + try { + const response = await containersApi.deployment( + params.containerId, + deploymentId, + ); + setSelectedDeployment(response.data); + } catch (detailsError) { + setDeploymentDetailError( + detailsError instanceof Error + ? detailsError.message + : "Failed to load deployment details", + ); + } finally { + setDeploymentDetailLoading(false); + } + }, + [params.containerId], + ); + + useEffect(() => { + if (containerRecord) void hydrateDeployments(); + }, [containerRecord, hydrateDeployments]); + const refreshRuntimeMetrics = useCallback(async () => { const container = containerRecordRef.current; if (!container || container.status !== "RUNNING") return; @@ -757,7 +870,11 @@ export default function AppContainerDetailPage() { serverIp: detail.server.ip, metrics: buildMetrics(detail, container), runtime: createRuntimeData(container, detail), - deployments: createDeploymentsData(container, detail), + deployments: createDeploymentsData( + container, + detail, + deploymentHistoryRef.current, + ), advanced: createAdvancedData(container, detail), environment: createEnvironmentData({ container, @@ -940,6 +1057,7 @@ export default function AppContainerDetailPage() { return; } + setActionError(""); window.open(appDetail.openUrl, "_blank", "noopener,noreferrer"); }, [appDetail]); @@ -1432,6 +1550,15 @@ export default function AppContainerDetailPage() { current?.onConfirm(); }} /> + { + setSelectedDeployment(null); + setDeploymentDetailError(null); + }} + /> ) : activeTab === "terminal" ? null : activeTab === "deployments" ? ( - + ) : activeTab === "advanced" ? ( ( path: string, @@ -754,7 +754,7 @@ function del(path: string, body?: unknown, config?: { timeoutMs?: number }) { ); } -// ─── WebSocket helper ───────────────────────────────────────────────────────── +// ─── WebSocket helper ───────────────────────────────────────────────────────── export function createWs(path: string): WebSocket { const token = getToken(); @@ -764,7 +764,7 @@ export function createWs(path: string): WebSocket { ); } -// ─── Auth ───────────────────────────────────────────────────────────────────── +// ─── Auth ───────────────────────────────────────────────────────────────────── export const auth = { registrationStatus: () => @@ -858,7 +858,7 @@ export const auth = { ), }; -// ─── Users ─────────────────────────────────────────────────────────────────── +// ─── Users ─────────────────────────────────────────────────────────────────── export type UserRole = "SUPER_ADMIN" | "OPERATOR" | "DEVELOPER" | "VIEWER"; @@ -1244,7 +1244,7 @@ export const gitProvidersApi = { ), }; -// ─── Servers ────────────────────────────────────────────────────────────────── +// ─── Servers ────────────────────────────────────────────────────────────────── export interface Server { id: string; @@ -1536,7 +1536,7 @@ export const servers = { ), }; -// ─── Containers ─────────────────────────────────────────────────────────────── +// ─── Containers ─────────────────────────────────────────────────────────────── export interface Container { id: string; @@ -1613,6 +1613,40 @@ export interface ContainerDetails { processes: ContainerProcess[]; } +export type DeploymentStatus = + | "QUEUED" + | "RUNNING" + | "SUCCESS" + | "FAILED" + | "CANCELLED" + | "ROLLED_BACK"; + +export type DeploymentTrigger = + | "MANUAL" + | "GIT_WEBHOOK" + | "REBUILD" + | "ROLLBACK" + | "APP_INSTALLER"; + +export interface DeploymentRecord { + id: string; + containerId: string; + serverId?: string; + status: DeploymentStatus; + trigger: DeploymentTrigger; + version: string | null; + commitSha: string | null; + branch: string | null; + image: string | null; + imageDigest: string | null; + configSnapshot?: Record; + error: string | null; + startedAt: string | null; + completedAt: string | null; + createdAt: string; + user: { id: string; name: string } | null; +} + export interface ContainerMetrics { container: { id: string; @@ -1786,6 +1820,35 @@ export const containers = { }, get: (id: string) => get<{ success: boolean; data: Container }>(`/containers/${id}`), + deployments: (id: string, params?: { page?: number; pageSize?: number }) => { + const qs = new URLSearchParams(); + if (params?.page) qs.set("page", String(params.page)); + if (params?.pageSize) qs.set("pageSize", String(params.pageSize)); + const query = qs.toString(); + return get<{ + success: boolean; + data: { + items: DeploymentRecord[]; + total: number; + page: number; + pageSize: number; + }; + }>(`/containers/${id}/deployments${query ? `?${query}` : ""}`); + }, + rollbackDeployment: (containerId: string, deploymentId: string) => + post<{ + success: boolean; + data: Container; + meta?: { + rollbackDeploymentId: string; + targetDeploymentId: string; + }; + message?: string; + }>(`/containers/${containerId}/deployments/${deploymentId}/rollback`, {}), + deployment: (containerId: string, deploymentId: string) => + get<{ success: boolean; data: DeploymentRecord }>( + `/containers/${containerId}/deployments/${deploymentId}`, + ), deploy: (body: ContainerDeployBody) => post<{ success: boolean; @@ -2003,7 +2066,7 @@ export const containers = { ), }; -// ─── Metrics ────────────────────────────────────────────────────────────────── +// ─── Metrics ────────────────────────────────────────────────────────────────── export interface DashboardOverview { servers: { total: number; online: number }; @@ -2059,7 +2122,7 @@ export const metrics = { ), }; -// ─── Domains ────────────────────────────────────────────────────────────────── +// ─── Domains ────────────────────────────────────────────────────────────────── export interface Domain { id: string; @@ -2450,7 +2513,7 @@ export const security = { ), }; -// ─── Apps ──────────────────────────────────────────────────────────────────── +// ─── Apps ──────────────────────────────────────────────────────────────────── export interface AppTemplate { id: string; @@ -2649,7 +2712,7 @@ export const apps = { }), }; -// ─── Backups ────────────────────────────────────────────────────────────────── +// ─── Backups ────────────────────────────────────────────────────────────────── export interface Backup { id: string; @@ -2750,7 +2813,7 @@ export const backups = { delete: (id: string) => del<{ success: boolean }>(`/backups/${id}`), }; -// ─── Logs ───────────────────────────────────────────────────────────────────── +// ─── Logs ───────────────────────────────────────────────────────────────────── export const logs = { list: (params?: { @@ -2775,7 +2838,7 @@ export const logs = { }, }; -// ─── Terminal ───────────────────────────────────────────────────────────────── +// ─── Terminal ───────────────────────────────────────────────────────────────── export const terminal = { sessions: () => @@ -2847,4 +2910,3 @@ export const apiClient = { terminal, }; export default apiClient; - diff --git a/src/server/index.ts b/src/server/index.ts index d5cdb10..65090b4 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -5,6 +5,7 @@ import jwt from "@fastify/jwt"; import rateLimit from "@fastify/rate-limit"; import websocket from "@fastify/websocket"; import prisma from "./lib/prisma"; +import { validateEncryptionConfiguration } from "./lib/crypto"; import { authRoutes } from "./routes/auth"; import { serverRoutes } from "./routes/servers"; @@ -109,10 +110,11 @@ async function ensureDatabaseConnection() { async function start() { const jwtSecret = getJwtSecretOrThrow(); + validateEncryptionConfiguration(); await ensureDatabaseConnection(); setDefaultSecurityHeaders(); - // ── Plugins ────────────────────────────────────────── + // ── Plugins ────────────────────────────────────────── await app.register(cors, { origin: getCorsOrigins(), credentials: true, @@ -143,7 +145,7 @@ async function start() { await app.register(websocket); - // ── Health check ────────────────────────────────────── + // ── Health check ────────────────────────────────────── app.get("/health", { config: { rateLimit: false } }, async () => ({ status: "ok", uptime: process.uptime(), @@ -151,7 +153,7 @@ async function start() { version: process.env.NEXT_PUBLIC_VERSION || "unknown", })); - // ── API Routes ──────────────────────────────────────── + // ── API Routes ──────────────────────────────────────── const API_PREFIX = "/api/v1"; await app.register(authRoutes, { prefix: `${API_PREFIX}/auth` }); @@ -183,7 +185,7 @@ async function start() { prefix: `${API_PREFIX}/notifications`, }); - // ── Error handler ───────────────────────────────────── + // ── Error handler ───────────────────────────────────── app.setErrorHandler((error: FastifyError, _req, reply) => { app.log.error(error); const code = error.statusCode || 500; @@ -194,14 +196,14 @@ async function start() { }); }); - // ── Start ───────────────────────────────────────────── + // ── Start ───────────────────────────────────────────── try { await app.listen({ port: PORT, host: HOST }); startS3StorageRetentionScheduler(); console.log( - `\n🚀 Doktainer Server Backend running on http://${HOST}:${PORT}`, + `\n🚀 Doktainer Server Backend running on http://${HOST}:${PORT}`, ); - console.log(`📋 Health Check: http://localhost:${PORT}/health\n`); + console.log(`📋 Health Check: http://localhost:${PORT}/health\n`); } catch (err) { const error = err as NodeJS.ErrnoException; if (error.code === "EADDRINUSE") { @@ -227,4 +229,3 @@ if (require.main === module) { } export { app, start }; - diff --git a/src/server/lib/crypto.ts b/src/server/lib/crypto.ts index 2cff163..23c4d75 100644 --- a/src/server/lib/crypto.ts +++ b/src/server/lib/crypto.ts @@ -1,11 +1,56 @@ -import crypto from 'crypto'; +import crypto from "crypto"; -const ALGORITHM = 'aes-256-gcm'; +const ALGORITHM = "aes-256-gcm"; +export const DEVELOPMENT_ENCRYPTION_KEY = + "dev-32-char-encryption-key-here!"; +const REJECTED_PRODUCTION_KEYS = new Set([ + DEVELOPMENT_ENCRYPTION_KEY, + "change-me", + "changeme", + "replace-me", + "replace-with-a-secure-random-key", +]); + +export function getEncryptionKeyOrThrow(env = process.env): string { + const configuredKey = env.ENCRYPTION_KEY?.trim(); + const isProduction = env.NODE_ENV === "production"; + + if (!configuredKey) { + if (isProduction) { + throw new Error( + "ENCRYPTION_KEY must be configured before starting Doktainer in production", + ); + } + return DEVELOPMENT_ENCRYPTION_KEY; + } + + if (configuredKey.length < 32) { + throw new Error("ENCRYPTION_KEY must be at least 32 characters long"); + } + + if ( + isProduction && + (REJECTED_PRODUCTION_KEYS.has(configuredKey.toLowerCase()) || + /^dev[-_]/i.test(configuredKey) || + new Set(configuredKey).size < 8 || + /^(.)\1+$/.test(configuredKey)) + ) { + throw new Error( + "ENCRYPTION_KEY must use a strong random value and must not use a development or placeholder value in production", + ); + } + + return configuredKey; +} + +export function validateEncryptionConfiguration(env = process.env): void { + getEncryptionKeyOrThrow(env); +} function getKey(): Buffer { - const key = process.env.ENCRYPTION_KEY || 'dev-32-char-encryption-key-here!'; + const key = getEncryptionKeyOrThrow(); // Derive a 32-byte key using SHA-256 - return crypto.createHash('sha256').update(key).digest(); + return crypto.createHash("sha256").update(key).digest(); } /** @@ -17,31 +62,33 @@ export function encrypt(plaintext: string): string { const cipher = crypto.createCipheriv(ALGORITHM, getKey(), iv); const encrypted = Buffer.concat([ - cipher.update(plaintext, 'utf8'), + cipher.update(plaintext, "utf8"), cipher.final(), ]); const authTag = cipher.getAuthTag(); return [ - iv.toString('base64'), - authTag.toString('base64'), - encrypted.toString('base64'), - ].join(':'); + iv.toString("base64"), + authTag.toString("base64"), + encrypted.toString("base64"), + ].join(":"); } /** * Decrypt an encrypted string produced by `encrypt()` */ export function decrypt(encoded: string): string { - const [ivB64, tagB64, dataB64] = encoded.split(':'); - if (!ivB64 || !tagB64 || !dataB64) throw new Error('Invalid encrypted format'); + const [ivB64, tagB64, dataB64] = encoded.split(":"); + if (!ivB64 || !tagB64 || !dataB64) throw new Error("Invalid encrypted format"); - const iv = Buffer.from(ivB64, 'base64'); - const authTag = Buffer.from(tagB64, 'base64'); - const data = Buffer.from(dataB64, 'base64'); + const iv = Buffer.from(ivB64, "base64"); + const authTag = Buffer.from(tagB64, "base64"); + const data = Buffer.from(dataB64, "base64"); const decipher = crypto.createDecipheriv(ALGORITHM, getKey(), iv); decipher.setAuthTag(authTag); - return Buffer.concat([decipher.update(data), decipher.final()]).toString('utf8'); + return Buffer.concat([decipher.update(data), decipher.final()]).toString( + "utf8", + ); } diff --git a/src/server/routes/apps.ts b/src/server/routes/apps.ts index e184917..1b1d7b7 100644 --- a/src/server/routes/apps.ts +++ b/src/server/routes/apps.ts @@ -7,6 +7,11 @@ import { auditLog } from "../services/audit.service"; import { dispatchRuntimeNotification } from "../services/notification.service"; import * as ssh from "../services/ssh.service"; import { rebuildAppInstall } from "../services/app-install-rebuild.service"; +import { + createDeployment, + updateDeployment, +} from "../services/deployment.service"; +import { sanitizeDeploymentError } from "../services/deployment-error.service"; import { APP_TEMPLATES, AppTemplate, @@ -98,48 +103,6 @@ type DockerInspectRuntime = { }>; }; -type DockerPortBindings = Record< - string, - Array<{ HostIp?: string; HostPort?: string }> | null ->; - -function formatEnvLines(env?: string[] | null): string { - return (env ?? []).filter(Boolean).join("\n"); -} - -function formatMountBindings(mounts?: DockerInspectRuntime["Mounts"]): string { - return (mounts ?? []) - .map((mount) => { - if (!mount.Source || !mount.Destination) return ""; - return `${mount.Source}:${mount.Destination}${mount.RW === false ? ":ro" : ""}`; - }) - .filter(Boolean) - .join(","); -} - -function formatPortBindings(bindings?: DockerPortBindings): string { - if (!bindings) return ""; - - return Object.entries(bindings) - .flatMap(([containerPortSpec, hostBindings]) => { - if (!hostBindings || hostBindings.length === 0) return []; - const [containerPort, protocol = "tcp"] = containerPortSpec.split("/"); - - return hostBindings - .map((binding) => { - const hostPort = binding.HostPort?.trim(); - if (!hostPort) return ""; - return `${hostPort}:${containerPort}${protocol !== "tcp" ? `/${protocol}` : ""}`; - }) - .filter(Boolean); - }) - .join(","); -} - -function formatCommandParts(cmd?: string[] | null): string { - return (cmd ?? []).filter(Boolean).join(" ").trim(); -} - function parseCsvBindings(value: string): string[] { return value .split(",") @@ -1045,16 +1008,88 @@ export async function appsRoutes(app: FastifyInstance) { }); } - const install = await prisma.appInstall.create({ - data: { - appId: finalAppId, - appName: finalAppName, - serverId: server.id, - containerName, - port: finalPorts || null, - status: "INSTALLING", + const { install, provisionalContainer } = await prisma.$transaction( + async (tx) => { + const createdInstall = await tx.appInstall.create({ + data: { + appId: finalAppId, + appName: finalAppName, + serverId: server.id, + containerName, + port: finalPorts || null, + status: "INSTALLING", + }, + }); + const createdContainer = await tx.container.create({ + data: { + name: containerName, + image: finalImage, + serverId: server.id, + environmentId: selectedEnvironmentId, + status: "STARTING", + sourceType: "APP_INSTALLER", + deployMode: "IMAGE", + ports: JSON.parse(JSON.stringify(parseCsvBindings(finalPorts))), + envVars: JSON.parse( + JSON.stringify(parseEnvironmentVariables(finalEnv)), + ), + volumes: JSON.parse( + JSON.stringify(parseCsvBindings(finalVolumes)), + ), + restartPolicy: finalRestartPolicy, + }, + }); + + return { + install: createdInstall, + provisionalContainer: createdContainer, + }; }, - }); + ); + const deploymentStartedAt = new Date(); + let deployment; + try { + deployment = await createDeployment({ + containerId: provisionalContainer.id, + organizationId: req.organizationId!, + serverId: server.id, + userId: req.userId, + status: "RUNNING", + trigger: "APP_INSTALLER", + version: finalImage, + image: finalImage, + configSnapshot: { + sourceType: "APP_INSTALLER", + deployMode: "IMAGE", + image: finalImage, + ports: finalPorts, + env: finalEnv, + volumes: finalVolumes, + network: finalNetwork, + restartPolicy: finalRestartPolicy, + command: finalCommand, + appId: finalAppId, + appName: finalAppName, + presetId: body.data.presetId ?? null, + }, + startedAt: deploymentStartedAt, + }); + } catch (error) { + const message = sanitizeDeploymentError(error, { + fallback: "Failed to create deployment history", + }); + await prisma.$transaction([ + prisma.container.update({ + where: { id: provisionalContainer.id }, + data: { status: "ERROR" }, + }), + prisma.appInstall.update({ + where: { id: install.id }, + data: { status: "FAILED", error: message }, + }), + ]); + return reply.status(500).send({ success: false, error: message }); + } setImmediate(async () => { try { @@ -1069,6 +1104,13 @@ export async function appsRoutes(app: FastifyInstance) { command: finalCommand, mountValidation: resolveAppMountValidation(finalAppId), }); + await prisma.container.update({ + where: { id: provisionalContainer.id }, + data: { + dockerId: dockerId.trim().slice(0, 12) || null, + status: "STARTING", + }, + }); await upsertAppInstallerContainer({ serverId: server.id, @@ -1082,6 +1124,32 @@ export async function appsRoutes(app: FastifyInstance) { restartPolicy: finalRestartPolicy, }); + const deployedContainer = await prisma.container.findUnique({ + where: { id: provisionalContainer.id }, + select: { id: true, image: true }, + }); + if (deployedContainer) { + await updateDeployment(deployment.id, { + status: "SUCCESS", + image: deployedContainer.image || finalImage, + configSnapshot: { + sourceType: "APP_INSTALLER", + deployMode: "IMAGE", + image: deployedContainer.image || finalImage, + ports: finalPorts, + env: finalEnv, + volumes: finalVolumes, + network: finalNetwork, + restartPolicy: finalRestartPolicy, + command: finalCommand, + appId: finalAppId, + appName: finalAppName, + presetId: body.data.presetId ?? null, + }, + completedAt: new Date(), + }); + } + await prisma.appInstall.update({ where: { id: install.id }, data: { status: "RUNNING" }, @@ -1125,11 +1193,28 @@ export async function appsRoutes(app: FastifyInstance) { }, }); } - } catch (err: any) { - await prisma.appInstall.update({ - where: { id: install.id }, - data: { status: "FAILED", error: err.message }, + } catch (err: unknown) { + const errorMessage = sanitizeDeploymentError(err, { + fallback: "App installation failed", }); + await prisma.$transaction([ + prisma.appInstall.update({ + where: { id: install.id }, + data: { status: "FAILED", error: errorMessage }, + }), + prisma.container.update({ + where: { id: provisionalContainer.id }, + data: { status: "ERROR" }, + }), + prisma.deployment.update({ + where: { id: deployment.id }, + data: { + status: "FAILED", + error: errorMessage, + completedAt: new Date(), + }, + }), + ]); await auditLog({ userId: req.userId, @@ -1138,7 +1223,7 @@ export async function appsRoutes(app: FastifyInstance) { action: "APP_INSTALL_FAILED", category: "SYSTEM", level: "ERROR", - message: `App "${finalAppName}" install failed on "${server.name}": ${err.message}`, + message: `App "${finalAppName}" install failed on "${server.name}": ${errorMessage}`, meta: { mode: body.data.mode, image: finalImage, @@ -1151,7 +1236,7 @@ export async function appsRoutes(app: FastifyInstance) { organizationId: req.organizationId, action: "app_build_error", title: `App deploy failed: ${finalAppName}`, - message: `App ${finalAppName} failed to deploy on ${server.name}. ${err.message}`, + message: `App ${finalAppName} failed to deploy on ${server.name}. ${errorMessage}`, serverId: server.id, resourceType: "app_install", resourceId: install.id, @@ -1164,7 +1249,7 @@ export async function appsRoutes(app: FastifyInstance) { presetId: body.data.presetId ?? null, serverId: server.id, serverName: server.name, - error: err.message, + error: errorMessage, }, }); } diff --git a/src/server/routes/containers.ts b/src/server/routes/containers.ts index 5cb1190..dbb8326 100644 --- a/src/server/routes/containers.ts +++ b/src/server/routes/containers.ts @@ -25,6 +25,19 @@ import { import * as ssh from "../services/ssh.service"; import { withCommandLogSink } from "../services/ssh.service"; import { rebuildAppInstall } from "../services/app-install-rebuild.service"; +import { + createDeployment, + getDeployment, + listDeployments, + updateDeployment, +} from "../services/deployment.service"; +import { rollbackContainerToDeployment } from "../services/deployment-rollback.service"; +import { + acquireDeploymentLock, + releaseDeploymentLock, + startDeploymentLockHeartbeat, +} from "../services/deployment-lock.service"; +import { sanitizeDeploymentError } from "../services/deployment-error.service"; const DeploySourceTypeSchema = z.enum([ "APP_INSTALLER", @@ -707,8 +720,7 @@ export function resolveContainerPathToHostMountPath( const matchedMount = mounts .filter( (mount) => - isUsableMountPath(mount.Source) && - isUsableMountPath(mount.Destination), + isUsableMountPath(mount.Source) && isUsableMountPath(mount.Destination), ) .map((mount) => ({ source: normalizeRemotePath(mount.Source!), @@ -732,7 +744,10 @@ export function resolveContainerPathToHostMountPath( return pathPosix.normalize(pathPosix.join(matchedMount.source, relativePath)); } -function getWritableMountRootsForPath(targetPath: string, mounts: DockerInspectMount[]) { +function getWritableMountRootsForPath( + targetPath: string, + mounts: DockerInspectMount[], +) { const normalizedTargetPath = normalizeRemotePath(targetPath); return mounts @@ -944,6 +959,7 @@ function resolveGitBuildType(input: z.infer) { } type DockerInspectRuntime = { + Image?: string; Config?: { Image?: string; Env?: string[]; @@ -1564,7 +1580,7 @@ function parseInjectedResponsePayload(response: { function runInjectedContainerJob(input: { app: FastifyInstance; - job: ReturnType; + job: Awaited>; method: "POST"; url: string; headers: Record; @@ -1574,19 +1590,22 @@ function runInjectedContainerJob(input: { void Promise.resolve().then(async () => { const abortController = new AbortController(); if (isProcessJobCancelling(input.job)) { - updateProcessJob(input.job, { + await updateProcessJob(input.job, { status: "cancelled", error: input.job.cancelReason ?? "Job cancelled", }); - appendProcessJobLog(input.job, "[job] Cancelled"); + await appendProcessJobLog(input.job, "[job] Cancelled"); return; } - updateProcessJob(input.job, { + await updateProcessJob(input.job, { status: "running", cancel: (reason) => abortController.abort(reason), }); - appendProcessJobLog(input.job, `[job] Started ${input.job.type}`); + await appendProcessJobLog( + input.job, + `[job] Started ${input.job.type}`, + ); try { const response = await withCommandLogSink( @@ -1601,7 +1620,10 @@ function runInjectedContainerJob(input: { await input.app.inject({ method: input.method, url: input.url, - headers: buildInternalJobHeaders(input.headers), + headers: { + ...buildInternalJobHeaders(input.headers), + "x-process-job-sync": "true", + }, payload: input.payload, }), { @@ -1610,12 +1632,12 @@ function runInjectedContainerJob(input: { }, ); - if (abortController.signal.aborted || isProcessJobCancelling(input.job)) { - updateProcessJob(input.job, { + if (isProcessJobCancelling(input.job)) { + await updateProcessJob(input.job, { status: "cancelled", error: input.job.cancelReason ?? "Job cancelled", }); - appendProcessJobLog(input.job, "[job] Cancelled"); + await appendProcessJobLog(input.job, "[job] Cancelled"); return; } @@ -1628,29 +1650,56 @@ function runInjectedContainerJob(input: { throw new Error(error || `Job failed with HTTP ${response.statusCode}`); } - updateProcessJob(input.job, { status: "success", result: payload }); - appendProcessJobLog(input.job, "[job] Completed successfully"); + await updateProcessJob(input.job, { + status: "success", + result: payload, + }); + await appendProcessJobLog(input.job, "[job] Completed successfully"); } catch (error) { const message = error instanceof Error ? error.message : "Job failed"; if ( - abortController.signal.aborted || isProcessJobCancelling(input.job) || message === "Command cancelled" ) { - updateProcessJob(input.job, { + await updateProcessJob(input.job, { status: "cancelled", error: input.job.cancelReason ?? "Job cancelled", }); - appendProcessJobLog(input.job, "[job] Cancelled"); + await appendProcessJobLog(input.job, "[job] Cancelled"); return; } - - updateProcessJob(input.job, { status: "error", error: message }); - appendProcessJobLog(input.job, `[job] Failed: ${message}`); + await updateProcessJob(input.job, { + status: "error", + error: message, + }); + await appendProcessJobLog(input.job, `[job] Failed: ${message}`); } }); } +function buildDeploymentSnapshot( + body: z.infer, + runtimeImage?: string, +) { + return { + sourceType: body.sourceType, + deployMode: body.deployMode, + buildType: body.buildType, + image: runtimeImage || body.image, + ports: body.ports, + env: body.env, + volumes: body.volumes, + restartPolicy: body.restartPolicy, + repoUrl: body.repoUrl, + repoBranch: body.repoBranch, + buildPath: body.buildPath, + composeFilePath: body.composeFilePath, + dockerfilePath: body.dockerfilePath, + dockerContextPath: body.dockerContextPath, + imageTag: body.imageTag, + }; +} + export async function containerRoutes(app: FastifyInstance) { const containerReadAccess = [ authenticate, @@ -1662,6 +1711,143 @@ export async function containerRoutes(app: FastifyInstance) { ]; // GET /containers — all containers from DB + const recordDeployment = async (input: { + container: { id: string; serverId: string; image: string }; + body: z.infer; + trigger: "MANUAL" | "GIT_WEBHOOK" | "APP_INSTALLER"; + userId?: string; + organizationId: string; + }) => { + const body = input.body; + return createDeployment({ + containerId: input.container.id, + organizationId: input.organizationId, + serverId: input.container.serverId, + userId: input.userId, + status: "SUCCESS", + trigger: input.trigger, + version: body.imageTag || body.repoBranch || input.container.image, + branch: body.repoBranch || null, + image: input.container.image, + configSnapshot: buildDeploymentSnapshot(body, input.container.image), + startedAt: new Date(), + completedAt: new Date(), + }); + }; + + const recordFailedDeployment = async (input: { + container: { id: string; serverId: string; image: string }; + body: z.infer; + trigger: "MANUAL" | "GIT_WEBHOOK" | "APP_INSTALLER"; + userId?: string; + organizationId: string; + error: string; + }) => + createDeployment({ + containerId: input.container.id, + organizationId: input.organizationId, + serverId: input.container.serverId, + userId: input.userId, + status: "FAILED", + trigger: input.trigger, + version: input.body.imageTag || input.body.repoBranch || input.container.image, + branch: input.body.repoBranch || null, + image: input.container.image, + configSnapshot: buildDeploymentSnapshot(input.body, input.container.image), + error: input.error, + startedAt: new Date(), + completedAt: new Date(), + }); + + app.get( + "/:id/deployments", + { preHandler: containerReadAccess }, + async (req, reply) => { + const { id } = req.params as { id: string }; + const query = req.query as { page?: string; pageSize?: string }; + const page = Math.max(1, Number.parseInt(query.page ?? "1", 10) || 1); + const pageSize = Math.min( + 50, + Math.max(1, Number.parseInt(query.pageSize ?? "20", 10) || 20), + ); + const container = await prisma.container.findFirst({ + where: { id, server: { organizationId: req.organizationId! } }, + select: { id: true }, + }); + if (!container) { + return reply + .status(404) + .send({ success: false, error: "Container not found" }); + } + + const data = await listDeployments({ + containerId: id, + organizationId: req.organizationId!, + page, + pageSize, + }); + return reply.send({ success: true, data }); + }, + ); + + app.get( + "/:id/deployments/:deploymentId", + { preHandler: containerReadAccess }, + async (req, reply) => { + const { id, deploymentId } = req.params as { + id: string; + deploymentId: string; + }; + const deployment = await getDeployment({ + containerId: id, + deploymentId, + organizationId: req.organizationId!, + }); + if (!deployment) { + return reply + .status(404) + .send({ success: false, error: "Deployment not found" }); + } + return reply.send({ success: true, data: deployment }); + }, + ); + + app.post( + "/:id/deployments/:deploymentId/rollback", + { preHandler: containerWriteAccess }, + async (req, reply) => { + const { id, deploymentId } = req.params as { + id: string; + deploymentId: string; + }; + + try { + const result = await rollbackContainerToDeployment({ + containerId: id, + deploymentId, + organizationId: req.organizationId!, + userId: req.userId, + }); + return reply.send({ + success: true, + data: result.updated, + meta: { + rollbackDeploymentId: result.deploymentId, + targetDeploymentId: result.targetDeploymentId, + }, + message: "Container rolled back successfully", + }); + } catch (error) { + return reply.status(400).send({ + success: false, + error: sanitizeDeploymentError(error, { + fallback: "Rollback failed", + }), + }); + } + }, + ); + app.get( "/jobs/:jobId", { preHandler: containerReadAccess }, @@ -1700,7 +1886,7 @@ export async function containerRoutes(app: FastifyInstance) { .send({ success: false, error: "Job not found" }); } - const cancelled = cancelProcessJob(job); + const cancelled = await cancelProcessJob(job); return reply.send({ success: true, @@ -1792,7 +1978,7 @@ export async function containerRoutes(app: FastifyInstance) { .send({ success: false, error: validationError }); } - const job = createProcessJob({ + const job = await createProcessJob({ type: "container_deploy", userId: req.userId, organizationId: req.organizationId, @@ -2196,10 +2382,10 @@ export async function containerRoutes(app: FastifyInstance) { container.dockerId || container.name, ); canWriteProjectEnv = - getWritableMountRootsForPath( - normalizedEnvPath, - [...getInspectMounts(inspect), ...storedMounts], - ).length > 0; + getWritableMountRootsForPath(normalizedEnvPath, [ + ...getInspectMounts(inspect), + ...storedMounts, + ]).length > 0; } catch { canWriteProjectEnv = false; } @@ -3082,7 +3268,6 @@ export async function containerRoutes(app: FastifyInstance) { volumes, network: selectedNetworkName, }); - const updatedContainer = await prisma.container.update({ where: { id: container.id }, data: { @@ -3092,6 +3277,14 @@ export async function containerRoutes(app: FastifyInstance) { include: { server: { select: { name: true, ip: true } } }, }); + await recordDeployment({ + container: updatedContainer, + body: body.data, + trigger: "MANUAL", + userId: req.userId, + organizationId: req.organizationId!, + }); + await auditLog({ userId: req.userId, serverId, @@ -3107,14 +3300,22 @@ export async function containerRoutes(app: FastifyInstance) { message: "Deployment completed successfully", }); } catch (err) { - const errorMessage = redactSecret( - err instanceof Error ? err.message : String(err), - accessToken, - ); + const errorMessage = sanitizeDeploymentError(err, { + fallback: "Container deployment failed", + secrets: [accessToken], + }); await prisma.container.update({ where: { id: container.id }, data: { status: "ERROR" }, }); + await recordFailedDeployment({ + container, + body: body.data, + trigger: "MANUAL", + userId: req.userId, + organizationId: req.organizationId!, + error: errorMessage, + }).catch(() => undefined); await auditLog({ userId: req.userId, serverId, @@ -3150,6 +3351,17 @@ export async function containerRoutes(app: FastifyInstance) { dockerId: dockerId.trim().slice(0, 12), }, }); + await recordDeployment({ + container: { + id: container.id, + serverId: container.serverId, + image: container.image, + }, + body: body.data, + trigger: "MANUAL", + userId: req.userId, + organizationId: req.organizationId!, + }); await auditLog({ userId: req.userId, serverId, @@ -3160,14 +3372,22 @@ export async function containerRoutes(app: FastifyInstance) { }); }) .catch(async (err) => { - const errorMessage = redactSecret( - err instanceof Error ? err.message : String(err), - accessToken, - ); + const errorMessage = sanitizeDeploymentError(err, { + fallback: "Container deployment failed", + secrets: [accessToken], + }); await prisma.container.update({ where: { id: container.id }, data: { status: "ERROR" }, }); + await recordFailedDeployment({ + container, + body: body.data, + trigger: "MANUAL", + userId: req.userId, + organizationId: req.organizationId!, + error: errorMessage, + }).catch(() => undefined); await auditLog({ userId: req.userId, serverId, @@ -3290,6 +3510,23 @@ export async function containerRoutes(app: FastifyInstance) { include: { server: { select: { name: true, ip: true } } }, }); + await Promise.all( + responseContainers.map((matchedContainer) => + recordDeployment({ + container: matchedContainer, + body: body.data, + trigger: + body.data.sourceType === "APP_INSTALLER" + ? "APP_INSTALLER" + : "MANUAL", + userId: req.userId, + organizationId: req.organizationId!, + }), + ), + ).catch((error) => { + app.log.error({ error }, "Failed to persist deployment history"); + }); + await auditLog({ userId: req.userId, serverId, @@ -3315,11 +3552,32 @@ export async function containerRoutes(app: FastifyInstance) { : "Deployment completed successfully", }); } catch (error) { - const errorMessage = redactSecret( + const errorMessage = sanitizeDeploymentError( ssh.formatDeploymentErrorMessage(error), - accessToken, + { + fallback: "Container deployment failed", + secrets: [accessToken], + }, ); + const failedContainer = await prisma.container.findFirst({ + where: { + serverId, + name: { equals: rest.name, mode: "insensitive" }, + }, + select: { id: true, serverId: true, image: true }, + }); + if (failedContainer) { + await recordFailedDeployment({ + container: failedContainer, + body: body.data, + trigger: body.data.sourceType === "APP_INSTALLER" ? "APP_INSTALLER" : "MANUAL", + userId: req.userId, + organizationId: req.organizationId!, + error: errorMessage, + }).catch(() => undefined); + } + await auditLog({ userId: req.userId, serverId, @@ -3501,7 +3759,7 @@ export async function containerRoutes(app: FastifyInstance) { .send({ success: false, error: "Container not found" }); } - const job = createProcessJob({ + const job = await createProcessJob({ type: "container_rebuild", userId: req.userId, organizationId: req.organizationId, @@ -3545,6 +3803,13 @@ export async function containerRoutes(app: FastifyInstance) { .send({ success: false, error: "Container not found" }); } + let gitDeploymentId: string | null = null; + let gitDeploymentLock: { token: string } | null = null; + let gitDeploymentHeartbeat: ReturnType< + typeof startDeploymentLockHeartbeat + > | null = null; + let gitAccessToken: string | undefined; + try { if (container.sourceType === "APP_INSTALLER") { const install = await prisma.appInstall.findFirst({ @@ -3626,6 +3891,50 @@ export async function containerRoutes(app: FastifyInstance) { const accessToken = container.deploymentSource.accessTokenEnc ? decrypt(container.deploymentSource.accessTokenEnc) : undefined; + gitAccessToken = accessToken; + const gitDeploymentSnapshot = { + sourceType: container.sourceType, + deployMode: container.deployMode, + buildType, + image: container.image, + ports: runtimeConfig.ports, + env: runtimeConfig.env, + volumes: runtimeConfig.volumes, + network: runtimeConfig.network, + restartPolicy: runtimeConfig.restartPolicy, + repoUrl: container.deploymentSource.repoUrl, + repoBranch: container.deploymentSource.repoBranch, + buildPath: container.deploymentSource.buildPath, + composeFilePath: container.deploymentSource.composeFilePath, + dockerfilePath: container.deploymentSource.dockerfilePath, + dockerContextPath: + container.deploymentSource.dockerContextPath, + imageTag: container.deploymentSource.imageTag, + deploymentPath: container.deploymentSource.deploymentPath, + }; + + gitDeploymentLock = await acquireDeploymentLock({ + containerId: container.id, + }); + gitDeploymentHeartbeat = startDeploymentLockHeartbeat({ + containerId: container.id, + token: gitDeploymentLock.token, + }); + const runningDeployment = await createDeployment({ + containerId: container.id, + organizationId: req.organizationId!, + serverId: container.serverId, + userId: req.userId, + status: "RUNNING", + trigger: "REBUILD", + version: + container.deploymentSource.repoBranch?.trim() || container.image, + branch: container.deploymentSource.repoBranch?.trim() || null, + image: container.image, + configSnapshot: gitDeploymentSnapshot, + startedAt: new Date(), + }); + gitDeploymentId = runningDeployment.id; if (buildType !== "COMPOSE") { for (const candidate of [container.dockerId, container.name]) { @@ -3650,7 +3959,9 @@ export async function containerRoutes(app: FastifyInstance) { } } - await ssh.deployContainerFromGitSource(container.server, { + const gitDeploymentResult = await ssh.deployContainerFromGitSource( + container.server, + { projectName: deploymentProjectName, repoUrl: container.deploymentSource.repoUrl, branch: toOptionalValue(container.deploymentSource.repoBranch), @@ -3686,7 +3997,9 @@ export async function containerRoutes(app: FastifyInstance) { deploymentPath: toOptionalValue( container.deploymentSource.deploymentPath, ), - }); + }, + ); + gitDeploymentHeartbeat.assertOwned(); await syncContainersForServers( [container.server], @@ -3740,6 +4053,46 @@ export async function containerRoutes(app: FastifyInstance) { (candidate) => candidate.name.toLowerCase() === container.name.toLowerCase(), ) ?? responseContainers[0]; + if (!updatedContainer) { + throw new Error( + "Redeploy from Git finished but no primary container could be resolved", + ); + } + + gitDeploymentHeartbeat.assertOwned(); + + let imageDigest: string | null = null; + try { + const inspect = (await ssh.dockerInspect( + container.server, + updatedContainer.dockerId || updatedContainer.name, + )) as DockerInspectRuntime; + imageDigest = + typeof inspect.Image === "string" && inspect.Image.trim() + ? inspect.Image.trim() + : null; + } catch { + // The canonical runtime image is still persisted even when Docker + // does not return an image ID after the successful health gate. + } + + await updateDeployment(runningDeployment.id, { + status: "SUCCESS", + completedAt: new Date(), + image: updatedContainer.image, + imageDigest, + commitSha: gitDeploymentResult.commitSha, + branch: container.deploymentSource.repoBranch?.trim() || null, + configSnapshot: { + ...gitDeploymentSnapshot, + image: updatedContainer.image, + imageDigest, + commitSha: gitDeploymentResult.commitSha, + matchedContainerIds: responseContainers.map( + (candidate) => candidate.id, + ), + }, + }); await auditLog({ userId: req.userId, @@ -3748,6 +4101,16 @@ export async function containerRoutes(app: FastifyInstance) { category: "CONTAINER", level: "SUCCESS", message: `Container "${container.name}" redeployed from Git on ${container.server.name}`, + meta: { + deploymentId: runningDeployment.id, + commitSha: gitDeploymentResult.commitSha, + branch: container.deploymentSource.repoBranch, + image: updatedContainer.image, + imageDigest, + matchedContainerIds: responseContainers.map( + (candidate) => candidate.id, + ), + }, }); return reply.send({ @@ -3759,11 +4122,45 @@ export async function containerRoutes(app: FastifyInstance) { : "Container redeployed successfully from Git", }); } catch (err: unknown) { + const errorMessage = gitDeploymentId + ? sanitizeDeploymentError(ssh.formatDeploymentErrorMessage(err), { + fallback: "Git redeploy failed", + secrets: [gitAccessToken], + }) + : err instanceof Error + ? err.message + : "Failed to rebuild container"; + + if (gitDeploymentId) { + await updateDeployment(gitDeploymentId, { + status: "FAILED", + error: errorMessage, + completedAt: new Date(), + }).catch(() => undefined); + await auditLog({ + userId: req.userId, + organizationId: req.organizationId, + serverId: container.serverId, + action: "CONTAINER_REDEPLOY_GIT", + category: "CONTAINER", + level: "ERROR", + message: `Git redeploy failed for container "${container.name}": ${errorMessage}`, + meta: { deploymentId: gitDeploymentId }, + }).catch(() => undefined); + } + return reply.status(500).send({ success: false, - error: - err instanceof Error ? err.message : "Failed to rebuild container", + error: errorMessage, }); + } finally { + gitDeploymentHeartbeat?.stop(); + if (gitDeploymentLock) { + await releaseDeploymentLock({ + containerId: container.id, + token: gitDeploymentLock.token, + }).catch(() => undefined); + } } }, ); diff --git a/src/server/routes/server-action-errors.ts b/src/server/routes/server-action-errors.ts new file mode 100644 index 0000000..9020d2c --- /dev/null +++ b/src/server/routes/server-action-errors.ts @@ -0,0 +1,23 @@ +function getErrorText(error: unknown) { + return error instanceof Error ? error.message : String(error ?? ""); +} + +export function formatServerActionError( + error: unknown, + fallbackMessage: string, +) { + const message = getErrorText(error); + + const portMatch = message.match(/(?:0\.0\.0\.0|:::):(?:80|443).*?(?:address already in use|already in use)/i); + if (portMatch || /address already in use/i.test(message)) { + return "The service could not start because port 80 or 443 is already in use by another host process or Docker container."; + } + + if (/syntax error|unexpected token|command not found|bad substitution/i.test(message)) { + return "The remote host rejected the generated shell command before it could finish. Refresh the server snapshot and retry the action."; + } + + return fallbackMessage; +} + + diff --git a/src/server/services/app-install-rebuild.service.ts b/src/server/services/app-install-rebuild.service.ts index da06ab9..0bf05d7 100644 --- a/src/server/services/app-install-rebuild.service.ts +++ b/src/server/services/app-install-rebuild.service.ts @@ -4,6 +4,13 @@ import { auditLog } from "./audit.service"; import { dispatchRuntimeNotification } from "./notification.service"; import * as ssh from "./ssh.service"; import { APP_TEMPLATES } from "../config/app-templates"; +import { createDeployment, updateDeployment } from "./deployment.service"; +import { + acquireDeploymentLock, + DeploymentLockConflictError, + releaseDeploymentLock, + startDeploymentLockHeartbeat, +} from "./deployment-lock.service"; type DockerInspectRuntime = { Config?: { @@ -176,15 +183,51 @@ export async function rebuildAppInstall(options: { throw new Error("Unable to determine which image should be rebuilt"); } - await prisma.appInstall.update({ - where: { id: install.id }, - data: { status: "INSTALLING", error: null }, - }); - let pulledLatestImage = false; let usedImageCache = false; + let deploymentId: string | null = null; + let deploymentLockToken: string | null = null; + let deploymentLockContainerId: string | null = null; + let deploymentLockHeartbeat: ReturnType< + typeof startDeploymentLockHeartbeat + > | null = null; try { + const currentContainer = await prisma.container.findFirst({ + where: { + serverId: install.serverId, + name: { equals: install.containerName, mode: "insensitive" }, + }, + select: { id: true }, + }); + if (currentContainer) { + const lock = await acquireDeploymentLock({ containerId: currentContainer.id }); + deploymentLockToken = lock.token; + deploymentLockContainerId = currentContainer.id; + deploymentLockHeartbeat = startDeploymentLockHeartbeat({ + containerId: currentContainer.id, + token: lock.token, + }); + const deployment = await createDeployment({ + containerId: currentContainer.id, + organizationId: options.organizationId ?? install.server.organizationId, + serverId: install.serverId, + userId: options.userId, + status: "RUNNING", + trigger: "REBUILD", + version: runtimeConfig.image, + image: runtimeConfig.image, + configSnapshot: runtimeConfig, + startedAt: new Date(), + }); + deploymentId = deployment.id; + } + + await prisma.appInstall.update({ + where: { id: install.id }, + data: { status: "INSTALLING", error: null }, + }); + try { await ssh.dockerPullImage(install.server, runtimeConfig.image); pulledLatestImage = true; @@ -218,6 +261,8 @@ export async function rebuildAppInstall(options: { const normalizedDockerId = dockerId.trim().slice(0, 12); + deploymentLockHeartbeat?.assertOwned(); + await prisma.container.updateMany({ where: { serverId: install.serverId, @@ -238,6 +283,14 @@ export async function rebuildAppInstall(options: { }, }); + if (deploymentId) { + await updateDeployment(deploymentId, { + status: "SUCCESS", + completedAt: new Date(), + image: runtimeConfig.image, + }); + } + await auditLog({ userId: options.userId, serverId: install.serverId, @@ -283,6 +336,9 @@ export async function rebuildAppInstall(options: { usedImageCache, }; } catch (error) { + if (error instanceof DeploymentLockConflictError) { + throw error; + } const message = error instanceof Error ? error.message : "Failed to rebuild app"; @@ -299,6 +355,15 @@ export async function rebuildAppInstall(options: { data: { status: "ERROR" }, }); + if (deploymentId) { + await updateDeployment(deploymentId, { + status: "FAILED", + error: message, + completedAt: new Date(), + image: runtimeConfig.image, + }).catch(() => undefined); + } + await auditLog({ userId: options.userId, serverId: install.serverId, @@ -335,5 +400,13 @@ export async function rebuildAppInstall(options: { } throw error instanceof Error ? error : new Error(message); + } finally { + deploymentLockHeartbeat?.stop(); + if (deploymentLockToken && deploymentLockContainerId) { + await releaseDeploymentLock({ + containerId: deploymentLockContainerId, + token: deploymentLockToken, + }).catch(() => undefined); + } } } diff --git a/src/server/services/container-health.service.ts b/src/server/services/container-health.service.ts new file mode 100644 index 0000000..6a601b9 --- /dev/null +++ b/src/server/services/container-health.service.ts @@ -0,0 +1,126 @@ +import type { Server } from "@prisma/client"; +import * as ssh from "./ssh.service"; +import { sanitizeDeploymentError } from "./deployment-error.service"; + +const DEFAULT_TIMEOUT_MS = 30_000; +const DEFAULT_INTERVAL_MS = 2_000; + +type DockerInspectHealth = { + State?: { + Status?: string; + Health?: { Status?: string }; + }; +}; + +export type ContainerHealthResult = { + healthy: boolean; + status: + | "healthy" + | "unhealthy" + | "starting" + | "not_running" + | "unknown"; + reason: string; + attempts: number; + durationMs: number; +}; + +export function evaluateDockerHealth( + inspect: DockerInspectHealth | null | undefined, +): Omit { + const state = inspect?.State?.Status?.toLowerCase(); + if (state === "created" || state === "restarting") { + return { + healthy: false, + status: "starting", + reason: `Container runtime state is ${state}`, + }; + } + if (state !== "running") { + return { + healthy: false, + status: "not_running", + reason: `Container runtime state is ${state || "unknown"}`, + }; + } + + const health = inspect?.State?.Health?.Status?.toLowerCase(); + if (health === "unhealthy") { + return { + healthy: false, + status: "unhealthy", + reason: "Docker health check reported unhealthy", + }; + } + if (health === "starting") { + return { + healthy: false, + status: "starting", + reason: "Docker health check is still starting", + }; + } + if (health === "healthy") { + return { + healthy: true, + status: "healthy", + reason: "Docker health check reported healthy", + }; + } + + return { + healthy: true, + status: "healthy", + reason: "Container is running and the image has no Docker health check", + }; +} + +export async function waitForDockerHealth(input: { + server: Server; + containerRef: string; + timeoutMs?: number; + intervalMs?: number; +}): Promise { + const startedAt = Date.now(); + const timeoutMs = Math.max(1_000, input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const intervalMs = Math.max(250, input.intervalMs ?? DEFAULT_INTERVAL_MS); + let attempts = 0; + let lastReason = "Docker health check did not complete"; + + while (Date.now() - startedAt <= timeoutMs) { + attempts += 1; + try { + const result = evaluateDockerHealth( + (await ssh.dockerInspect( + input.server, + input.containerRef, + )) as DockerInspectHealth, + ); + lastReason = result.reason; + if (result.healthy) { + return { ...result, attempts, durationMs: Date.now() - startedAt }; + } + if ( + result.status === "unhealthy" || + result.status === "not_running" + ) { + return { ...result, attempts, durationMs: Date.now() - startedAt }; + } + } catch (error) { + lastReason = sanitizeDeploymentError(error, { + fallback: "Docker health inspection failed", + }); + } + + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + + return { + healthy: false, + status: "unknown", + reason: `Docker health check timed out after ${timeoutMs}ms: ${lastReason}`, + attempts, + durationMs: Date.now() - startedAt, + }; +} + + diff --git a/src/server/services/deployment-error.service.ts b/src/server/services/deployment-error.service.ts new file mode 100644 index 0000000..0314d66 --- /dev/null +++ b/src/server/services/deployment-error.service.ts @@ -0,0 +1,76 @@ +const SENSITIVE_ASSIGNMENT_PATTERN = + /\b(password|passwd|secret|token|api[-_]?key|private[-_]?key|credential|authorization)\s*[:=]\s*(?:"[^"]*"|'[^']*'|[^\s,;]+)/gi; +const URL_CREDENTIAL_PATTERN = + /([a-z][a-z0-9+.-]*:\/\/)([^/\s:@]+):([^@\s/]+)@/gi; +const SENSITIVE_QUERY_PATTERN = + /([?&](?:access_token|token|api_key|key|secret)=)[^&\s]+/gi; +const PRIVATE_KEY_PATTERN = + /-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/gi; + +function errorText(error: unknown) { + return error instanceof Error ? error.message : String(error ?? ""); +} + +function redactKnownSecrets(message: string, secrets: Array) { + return secrets.reduce((current, secret) => { + const normalized = secret?.trim(); + return normalized ? current.split(normalized).join("[REDACTED]") : current; + }, message); +} + +export function redactDeploymentErrorDetails( + error: unknown, + secrets: Array = [], +) { + return redactKnownSecrets(errorText(error), secrets) + .replace(PRIVATE_KEY_PATTERN, "[REDACTED PRIVATE KEY]") + .replace( + /\bauthorization\s*:\s*(?:Bearer|Basic)\s+\S+/gi, + "authorization: [REDACTED]", + ) + .replace( + /\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, + "[REDACTED AUTHORIZATION]", + ) + .replace(URL_CREDENTIAL_PATTERN, "$1[REDACTED]@") + .replace(SENSITIVE_QUERY_PATTERN, "$1[REDACTED]") + .replace(SENSITIVE_ASSIGNMENT_PATTERN, "$1=[REDACTED]") + .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, "") + .trim(); +} + +export function sanitizeDeploymentError( + error: unknown, + options: { + fallback?: string; + secrets?: Array; + } = {}, +) { + const fallback = options.fallback || "Deployment operation failed"; + const message = redactDeploymentErrorDetails(error, options.secrets); + if (!message) return fallback; + + if (/address already in use|port is already allocated|bind.*failed/i.test(message)) { + return "The deployment could not start because a required host port is already in use."; + } + + if (/no such image|pull access denied|manifest unknown/i.test(message)) { + return "The deployment image could not be resolved or pulled from its registry."; + } + + if (/no such container|container .* not found/i.test(message)) { + return "The target container could not be found on the selected server."; + } + + if ( + /command failed|docker (?:run|pull|inspect|rename|stop|rm)|bash:|sh:|stderr|stdout|prisma|stack|ssh/i.test( + message, + ) + ) { + return fallback; + } + + return message.slice(0, 800); +} + + diff --git a/src/server/services/deployment-lock.service.ts b/src/server/services/deployment-lock.service.ts new file mode 100644 index 0000000..3bf0a02 --- /dev/null +++ b/src/server/services/deployment-lock.service.ts @@ -0,0 +1,130 @@ +import { randomUUID } from "node:crypto"; +import { Prisma } from "@prisma/client"; +import prisma from "../lib/prisma"; + +export const DEFAULT_DEPLOYMENT_LOCK_TTL_MS = 15 * 60 * 1000; +export const DEFAULT_DEPLOYMENT_LOCK_HEARTBEAT_MS = 60 * 1000; + +export class DeploymentLockConflictError extends Error { + constructor() { + super("Another deployment operation is already running for this container"); + this.name = "DeploymentLockConflictError"; + } +} + +export async function acquireDeploymentLock(input: { + containerId: string; + ttlMs?: number; +}) { + const token = randomUUID(); + const expiresAt = new Date( + Date.now() + + Math.max(30_000, input.ttlMs ?? DEFAULT_DEPLOYMENT_LOCK_TTL_MS), + ); + + try { + return await prisma.$transaction(async (tx) => { + const reclaimed = await tx.deploymentLock.updateMany({ + where: { containerId: input.containerId, expiresAt: { lte: new Date() } }, + data: { token, expiresAt }, + }); + + if (reclaimed.count > 0) return { token, expiresAt }; + + await tx.deploymentLock.create({ + data: { containerId: input.containerId, token, expiresAt }, + }); + return { token, expiresAt }; + }); + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { + throw new DeploymentLockConflictError(); + } + throw error; + } +} + +export async function releaseDeploymentLock(input: { + containerId: string; + token: string; +}) { + await prisma.deploymentLock.deleteMany({ + where: { containerId: input.containerId, token: input.token }, + }); +} + +export async function renewDeploymentLock(input: { + containerId: string; + token: string; + ttlMs?: number; +}) { + const result = await prisma.deploymentLock.updateMany({ + where: { containerId: input.containerId, token: input.token, expiresAt: { gt: new Date() } }, + data: { + expiresAt: new Date( + Date.now() + + Math.max(30_000, input.ttlMs ?? DEFAULT_DEPLOYMENT_LOCK_TTL_MS), + ), + }, + }); + if (result.count !== 1) throw new DeploymentLockConflictError(); +} + +export function startDeploymentLockHeartbeat( + input: { + containerId: string; + token: string; + ttlMs?: number; + intervalMs?: number; + }, + renew = renewDeploymentLock, +) { + const ttlMs = Math.max( + 30_000, + input.ttlMs ?? DEFAULT_DEPLOYMENT_LOCK_TTL_MS, + ); + const intervalMs = Math.min( + Math.max(10, input.intervalMs ?? DEFAULT_DEPLOYMENT_LOCK_HEARTBEAT_MS), + Math.max(10, Math.floor(ttlMs / 3)), + ); + let stopped = false; + let renewing = false; + let ownershipError: Error | null = null; + + const timer = setInterval(() => { + if (stopped || renewing || ownershipError) return; + renewing = true; + void renew({ + containerId: input.containerId, + token: input.token, + ttlMs, + }) + .catch((error: unknown) => { + ownershipError = + error instanceof Error + ? error + : new DeploymentLockConflictError(); + }) + .finally(() => { + renewing = false; + }); + }, intervalMs); + timer.unref?.(); + + return { + assertOwned() { + if (ownershipError) { + throw new DeploymentLockConflictError(); + } + }, + hasLostOwnership() { + return ownershipError !== null; + }, + stop() { + stopped = true; + clearInterval(timer); + }, + }; +} + + diff --git a/src/server/services/deployment-rollback.service.ts b/src/server/services/deployment-rollback.service.ts new file mode 100644 index 0000000..8e32384 --- /dev/null +++ b/src/server/services/deployment-rollback.service.ts @@ -0,0 +1,714 @@ +import prisma from "../lib/prisma"; +import { auditLog } from "./audit.service"; +import { + createDeployment, + getRollbackSnapshot, + updateDeployment, + type DeploymentSnapshot, +} from "./deployment.service"; +import * as ssh from "./ssh.service"; +import { waitForDockerHealth } from "./container-health.service"; +import { resolveDeploymentStrategy } from "./deployment-strategy"; +import { + acquireDeploymentLock, + releaseDeploymentLock, + startDeploymentLockHeartbeat, +} from "./deployment-lock.service"; +import { sanitizeDeploymentError } from "./deployment-error.service"; + +type RollbackRuntime = { + image: string; + ports: string; + env: string; + volumes: string; + network: string; + restartPolicy: string; + entrypoint?: string; + commandArgs?: string[]; + command: string; +}; + +type DockerInspectRuntime = { + Config?: { + Image?: string; + Env?: string[]; + Entrypoint?: string[] | string | null; + Cmd?: string[] | null; + }; + HostConfig?: { + RestartPolicy?: { Name?: string | null }; + PortBindings?: Record< + string, + Array<{ HostPort?: string }> | null + >; + NetworkMode?: string | null; + }; + Mounts?: Array<{ + Source?: string; + Destination?: string; + RW?: boolean; + }>; +}; + +type DockerPortBindings = NonNullable< + NonNullable["PortBindings"] +>; + +type RollbackServer = Parameters[0]; + +type RollbackRuntimeDependencies = { + runContainer: typeof ssh.runContainer; + stopAndRemoveContainer: typeof stopAndRemoveContainer; + dockerRename: typeof ssh.dockerRename; + waitForDockerHealth: typeof waitForDockerHealth; +}; + +const rollbackRuntimeDependencies: RollbackRuntimeDependencies = { + runContainer: ssh.runContainer, + stopAndRemoveContainer, + dockerRename: ssh.dockerRename, + waitForDockerHealth, +}; + +function snapshotString(snapshot: DeploymentSnapshot, key: string, fallback = "") { + const value = snapshot[key]; + return typeof value === "string" ? value : fallback; +} + +export function normalizeRollbackPortMappings(value: unknown) { + const entries = Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : typeof value === "string" + ? value.split(",") + : []; + const normalized = new Map(); + + for (const rawEntry of entries) { + const entry = rawEntry.trim(); + if (!entry) continue; + + if (!entry.includes("->")) { + normalized.set(entry, entry); + continue; + } + + const [publishedRaw, containerRaw] = entry.split("->", 2); + const published = publishedRaw?.trim(); + const containerSpec = containerRaw?.trim(); + if (!published || !containerSpec) continue; + + const [containerPort, protocol = "tcp"] = containerSpec.split("/"); + const hostPort = published.match(/(?:^|:)(\d+)$/)?.[1]; + if (!hostPort || !/^\d+$/.test(containerPort ?? "")) continue; + + const hostIpMatch = published.match( + /^((?:\d{1,3}\.){3}\d{1,3}):\d+$/, + ); + const hostIp = + hostIpMatch?.[1] && hostIpMatch[1] !== "0.0.0.0" + ? `${hostIpMatch[1]}:` + : ""; + const mapping = `${hostIp}${hostPort}:${containerPort}${ + protocol !== "tcp" ? `/${protocol}` : "" + }`; + normalized.set(`${hostPort}:${containerPort}/${protocol}`, mapping); + } + + return [...normalized.values()].join(","); +} + +function storedList(value: unknown) { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function formatPortBindings(bindings?: DockerPortBindings) { + if (!bindings) return ""; + + return Object.entries(bindings) + .flatMap(([containerPortSpec, hostBindings]) => { + if (!hostBindings?.length) return []; + const [containerPort, protocol = "tcp"] = containerPortSpec.split("/"); + return hostBindings.flatMap((binding) => { + const hostPort = binding.HostPort?.trim(); + if (!hostPort) return []; + return `${hostPort}:${containerPort}${protocol !== "tcp" ? `/${protocol}` : ""}`; + }); + }) + .join(","); +} + +function formatMountBindings(mounts?: DockerInspectRuntime["Mounts"]) { + return (mounts ?? []) + .flatMap((mount) => { + if (!mount.Source || !mount.Destination) return []; + return `${mount.Source}:${mount.Destination}${mount.RW === false ? ":ro" : ""}`; + }) + .join(","); +} + +function formatEntrypoint(value?: string[] | string | null) { + if (typeof value === "string") return value.trim() || undefined; + return value?.[0]?.trim() || undefined; +} + +function snapshotRuntime( + snapshot: DeploymentSnapshot, + fallbackImage?: string | null, +): RollbackRuntime { + const image = (snapshotString(snapshot, "image") || fallbackImage || "").trim(); + if (!image) throw new Error("Rollback artifact does not contain an image"); + + return { + image, + ports: normalizeRollbackPortMappings(snapshot.ports), + env: snapshotString(snapshot, "env"), + volumes: snapshotString(snapshot, "volumes"), + network: snapshotString(snapshot, "network", "bridge"), + restartPolicy: snapshotString(snapshot, "restartPolicy", "unless-stopped"), + entrypoint: snapshotString(snapshot, "entrypoint") || undefined, + commandArgs: storedList(snapshot.commandArgs), + command: snapshotString(snapshot, "command"), + }; +} + +type RollbackImageDependencies = { + dockerInspect: typeof ssh.dockerInspect; + dockerPullImage: typeof ssh.dockerPullImage; +}; + +const rollbackImageDependencies: RollbackImageDependencies = { + dockerInspect: ssh.dockerInspect, + dockerPullImage: ssh.dockerPullImage, +}; + +async function pullOrUseLocalImage( + server: RollbackServer, + image: string, + dependencies: RollbackImageDependencies, +) { + try { + await dependencies.dockerPullImage(server, image); + return image; + } catch (pullError) { + try { + await dependencies.dockerInspect(server, image); + return image; + } catch { + throw pullError; + } + } +} + +export async function resolveRollbackImageReference( + input: { + server: RollbackServer; + image: string; + imageDigest?: string | null; + }, + dependencies: RollbackImageDependencies = rollbackImageDependencies, +) { + const image = input.image.trim(); + const digest = input.imageDigest?.trim() ?? ""; + + // Existing records store Docker's local image ID (`sha256:...`) in + // imageDigest. It is not a registry manifest digest and must never be + // appended to an image name as `image@sha256:...`. + if (/^sha256:[a-f0-9]{64}$/i.test(digest)) { + try { + await dependencies.dockerInspect(input.server, digest); + return digest; + } catch { + // The local immutable artifact may have been pruned. Fall back to the + // stored image reference so legacy records remain usable when possible. + } + } + + if (digest.includes("@sha256:")) { + try { + return await pullOrUseLocalImage(input.server, digest, dependencies); + } catch { + // Fall back to the image reference below for older/incomplete records. + } + } + + return pullOrUseLocalImage(input.server, image, dependencies); +} + +async function resolvePreviousRuntime(input: { + container: { + name: string; + dockerId: string | null; + image: string; + ports: unknown; + envVars: unknown; + volumes: unknown; + restartPolicy: string; + server: RollbackServer; + }; +}): Promise { + try { + const inspect = (await ssh.dockerInspect( + input.container.server, + input.container.dockerId || input.container.name, + )) as DockerInspectRuntime; + + return { + image: inspect.Config?.Image?.trim() || input.container.image, + ports: formatPortBindings(inspect.HostConfig?.PortBindings), + env: (inspect.Config?.Env ?? []).filter(Boolean).join("\n"), + volumes: formatMountBindings(inspect.Mounts), + network: inspect.HostConfig?.NetworkMode?.trim() || "bridge", + restartPolicy: + inspect.HostConfig?.RestartPolicy?.Name?.trim() || + input.container.restartPolicy || + "unless-stopped", + entrypoint: formatEntrypoint(inspect.Config?.Entrypoint), + commandArgs: inspect.Config?.Cmd?.filter( + (argument): argument is string => typeof argument === "string", + ), + command: "", + }; + } catch { + return { + image: input.container.image, + ports: storedList(input.container.ports).join(","), + env: storedList(input.container.envVars).join("\n"), + volumes: storedList(input.container.volumes).join(","), + network: "bridge", + restartPolicy: input.container.restartPolicy || "unless-stopped", + command: "", + }; + } +} + +async function stopAndRemoveContainer( + server: Parameters[0], + candidates: string[], +) { + for (const candidate of candidates.filter(Boolean)) { + try { + await ssh.dockerAction(server, candidate, "stop"); + } catch { + // The runtime may already be stopped; removal below is still attempted. + } + + try { + await ssh.dockerAction(server, candidate, "rm"); + return; + } catch (error) { + if (!/not found|no such container/i.test(String(error))) throw error; + } + } +} + +async function waitForRuntimeHealth(input: { + server: RollbackServer; + containerRef: string; + runtime: RollbackRuntime; + label: string; + dependencies: RollbackRuntimeDependencies; +}) { + const health = await input.dependencies.waitForDockerHealth({ + server: input.server, + containerRef: input.containerRef, + }); + if (!health.healthy) { + throw new Error(`${input.label} runtime check failed: ${health.reason}`); + } +} + +function validateRuntimeBeforeMutation(input: { + name: string; + runtime: RollbackRuntime; + ports: string; +}) { + ssh.buildDockerRunCommand({ + name: input.name, + image: input.runtime.image, + ports: input.ports, + env: input.runtime.env, + volumes: input.runtime.volumes, + network: input.runtime.network, + restartPolicy: input.runtime.restartPolicy, + entrypoint: input.runtime.entrypoint, + commandArgs: input.runtime.commandArgs, + command: input.runtime.command, + }); +} + +export class RollbackRuntimeError extends Error { + constructor( + message: string, + readonly recoveryDockerId: string | null, + readonly recoveryMode: "UNCHANGED" | "RECREATED" | "FAILED", + readonly recoveryError: string | null = null, + ) { + super( + recoveryError + ? `${message} Previous runtime recovery failed: ${recoveryError}` + : message, + ); + this.name = "RollbackRuntimeError"; + } +} + +export async function replaceRuntimeForRollback( + input: { + server: RollbackServer; + containerName: string; + currentContainerRefs: string[]; + temporaryName: string; + strategy: ReturnType; + targetRuntime: RollbackRuntime; + previousRuntime: RollbackRuntime; + finalize?: (result: { + dockerId: string; + runtimeRef: string; + }) => Promise; + }, + dependencies: RollbackRuntimeDependencies = rollbackRuntimeDependencies, +) { + const targetName = + input.strategy === "ATOMIC_RENAME" + ? input.temporaryName + : input.containerName; + const targetPorts = + input.strategy === "ATOMIC_RENAME" ? "" : input.targetRuntime.ports; + + // Both the target and recovery commands must be valid before the active + // runtime is touched. This prevents a deterministic validation error from + // causing avoidable downtime. + validateRuntimeBeforeMutation({ + name: targetName, + runtime: input.targetRuntime, + ports: targetPorts, + }); + validateRuntimeBeforeMutation({ + name: input.containerName, + runtime: input.previousRuntime, + ports: input.previousRuntime.ports, + }); + + let previousRuntimeTouched = false; + let candidateRef: string | null = null; + + try { + if (input.strategy === "RECREATE_WITH_RECOVERY") { + previousRuntimeTouched = true; + await dependencies.stopAndRemoveContainer( + input.server, + input.currentContainerRefs, + ); + } + + const dockerId = await dependencies.runContainer(input.server, { + name: targetName, + image: input.targetRuntime.image, + ports: targetPorts, + env: input.targetRuntime.env, + volumes: input.targetRuntime.volumes, + network: input.targetRuntime.network, + restartPolicy: input.targetRuntime.restartPolicy, + entrypoint: input.targetRuntime.entrypoint, + commandArgs: input.targetRuntime.commandArgs, + command: input.targetRuntime.command, + }); + candidateRef = dockerId.trim() || targetName; + + await waitForRuntimeHealth({ + server: input.server, + containerRef: candidateRef, + runtime: input.targetRuntime, + label: "Rollback", + dependencies, + }); + + if (input.strategy === "ATOMIC_RENAME") { + previousRuntimeTouched = true; + await dependencies.stopAndRemoveContainer( + input.server, + input.currentContainerRefs, + ); + await dependencies.dockerRename( + input.server, + input.temporaryName, + input.containerName, + ); + } + + const result = input.finalize + ? await input.finalize({ + dockerId: dockerId.trim(), + runtimeRef: candidateRef, + }) + : (undefined as TResult); + + return { dockerId: dockerId.trim(), runtimeRef: candidateRef, result }; + } catch (error) { + const message = sanitizeDeploymentError(error, { + fallback: "Rollback runtime replacement failed", + }); + + if (!previousRuntimeTouched) { + try { + await dependencies.stopAndRemoveContainer(input.server, [ + candidateRef ?? "", + input.temporaryName, + ]); + const previousRef = + input.currentContainerRefs.find((value) => value.trim()) ?? + input.containerName; + await waitForRuntimeHealth({ + server: input.server, + containerRef: previousRef, + runtime: input.previousRuntime, + label: "Previous runtime", + dependencies, + }); + throw new RollbackRuntimeError(message, previousRef, "UNCHANGED"); + } catch (recoveryError) { + if (recoveryError instanceof RollbackRuntimeError) throw recoveryError; + throw new RollbackRuntimeError( + message, + null, + "FAILED", + recoveryError instanceof Error + ? sanitizeDeploymentError(recoveryError, { + fallback: "Previous runtime recovery failed", + }) + : "Previous runtime recovery failed", + ); + } + } + + try { + await dependencies.stopAndRemoveContainer(input.server, [ + candidateRef ?? "", + input.temporaryName, + input.containerName, + ]); + const recoveryDockerId = await dependencies.runContainer(input.server, { + name: input.containerName, + image: input.previousRuntime.image, + ports: input.previousRuntime.ports, + env: input.previousRuntime.env, + volumes: input.previousRuntime.volumes, + network: input.previousRuntime.network, + restartPolicy: input.previousRuntime.restartPolicy, + entrypoint: input.previousRuntime.entrypoint, + commandArgs: input.previousRuntime.commandArgs, + command: input.previousRuntime.command, + }); + const recoveryRef = recoveryDockerId.trim() || input.containerName; + await waitForRuntimeHealth({ + server: input.server, + containerRef: recoveryRef, + runtime: input.previousRuntime, + label: "Previous runtime", + dependencies, + }); + throw new RollbackRuntimeError( + message, + recoveryDockerId.trim(), + "RECREATED", + ); + } catch (recoveryError) { + if (recoveryError instanceof RollbackRuntimeError) throw recoveryError; + throw new RollbackRuntimeError( + message, + null, + "FAILED", + recoveryError instanceof Error + ? sanitizeDeploymentError(recoveryError, { + fallback: "Previous runtime recovery failed", + }) + : "Previous runtime recovery failed", + ); + } + } +} + +export async function rollbackContainerToDeployment(input: { + containerId: string; + deploymentId: string; + organizationId: string; + userId?: string; +}) { + const container = await prisma.container.findFirst({ + where: { + id: input.containerId, + server: { organizationId: input.organizationId }, + }, + include: { server: true }, + }); + if (!container) throw new Error("Container not found"); + + const target = await getRollbackSnapshot(input); + if (!target) { + throw new Error("Only a successful deployment in this container can be rolled back"); + } + if (target.serverId !== container.serverId) { + throw new Error("Rollback target belongs to a different server"); + } + + const runningDeployment = await prisma.deployment.findFirst({ + where: { + containerId: container.id, + organizationId: input.organizationId, + status: "RUNNING", + }, + select: { id: true }, + }); + if (runningDeployment) { + throw new Error("Another deployment operation is already running for this container"); + } + + const runtime = snapshotRuntime(target.snapshot, target.image); + const strategy = resolveDeploymentStrategy(runtime.ports); + const lock = await acquireDeploymentLock({ containerId: container.id }); + const lockHeartbeat = startDeploymentLockHeartbeat({ + containerId: container.id, + token: lock.token, + }); + const rollback = await createDeployment({ + containerId: container.id, + organizationId: input.organizationId, + serverId: container.serverId, + userId: input.userId, + status: "RUNNING", + trigger: "ROLLBACK", + version: target.version ?? runtime.image, + image: runtime.image, + imageDigest: target.imageDigest, + configSnapshot: { ...target.snapshot, deploymentStrategy: strategy }, + startedAt: new Date(), + }).catch(async (error) => { + lockHeartbeat.stop(); + await releaseDeploymentLock({ containerId: container.id, token: lock.token }); + throw error; + }); + + const previousRuntime = await resolvePreviousRuntime({ + container, + }); + + try { + const image = await resolveRollbackImageReference({ + server: container.server, + image: runtime.image, + imageDigest: target.imageDigest, + }); + runtime.image = image; + + const atomicName = `${container.name}-rollback-${rollback.id.slice(-8)}`.slice(0, 128); + const replacement = await replaceRuntimeForRollback({ + server: container.server, + containerName: container.name, + currentContainerRefs: [container.dockerId ?? "", container.name], + temporaryName: atomicName, + strategy, + targetRuntime: runtime, + previousRuntime, + finalize: async ({ dockerId }) => { + lockHeartbeat.assertOwned(); + const updated = await prisma.container.update({ + where: { id: container.id }, + data: { + image, + status: "RUNNING", + dockerId: dockerId.trim().slice(0, 12) || null, + }, + include: { server: { select: { name: true, ip: true } } }, + }); + + await updateDeployment(rollback.id, { + status: "SUCCESS", + completedAt: new Date(), + image, + imageDigest: target.imageDigest, + }); + await auditLog({ + userId: input.userId, + organizationId: input.organizationId, + serverId: container.serverId, + action: "CONTAINER_ROLLBACK", + category: "CONTAINER", + level: "SUCCESS", + message: `Container "${container.name}" rolled back to deployment ${target.id}`, + meta: { deploymentId: rollback.id, targetDeploymentId: target.id, image }, + }); + + return updated; + }, + }); + + return { + updated: replacement.result, + deploymentId: rollback.id, + targetDeploymentId: target.id, + }; + } catch (error) { + const message = sanitizeDeploymentError(error, { + fallback: "Rollback failed", + }); + const runtimeError = + error instanceof RollbackRuntimeError ? error : null; + if (runtimeError?.recoveryMode !== "FAILED") { + await prisma.container.update({ + where: { id: container.id }, + data: { + status: "RUNNING", + image: previousRuntime.image, + dockerId: + runtimeError?.recoveryDockerId?.trim().slice(0, 12) || + container.dockerId, + }, + }); + await auditLog({ + userId: input.userId, + organizationId: input.organizationId, + serverId: container.serverId, + action: "CONTAINER_ROLLBACK_RECOVERY", + category: "CONTAINER", + level: "WARNING", + message: `Previous runtime for container "${container.name}" was recovered after rollback failure`, + meta: { + deploymentId: rollback.id, + recovery: + runtimeError?.recoveryMode === "UNCHANGED" + ? "UNCHANGED_HEALTHY" + : "RECREATED_HEALTHY", + }, + }); + } else { + await prisma.container.update({ + where: { id: container.id }, + data: { status: "ERROR" }, + }); + await auditLog({ + userId: input.userId, + organizationId: input.organizationId, + serverId: container.serverId, + action: "CONTAINER_ROLLBACK_RECOVERY", + category: "CONTAINER", + level: "ERROR", + message: `Previous runtime recovery failed for container "${container.name}"`, + meta: { deploymentId: rollback.id, recovery: "FAILED" }, + }).catch(() => undefined); + } + + await updateDeployment(rollback.id, { + status: "FAILED", + error: message, + completedAt: new Date(), + }).catch(() => undefined); + throw new Error(message); + } finally { + lockHeartbeat.stop(); + await releaseDeploymentLock({ containerId: container.id, token: lock.token }); + } +} + + diff --git a/src/server/services/deployment-strategy.ts b/src/server/services/deployment-strategy.ts new file mode 100644 index 0000000..c701f16 --- /dev/null +++ b/src/server/services/deployment-strategy.ts @@ -0,0 +1,7 @@ +export type DeploymentStrategy = "ATOMIC_RENAME" | "RECREATE_WITH_RECOVERY"; + +export function resolveDeploymentStrategy(ports: string): DeploymentStrategy { + return ports.trim() ? "RECREATE_WITH_RECOVERY" : "ATOMIC_RENAME"; +} + + diff --git a/src/server/services/deployment.service.ts b/src/server/services/deployment.service.ts new file mode 100644 index 0000000..ca9676a --- /dev/null +++ b/src/server/services/deployment.service.ts @@ -0,0 +1,254 @@ +import { + DeploymentStatus, + DeploymentTrigger, + Prisma, +} from "@prisma/client"; +import prisma from "../lib/prisma"; +import { decrypt, encrypt } from "../lib/crypto"; + +const SENSITIVE_KEY_PATTERN = /(pass(word)?|secret|token|api[-_]?key|private[-_]?key|credential)/i; + +function redactEnvironmentLine(line: string) { + const separator = line.indexOf("="); + if (separator < 1) return line; + + const key = line.slice(0, separator).trim(); + return SENSITIVE_KEY_PATTERN.test(key) + ? `${key}=` + : line; +} + +function sanitizeSnapshotValue(value: unknown, key?: string): unknown { + if (key && SENSITIVE_KEY_PATTERN.test(key)) return ""; + + if (typeof value === "string") { + if (key?.toLowerCase().includes("env")) { + return value + .split(/\r?\n/) + .map(redactEnvironmentLine) + .join("\n"); + } + return value; + } + + if (Array.isArray(value)) { + return value.map((item) => sanitizeSnapshotValue(item)); + } + + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([entryKey, entryValue]) => [ + entryKey, + sanitizeSnapshotValue(entryValue, entryKey), + ]), + ); + } + + return value; +} + +export type DeploymentSnapshot = Record; + +export type CreateDeploymentInput = { + containerId: string; + organizationId: string; + serverId: string; + userId?: string; + status: DeploymentStatus; + trigger: DeploymentTrigger; + version?: string | null; + commitSha?: string | null; + branch?: string | null; + image?: string | null; + imageDigest?: string | null; + configSnapshot: DeploymentSnapshot; + error?: string | null; + startedAt?: Date | null; + completedAt?: Date | null; +}; + +export function sanitizeDeploymentSnapshot( + snapshot: DeploymentSnapshot, +): Prisma.InputJsonValue { + return sanitizeSnapshotValue(snapshot) as Prisma.InputJsonValue; +} + +export async function createDeployment(input: CreateDeploymentInput) { + return prisma.deployment.create({ + data: { + containerId: input.containerId, + organizationId: input.organizationId, + serverId: input.serverId, + userId: input.userId ?? null, + status: input.status, + trigger: input.trigger, + version: input.version ?? null, + commitSha: input.commitSha ?? null, + branch: input.branch ?? null, + image: input.image ?? null, + imageDigest: input.imageDigest ?? null, + configSnapshot: sanitizeDeploymentSnapshot(input.configSnapshot), + rollbackSnapshotEnc: encrypt(JSON.stringify(input.configSnapshot)), + error: input.error ?? null, + startedAt: input.startedAt ?? null, + completedAt: input.completedAt ?? null, + }, + }); +} + +export async function getRollbackSnapshot(input: { + containerId: string; + deploymentId: string; + organizationId: string; +}) { + const deployment = await prisma.deployment.findFirst({ + where: { + id: input.deploymentId, + containerId: input.containerId, + organizationId: input.organizationId, + status: "SUCCESS", + }, + select: { + id: true, + containerId: true, + serverId: true, + version: true, + image: true, + imageDigest: true, + configSnapshot: true, + rollbackSnapshotEnc: true, + }, + }); + + if (!deployment) return null; + if (!deployment.rollbackSnapshotEnc) { + throw new Error( + "This deployment has no encrypted rollback artifact. It cannot be restored safely.", + ); + } + + let snapshot: DeploymentSnapshot; + try { + const parsed: unknown = JSON.parse(decrypt(deployment.rollbackSnapshotEnc)); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("Invalid rollback snapshot"); + } + snapshot = parsed as DeploymentSnapshot; + } catch { + throw new Error("Rollback artifact is invalid or cannot be decrypted"); + } + + return { ...deployment, snapshot }; +} + +export async function updateDeployment( + id: string, + patch: { + status: DeploymentStatus; + error?: string | null; + completedAt?: Date | null; + image?: string | null; + imageDigest?: string | null; + commitSha?: string | null; + branch?: string | null; + configSnapshot?: DeploymentSnapshot; + }, +) { + return prisma.deployment.update({ + where: { id }, + data: { + status: patch.status, + error: patch.error ?? null, + completedAt: patch.completedAt ?? null, + ...(patch.image === undefined ? {} : { image: patch.image }), + ...(patch.imageDigest === undefined + ? {} + : { imageDigest: patch.imageDigest }), + ...(patch.commitSha === undefined ? {} : { commitSha: patch.commitSha }), + ...(patch.branch === undefined ? {} : { branch: patch.branch }), + ...(patch.configSnapshot === undefined + ? {} + : { + configSnapshot: sanitizeDeploymentSnapshot(patch.configSnapshot), + rollbackSnapshotEnc: encrypt( + JSON.stringify(patch.configSnapshot), + ), + }), + }, + }); +} + +export async function listDeployments(input: { + containerId: string; + organizationId: string; + page: number; + pageSize: number; +}) { + const skip = (input.page - 1) * input.pageSize; + const where = { + containerId: input.containerId, + organizationId: input.organizationId, + }; + + const [items, total] = await prisma.$transaction([ + prisma.deployment.findMany({ + where, + orderBy: { createdAt: "desc" }, + skip, + take: input.pageSize, + select: { + id: true, + containerId: true, + status: true, + trigger: true, + version: true, + commitSha: true, + branch: true, + image: true, + imageDigest: true, + error: true, + startedAt: true, + completedAt: true, + createdAt: true, + user: { select: { id: true, name: true } }, + }, + }), + prisma.deployment.count({ where }), + ]); + + return { items, total, page: input.page, pageSize: input.pageSize }; +} + +export async function getDeployment(input: { + containerId: string; + deploymentId: string; + organizationId: string; +}) { + return prisma.deployment.findFirst({ + where: { + id: input.deploymentId, + containerId: input.containerId, + organizationId: input.organizationId, + }, + select: { + id: true, + containerId: true, + serverId: true, + status: true, + trigger: true, + version: true, + commitSha: true, + branch: true, + image: true, + imageDigest: true, + configSnapshot: true, + error: true, + startedAt: true, + completedAt: true, + createdAt: true, + user: { select: { id: true, name: true } }, + }, + }); +} + + diff --git a/src/server/services/ssh-services/docker-containers.ts b/src/server/services/ssh-services/docker-containers.ts index f26d2ce..165bace 100644 --- a/src/server/services/ssh-services/docker-containers.ts +++ b/src/server/services/ssh-services/docker-containers.ts @@ -1,8 +1,8 @@ import yaml from "js-yaml"; import { randomBytes } from "crypto"; import { posix as pathPosix } from "path"; -import { Server } from "@prisma/client"; import { TextDecoder } from "util"; +import { Server } from "@prisma/client"; import { execStrict } from "./commands"; import { execDocker, execDockerStrict } from "./internal/docker"; import { privilegedCommand } from "./internal/privilege"; @@ -1140,6 +1140,18 @@ export async function listContainerFiles( }; } +export async function dockerRename( + server: Server, + containerId: string, + newName: string, +): Promise { + await execDockerStrict( + server, + `docker rename ${escapeShellArg(containerId)} ${escapeShellArg(newName)}`, + shortDockerCommandTimeout(DOCKER_ACTION_TIMEOUT_MS), + ); +} + export async function getContainerMainProcessWorkingDirectory( server: Server, containerId: string, @@ -2518,6 +2530,7 @@ export async function deployContainerFromGitSource( dockerId?: string; imageTag?: string; composeFilePath?: string; + commitSha: string; }> { const projectName = sanitizeProjectName(opts.projectName); const deploymentPath = @@ -2572,6 +2585,24 @@ export async function deployContainerFromGitSource( throw new Error(formatDeploymentErrorMessage(error)); } + let commitSha = ""; + try { + commitSha = ( + await execStrict( + server, + privilegedCommand( + server, + `git -C ${escapeShellArg(deploymentPath)} rev-parse HEAD`, + ), + shortDockerCommandTimeout(DOCKER_INSPECT_TIMEOUT_MS), + ) + ).trim(); + } catch (error) { + throw new Error( + `Repository was cloned but its commit SHA could not be resolved: ${formatDeploymentErrorMessage(error)}`, + ); + } + if (buildType === "COMPOSE") { try { await writeComposeEnvFiles({ @@ -2608,6 +2639,7 @@ export async function deployContainerFromGitSource( return { deploymentPath, composeFilePath, + commitSha, }; } @@ -2708,6 +2740,7 @@ export async function deployContainerFromGitSource( deploymentPath, dockerId, imageTag, + commitSha, }; } @@ -2813,6 +2846,7 @@ export async function deployContainerFromGitSource( deploymentPath, dockerId, imageTag, + commitSha, }; } @@ -2851,6 +2885,7 @@ export async function deployContainerFromGitSource( deploymentPath, dockerId, imageTag, + commitSha, }; } @@ -2916,5 +2951,6 @@ export async function deployContainerFromGitSource( deploymentPath, dockerId, imageTag, + commitSha, }; } diff --git a/tests/backend/container-health.test.ts b/tests/backend/container-health.test.ts new file mode 100644 index 0000000..70e4847 --- /dev/null +++ b/tests/backend/container-health.test.ts @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { evaluateDockerHealth } from "../../src/server/services/container-health.service"; + +test("docker health gate accepts a running image without a healthcheck", () => { + const result = evaluateDockerHealth({ State: { Status: "running" } }); + assert.equal(result.healthy, true); + assert.equal(result.status, "healthy"); +}); + +test("docker health gate rejects unhealthy containers", () => { + const result = evaluateDockerHealth({ + State: { Status: "running", Health: { Status: "unhealthy" } }, + }); + assert.equal(result.healthy, false); + assert.equal(result.status, "unhealthy"); +}); + +test("docker health gate rejects containers that are not running", () => { + const result = evaluateDockerHealth({ State: { Status: "exited" } }); + assert.equal(result.healthy, false); + assert.equal(result.status, "not_running"); +}); + +test("docker runtime verification keeps polling while a container is restarting", () => { + const result = evaluateDockerHealth({ State: { Status: "restarting" } }); + assert.equal(result.healthy, false); + assert.equal(result.status, "starting"); +}); + + diff --git a/tests/backend/deployment-concurrency.test.ts b/tests/backend/deployment-concurrency.test.ts new file mode 100644 index 0000000..5182f0e --- /dev/null +++ b/tests/backend/deployment-concurrency.test.ts @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + DeploymentLockConflictError, + startDeploymentLockHeartbeat, +} from "../../src/server/services/deployment-lock.service"; + +test("deployment lock heartbeat renews with the owner token and detects ownership loss", async () => { + const renewals: Array<{ containerId: string; token: string }> = []; + let shouldFail = false; + const heartbeat = startDeploymentLockHeartbeat( + { + containerId: "container-1", + token: "owner-token", + ttlMs: 30_000, + intervalMs: 10, + }, + async (input) => { + renewals.push({ + containerId: input.containerId, + token: input.token, + }); + if (shouldFail) throw new DeploymentLockConflictError(); + }, + ); + + await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(renewals.length >= 1); + assert.deepEqual(renewals[0], { + containerId: "container-1", + token: "owner-token", + }); + heartbeat.assertOwned(); + + shouldFail = true; + await new Promise((resolve) => setTimeout(resolve, 25)); + assert.equal(heartbeat.hasLostOwnership(), true); + assert.throws(() => heartbeat.assertOwned(), DeploymentLockConflictError); + heartbeat.stop(); +}); + + diff --git a/tests/backend/deployment-history.test.ts b/tests/backend/deployment-history.test.ts new file mode 100644 index 0000000..ff5fc88 --- /dev/null +++ b/tests/backend/deployment-history.test.ts @@ -0,0 +1,22 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { sanitizeDeploymentSnapshot } from "../../src/server/services/deployment.service"; + +test("deployment snapshots redact sensitive configuration values", () => { + const snapshot = sanitizeDeploymentSnapshot({ + env: "APP_ENV=production\nDATABASE_PASSWORD=super-secret\nPUBLIC_URL=https://example.test", + accessToken: "provider-token", + nested: { apiKey: "another-secret" }, + image: "nginx:latest", + }) as Record; + + assert.equal( + snapshot.env, + "APP_ENV=production\nDATABASE_PASSWORD=\nPUBLIC_URL=https://example.test", + ); + assert.equal(snapshot.accessToken, ""); + assert.deepEqual(snapshot.nested, { apiKey: "" }); + assert.equal(snapshot.image, "nginx:latest"); +}); + + diff --git a/tests/backend/deployment-lifecycle-regressions.test.ts b/tests/backend/deployment-lifecycle-regressions.test.ts new file mode 100644 index 0000000..11eeb6a --- /dev/null +++ b/tests/backend/deployment-lifecycle-regressions.test.ts @@ -0,0 +1,85 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const CONTAINER_ROUTES = "src/server/routes/containers.ts"; +const APP_ROUTES = "src/server/routes/apps.ts"; +const DOCKER_CONTAINERS = + "src/server/services/ssh-services/docker-containers.ts"; + +function readSource(path: string) { + return readFileSync(path, "utf8"); +} + +function sourceBlock(source: string, startPattern: string, endPattern: string) { + const start = source.indexOf(startPattern); + assert.notEqual(start, -1, `Missing source block start: ${startPattern}`); + + const end = source.indexOf(endPattern, start); + assert.notEqual(end, -1, `Missing source block end: ${endPattern}`); + + return source.slice(start, end); +} + +test("Git rebuild follows the deployment history lifecycle", () => { + const source = readSource(CONTAINER_ROUTES); + const rebuildRoute = sourceBlock( + source, + '"/:id/rebuild"', + "GET /containers/:id/logs", + ); + + const acquireIndex = rebuildRoute.indexOf("acquireDeploymentLock"); + const runningIndex = rebuildRoute.indexOf('status: "RUNNING"'); + const mutationIndex = rebuildRoute.indexOf("deployContainerFromGitSource"); + const successIndex = rebuildRoute.indexOf('status: "SUCCESS"', mutationIndex); + + assert.ok(acquireIndex >= 0); + assert.ok(runningIndex > acquireIndex); + assert.ok(mutationIndex > runningIndex); + assert.ok(successIndex > mutationIndex); + assert.doesNotMatch(rebuildRoute, /waitForDeploymentHealth/); + assert.match(rebuildRoute, /status: "FAILED"/); + assert.match(rebuildRoute, /releaseDeploymentLock/); + assert.match(rebuildRoute, /commitSha: gitDeploymentResult\.commitSha/); + assert.match(rebuildRoute, /imageDigest/); +}); + +test("Fresh App Installer creates a provisional parent and RUNNING deployment before Docker mutation", () => { + const source = readSource(APP_ROUTES); + const installRoute = sourceBlock( + source, + 'app.post("/install"', + '"/installs/:id/action"', + ); + + const provisionalIndex = installRoute.indexOf( + "const createdContainer = await tx.container.create", + ); + const runningIndex = installRoute.indexOf('status: "RUNNING"'); + const mutationIndex = installRoute.indexOf( + "const dockerId = await ssh.runContainer", + ); + + assert.ok(provisionalIndex >= 0); + assert.ok(runningIndex > provisionalIndex); + assert.ok(mutationIndex > runningIndex); + assert.match(installRoute, /where: \{ id: deployment\.id \}/); + assert.match(installRoute, /status: "FAILED"/); + assert.match(installRoute, /where: \{ id: provisionalContainer\.id \}/); +}); + +test("Git deployment resolves and returns the cloned commit SHA", () => { + const source = readSource(DOCKER_CONTAINERS); + const start = source.indexOf( + "export async function deployContainerFromGitSource", + ); + assert.notEqual(start, -1); + const gitDeployFunction = source.slice(start); + + assert.match(gitDeployFunction, /git -C .* rev-parse HEAD/); + assert.match(gitDeployFunction, /commitSha: string/); + assert.match(gitDeployFunction, /commitSha,/); +}); + + diff --git a/tests/backend/deployment-reliability.test.ts b/tests/backend/deployment-reliability.test.ts new file mode 100644 index 0000000..ab28f9f --- /dev/null +++ b/tests/backend/deployment-reliability.test.ts @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; +import { sanitizeDeploymentSnapshot } from "../../src/server/services/deployment.service"; + +test("deployment snapshot keeps rollback-safe runtime fields sanitized", () => { + const snapshot = sanitizeDeploymentSnapshot({ + image: "example/app@sha256:abc", + ports: "8080:80", + env: "APP_ENV=production\nAPI_TOKEN=secret", + volumes: "/data:/data", + }) as Record; + + assert.equal(snapshot.image, "example/app@sha256:abc"); + assert.equal(snapshot.env, "APP_ENV=production\nAPI_TOKEN="); +}); + +test("deployment migrations include the history and rollback schema", () => { + const migrationsPath = join(process.cwd(), "prisma", "migrations"); + const migrations = readdirSync(migrationsPath); + assert.ok(migrations.some((name) => name.includes("add_deployment_history"))); + assert.ok(migrations.some((name) => name.includes("add_deployment_rollback_snapshot"))); + assert.ok(migrations.some((name) => name.includes("add_deployment_locks"))); + assert.equal( + migrations.some((name) => + /deployment_attempt|process_job|service_lease|deployment_health_policy/.test( + name, + ), + ), + false, + ); + + const schema = readFileSync(join(process.cwd(), "prisma", "schema.prisma"), "utf8"); + assert.match(schema, /model Deployment\s*\{/); + assert.match(schema, /rollbackSnapshotEnc\s+String\?/); + assert.doesNotMatch(schema, /model DeploymentAttempt\s*\{/); + assert.doesNotMatch(schema, /model ProcessJob\s*\{/); + assert.doesNotMatch(schema, /model ServiceLease\s*\{/); +}); + + diff --git a/tests/backend/deployment-rollback-runtime.test.ts b/tests/backend/deployment-rollback-runtime.test.ts new file mode 100644 index 0000000..e2ce183 --- /dev/null +++ b/tests/backend/deployment-rollback-runtime.test.ts @@ -0,0 +1,275 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + normalizeRollbackPortMappings, + replaceRuntimeForRollback, + resolveRollbackImageReference, + RollbackRuntimeError, +} from "../../src/server/services/deployment-rollback.service"; + +type RollbackInput = Parameters[0]; +type RollbackDependencies = NonNullable< + Parameters[1] +>; + +const server = {} as RollbackInput["server"]; + +test("normalizes Docker display port mappings for rollback", () => { + assert.equal( + normalizeRollbackPortMappings( + "0.0.0.0:8080->80/tcp, [::]:8080->80/tcp, 127.0.0.1:8443->443/tcp", + ), + "8080:80,127.0.0.1:8443:443", + ); + assert.equal( + normalizeRollbackPortMappings(["8081:81", "9000->90/udp"]), + "8081:81,9000:90/udp", + ); +}); + +test("uses a locally available Docker image ID without creating an invalid registry reference", async () => { + const inspected: string[] = []; + const pulled: string[] = []; + + const image = await resolveRollbackImageReference( + { + server, + image: "example/app:latest", + imageDigest: `sha256:${"a".repeat(64)}`, + }, + { + dockerInspect: async (_server, imageRef) => { + inspected.push(imageRef); + return {}; + }, + dockerPullImage: async (_server, imageRef) => { + pulled.push(imageRef); + }, + }, + ); + + assert.equal(image, `sha256:${"a".repeat(64)}`); + assert.deepEqual(inspected, [`sha256:${"a".repeat(64)}`]); + assert.deepEqual(pulled, []); +}); + +test("falls back to the stored image when a historical local image ID was pruned", async () => { + const pulled: string[] = []; + + const image = await resolveRollbackImageReference( + { + server, + image: "example/app:stable", + imageDigest: `sha256:${"b".repeat(64)}`, + }, + { + dockerInspect: async () => { + throw new Error("image not found"); + }, + dockerPullImage: async (_server, imageRef) => { + pulled.push(imageRef); + }, + }, + ); + + assert.equal(image, "example/app:stable"); + assert.deepEqual(pulled, ["example/app:stable"]); +}); + +function runtime(overrides: Partial = {}) { + return { + image: "example/app:target", + ports: "", + env: "APP_ENV=production", + volumes: "/data:/data", + network: "bridge", + restartPolicy: "unless-stopped", + command: "", + ...overrides, + }; +} + +function dependencies( + overrides: Partial = {}, +): RollbackDependencies { + return { + runContainer: async () => "new-container-id", + stopAndRemoveContainer: async () => undefined, + dockerRename: async () => undefined, + waitForDockerHealth: async () => ({ + healthy: true, + reason: "running", + status: "healthy", + attempts: 1, + durationMs: 0, + }), + ...overrides, + }; +} + +test("published-port rollback removes the active runtime before claiming its name and host port", async () => { + const events: string[] = []; + + await replaceRuntimeForRollback( + { + server, + containerName: "app", + currentContainerRefs: ["old-container-id", "app"], + temporaryName: "app-rollback", + strategy: "RECREATE_WITH_RECOVERY", + targetRuntime: runtime({ ports: "8080:80" }), + previousRuntime: runtime({ + image: "example/app:previous", + ports: "8080:80", + }), + }, + dependencies({ + stopAndRemoveContainer: async (_server, refs) => { + events.push(`remove:${refs.join(",")}`); + }, + runContainer: async (_server, options) => { + events.push(`run:${options.name}:${options.ports}`); + return "new-container-id"; + }, + waitForDockerHealth: async () => { + events.push("runtime-check"); + return { + healthy: true, + reason: "running", + status: "healthy", + attempts: 1, + durationMs: 0, + }; + }, + }), + ); + + assert.deepEqual(events, [ + "remove:old-container-id,app", + "run:app:8080:80", + "runtime-check", + ]); +}); + +test("failed published-port rollback recreates and verifies the previous runtime", async () => { + const events: string[] = []; + let runCount = 0; + + await assert.rejects( + replaceRuntimeForRollback( + { + server, + containerName: "app", + currentContainerRefs: ["old-container-id", "app"], + temporaryName: "app-rollback", + strategy: "RECREATE_WITH_RECOVERY", + targetRuntime: runtime({ ports: "8080:80" }), + previousRuntime: runtime({ + image: "example/app:previous", + ports: "8080:80", + }), + }, + dependencies({ + stopAndRemoveContainer: async (_server, refs) => { + events.push(`remove:${refs.join(",")}`); + }, + runContainer: async (_server, options) => { + runCount += 1; + events.push(`run:${options.image}`); + return runCount === 1 ? "failed-target-id" : "recovered-container-id"; + }, + waitForDockerHealth: async ({ containerRef }) => { + events.push(`runtime-check:${containerRef}`); + if (containerRef === "failed-target-id") { + return { + healthy: false, + reason: "not running", + status: "not_running", + attempts: 1, + durationMs: 0, + }; + } + return { + healthy: true, + reason: "running", + status: "healthy", + attempts: 1, + durationMs: 0, + }; + }, + }), + ), + (error: unknown) => { + assert.ok(error instanceof RollbackRuntimeError); + assert.equal(error.recoveryMode, "RECREATED"); + assert.equal(error.recoveryDockerId, "recovered-container-id"); + return true; + }, + ); + + assert.deepEqual(events, [ + "remove:old-container-id,app", + "run:example/app:target", + "runtime-check:failed-target-id", + "remove:failed-target-id,app-rollback,app", + "run:example/app:previous", + "runtime-check:recovered-container-id", + ]); +}); + +test("failed atomic candidate leaves the active runtime in place", async () => { + const removedRefs: string[][] = []; + let runCount = 0; + + await assert.rejects( + replaceRuntimeForRollback( + { + server, + containerName: "app", + currentContainerRefs: ["old-container-id", "app"], + temporaryName: "app-rollback", + strategy: "ATOMIC_RENAME", + targetRuntime: runtime(), + previousRuntime: runtime({ image: "example/app:previous" }), + }, + dependencies({ + stopAndRemoveContainer: async (_server, refs) => { + removedRefs.push(refs); + }, + runContainer: async () => { + runCount += 1; + return "failed-target-id"; + }, + waitForDockerHealth: async ({ containerRef }) => { + if (containerRef === "failed-target-id") { + return { + healthy: false, + reason: "not running", + status: "not_running", + attempts: 1, + durationMs: 0, + }; + } + return { + healthy: true, + reason: "running", + status: "healthy", + attempts: 1, + durationMs: 0, + }; + }, + }), + ), + (error: unknown) => { + assert.ok(error instanceof RollbackRuntimeError); + assert.equal(error.recoveryMode, "UNCHANGED"); + assert.equal(error.recoveryDockerId, "old-container-id"); + return true; + }, + ); + + assert.equal(runCount, 1); + assert.deepEqual(removedRefs, [["failed-target-id", "app-rollback"]]); +}); + + diff --git a/tests/backend/deployment-security.test.ts b/tests/backend/deployment-security.test.ts new file mode 100644 index 0000000..d817a6b --- /dev/null +++ b/tests/backend/deployment-security.test.ts @@ -0,0 +1,141 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { + DEVELOPMENT_ENCRYPTION_KEY, + getEncryptionKeyOrThrow, +} from "../../src/server/lib/crypto"; +import { + redactDeploymentErrorDetails, + sanitizeDeploymentError, +} from "../../src/server/services/deployment-error.service"; + +test("production rejects a missing, short, development, or placeholder encryption key", () => { + assert.throws( + () => getEncryptionKeyOrThrow({ NODE_ENV: "production" }), + /must be configured/, + ); + assert.throws( + () => + getEncryptionKeyOrThrow({ + NODE_ENV: "production", + ENCRYPTION_KEY: "short", + }), + /at least 32 characters/, + ); + assert.throws( + () => + getEncryptionKeyOrThrow({ + NODE_ENV: "production", + ENCRYPTION_KEY: DEVELOPMENT_ENCRYPTION_KEY, + }), + /strong random value/, + ); + assert.throws( + () => + getEncryptionKeyOrThrow({ + NODE_ENV: "production", + ENCRYPTION_KEY: + "dev-this-key-is-long-enough-but-still-not-production", + }), + /strong random value/, + ); + assert.throws( + () => + getEncryptionKeyOrThrow({ + NODE_ENV: "production", + ENCRYPTION_KEY: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }), + /strong random value/, + ); +}); + +test("development keeps the legacy fallback while production accepts a strong key", () => { + assert.equal( + getEncryptionKeyOrThrow({ NODE_ENV: "development" }), + DEVELOPMENT_ENCRYPTION_KEY, + ); + assert.equal( + getEncryptionKeyOrThrow({ + NODE_ENV: "production", + ENCRYPTION_KEY: + "47c2ecffbb4a488eb9a8d1939b54810f1db621bccf1bbf87c87a07de1e6e42d1", + }), + "47c2ecffbb4a488eb9a8d1939b54810f1db621bccf1bbf87c87a07de1e6e42d1", + ); +}); + +test("backend startup validates encryption configuration before connecting to the database", () => { + const source = readFileSync("src/server/index.ts", "utf8"); + const validationIndex = source.indexOf("validateEncryptionConfiguration();"); + const databaseIndex = source.indexOf("await ensureDatabaseConnection();"); + + assert.ok(validationIndex >= 0); + assert.ok(databaseIndex > validationIndex); +}); + +test("deployment error redaction removes credentials, authorization, query secrets, and private keys", () => { + const message = redactDeploymentErrorDetails( + new Error( + [ + "Command failed with password=super-secret", + "Authorization: Bearer abc.def.ghi", + "https://deploy-user:deploy-pass@example.test/repo.git?access_token=query-token", + "-----BEGIN PRIVATE KEY-----", + "private-material", + "-----END PRIVATE KEY-----", + ].join("\n"), + ), + ["super-secret"], + ); + + assert.doesNotMatch( + message, + /super-secret|abc\.def\.ghi|deploy-user|deploy-pass|query-token|private-material/, + ); + assert.match(message, /REDACTED/); +}); + +test("deployment errors normalize raw command details", () => { + assert.equal( + sanitizeDeploymentError( + new Error( + "Command failed: docker run -e API_TOKEN=secret example/app; stderr: access denied", + ), + { fallback: "Git redeploy failed" }, + ), + "Git redeploy failed", + ); + +}); + +test("deployment and rollback boundaries use the centralized sanitizer before persistence and API responses", () => { + const rollbackService = readFileSync( + "src/server/services/deployment-rollback.service.ts", + "utf8", + ); + const containerRoutes = readFileSync( + "src/server/routes/containers.ts", + "utf8", + ); + const containerHealth = readFileSync( + "src/server/services/container-health.service.ts", + "utf8", + ); + + assert.match( + rollbackService, + /const message = sanitizeDeploymentError\(error,[\s\S]*?updateDeployment\(rollback\.id,[\s\S]*?error: message/, + ); + assert.match( + containerRoutes, + /"\/:id\/deployments\/:deploymentId\/rollback"[\s\S]*?error: sanitizeDeploymentError\(error/, + ); + assert.match( + containerRoutes, + /sanitizeDeploymentError\(ssh\.formatDeploymentErrorMessage\(err\)/, + ); + assert.match(containerHealth, /lastReason = sanitizeDeploymentError\(error/); +}); + + diff --git a/tests/backend/deployment-strategy.test.ts b/tests/backend/deployment-strategy.test.ts new file mode 100644 index 0000000..4757613 --- /dev/null +++ b/tests/backend/deployment-strategy.test.ts @@ -0,0 +1,16 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { resolveDeploymentStrategy } from "../../src/server/services/deployment-strategy"; + +describe("deployment strategy", () => { + it("uses atomic rename when no published ports are configured", () => { + assert.equal(resolveDeploymentStrategy(""), "ATOMIC_RENAME"); + assert.equal(resolveDeploymentStrategy(" "), "ATOMIC_RENAME"); + }); + + it("uses recreate with recovery when published ports are configured", () => { + assert.equal(resolveDeploymentStrategy("8080:80"), "RECREATE_WITH_RECOVERY"); + }); +}); + +