fix(deploy): normalize Docker access errors and harden SSE parsing
- handle malformed plain-text SSE payloads safely - normalize Docker socket permission errors - verify authoritative deployment job status - preserve deployment logs on failure
This commit is contained in:
1 parent
323596c586
commit
73b4b6255d
8 files changed
+182
-31
No files matched your search
@@ -789,6 +789,7 @@ export default function DeployContainerModal({
|
||||
let processLogLines = [
|
||||
`[deploy] Starting deployment for ${form.name || "new container"}`,
|
||||
];
|
||||
let latestTerminalLogs = [...processLogLines];
|
||||
|
||||
try {
|
||||
if (!choice) {
|
||||
@@ -800,7 +801,7 @@ export default function DeployContainerModal({
|
||||
`[deploy] Source: ${choice}`,
|
||||
`[deploy] Server ID: ${form.serverId}`,
|
||||
];
|
||||
let latestTerminalLogs = [...processLogLines];
|
||||
latestTerminalLogs = [...processLogLines];
|
||||
|
||||
const updateProcessTerminal = (
|
||||
lines: string[],
|
||||
@@ -1010,6 +1011,12 @@ export default function DeployContainerModal({
|
||||
},
|
||||
});
|
||||
|
||||
// The status event can race with stream closure or be altered by a
|
||||
// reverse proxy. The job endpoint is the authoritative final result.
|
||||
const settledJob = await containersApi.getJob(job.id);
|
||||
finalStatus = settledJob.data.status;
|
||||
finalError = settledJob.data.error ?? settledJob.data.cancelReason;
|
||||
|
||||
if (cancelRequestedRef.current && finalStatus !== "cancelled") {
|
||||
const cancelledJob = currentJobIdRef.current
|
||||
? await containersApi.getJob(currentJobIdRef.current)
|
||||
@@ -1029,6 +1036,12 @@ export default function DeployContainerModal({
|
||||
throw new Error(finalError || "Deployment cancelled");
|
||||
}
|
||||
|
||||
if (finalStatus !== "success") {
|
||||
throw new Error(
|
||||
"The deployment log stream ended before the server reported a final result. Check the job status and retry the deployment.",
|
||||
);
|
||||
}
|
||||
|
||||
updateDeployTimelineModal({
|
||||
onProcessUpdate,
|
||||
activeStep: 3,
|
||||
@@ -1067,6 +1080,7 @@ export default function DeployContainerModal({
|
||||
onProcessUpdate,
|
||||
name: form.name,
|
||||
message,
|
||||
terminalLogs: latestTerminalLogs,
|
||||
});
|
||||
onProcessUpdate?.({ cancelAction: undefined });
|
||||
} finally {
|
||||
|
||||
@@ -120,14 +120,17 @@ export function failDeployTimelineModal({
|
||||
onProcessUpdate,
|
||||
name,
|
||||
message,
|
||||
terminalLogs,
|
||||
}: {
|
||||
onProcessUpdate?: ProcessLogUpdateHandler;
|
||||
name: string;
|
||||
message: string;
|
||||
terminalLogs?: string[];
|
||||
}) {
|
||||
onProcessUpdate?.({
|
||||
timelineLogs: createFailedDeployTimeline(2),
|
||||
terminalLogs: [
|
||||
...(terminalLogs ?? []),
|
||||
`[deploy] Deployment failed for ${name || "new container"}`,
|
||||
`[error] ${message}`,
|
||||
],
|
||||
|
||||
+14
-25
@@ -19,6 +19,7 @@ import {
|
||||
getStoredOrganizationId,
|
||||
setStoredOrganizationId,
|
||||
} from "@/lib/organization-state";
|
||||
import { parseServerSentEventBlock } from "@/lib/server-sent-events";
|
||||
|
||||
const DEFAULT_API_PORT = process.env.NEXT_PUBLIC_API_PORT || "4000";
|
||||
|
||||
@@ -1814,28 +1815,6 @@ export interface ProcessJobRecord {
|
||||
cancelReason?: string;
|
||||
}
|
||||
|
||||
function parseServerSentEventBlock(block: string) {
|
||||
let event = "message";
|
||||
const dataLines: string[] = [];
|
||||
|
||||
for (const line of block.split("\n")) {
|
||||
if (line.startsWith("event:")) {
|
||||
event = line.slice("event:".length).trim();
|
||||
} else if (line.startsWith("data:")) {
|
||||
dataLines.push(line.slice("data:".length).trimStart());
|
||||
}
|
||||
}
|
||||
|
||||
if (dataLines.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
event,
|
||||
data: JSON.parse(dataLines.join("\n")) as unknown,
|
||||
};
|
||||
}
|
||||
|
||||
export const containers = {
|
||||
list: (params?: { serverId?: string; status?: string }) => {
|
||||
const qs = new URLSearchParams();
|
||||
@@ -1937,16 +1916,26 @@ export const containers = {
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const blocks = buffer.split(/\n\n/);
|
||||
const blocks = buffer.split(/\r?\n\r?\n/);
|
||||
buffer = blocks.pop() ?? "";
|
||||
|
||||
for (const block of blocks) {
|
||||
const parsed = parseServerSentEventBlock(block.trim());
|
||||
if (!parsed) continue;
|
||||
|
||||
if (parsed.event === "log") {
|
||||
if (
|
||||
parsed.event === "log" &&
|
||||
typeof parsed.data === "object" &&
|
||||
parsed.data !== null &&
|
||||
typeof (parsed.data as ProcessJobLogEntry).message === "string"
|
||||
) {
|
||||
handlers.onLog?.(parsed.data as ProcessJobLogEntry);
|
||||
} else if (parsed.event === "status") {
|
||||
} else if (
|
||||
parsed.event === "status" &&
|
||||
typeof parsed.data === "object" &&
|
||||
parsed.data !== null &&
|
||||
typeof (parsed.data as ProcessJobRecord).status === "string"
|
||||
) {
|
||||
handlers.onStatus?.(parsed.data as ProcessJobRecord);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
export interface ParsedServerSentEvent {
|
||||
event: string;
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one SSE block without allowing a malformed upstream payload to abort
|
||||
* the whole log stream. Some reverse proxies can surface a plain-text backend
|
||||
* error in an otherwise valid event stream.
|
||||
*/
|
||||
export function parseServerSentEventBlock(
|
||||
block: string,
|
||||
): ParsedServerSentEvent | null {
|
||||
let event = "message";
|
||||
const dataLines: string[] = [];
|
||||
|
||||
for (const line of block.split(/\r?\n/)) {
|
||||
if (line.startsWith("event:")) {
|
||||
event = line.slice("event:".length).trim();
|
||||
} else if (line.startsWith("data:")) {
|
||||
dataLines.push(line.slice("data:".length).trimStart());
|
||||
}
|
||||
}
|
||||
|
||||
if (dataLines.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rawData = dataLines.join("\n");
|
||||
try {
|
||||
return { event, data: JSON.parse(rawData) as unknown };
|
||||
} catch {
|
||||
return { event, data: rawData };
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,12 @@ import { posix as pathPosix } from "path";
|
||||
import { TextDecoder } from "util";
|
||||
import { Server } from "@prisma/client";
|
||||
import { execStrict } from "./commands";
|
||||
import { execDocker, execDockerStrict } from "./internal/docker";
|
||||
import {
|
||||
DOCKER_ACCESS_DENIED_MESSAGE,
|
||||
execDocker,
|
||||
execDockerStrict,
|
||||
parseDockerJson,
|
||||
} from "./internal/docker";
|
||||
import { privilegedCommand } from "./internal/privilege";
|
||||
import { escapeShellArg } from "./internal/shell";
|
||||
|
||||
@@ -576,6 +581,15 @@ export function formatDeploymentErrorMessage(error: unknown): string {
|
||||
const cleaned = trimCommandErrorNoise(raw);
|
||||
const lower = cleaned.toLowerCase();
|
||||
|
||||
if (
|
||||
lower.includes("permission denied") &&
|
||||
(lower.includes("docker.sock") ||
|
||||
lower.includes("docker daemon") ||
|
||||
lower.includes("docker api"))
|
||||
) {
|
||||
return DOCKER_ACCESS_DENIED_MESSAGE;
|
||||
}
|
||||
|
||||
if (
|
||||
lower.includes("could not read username for 'https://") ||
|
||||
lower.includes("terminal prompts disabled") ||
|
||||
@@ -1003,7 +1017,10 @@ export async function dockerInspect(
|
||||
`docker inspect ${escapeShellArg(containerId)}`,
|
||||
shortDockerCommandTimeout(DOCKER_INSPECT_TIMEOUT_MS),
|
||||
);
|
||||
const parsed = JSON.parse(stdout) as DockerContainerInspect[];
|
||||
const parsed = parseDockerJson<DockerContainerInspect[]>(
|
||||
stdout,
|
||||
"Docker container inspection",
|
||||
);
|
||||
return parsed[0] ?? {};
|
||||
}
|
||||
|
||||
@@ -1016,14 +1033,14 @@ export async function dockerStats(
|
||||
`docker stats --no-stream --format '{{json .}}' ${escapeShellArg(containerId)}`,
|
||||
shortDockerCommandTimeout(DOCKER_STATS_TIMEOUT_MS),
|
||||
);
|
||||
const parsed = JSON.parse(stdout.trim()) as {
|
||||
const parsed = parseDockerJson<{
|
||||
CPUPerc?: string;
|
||||
MemPerc?: string;
|
||||
MemUsage?: string;
|
||||
NetIO?: string;
|
||||
BlockIO?: string;
|
||||
PIDs?: string;
|
||||
};
|
||||
}>(stdout, "Docker container statistics");
|
||||
|
||||
return {
|
||||
cpuPercent: parseFloat((parsed.CPUPerc || "0").replace("%", "")) || 0,
|
||||
@@ -1848,7 +1865,10 @@ async function inspectImageExposedPorts(server: Server, imageTag: string) {
|
||||
return [] as string[];
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(normalized) as Record<string, unknown> | null;
|
||||
const parsed = parseDockerJson<Record<string, unknown> | null>(
|
||||
normalized,
|
||||
"Docker image inspection",
|
||||
);
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
return [] as string[];
|
||||
}
|
||||
|
||||
@@ -4,6 +4,9 @@ import type { SSHExecCommandResponse } from "node-ssh";
|
||||
import { exec } from "../commands";
|
||||
import { detectServerPlatform } from "../platform";
|
||||
|
||||
export const DOCKER_ACCESS_DENIED_MESSAGE =
|
||||
"Docker access was denied for the configured SSH user. Grant that user access to the Docker socket (usually by adding it to the docker group), or configure passwordless sudo for Docker commands, then reconnect and retry the deployment.";
|
||||
|
||||
function shouldRetryDockerWithSudo(output: string): boolean {
|
||||
const lower = output.toLowerCase();
|
||||
return (
|
||||
@@ -37,6 +40,30 @@ export async function execDocker(
|
||||
return sudoResult.code === 0 ? sudoResult : directResult;
|
||||
}
|
||||
|
||||
export function parseDockerJson<T>(output: string, operation: string): T {
|
||||
const normalized = output.trim();
|
||||
const lower = normalized.toLowerCase();
|
||||
|
||||
if (
|
||||
lower.includes("permission denied") &&
|
||||
(lower.includes("docker.sock") ||
|
||||
lower.includes("docker daemon") ||
|
||||
lower.includes("docker api"))
|
||||
) {
|
||||
throw new Error(DOCKER_ACCESS_DENIED_MESSAGE);
|
||||
}
|
||||
|
||||
if (!normalized) {
|
||||
throw new Error(`${operation} returned an empty response from Docker`);
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(normalized) as T;
|
||||
} catch {
|
||||
throw new Error(`${operation} returned an invalid response from Docker`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function execDockerStrict(
|
||||
server: Server,
|
||||
command: string,
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { formatDeploymentErrorMessage } from "../../src/server/services/ssh-services/docker-containers.ts";
|
||||
import {
|
||||
DOCKER_ACCESS_DENIED_MESSAGE,
|
||||
parseDockerJson,
|
||||
} from "../../src/server/services/ssh-services/internal/docker.ts";
|
||||
|
||||
test("Docker socket permission failures have an actionable deployment error", () => {
|
||||
const message = formatDeploymentErrorMessage(
|
||||
new Error(
|
||||
"permission denied while trying to connect to the docker API at unix:///var/run/docker.sock",
|
||||
),
|
||||
);
|
||||
|
||||
assert.match(message, /Docker access was denied/i);
|
||||
assert.match(message, /docker group/i);
|
||||
assert.doesNotMatch(message, /Unexpected token|not valid JSON/i);
|
||||
});
|
||||
|
||||
test("Docker JSON parsing reports socket permission failures before JSON errors", () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseDockerJson(
|
||||
"permission denied while trying to connect to the docker API at unix:///var/run/docker.sock",
|
||||
"Docker image inspection",
|
||||
),
|
||||
{ message: DOCKER_ACCESS_DENIED_MESSAGE },
|
||||
);
|
||||
});
|
||||
|
||||
test("Docker JSON parsing hides malformed raw output", () => {
|
||||
assert.throws(
|
||||
() => parseDockerJson("not-json", "Docker image inspection"),
|
||||
{ message: "Docker image inspection returned an invalid response from Docker" },
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { parseServerSentEventBlock } from "../src/lib/server-sent-events.ts";
|
||||
|
||||
test("SSE parser reads JSON events with CRLF delimiters", () => {
|
||||
assert.deepEqual(
|
||||
parseServerSentEventBlock(
|
||||
'event: log\r\ndata: {"id":1,"message":"building"}',
|
||||
),
|
||||
{ event: "log", data: { id: 1, message: "building" } },
|
||||
);
|
||||
});
|
||||
|
||||
test("SSE parser preserves malformed plain-text data instead of throwing", () => {
|
||||
assert.deepEqual(
|
||||
parseServerSentEventBlock(
|
||||
"event: error\ndata: permission denied while trying to connect to /var/run/docker.sock",
|
||||
),
|
||||
{
|
||||
event: "error",
|
||||
data: "permission denied while trying to connect to /var/run/docker.sock",
|
||||
},
|
||||
);
|
||||
});
|
||||
Reference in new issue
Block a user