From 58d664d7a769d03e114b56882a3cbd0e4ce4908b Mon Sep 17 00:00:00 2001 From: sajdakabir Date: Wed, 6 May 2026 22:54:46 +0530 Subject: [PATCH 1/2] fix(deploy): notify on container crash loop after deployment (cherry picked from commit e1a1f447246a8fd24be1a70214ab3c57774b2b46) --- packages/server/src/services/application.ts | 22 +++++- packages/server/src/utils/docker/utils.ts | 83 +++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/packages/server/src/services/application.ts b/packages/server/src/services/application.ts index c8ebb3be77..ce3d856c8b 100644 --- a/packages/server/src/services/application.ts +++ b/packages/server/src/services/application.ts @@ -30,7 +30,7 @@ import { createTraefikConfig } from "@dokploy/server/utils/traefik/application"; import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; import type { z } from "zod"; -import { encodeBase64 } from "../utils/docker/utils"; +import { encodeBase64, waitForSwarmServiceStable } from "../utils/docker/utils"; import { getDokployUrl } from "./admin"; import { createDeployment, @@ -218,6 +218,16 @@ export const deployApplication = async ({ } await mechanizeDockerContainer(application); + + const stability = await waitForSwarmServiceStable(application.appName, { + serverId, + }); + if (!stability.stable) { + throw new Error( + `Container did not stay running after deployment: ${stability.reason}`, + ); + } + await updateDeploymentStatus(deployment.deploymentId, "done"); await updateApplicationStatus(applicationId, "done"); @@ -309,6 +319,16 @@ export const rebuildApplication = async ({ await execAsync(commandWithLog); } await mechanizeDockerContainer(application); + + const stability = await waitForSwarmServiceStable(application.appName, { + serverId, + }); + if (!stability.stable) { + throw new Error( + `Container did not stay running after rebuild: ${stability.reason}`, + ); + } + await updateDeploymentStatus(deployment.deploymentId, "done"); await updateApplicationStatus(applicationId, "done"); diff --git a/packages/server/src/utils/docker/utils.ts b/packages/server/src/utils/docker/utils.ts index 391d53aac4..442bf8d5b1 100644 --- a/packages/server/src/utils/docker/utils.ts +++ b/packages/server/src/utils/docker/utils.ts @@ -869,6 +869,89 @@ const getSwarmServiceContainerId = async ( } }; +export type SwarmStabilityResult = + | { stable: true } + | { stable: false; reason: string }; + +export const waitForSwarmServiceStable = async ( + appName: string, + { + serverId, + windowMs = 60_000, + pollMs = 5_000, + }: { serverId?: string | null; windowMs?: number; pollMs?: number } = {}, +): Promise => { + const remoteDocker = await getRemoteDocker(serverId); + const deadline = Date.now() + windowMs; + let everRunning = false; + let lastReason = "Service did not reach running state"; + + while (Date.now() < deadline) { + try { + const tasks = await remoteDocker.listTasks({ + filters: JSON.stringify({ service: [appName] }), + }); + + const sorted = [...tasks].sort((a, b) => { + const at = new Date(a.UpdatedAt ?? 0).getTime(); + const bt = new Date(b.UpdatedAt ?? 0).getTime(); + return bt - at; + }); + const latest = sorted[0]; + const state = latest?.Status?.State; + const message = latest?.Status?.Err || latest?.Status?.Message || ""; + + if (state === "failed" || state === "rejected") { + return { + stable: false, + reason: message + ? `Task ${state}: ${message}` + : `Task entered ${state} state`, + }; + } + + const runningCount = sorted.filter( + (t) => t.Status?.State === "running", + ).length; + const startingCount = sorted.filter((t) => + [ + "new", + "pending", + "assigned", + "accepted", + "preparing", + "starting", + ].includes(t.Status?.State ?? ""), + ).length; + + if (runningCount > 0) { + everRunning = true; + } else if (everRunning && startingCount > 0) { + return { + stable: false, + reason: message + ? `Container restarted after running: ${message}` + : "Container restarted after reaching running state", + }; + } + + lastReason = message + ? `Latest task state: ${state ?? "unknown"} (${message})` + : `Latest task state: ${state ?? "unknown"}`; + } catch (error) { + lastReason = + error instanceof Error ? error.message : "Failed to inspect service"; + } + + await new Promise((resolve) => setTimeout(resolve, pollMs)); + } + + if (everRunning) { + return { stable: true }; + } + return { stable: false, reason: lastReason }; +}; + export const checkPostgresHealth = async (): Promise => { const serviceCheck = await checkSwarmServiceRunning("dokploy-postgres"); if (serviceCheck.status === "unhealthy") { From 20cb050a93a6f37bb5a278d59d1dc6a3672e0e22 Mon Sep 17 00:00:00 2001 From: AminDhouib Date: Sun, 12 Jul 2026 10:52:41 -0400 Subject: [PATCH 2/2] test: adapt deploy tests for post-deploy stability window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - application.command.test.ts: stub waitForSwarmServiceStable (partial mock, rest of docker/utils stays real) — command-generation tests have no real swarm service to poll, so the new 60s stability window timed them out. - application.real.test.ts: raise REAL_TEST_TIMEOUT 180s -> 300s to absorb the fixed 60s stability window each successful real deploy now includes. Fork-side adaptation for the port of Dokploy/dokploy#4357 (upstream's own pr-check(test) was red for exactly this reason). --- .../__test__/deploy/application.command.test.ts | 14 ++++++++++++++ .../__test__/deploy/application.real.test.ts | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/apps/dokploy/__test__/deploy/application.command.test.ts b/apps/dokploy/__test__/deploy/application.command.test.ts index 1a33489b57..02009e8345 100644 --- a/apps/dokploy/__test__/deploy/application.command.test.ts +++ b/apps/dokploy/__test__/deploy/application.command.test.ts @@ -103,6 +103,20 @@ vi.mock("@dokploy/server/services/rollbacks", () => ({ createRollback: vi.fn(), })); +// deployApplication now verifies post-deploy service stability by polling +// Docker Swarm for a 60s window (waitForSwarmServiceStable). These are +// command-generation tests with no real Docker service running, so stub it +// to report a stable service; keep the rest of docker/utils real. +vi.mock("@dokploy/server/utils/docker/utils", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/docker/utils") + >("@dokploy/server/utils/docker/utils"); + return { + ...actual, + waitForSwarmServiceStable: vi.fn().mockResolvedValue({ stable: true }), + }; +}); + import { db } from "@dokploy/server/db"; import { cloneGitRepository } from "@dokploy/server/utils/providers/git"; diff --git a/apps/dokploy/__test__/deploy/application.real.test.ts b/apps/dokploy/__test__/deploy/application.real.test.ts index 4adff6f075..633e9ad09d 100644 --- a/apps/dokploy/__test__/deploy/application.real.test.ts +++ b/apps/dokploy/__test__/deploy/application.real.test.ts @@ -6,7 +6,10 @@ import { execAsync } from "@dokploy/server/utils/process/execAsync"; import { format } from "date-fns"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const REAL_TEST_TIMEOUT = 180000; // 3 minutes +// 5 minutes: deployApplication now waits a fixed 60s post-deploy stability +// window (waitForSwarmServiceStable) on every successful deploy, on top of +// the real nixpacks/railpack build time. +const REAL_TEST_TIMEOUT = 300000; // Mock ONLY database and notifications vi.mock("@dokploy/server/db", () => {