Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions apps/dokploy/__test__/deploy/application.command.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,20 @@ vi.mock("@dokploy/server/services/rollbacks", () => ({
createRollback: vi.fn(),
}));

// deployApplication now verifies post-deploy service stability by polling
// Docker Swarm for a 60s window (waitForSwarmServiceStable). These are
// command-generation tests with no real Docker service running, so stub it
// to report a stable service; keep the rest of docker/utils real.
vi.mock("@dokploy/server/utils/docker/utils", async () => {
const actual = await vi.importActual<
typeof import("@dokploy/server/utils/docker/utils")
>("@dokploy/server/utils/docker/utils");
return {
...actual,
waitForSwarmServiceStable: vi.fn().mockResolvedValue({ stable: true }),
};
});

import { db } from "@dokploy/server/db";
import { cloneGitRepository } from "@dokploy/server/utils/providers/git";

Expand Down
5 changes: 4 additions & 1 deletion apps/dokploy/__test__/deploy/application.real.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ import { execAsync } from "@dokploy/server/utils/process/execAsync";
import { format } from "date-fns";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";

const REAL_TEST_TIMEOUT = 180000; // 3 minutes
// 5 minutes: deployApplication now waits a fixed 60s post-deploy stability
// window (waitForSwarmServiceStable) on every successful deploy, on top of
// the real nixpacks/railpack build time.
const REAL_TEST_TIMEOUT = 300000;

// Mock ONLY database and notifications
vi.mock("@dokploy/server/db", () => {
Expand Down
22 changes: 21 additions & 1 deletion packages/server/src/services/application.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import { createTraefikConfig } from "@dokploy/server/utils/traefik/application";
import { TRPCError } from "@trpc/server";
import { eq } from "drizzle-orm";
import type { z } from "zod";
import { encodeBase64 } from "../utils/docker/utils";
import { encodeBase64, waitForSwarmServiceStable } from "../utils/docker/utils";
import { getDokployUrl } from "./admin";
import {
createDeployment,
Expand Down Expand Up @@ -218,6 +218,16 @@ export const deployApplication = async ({
}

await mechanizeDockerContainer(application);

const stability = await waitForSwarmServiceStable(application.appName, {
serverId,
});
if (!stability.stable) {
throw new Error(
`Container did not stay running after deployment: ${stability.reason}`,
);
}

await updateDeploymentStatus(deployment.deploymentId, "done");
await updateApplicationStatus(applicationId, "done");

Expand Down Expand Up @@ -309,6 +319,16 @@ export const rebuildApplication = async ({
await execAsync(commandWithLog);
}
await mechanizeDockerContainer(application);

const stability = await waitForSwarmServiceStable(application.appName, {
serverId,
});
if (!stability.stable) {
throw new Error(
`Container did not stay running after rebuild: ${stability.reason}`,
);
}

await updateDeploymentStatus(deployment.deploymentId, "done");
await updateApplicationStatus(applicationId, "done");

Expand Down
83 changes: 83 additions & 0 deletions packages/server/src/utils/docker/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -869,6 +869,89 @@ const getSwarmServiceContainerId = async (
}
};

export type SwarmStabilityResult =
| { stable: true }
| { stable: false; reason: string };

export const waitForSwarmServiceStable = async (
appName: string,
{
serverId,
windowMs = 60_000,
pollMs = 5_000,
}: { serverId?: string | null; windowMs?: number; pollMs?: number } = {},
): Promise<SwarmStabilityResult> => {
const remoteDocker = await getRemoteDocker(serverId);
const deadline = Date.now() + windowMs;
let everRunning = false;
let lastReason = "Service did not reach running state";

while (Date.now() < deadline) {
try {
const tasks = await remoteDocker.listTasks({
filters: JSON.stringify({ service: [appName] }),
});

const sorted = [...tasks].sort((a, b) => {
const at = new Date(a.UpdatedAt ?? 0).getTime();
const bt = new Date(b.UpdatedAt ?? 0).getTime();
return bt - at;
});
const latest = sorted[0];
const state = latest?.Status?.State;
const message = latest?.Status?.Err || latest?.Status?.Message || "";

if (state === "failed" || state === "rejected") {
return {
stable: false,
reason: message
? `Task ${state}: ${message}`
: `Task entered ${state} state`,
};
}

const runningCount = sorted.filter(
(t) => t.Status?.State === "running",
).length;
const startingCount = sorted.filter((t) =>
[
"new",
"pending",
"assigned",
"accepted",
"preparing",
"starting",
].includes(t.Status?.State ?? ""),
).length;

if (runningCount > 0) {
everRunning = true;
} else if (everRunning && startingCount > 0) {
return {
stable: false,
reason: message
? `Container restarted after running: ${message}`
: "Container restarted after reaching running state",
};
}

lastReason = message
? `Latest task state: ${state ?? "unknown"} (${message})`
: `Latest task state: ${state ?? "unknown"}`;
} catch (error) {
lastReason =
error instanceof Error ? error.message : "Failed to inspect service";
}

await new Promise((resolve) => setTimeout(resolve, pollMs));
}

if (everRunning) {
return { stable: true };
}
return { stable: false, reason: lastReason };
};

export const checkPostgresHealth = async (): Promise<ServiceHealthStatus> => {
const serviceCheck = await checkSwarmServiceRunning("dokploy-postgres");
if (serviceCheck.status === "unhealthy") {
Expand Down
Loading