Skip to content

Commit 2e2fff2

Browse files
committed
test(webapp): bring mollifier integration tests on phase-3 in line with phase-3 semantics
The three "dual-write" tests inherited from phase-1 were asserting invariants that phase-3 deliberately abandoned when the mollify path moved from "buffer.accept + engine.trigger" to "buffer.accept + synthetic result, drainer replays later": - `mollify action triggers dual-write` — rewritten to assert the new contract: synthetic `MollifySyntheticResult` (run.friendlyId, isCached:false, notice.code = "mollifier.queued"), buffer.accept fires with the canonical engine.trigger snapshot, NO Postgres row (the run materialises only when the drainer replays). - `engine.trigger throwing AFTER buffer.accept` — deleted. Phase-3 never invokes engine.trigger on the mollify path, so the scenario is structurally impossible. - `debounce match produces an orphan buffer entry` — deleted. Phase-3's C1 debounce bypass at the gate (returns pass_through for debounce triggers) means the mollify branch is never entered for debounced requests. The C1 invariant is pinned at mollifierGate.test.ts:440; duplicating it at the trigger-task layer adds nothing. Net: 6 mollifier integration tests → 4, all 4 passing, no coverage gap (gate-level + drainer-handler-level tests own the deleted scenarios' invariants).
1 parent b608ef2 commit 2e2fff2

1 file changed

Lines changed: 42 additions & 253 deletions

File tree

apps/webapp/test/engine/triggerTask.test.ts

Lines changed: 42 additions & 253 deletions
Original file line numberDiff line numberDiff line change
@@ -1269,8 +1269,17 @@ describe("RunEngineTriggerTaskService", () => {
12691269
);
12701270

12711271
containerTest(
1272-
"mollifier · mollify action triggers dual-write (buffer.accept + engine.trigger)",
1272+
"mollifier · mollify action writes to buffer and returns synthetic result (no Postgres row)",
12731273
async ({ prisma, redisOptions }) => {
1274+
// Phase 3 semantics: when the gate decides mollify, the call site
1275+
// invokes `mollifyTrigger` which writes the engine.trigger snapshot
1276+
// to the buffer and returns a synthesised `MollifySyntheticResult`
1277+
// (run.friendlyId + notice + isCached:false). `engine.trigger` is
1278+
// NEVER invoked on this path — the run materialises in Postgres
1279+
// later, when the drainer replays the snapshot. The replay is
1280+
// covered by `mollifierDrainerHandler.test.ts`; this test pins the
1281+
// call-site integration: synthetic result + buffer write + no
1282+
// Postgres side effect.
12741283
const engine = new RunEngine({
12751284
prisma,
12761285
worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 },
@@ -1319,25 +1328,44 @@ describe("RunEngineTriggerTaskService", () => {
13191328
body: { payload: { hello: "world" } },
13201329
});
13211330

1322-
// engine.trigger ran — Postgres has the run
1331+
// Synthetic result is returned with the `mollifier.queued` notice
1332+
// (the call-site casts the synthetic shape to `TriggerTaskServiceResult`;
1333+
// at runtime the `notice` and `isCached: false` fields are present
1334+
// and read by the api.v1.tasks.$taskId.trigger.ts route handler).
13231335
expect(result).toBeDefined();
13241336
expect(result?.run.friendlyId).toBeDefined();
1325-
const pgRun = await prisma.taskRun.findFirst({ where: { id: result!.run.id } });
1326-
expect(pgRun).not.toBeNull();
1327-
expect(pgRun!.friendlyId).toBe(result!.run.friendlyId);
1328-
1329-
// buffer.accept ran — Redis has the audit copy under the same friendlyId
1337+
const synthetic = result as unknown as {
1338+
run: { friendlyId: string };
1339+
isCached: false;
1340+
notice: { code: string; message: string; docs: string };
1341+
};
1342+
expect(synthetic.isCached).toBe(false);
1343+
expect(synthetic.notice.code).toBe("mollifier.queued");
1344+
expect(synthetic.notice.message).toBeTypeOf("string");
1345+
expect(synthetic.notice.docs).toBeTypeOf("string");
1346+
1347+
// buffer.accept ran — Redis has the canonical engine.trigger snapshot
1348+
// under the synthesised friendlyId. The drainer will read this and
1349+
// replay it through engine.trigger to materialise the run.
13301350
expect(buffer.accepted).toHaveLength(1);
13311351
expect(buffer.accepted[0]!.runId).toBe(result!.run.friendlyId);
13321352
expect(buffer.accepted[0]!.envId).toBe(authenticatedEnvironment.id);
13331353
expect(buffer.accepted[0]!.orgId).toBe(authenticatedEnvironment.organizationId);
1334-
1335-
// payload is the canonical replay shape
1336-
const payload = JSON.parse(buffer.accepted[0]!.payload);
1337-
expect(payload.runFriendlyId).toBe(result!.run.friendlyId);
1338-
expect(payload.taskId).toBe(taskIdentifier);
1339-
expect(payload.envId).toBe(authenticatedEnvironment.id);
1340-
expect(payload.body).toEqual({ payload: { hello: "world" } });
1354+
// Payload is a JSON-serialised MollifierSnapshot (the engine.trigger
1355+
// input). Schema is internal to the engine, so we only assert that
1356+
// it parses and references the friendlyId — anything more specific
1357+
// would couple the mollifier-layer test to engine-layer fields.
1358+
expect(() => JSON.parse(buffer.accepted[0]!.payload)).not.toThrow();
1359+
1360+
// Postgres has NOT been written: engine.trigger was never called on
1361+
// the mollify path. The run materialises only when the drainer
1362+
// replays the snapshot. Regression intent: if a future change makes
1363+
// the mollify branch fall through to engine.trigger (re-introducing
1364+
// phase-1 dual-write), this assertion fails loudly.
1365+
const pgRun = await prisma.taskRun.findFirst({
1366+
where: { friendlyId: result!.run.friendlyId },
1367+
});
1368+
expect(pgRun).toBeNull();
13411369

13421370
await engine.quit();
13431371
},
@@ -1398,108 +1426,6 @@ describe("RunEngineTriggerTaskService", () => {
13981426
},
13991427
);
14001428

1401-
containerTest(
1402-
"mollifier · engine.trigger throwing AFTER buffer.accept leaves an orphan entry (documented behaviour)",
1403-
async ({ prisma, redisOptions }) => {
1404-
// SCENARIO: dual-write where buffer.accept succeeds but engine.trigger
1405-
// throws. The throw propagates to the caller (correct: customer sees
1406-
// the same 4xx as today), and the buffer entry remains as an "orphan"
1407-
// — Phase 1's no-op drainer will pop+ack it on its next poll, so the
1408-
// orphan is bounded (~drainer pollIntervalMs) but observable in the
1409-
// audit trail (mollifier.buffered with no matching TaskRun).
1410-
//
1411-
// Why engine.trigger can throw post-buffer:
1412-
// - RunDuplicateIdempotencyKeyError (Prisma P2002 on idempotencyKey):
1413-
// a concurrent non-mollified trigger with the same idempotencyKey
1414-
// wins the DB UNIQUE constraint between IdempotencyKeyConcern's
1415-
// pre-check and engine.trigger's INSERT.
1416-
// - RunOneTimeUseTokenError (Prisma P2002 on oneTimeUseToken).
1417-
// - Transient Prisma errors (FK constraint, connection drop, etc.).
1418-
//
1419-
// Why we don't "fix" this race in Phase 1:
1420-
// The customer correctly gets the error. State eventually converges
1421-
// (drainer pops the orphan). The audit-trail explicitly surfaces
1422-
// "buffered without TaskRun" entries to operators. A real fix is
1423-
// Phase 2's responsibility once the buffer becomes the primary write
1424-
// — at that point we add the mollifier-specific idempotency index.
1425-
//
1426-
// This test pins the current ordering: buffer.accept fires synchronously
1427-
// BEFORE engine.trigger, and engine.trigger failure does NOT roll back
1428-
// the buffer write. Any future change that reverses the order or adds
1429-
// a silent rollback will fail this assertion and force a design
1430-
// decision rather than a silent behaviour change.
1431-
1432-
const engine = new RunEngine({
1433-
prisma,
1434-
worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 },
1435-
queue: { redis: redisOptions },
1436-
runLock: { redis: redisOptions },
1437-
machines: {
1438-
defaultMachine: "small-1x",
1439-
machines: { "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 } },
1440-
baseCostInCents: 0.0005,
1441-
},
1442-
tracer: trace.getTracer("test", "0.0.0"),
1443-
});
1444-
1445-
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
1446-
const taskIdentifier = "test-task";
1447-
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
1448-
1449-
const buffer = new CapturingMollifierBuffer();
1450-
1451-
// Force engine.trigger to throw on this single call. We spy AFTER
1452-
// setupBackgroundWorker so the worker setup still uses the real
1453-
// engine.trigger (which has its own engine.trigger-ish calls for
1454-
// worker bootstrap — though in practice setupBackgroundWorker doesn't
1455-
// call trigger).
1456-
const simulatedFailure = new Error("simulated engine.trigger failure post-buffer");
1457-
vi.spyOn(engine, "trigger").mockRejectedValueOnce(simulatedFailure);
1458-
1459-
const triggerTaskService = new RunEngineTriggerTaskService({
1460-
engine,
1461-
prisma,
1462-
payloadProcessor: new MockPayloadProcessor(),
1463-
queueConcern: new DefaultQueueManager(prisma, engine),
1464-
idempotencyKeyConcern: new IdempotencyKeyConcern(prisma, engine, new MockTraceEventConcern()),
1465-
validator: new MockTriggerTaskValidator(),
1466-
traceEventConcern: new MockTraceEventConcern(),
1467-
tracer: trace.getTracer("test", "0.0.0"),
1468-
metadataMaximumSize: 1024 * 1024,
1469-
evaluateGate: async () => ({
1470-
action: "mollify",
1471-
decision: {
1472-
divert: true,
1473-
reason: "per_env_rate",
1474-
count: 150,
1475-
threshold: 100,
1476-
windowMs: 200,
1477-
holdMs: 500,
1478-
},
1479-
}),
1480-
getMollifierBuffer: () => buffer as never,
1481-
isMollifierGloballyEnabled: () => true,
1482-
});
1483-
1484-
await expect(
1485-
triggerTaskService.call({
1486-
taskId: taskIdentifier,
1487-
environment: authenticatedEnvironment,
1488-
body: { payload: { test: "x" } },
1489-
}),
1490-
).rejects.toThrow(/simulated engine.trigger failure post-buffer/);
1491-
1492-
// The buffer write happened BEFORE engine.trigger threw. The orphan
1493-
// remains; the audit-trail will surface it (mollifier.buffered with
1494-
// no matching TaskRun row). Phase 1's no-op drainer cleans it up.
1495-
expect(buffer.accepted).toHaveLength(1);
1496-
const orphanPayload = JSON.parse(buffer.accepted[0]!.payload);
1497-
expect(orphanPayload.taskId).toBe(taskIdentifier);
1498-
1499-
await engine.quit();
1500-
},
1501-
);
1502-
15031429
containerTest(
15041430
"mollifier · idempotency-key match short-circuits BEFORE the gate is consulted",
15051431
async ({ prisma, redisOptions }) => {
@@ -1607,143 +1533,6 @@ describe("RunEngineTriggerTaskService", () => {
16071533
},
16081534
);
16091535

1610-
containerTest(
1611-
"mollifier · debounce match produces an orphan buffer entry (documented behaviour)",
1612-
async ({ prisma, redisOptions }) => {
1613-
// SCENARIO: a trigger with a debounce key arrives while a matching
1614-
// debounced run already exists. `debounceSystem.handleDebounce` runs
1615-
// INSIDE `engine.trigger` (line ~514 of run-engine/src/engine/index.ts),
1616-
// AFTER buffer.accept has already written the new friendlyId. The
1617-
// service correctly returns the existing run id to the customer, but
1618-
// the buffer is left with an orphan entry for the new friendlyId.
1619-
//
1620-
// Why this is acceptable in Phase 1:
1621-
// - Customer-facing behaviour is unchanged from today: they receive
1622-
// the existing run id, same as the non-mollified path.
1623-
// - The orphan is bounded — the drainer's no-op-ack handler pops
1624-
// and acks it on its next poll.
1625-
// - The audit-trail surfaces it: a `mollifier.buffered` log line
1626-
// with `runId` that has no matching TaskRun in Postgres.
1627-
//
1628-
// Why Phase 2 cares:
1629-
// - When the buffer becomes the primary write path, debounce can
1630-
// no longer be allowed to run AFTER buffer.accept. The drainer's
1631-
// engine.trigger replay would observe "existing" and skip the
1632-
// persist — the customer's synthesised 200 (with the new
1633-
// friendlyId) would never get a TaskRun, and the audit-trail
1634-
// divergence becomes a real data-loss bug.
1635-
// - Phase 2 must lift `handleDebounce` into the call site BEFORE
1636-
// buffer.accept:
1637-
// 1. handleDebounce → if existing, return existing run; do NOT
1638-
// touch the buffer.
1639-
// 2. Otherwise, accept with `claimId` threaded into the
1640-
// canonical payload so the drainer's replay can
1641-
// `registerDebouncedRun` after persisting.
1642-
//
1643-
// This test pins the current ordering. A future change that "fixes"
1644-
// it by lifting handleDebounce upfront will fail the orphan
1645-
// assertion below and force an explicit choice (update the test,
1646-
// remove this scenario, or stage the lift behind a flag).
1647-
1648-
const engine = new RunEngine({
1649-
prisma,
1650-
worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 },
1651-
queue: { redis: redisOptions },
1652-
runLock: { redis: redisOptions },
1653-
machines: {
1654-
defaultMachine: "small-1x",
1655-
machines: { "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 } },
1656-
baseCostInCents: 0.0005,
1657-
},
1658-
tracer: trace.getTracer("test", "0.0.0"),
1659-
});
1660-
1661-
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
1662-
const taskIdentifier = "test-task";
1663-
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
1664-
1665-
const idempotencyKeyConcern = new IdempotencyKeyConcern(
1666-
prisma,
1667-
engine,
1668-
new MockTraceEventConcern(),
1669-
);
1670-
1671-
// Setup: trigger with debounce — creates the existing run + Redis claim.
1672-
const baseline = new RunEngineTriggerTaskService({
1673-
engine,
1674-
prisma,
1675-
payloadProcessor: new MockPayloadProcessor(),
1676-
queueConcern: new DefaultQueueManager(prisma, engine),
1677-
idempotencyKeyConcern,
1678-
validator: new MockTriggerTaskValidator(),
1679-
traceEventConcern: new MockTraceEventConcern(),
1680-
tracer: trace.getTracer("test", "0.0.0"),
1681-
metadataMaximumSize: 1024 * 1024,
1682-
});
1683-
const first = await baseline.call({
1684-
taskId: taskIdentifier,
1685-
environment: authenticatedEnvironment,
1686-
body: {
1687-
payload: { test: "x" },
1688-
options: { debounce: { key: "regression-debounce-6", delay: "30s" } },
1689-
},
1690-
});
1691-
expect(first?.run.friendlyId).toBeDefined();
1692-
1693-
// Action: same debounce key, mollify-stub gate.
1694-
const buffer = new CapturingMollifierBuffer();
1695-
const mollifierService = new RunEngineTriggerTaskService({
1696-
engine,
1697-
prisma,
1698-
payloadProcessor: new MockPayloadProcessor(),
1699-
queueConcern: new DefaultQueueManager(prisma, engine),
1700-
idempotencyKeyConcern,
1701-
validator: new MockTriggerTaskValidator(),
1702-
traceEventConcern: new MockTraceEventConcern(),
1703-
tracer: trace.getTracer("test", "0.0.0"),
1704-
metadataMaximumSize: 1024 * 1024,
1705-
evaluateGate: async () => ({
1706-
action: "mollify",
1707-
decision: {
1708-
divert: true,
1709-
reason: "per_env_rate",
1710-
count: 150,
1711-
threshold: 100,
1712-
windowMs: 200,
1713-
holdMs: 500,
1714-
},
1715-
}),
1716-
getMollifierBuffer: () => buffer as never,
1717-
isMollifierGloballyEnabled: () => true,
1718-
});
1719-
1720-
const debounced = await mollifierService.call({
1721-
taskId: taskIdentifier,
1722-
environment: authenticatedEnvironment,
1723-
body: {
1724-
payload: { test: "x" },
1725-
options: { debounce: { key: "regression-debounce-6", delay: "30s" } },
1726-
},
1727-
});
1728-
1729-
// Customer-facing behaviour: the existing run is returned (correct).
1730-
expect(debounced).toBeDefined();
1731-
expect(debounced?.run.friendlyId).toBe(first?.run.friendlyId);
1732-
1733-
// Orphan: buffer.accept fired with the new friendlyId we generated
1734-
// upfront, and that friendlyId has no matching TaskRun in Postgres
1735-
// because engine.trigger returned the existing run via debounce.
1736-
expect(buffer.accepted).toHaveLength(1);
1737-
expect(buffer.accepted[0]!.runId).not.toBe(first?.run.friendlyId);
1738-
const orphanFriendlyId = buffer.accepted[0]!.runId;
1739-
const orphanRow = await prisma.taskRun.findFirst({
1740-
where: { friendlyId: orphanFriendlyId },
1741-
});
1742-
expect(orphanRow).toBeNull();
1743-
1744-
await engine.quit();
1745-
},
1746-
);
17471536
});
17481537

17491538
describe("DefaultQueueManager task metadata cache", () => {

0 commit comments

Comments
 (0)