Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bin/burnlist.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ Usage:
burnlist differential-testing sdk
burnlist streaming-diff <ensure-feed|capture|url|hook> ...
burnlist hooks [install|uninstall|status] [--agent codex,claude] [--untracked] (bare defaults to status)
burnlist oven <list|view|bind|unbind|bindings|create|update> ...
burnlist oven <list|view|bind|unbind|bindings|event|create|update> ...
burnlist new [--repo <path>]
burnlist show <id>[#<item>] [--repo <path>]
burnlist ready <id> [--repo <path>]
Expand Down
2 changes: 1 addition & 1 deletion ovens/differential-testing/engine/adapter-sdk.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ export {
readDifferentialTestingWorkerState,
} from "./worker-runtime.mjs";

export const DIFFERENTIAL_TESTING_ADAPTER_SDK_VERSION = 3;
export const DIFFERENTIAL_TESTING_ADAPTER_SDK_VERSION = 4;
49 changes: 43 additions & 6 deletions ovens/differential-testing/engine/adapter-sdk.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,16 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import test from "node:test";
import { readOvenEvents } from "../../../src/events/oven-event-store.mjs";

import {
DIFFERENTIAL_TESTING_ADAPTER_SDK_VERSION,
DIFFERENTIAL_TESTING_WORKER_STATE_SCHEMA,
createDifferentialTestingWorker,
} from "./adapter-sdk.mjs";

test("SDK v3 persists one state before inbox deletion and handles projection-only events", async () => {
assert.equal(DIFFERENTIAL_TESTING_ADAPTER_SDK_VERSION, 3);
test("SDK v4 persists one state before inbox deletion and handles projection-only events", async () => {
assert.equal(DIFFERENTIAL_TESTING_ADAPTER_SDK_VERSION, 4);
const root = await mkdtemp(join(tmpdir(), "burnlist-differential-worker-"));
const inbox = [];
const durableBeforeDelete = [];
Expand Down Expand Up @@ -53,13 +54,18 @@ test("SDK v3 persists one state before inbox deletion and handles projection-onl
assert.ok(durableBeforeDelete.every(Boolean));
assert.ok(projectionRuns >= 2);
assert.equal(JSON.parse(await readFile(worker.statePath, "utf8")).schema, DIFFERENTIAL_TESTING_WORKER_STATE_SCHEMA);
const events = readOvenEvents(root, { ovenIds: ["differential-testing"] });
assert.equal(events.length, 1);
assert.equal(events[0].kind, "iteration");
assert.equal(events[0].phase, "complete");
assert.equal(events[0].subjectId, first.scenarioId);
worker.close();
} finally {
await rm(root, { recursive: true, force: true });
}
});

test("SDK v3 serializes scenarios and coalesces one running successor", async () => {
test("SDK v4 serializes scenarios and coalesces one running successor", async () => {
const root = await mkdtemp(join(tmpdir(), "burnlist-differential-serial-"));
const inbox = [];
const releases = [];
Expand Down Expand Up @@ -99,13 +105,17 @@ test("SDK v3 serializes scenarios and coalesces one running successor", async ()
assert.equal(maxActive, 1);
assert.equal(worker.scenarioStatus(first.scenarioId).status, "complete");
assert.equal(worker.scenarioStatus(second.scenarioId).status, "complete");
const superseded = readOvenEvents(root, { ovenIds: ["differential-testing"] })
.find((event) => event.phase === "superseded");
assert.equal(superseded.payload.attempt, 1);
assert.equal(superseded.payload.requestId, first.requestId);
worker.close();
} finally {
await rm(root, { recursive: true, force: true });
}
});

test("SDK v3 retries transient work and recovers interrupted telemetry and projection", async () => {
test("SDK v4 retries transient work and recovers interrupted telemetry and projection", async () => {
const root = await mkdtemp(join(tmpdir(), "burnlist-differential-restart-"));
const request = job("1111111111111111", "a".repeat(64), 1);
let telemetryRuns = 0;
Expand Down Expand Up @@ -161,13 +171,16 @@ test("SDK v3 retries transient work and recovers interrupted telemetry and proje
assert.ok(projectionRuns >= 2);
assert.equal(worker.scenarioStatus(request.scenarioId).status, "complete");
assert.equal(worker.snapshot().projection.status, "complete");
const events = readOvenEvents(root, { ovenIds: ["differential-testing"] });
assert.equal(events.length, 2);
assert.deepEqual(new Set(events.map((event) => event.phase)), new Set(["retrying", "complete"]));
worker.close();
} finally {
await rm(root, { recursive: true, force: true });
}
});

test("SDK v3 quarantines an uncooperative timed-out runner and enforces one worker lock", async () => {
test("SDK v4 quarantines an uncooperative timed-out runner and enforces one worker lock", async () => {
const root = await mkdtemp(join(tmpdir(), "burnlist-differential-timeout-"));
const request = job("1111111111111111", "a".repeat(64), 1);
const inbox = [entry(eventFor(request, true))];
Expand All @@ -193,7 +206,7 @@ test("SDK v3 quarantines an uncooperative timed-out runner and enforces one work
}
});

test("SDK v3 rejects legacy state and delete callback configuration failures are fatal", async () => {
test("SDK v4 rejects legacy state and delete callback configuration failures are fatal", async () => {
const legacyRoot = await mkdtemp(join(tmpdir(), "burnlist-differential-legacy-"));
try {
await writeJson(join(legacyRoot, ".local", "differential-testing", "state.json"), {
Expand Down Expand Up @@ -225,6 +238,30 @@ test("SDK v3 rejects legacy state and delete callback configuration failures are
}
});

test("SDK v4 keeps canonical worker success when observational publication returns a rejection", async () => {
const root = await mkdtemp(join(tmpdir(), "burnlist-differential-event-error-"));
const request = job("1111111111111111", "a".repeat(64), 1);
const inbox = [entry(eventFor(request, true))];
const errors = [];
try {
const worker = fixtureWorker({
root,
inbox,
emitOvenEvent() { return Promise.reject(new Error("event store unavailable")); },
onOvenEventError(error, identity) { errors.push({ message: error.message, identity }); },
});
worker.start();
await worker.idle();
assert.equal(worker.scenarioStatus(request.scenarioId).status, "complete");
assert.equal(errors.length, 1);
assert.equal(errors[0].message, "emitOvenEvent must complete synchronously.");
assert.equal(errors[0].identity.scenarioId, request.scenarioId);
worker.close();
} finally {
await rm(root, { recursive: true, force: true });
}
});

function fixtureWorker({ root, inbox, ...overrides }) {
return createDifferentialTestingWorker({
root,
Expand Down
39 changes: 39 additions & 0 deletions ovens/differential-testing/engine/worker-runtime.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
writeFileSync,
} from "node:fs";
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
import { publishOvenEvent as publishRepoOvenEvent } from "../../../src/events/oven-event-store.mjs";

export const DIFFERENTIAL_TESTING_WORKER_STATE_SCHEMA = "burnlist-differential-testing-worker-state@1";

Expand Down Expand Up @@ -75,6 +76,8 @@ export function createDifferentialTestingWorker({
classifyTelemetryError,
project,
onFatal = () => {},
emitOvenEvent = null,
onOvenEventError = () => {},
now = () => new Date().toISOString(),
pollIntervalMs = 250,
telemetryTimeoutMs = 5 * 60_000,
Expand Down Expand Up @@ -108,6 +111,7 @@ export function createDifferentialTestingWorker({
[classifyTelemetryError, "classifyTelemetryError"],
[project, "project"],
[onFatal, "onFatal"],
[onOvenEventError, "onOvenEventError"],
]) {
if (typeof callback !== "function") throw new Error(`${label} is required.`);
}
Expand All @@ -134,6 +138,8 @@ export function createDifferentialTestingWorker({
}

const repoRoot = resolve(root);
if (emitOvenEvent !== null && typeof emitOvenEvent !== "function") throw new Error("emitOvenEvent must be a function or null.");
const eventPublisher = emitOvenEvent ?? ((event) => publishRepoOvenEvent(repoRoot, event));
const storeRoot = containedPath(repoRoot, storeDirectory, "Differential Testing store");
const statePath = containedPath(storeRoot, stateFile, "Differential Testing worker state");
const scratchRoot = resolve(storeRoot, ".scratch");
Expand Down Expand Up @@ -396,10 +402,13 @@ export function createDifferentialTestingWorker({
run: { id: runId, scratchDirectory: displayPath(repoRoot, scratchDirectory) },
});
state.telemetry.active = { scenarioId, requestId: request.requestId, runId, startedAt };
const iterationAttempt = scenario.attempts;
queueProjection("telemetry-running", scenarioId);
bumpAndPersist();
let preserveScratch = false;
let quarantined = false;
let iterationPhase = "failed";
let iterationError = null;
try {
const staged = await runWithTimeout(runTelemetry, {
root: repoRoot,
Expand All @@ -408,6 +417,7 @@ export function createDifferentialTestingWorker({
request: structuredClone(request),
}, telemetryTimeoutMs, telemetryAbortGraceMs);
if (scenario.pendingRequest) {
iterationPhase = "superseded";
queuePendingScenario(scenario, { superseded: true }, timestamp(now(), "successor timestamp"));
if (!state.telemetry.queue.includes(scenarioId)) state.telemetry.queue.push(scenarioId);
} else {
Expand All @@ -430,13 +440,16 @@ export function createDifferentialTestingWorker({
scenario.nextAttemptAt = null;
scenario.publication = structuredClone(publication);
scenario.run = { ...scenario.run, exitCode: staged.exitCode ?? null };
iterationPhase = "complete";
}
} catch (error) {
iterationError = String(error?.message ?? error).slice(0, 1_000);
preserveScratch = error?.preserveScratch === true;
if (error?.workerFatal === true) {
quarantined = true;
throw error;
} else if (scenario.pendingRequest) {
iterationPhase = "superseded";
queuePendingScenario(scenario, { superseded: true, discardedError: error?.message ?? String(error) }, timestamp(now(), "successor timestamp"));
if (!state.telemetry.queue.includes(scenarioId)) state.telemetry.queue.push(scenarioId);
} else {
Expand All @@ -453,11 +466,13 @@ export function createDifferentialTestingWorker({
scenario.run = { ...scenario.run, exitCode: error?.exitCode ?? null };
if (classification === "permanent" || scenario.attempts >= telemetryMaxAttempts) {
scenario.status = "failed";
iterationPhase = "failed";
scenario.finishedAt = timestamp(now(), "telemetry failure timestamp");
scenario.nextAttemptAt = null;
if (classification !== "permanent") scenario.error += ` Telemetry retry exhausted after ${scenario.attempts} attempts.`;
} else {
scenario.status = "retrying";
iterationPhase = "retrying";
scenario.startedAt = null;
scenario.finishedAt = null;
scenario.nextAttemptAt = timestampAfter(now(), retryDelay(scenario.attempts, telemetryRetryBaseMs, telemetryRetryMaxMs));
Expand All @@ -471,6 +486,30 @@ export function createDifferentialTestingWorker({
scenario.updatedAt = timestamp(now(), "scenario update timestamp");
queueProjection(`telemetry-${scenario.status}`, scenarioId);
bumpAndPersist();
try {
const emitted = eventPublisher({
ovenId: "differential-testing",
subjectId: scenarioId,
kind: "iteration",
phase: iterationPhase,
cursor: runId,
occurredAt: scenario.updatedAt,
payload: {
requestId: request.requestId,
runId,
attempt: iterationAttempt,
status: iterationPhase,
published: iterationPhase === "complete",
...(iterationError ? { error: iterationError } : {}),
},
});
if (emitted && typeof emitted.then === "function") {
void Promise.resolve(emitted).catch(() => {});
throw new Error("emitOvenEvent must complete synchronously.");
}
} catch (error) {
try { onOvenEventError(error, { scenarioId, requestId: request.requestId, runId, phase: iterationPhase }); } catch {}
}
}
}
}
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"burnlist": "bin/burnlist.mjs"
},
"exports": {
"./oven-events": "./src/events/oven-events.mjs",
"./differential-testing": "./ovens/differential-testing/engine/adapter-sdk.mjs",
"./differential-testing/contract": "./ovens/differential-testing/engine/contract.mjs",
"./differential-testing/transport": "./ovens/differential-testing/engine/transport.mjs"
Expand Down
7 changes: 6 additions & 1 deletion scripts/smoke-global-install.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,14 @@ try {
"createDifferentialTestingWorker",
"readDifferentialTestingWorkerState",
];
if (sdk.DIFFERENTIAL_TESTING_ADAPTER_SDK_VERSION !== 3
if (sdk.DIFFERENTIAL_TESTING_ADAPTER_SDK_VERSION !== 4
|| JSON.stringify(Object.keys(sdk).sort()) !== JSON.stringify(expected.sort())) process.exit(1);
`]);
run(process.execPath, ["--input-type=module", "--eval", `
const events = await import("burnlist/oven-events");
const expected = ["normalizeOvenEvent", "publishOvenEvent", "readOvenEvents"];
if (!expected.every((name) => typeof events[name] === "function")) process.exit(1);
`], { cwd: packageRoot });

run(cli, ["uninstall", "--global", "--purge"]);
for (const agentDirectory of [".claude", ".agents"]) {
Expand Down
5 changes: 5 additions & 0 deletions scripts/verify-package.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,15 @@ const required = [
"src/cli/skills-register.mjs",
"src/cli/oven-cli.mjs",
"src/cli/registry-cli.mjs",
"src/events/oven-event-contract.mjs",
"src/events/oven-event-feed.mjs",
"src/events/oven-event-store.mjs",
"src/events/oven-events.mjs",
"src/ovens/oven-contract.mjs",
"src/server/burnlist-dashboard-server.mjs",
"skills/burnlist/SKILL.md",
"skills/burnlist/references/burnlist-creation.md",
"skills/burnlist/references/oven-event-coordination.md",
"ovens/checklist/instructions.md",
"ovens/differential-testing/instructions.md",
"ovens/differential-testing/differential-testing.oven",
Expand Down
10 changes: 9 additions & 1 deletion scripts/verify-test-files.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
export const verificationTestFiles = [
"src/server/dashboard-routes.test.mjs",
"src/server/oven-event-routes.test.mjs",
"src/events/oven-event-feed.test.mjs",
"src/events/oven-events.test.mjs",
"src/server/custom-oven-view-routes.test.mjs",
"src/server/custom-oven-index.test.mjs",
"src/server/oven-routes.test.mjs",
Expand Down Expand Up @@ -52,7 +55,6 @@ export const verificationTestFiles = [
"src/cli/oven-cli-stdout.test.mjs",
"src/cli/oven-storage.test.mjs",
"src/cli/umbrella.test.mjs",
"src/cli/streaming-diff-cli.test.mjs",
"src/cli/hooks-config.test.mjs",
"ovens/streaming-diff/engine/streaming-diff-hook-adapters.test.mjs",
"src/server/oven-bindings.test.mjs",
Expand Down Expand Up @@ -80,3 +82,9 @@ export const verificationTestFiles = [
"dashboard/src/lib/project-open.test.mjs",
"dashboard/src/lib/oven-identity.test.mjs",
];

// These subprocess wall-clock assertions must not compete with the parallel
// test-file pool or host saturation can look like a hook timeout regression.
export const verificationSerialTestFiles = [
"src/cli/streaming-diff-cli.test.mjs",
];
10 changes: 7 additions & 3 deletions scripts/verify.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { readFileSync, readdirSync, statSync } from "node:fs";
import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { assertBuiltInOven, assertBuiltInOvenSet, assertSkillSet } from "./verify-oven-assertions.mjs";
import { verificationTestFiles } from "./verify-test-files.mjs";
import { verificationSerialTestFiles, verificationTestFiles } from "./verify-test-files.mjs";

const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const packageJson = JSON.parse(readFileSync(resolve(repoRoot, "package.json"), "utf8"));
Expand Down Expand Up @@ -217,8 +217,9 @@ function assertPublishablePackage() {
}
if (packageJson.exports?.["./differential-testing"] !== "./ovens/differential-testing/engine/adapter-sdk.mjs"
|| packageJson.exports?.["./differential-testing/contract"] !== "./ovens/differential-testing/engine/contract.mjs"
|| packageJson.exports?.["./differential-testing/transport"] !== "./ovens/differential-testing/engine/transport.mjs") {
console.error("package.json does not expose the stable Differential Testing worker, contract, and transport subpaths.");
|| packageJson.exports?.["./differential-testing/transport"] !== "./ovens/differential-testing/engine/transport.mjs"
|| packageJson.exports?.["./oven-events"] !== "./src/events/oven-events.mjs") {
console.error("package.json does not expose the stable Differential Testing and Oven event subpaths.");
process.exit(1);
}
if (packageJson.scripts?.postinstall !== "node scripts/register-skills.mjs") {
Expand Down Expand Up @@ -265,6 +266,7 @@ assertSourceExcludes("dashboard/src/App.tsx", "function Detail(", "Dashboard sti
assertSourceIncludes("dashboard/src/components/BurnOvens/BurnOvens.tsx", "New Oven", "Oven controls are missing.");
assertSourceIncludes("src/server/burnlist-dashboard-server.mjs", 'url.pathname === "/api/ovens"', "Oven API is missing.");
assertSourceIncludes("src/server/burnlist-dashboard-server.mjs", "/api\\/oven-data", "Read-only Oven data API is missing.");
assertSourceIncludes("src/server/burnlist-dashboard-server.mjs", 'url.pathname === "/api/events"', "Replayable generic Oven event API is missing.");
assertSourceIncludes("src/server/burnlist-dashboard-server.mjs", 'url.pathname === "/api/repo-map"', "Read-only repository map API is missing.");
assertSourceIncludes("src/server/repo-map.mjs", 'REPO_MAP_SCHEMA = "burnlist-repo-map@1"', "Repository map API does not expose its strict v1 schema.");
assertSourceIncludes("src/server/burnlist-dashboard-server.mjs", 'assertKnownKeys(value, new Set(["id", "name", "instructions"]), "Oven")', "Oven creation does not reject fields outside the strict Oven contract.");
Expand Down Expand Up @@ -379,6 +381,7 @@ assertSourceExcludes("dashboard/src/oven/differential-testing-render/differentia
assertSourceExcludes("src/server/burnlist-dashboard-server.mjs", "exact comparator when used", "Fallback Run Burn still requests the superseded manual comparator workflow.");
assertSourceExcludes("dashboard/src/components/BurnOvens/BurnOvens.tsx", "exact comparator when used", "React Run Burn still requests the superseded manual comparator workflow.");
assertSourceIncludes("skills/burnlist/SKILL.md", "references/burnlist-creation.md", "The Burnlist skill does not route creation work.");
assertSourceIncludes("skills/burnlist/SKILL.md", "references/oven-event-coordination.md", "The Burnlist skill does not route coordinator event work.");
assertSourceIncludes("src/cli/skills-register.mjs", 'join(home, ".claude", "skills")', "Global npm install does not use the Claude skill directory.");
assertSourceIncludes("src/cli/skills-register.mjs", 'join(home, ".agents", "skills")', "Global npm install does not use the Codex skill directory.");
assertSourceIncludes("bin/burnlist.mjs", "Usage:", "Burnlist CLI help is missing.");
Expand All @@ -398,6 +401,7 @@ assertDifferentialTestingContractAssets();
assertPublishablePackage();

run(process.execPath, ["--test", ...verificationTestFiles]);
for (const file of verificationSerialTestFiles) run(process.execPath, ["--test", file]);
run(process.execPath, ["dashboard/src/oven/test-support/run-oven-tests.mjs"]);

const {
Expand Down
Loading
Loading