From ad8b032b36b9a09810eb4cec015677985874067b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Halber?= Date: Thu, 4 Jun 2026 14:42:59 -0700 Subject: [PATCH 1/2] chore: remove smoke test --- packages/spark/src/clack-wizard.ts | 29 +----------- packages/spark/test/clack-wizard.test.ts | 58 +++--------------------- 2 files changed, 7 insertions(+), 80 deletions(-) diff --git a/packages/spark/src/clack-wizard.ts b/packages/spark/src/clack-wizard.ts index b395810..881c1f9 100644 --- a/packages/spark/src/clack-wizard.ts +++ b/packages/spark/src/clack-wizard.ts @@ -25,7 +25,6 @@ import { codingToolLabel, discoverCodingTools, runCodingTool, - smokeTestCodingTool, type CodingToolId, type CodingToolEvent, type CodingToolRunResult, @@ -124,10 +123,6 @@ export type WizardDeps = { export type CodingToolRuntime = { readonly discover: () => Promise; - readonly smokeTest: (args: { - readonly id: CodingToolId; - readonly cwd: string; - }) => Promise; readonly run: (args: { readonly id: CodingToolId; readonly cwd: string; @@ -529,28 +524,7 @@ async function preflightCodingTools( ): Promise { spinner.update(COPY.instrumentation.builtIn.determiningAvailable); try { - const statuses = await deps.codingTools.discover(); - return await Promise.all( - statuses.map(async (status) => { - if (!status.usable) return status; - - try { - await deps.codingTools.smokeTest({ - id: status.id, - cwd: deps.cwd, - }); - return status; - } catch (error) { - const message = - error instanceof Error ? error.message : String(error); - return { - ...status, - usable: false, - unavailableReason: message || "Smoke test failed.", - }; - } - }), - ); + return await deps.codingTools.discover(); } finally { spinner.clear(); } @@ -830,7 +804,6 @@ export function buildDefaultDeps(args: DefaultDepsArgs): WizardDeps { braintrustCli: createBraintrustCliRuntime({ env }), codingTools: { discover: discoverCodingTools, - smokeTest: smokeTestCodingTool, run: runCodingTool, }, }; diff --git a/packages/spark/test/clack-wizard.test.ts b/packages/spark/test/clack-wizard.test.ts index a228247..7ffdf1c 100644 --- a/packages/spark/test/clack-wizard.test.ts +++ b/packages/spark/test/clack-wizard.test.ts @@ -351,12 +351,6 @@ function buildDeps( authMode: "pro", }, ]), - smokeTest: () => - Promise.resolve({ - exitCode: 0, - signal: null, - finalText: "BRAINTRUST_SETUP_TOOL_OK", - }), run: ({ env, onEvent, prompt }) => { expect(env["BRAINTRUST_API_KEY"]).toBe("bt-secret-key"); expect(env["BT_WIZARD_RESULT_FILE"]).toBeDefined(); @@ -551,8 +545,6 @@ describe("runClackWizard", () => { it("preflights available coding agents before asking for instrumentation mode", async () => { const calls: string[] = []; - let activeSmokeTests = 0; - let maxActiveSmokeTests = 0; const { events } = createPrompts({ selects: [ "yes", @@ -585,18 +577,6 @@ describe("runClackWizard", () => { }, ]); }, - smokeTest: async ({ id }) => { - calls.push(`smoke:${id}`); - activeSmokeTests += 1; - maxActiveSmokeTests = Math.max(maxActiveSmokeTests, activeSmokeTests); - await new Promise((resolve) => setTimeout(resolve, 0)); - activeSmokeTests -= 1; - return { - exitCode: 0, - signal: null, - finalText: "BRAINTRUST_SETUP_TOOL_OK", - }; - }, run: ({ onEvent }) => { onEvent({ type: "completed", message: "Done" }); return Promise.resolve({ @@ -610,12 +590,7 @@ describe("runClackWizard", () => { await runClackWizard(deps); - expect(calls.slice(0, 3)).toEqual([ - "discover", - "smoke:claude", - "smoke:codex", - ]); - expect(maxActiveSmokeTests).toBe(2); + expect(calls).toEqual(["discover"]); expect(events).toContain(`select:${TOOL_SELECT_MESSAGE}`); const codingAgentSpinnerStart = events.indexOf( `spinner.start:${CODING_AGENT_SCAN_MESSAGE}`, @@ -1290,7 +1265,6 @@ describe("runClackWizard", () => { }); it("cancels cleanly when the user aborts the tool select", async () => { - const smokeCalls: string[] = []; const { events } = createPrompts({ selects: ["yes", "no", "first", CANCEL], }); @@ -1313,21 +1287,12 @@ describe("runClackWizard", () => { usable: true, }, ]), - smokeTest: ({ id }) => { - smokeCalls.push(id); - return Promise.resolve({ - exitCode: 0, - signal: null, - finalText: "BRAINTRUST_SETUP_TOOL_OK", - }); - }, run: () => Promise.reject(new Error("should not run")), }, }); await expect(runClackWizard(deps)).rejects.toThrow(WizardCancelledError); expect(events).toContain(`cancel:${WIZARD_CANCEL_MESSAGE}`); - expect(smokeCalls).toEqual(["claude", "codex"]); }); it("offers own-agent and manual setup when built-in coding agent execution is aborted", async () => { @@ -1357,14 +1322,6 @@ describe("runClackWizard", () => { usable: true, }, ]), - smokeTest: () => { - calls.push("smoke"); - return Promise.resolve({ - exitCode: 0, - signal: null, - finalText: "BRAINTRUST_SETUP_TOOL_OK", - }); - }, run: () => { calls.push("run"); return Promise.reject(new Error("should not run")); @@ -1384,7 +1341,7 @@ describe("runClackWizard", () => { expect(events).not.toContain( "taskLog:Running Claude Code to instrument your application:0:false", ); - expect(calls).toEqual(["smoke"]); + expect(calls).toEqual([]); }); it("asks before continuing when not in a git repo", async () => { @@ -1571,7 +1528,7 @@ describe("runClackWizard", () => { ).toBe(true); }); - it("omits built-in setup when preflight smoke tests fail", async () => { + it("omits built-in setup when coding agent discovery finds no usable tools", async () => { const { events } = createPrompts({ selects: ["yes", "no", "manual", "confirm", "checked", "confirmed"], }); @@ -1584,13 +1541,10 @@ describe("runClackWizard", () => { label: "Claude Code", command: "claude", installed: true, - usable: true, + usable: false, + unavailableReason: "Claude Code is not logged in.", }, ]), - smokeTest: () => - Promise.reject( - new Error("Claude Code could not complete a smoke prompt."), - ), run: () => Promise.reject(new Error("should not run")), }, }); @@ -1605,7 +1559,7 @@ describe("runClackWizard", () => { events.some( (event) => event.startsWith("warn:No usable coding agents found.") && - event.includes("Claude Code could not complete a smoke prompt."), + event.includes("Claude Code is not logged in."), ), ).toBe(true); expect(events).not.toContain(`select:${CODING_AGENT_PROCEED_MESSAGE}`); From 8f0f7ae8ffffdcf3046251de5922770543417aea Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 5 Jun 2026 11:58:50 +0200 Subject: [PATCH 2/2] undo & lower cost --- packages/spark/src/clack-wizard.ts | 29 +++++++++++- packages/spark/src/coding-tools.ts | 2 + packages/spark/src/coding-tools/claude.ts | 4 ++ packages/spark/src/coding-tools/codex.ts | 13 +++-- packages/spark/src/coding-tools/index.ts | 16 +++++++ packages/spark/test/clack-wizard.test.ts | 58 ++++++++++++++++++++--- packages/spark/test/coding-tools.test.ts | 42 ++++++++++++++++ 7 files changed, 154 insertions(+), 10 deletions(-) diff --git a/packages/spark/src/clack-wizard.ts b/packages/spark/src/clack-wizard.ts index 881c1f9..b395810 100644 --- a/packages/spark/src/clack-wizard.ts +++ b/packages/spark/src/clack-wizard.ts @@ -25,6 +25,7 @@ import { codingToolLabel, discoverCodingTools, runCodingTool, + smokeTestCodingTool, type CodingToolId, type CodingToolEvent, type CodingToolRunResult, @@ -123,6 +124,10 @@ export type WizardDeps = { export type CodingToolRuntime = { readonly discover: () => Promise; + readonly smokeTest: (args: { + readonly id: CodingToolId; + readonly cwd: string; + }) => Promise; readonly run: (args: { readonly id: CodingToolId; readonly cwd: string; @@ -524,7 +529,28 @@ async function preflightCodingTools( ): Promise { spinner.update(COPY.instrumentation.builtIn.determiningAvailable); try { - return await deps.codingTools.discover(); + const statuses = await deps.codingTools.discover(); + return await Promise.all( + statuses.map(async (status) => { + if (!status.usable) return status; + + try { + await deps.codingTools.smokeTest({ + id: status.id, + cwd: deps.cwd, + }); + return status; + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + return { + ...status, + usable: false, + unavailableReason: message || "Smoke test failed.", + }; + } + }), + ); } finally { spinner.clear(); } @@ -804,6 +830,7 @@ export function buildDefaultDeps(args: DefaultDepsArgs): WizardDeps { braintrustCli: createBraintrustCliRuntime({ env }), codingTools: { discover: discoverCodingTools, + smokeTest: smokeTestCodingTool, run: runCodingTool, }, }; diff --git a/packages/spark/src/coding-tools.ts b/packages/spark/src/coding-tools.ts index 662866a..9696411 100644 --- a/packages/spark/src/coding-tools.ts +++ b/packages/spark/src/coding-tools.ts @@ -1,6 +1,8 @@ export { buildClaudeCommandForTest, + buildClaudeSmokeCommandForTest, buildCodexCommandForTest, + buildCodexSmokeCommandForTest, codingToolLabel, discoverCodingTools, parseClaudeEventForTest, diff --git a/packages/spark/src/coding-tools/claude.ts b/packages/spark/src/coding-tools/claude.ts index da03286..b13bca7 100644 --- a/packages/spark/src/coding-tools/claude.ts +++ b/packages/spark/src/coding-tools/claude.ts @@ -23,6 +23,10 @@ export const claudeAdapter: Adapter = { command: commandPath, args: [ "-p", + "--model", + "haiku", + "--effort", + "low", "--verbose", "--output-format", "stream-json", diff --git a/packages/spark/src/coding-tools/codex.ts b/packages/spark/src/coding-tools/codex.ts index d2992e9..e509684 100644 --- a/packages/spark/src/coding-tools/codex.ts +++ b/packages/spark/src/coding-tools/codex.ts @@ -14,7 +14,14 @@ import { stringField, } from "./utils"; -const CODEX_MODEL_ARGS = [ +const CODEX_SMOKE_MODEL_ARGS = [ + "--model", + "gpt-5.4-mini", + "-c", + 'model_reasoning_effort="low"', +] as const; + +const CODEX_RUN_MODEL_ARGS = [ "--model", "gpt-5.5", "-c", @@ -33,7 +40,7 @@ export const codexAdapter: Adapter = { "--json", "--ephemeral", "--skip-git-repo-check", - ...CODEX_MODEL_ARGS, + ...CODEX_SMOKE_MODEL_ARGS, "-s", "read-only", "-C", @@ -50,7 +57,7 @@ export const codexAdapter: Adapter = { "--json", "--ephemeral", "--skip-git-repo-check", - ...CODEX_MODEL_ARGS, + ...CODEX_RUN_MODEL_ARGS, "-s", "danger-full-access", "--dangerously-bypass-approvals-and-sandbox", diff --git a/packages/spark/src/coding-tools/index.ts b/packages/spark/src/coding-tools/index.ts index ffc73f2..2ddaa16 100644 --- a/packages/spark/src/coding-tools/index.ts +++ b/packages/spark/src/coding-tools/index.ts @@ -100,6 +100,14 @@ export function buildClaudeCommandForTest(args: { return adapterFor("claude").runCommand(args); } +export function buildClaudeSmokeCommandForTest(args: { + readonly commandPath: string; + readonly cwd: string; + readonly prompt: string; +}): CommandSpec { + return adapterFor("claude").smokeCommand(args); +} + export function buildCodexCommandForTest(args: { readonly commandPath: string; readonly cwd: string; @@ -109,6 +117,14 @@ export function buildCodexCommandForTest(args: { return adapterFor("codex").runCommand(args); } +export function buildCodexSmokeCommandForTest(args: { + readonly commandPath: string; + readonly cwd: string; + readonly prompt: string; +}): CommandSpec { + return adapterFor("codex").smokeCommand(args); +} + export function parseClaudeEventForTest( value: unknown, cwd: string, diff --git a/packages/spark/test/clack-wizard.test.ts b/packages/spark/test/clack-wizard.test.ts index 7ffdf1c..a228247 100644 --- a/packages/spark/test/clack-wizard.test.ts +++ b/packages/spark/test/clack-wizard.test.ts @@ -351,6 +351,12 @@ function buildDeps( authMode: "pro", }, ]), + smokeTest: () => + Promise.resolve({ + exitCode: 0, + signal: null, + finalText: "BRAINTRUST_SETUP_TOOL_OK", + }), run: ({ env, onEvent, prompt }) => { expect(env["BRAINTRUST_API_KEY"]).toBe("bt-secret-key"); expect(env["BT_WIZARD_RESULT_FILE"]).toBeDefined(); @@ -545,6 +551,8 @@ describe("runClackWizard", () => { it("preflights available coding agents before asking for instrumentation mode", async () => { const calls: string[] = []; + let activeSmokeTests = 0; + let maxActiveSmokeTests = 0; const { events } = createPrompts({ selects: [ "yes", @@ -577,6 +585,18 @@ describe("runClackWizard", () => { }, ]); }, + smokeTest: async ({ id }) => { + calls.push(`smoke:${id}`); + activeSmokeTests += 1; + maxActiveSmokeTests = Math.max(maxActiveSmokeTests, activeSmokeTests); + await new Promise((resolve) => setTimeout(resolve, 0)); + activeSmokeTests -= 1; + return { + exitCode: 0, + signal: null, + finalText: "BRAINTRUST_SETUP_TOOL_OK", + }; + }, run: ({ onEvent }) => { onEvent({ type: "completed", message: "Done" }); return Promise.resolve({ @@ -590,7 +610,12 @@ describe("runClackWizard", () => { await runClackWizard(deps); - expect(calls).toEqual(["discover"]); + expect(calls.slice(0, 3)).toEqual([ + "discover", + "smoke:claude", + "smoke:codex", + ]); + expect(maxActiveSmokeTests).toBe(2); expect(events).toContain(`select:${TOOL_SELECT_MESSAGE}`); const codingAgentSpinnerStart = events.indexOf( `spinner.start:${CODING_AGENT_SCAN_MESSAGE}`, @@ -1265,6 +1290,7 @@ describe("runClackWizard", () => { }); it("cancels cleanly when the user aborts the tool select", async () => { + const smokeCalls: string[] = []; const { events } = createPrompts({ selects: ["yes", "no", "first", CANCEL], }); @@ -1287,12 +1313,21 @@ describe("runClackWizard", () => { usable: true, }, ]), + smokeTest: ({ id }) => { + smokeCalls.push(id); + return Promise.resolve({ + exitCode: 0, + signal: null, + finalText: "BRAINTRUST_SETUP_TOOL_OK", + }); + }, run: () => Promise.reject(new Error("should not run")), }, }); await expect(runClackWizard(deps)).rejects.toThrow(WizardCancelledError); expect(events).toContain(`cancel:${WIZARD_CANCEL_MESSAGE}`); + expect(smokeCalls).toEqual(["claude", "codex"]); }); it("offers own-agent and manual setup when built-in coding agent execution is aborted", async () => { @@ -1322,6 +1357,14 @@ describe("runClackWizard", () => { usable: true, }, ]), + smokeTest: () => { + calls.push("smoke"); + return Promise.resolve({ + exitCode: 0, + signal: null, + finalText: "BRAINTRUST_SETUP_TOOL_OK", + }); + }, run: () => { calls.push("run"); return Promise.reject(new Error("should not run")); @@ -1341,7 +1384,7 @@ describe("runClackWizard", () => { expect(events).not.toContain( "taskLog:Running Claude Code to instrument your application:0:false", ); - expect(calls).toEqual([]); + expect(calls).toEqual(["smoke"]); }); it("asks before continuing when not in a git repo", async () => { @@ -1528,7 +1571,7 @@ describe("runClackWizard", () => { ).toBe(true); }); - it("omits built-in setup when coding agent discovery finds no usable tools", async () => { + it("omits built-in setup when preflight smoke tests fail", async () => { const { events } = createPrompts({ selects: ["yes", "no", "manual", "confirm", "checked", "confirmed"], }); @@ -1541,10 +1584,13 @@ describe("runClackWizard", () => { label: "Claude Code", command: "claude", installed: true, - usable: false, - unavailableReason: "Claude Code is not logged in.", + usable: true, }, ]), + smokeTest: () => + Promise.reject( + new Error("Claude Code could not complete a smoke prompt."), + ), run: () => Promise.reject(new Error("should not run")), }, }); @@ -1559,7 +1605,7 @@ describe("runClackWizard", () => { events.some( (event) => event.startsWith("warn:No usable coding agents found.") && - event.includes("Claude Code is not logged in."), + event.includes("Claude Code could not complete a smoke prompt."), ), ).toBe(true); expect(events).not.toContain(`select:${CODING_AGENT_PROCEED_MESSAGE}`); diff --git a/packages/spark/test/coding-tools.test.ts b/packages/spark/test/coding-tools.test.ts index 87af9e3..30e1ded 100644 --- a/packages/spark/test/coding-tools.test.ts +++ b/packages/spark/test/coding-tools.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest"; import { buildClaudeCommandForTest, + buildClaudeSmokeCommandForTest, buildCodexCommandForTest, + buildCodexSmokeCommandForTest, parseClaudeEventForTest, parseClaudeEventsForTest, parseClaudeStatusForTest, @@ -115,6 +117,25 @@ describe("coding tool command construction", () => { expect(spec.stdin).toBe("prompt"); }); + it("builds low-cost Claude Code smoke command", () => { + const spec = buildClaudeSmokeCommandForTest({ + commandPath: "/bin/claude", + cwd: CWD, + prompt: "prompt", + }); + + expect(spec.command).toBe("/bin/claude"); + expect(spec.args).toContain("--model"); + expect(spec.args).toContain("haiku"); + expect(spec.args).toContain("--effort"); + expect(spec.args).toContain("low"); + expect(spec.args).toContain("--tools"); + expect(spec.args).toContain(""); + expect(spec.args).not.toContain("bypassPermissions"); + expect(spec.args).not.toContain("--dangerously-skip-permissions"); + expect(spec.stdin).toBe("prompt"); + }); + it("builds autonomous Codex command", () => { const spec = buildCodexCommandForTest({ commandPath: "/bin/codex", @@ -134,6 +155,27 @@ describe("coding tool command construction", () => { expect(spec.args.at(-1)).toBe("-"); expect(spec.stdin).toBe("prompt"); }); + + it("builds low-cost Codex smoke command", () => { + const spec = buildCodexSmokeCommandForTest({ + commandPath: "/bin/codex", + cwd: CWD, + prompt: "prompt", + }); + + expect(spec.command).toBe("/bin/codex"); + expect(spec.args).toContain("--json"); + expect(spec.args).toContain("--model"); + expect(spec.args).toContain("gpt-5.4-mini"); + expect(spec.args).toContain('model_reasoning_effort="low"'); + expect(spec.args).toContain("read-only"); + expect(spec.args).not.toContain("danger-full-access"); + expect(spec.args).not.toContain( + "--dangerously-bypass-approvals-and-sandbox", + ); + expect(spec.args.at(-1)).toBe("-"); + expect(spec.stdin).toBe("prompt"); + }); }); describe("coding tool event normalization", () => {