From 658b435674b4d28ae30211f8b78257dcb892d796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Halber?= Date: Fri, 29 May 2026 20:31:53 -0700 Subject: [PATCH 1/3] chore: do not prompt for updating bt if already up to date --- packages/spark/src/braintrust-cli.ts | 40 ++++++++++++ packages/spark/src/clack-copy.ts | 4 ++ packages/spark/src/clack-wizard.ts | 49 ++++++++++---- packages/spark/test/clack-wizard.test.ts | 82 ++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 12 deletions(-) diff --git a/packages/spark/src/braintrust-cli.ts b/packages/spark/src/braintrust-cli.ts index 1855c7c..d2aa55d 100644 --- a/packages/spark/src/braintrust-cli.ts +++ b/packages/spark/src/braintrust-cli.ts @@ -17,6 +17,10 @@ export type BraintrustCliContext = { readonly project?: string; }; +export type BraintrustCliUpdateCheck = { + readonly upToDate: boolean; +}; + export type BraintrustCliConfigureArgs = { readonly apiKey: string; readonly apiUrl: string; @@ -28,6 +32,9 @@ export type BraintrustCliConfigureArgs = { export type BraintrustCliRuntime = { readonly discover: () => Promise; readonly install: () => Promise; + readonly checkForUpdate: ( + commandPath: string, + ) => Promise; readonly update: (commandPath: string) => Promise; readonly status: (commandPath: string) => Promise; readonly loginAndSwitch: ( @@ -114,6 +121,20 @@ export function createBraintrustCliRuntime( await execChecked("Braintrust CLI install", spec, exec); }, + async checkForUpdate(commandPath) { + const result = await exec({ + command: commandPath, + args: ["self", "update", "--check", "--json"], + env, + }); + if (result.exitCode !== 0) { + throw new Error( + `Braintrust CLI update check failed with exit code ${result.exitCode}. ${summarizeCommandOutput(result)}`.trim(), + ); + } + return parseUpdateCheckJson(result.stdout); + }, + async update(commandPath) { await execChecked( "Braintrust CLI update", @@ -282,6 +303,25 @@ async function executableExists(path: string): Promise { ); } +function parseUpdateCheckJson(stdout: string): BraintrustCliUpdateCheck { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error("Braintrust CLI update check returned invalid JSON."); + } + if (!parsed || typeof parsed !== "object") { + throw new Error("Braintrust CLI update check returned invalid JSON."); + } + const upToDate = (parsed as Record)["up_to_date"]; + if (typeof upToDate !== "boolean") { + throw new Error( + "Braintrust CLI update check response was missing the `up_to_date` field.", + ); + } + return { upToDate }; +} + function parseStatusJson(stdout: string): BraintrustCliContext { let parsed: unknown; try { diff --git a/packages/spark/src/clack-copy.ts b/packages/spark/src/clack-copy.ts index b896875..cc0958d 100644 --- a/packages/spark/src/clack-copy.ts +++ b/packages/spark/src/clack-copy.ts @@ -93,6 +93,10 @@ export const CLACK_WIZARD_COPY = { configuringContext: "Configuring Braintrust CLI context...", updateFailed: (message: string) => `Could not update Braintrust CLI: ${message}`, + updateCheckFailed: (message: string) => + `Could not check for Braintrust CLI updates: ${message}`, + upToDate: (installedLabel: string) => + `Braintrust CLI is up to date (${installedLabel}).`, installFailed: (message: string) => `Could not install Braintrust CLI: ${message}`, configureFailed: (message: string) => diff --git a/packages/spark/src/clack-wizard.ts b/packages/spark/src/clack-wizard.ts index 3e26953..cdc79b6 100644 --- a/packages/spark/src/clack-wizard.ts +++ b/packages/spark/src/clack-wizard.ts @@ -394,23 +394,48 @@ async function handleBraintrustCliSetup( let commandPath = discovery.commandPath; if (discovery.installed) { - const shouldUpdate = await selectBoolean({ - message: COPY.braintrustCli.updateQuestion, - choices: COPY.braintrustCli.updateChoices, - yesFirst: true, - }); - if (shouldUpdate && commandPath) { - spinner.update(COPY.braintrustCli.updating); + if (commandPath) { + const installedLabel = + discovery.version ?? + commandPath ?? + COPY.braintrustCli.installedVersionUnknown; + + let upToDate = false; try { - await deps.braintrustCli.update(commandPath); - discovery = await deps.braintrustCli.discover(); - commandPath = discovery.commandPath ?? commandPath; + const check = await deps.braintrustCli.checkForUpdate(commandPath); + upToDate = check.upToDate; } catch (error) { - spinner.clear(); clack.log.warn( - COPY.braintrustCli.updateFailed(summarizeBraintrustCliError(error)), + COPY.braintrustCli.updateCheckFailed( + summarizeBraintrustCliError(error), + ), ); } + + if (upToDate) { + clack.log.info(COPY.braintrustCli.upToDate(installedLabel)); + } else { + const shouldUpdate = await selectBoolean({ + message: COPY.braintrustCli.updateQuestion, + choices: COPY.braintrustCli.updateChoices, + yesFirst: true, + }); + if (shouldUpdate) { + spinner.update(COPY.braintrustCli.updating); + try { + await deps.braintrustCli.update(commandPath); + discovery = await deps.braintrustCli.discover(); + commandPath = discovery.commandPath ?? commandPath; + } catch (error) { + spinner.clear(); + clack.log.warn( + COPY.braintrustCli.updateFailed( + summarizeBraintrustCliError(error), + ), + ); + } + } + } } } else { const shouldInstall = await selectBoolean({ diff --git a/packages/spark/test/clack-wizard.test.ts b/packages/spark/test/clack-wizard.test.ts index 5263918..f9c5ec2 100644 --- a/packages/spark/test/clack-wizard.test.ts +++ b/packages/spark/test/clack-wizard.test.ts @@ -21,6 +21,7 @@ import { type BraintrustCliDiscovery, type BraintrustCliContext, type BraintrustCliRuntime, + type BraintrustCliUpdateCheck, } from "../src/braintrust-cli"; import { type CodingToolRuntime, @@ -388,6 +389,9 @@ function createBraintrustCliStub( args: { readonly discoveries?: readonly BraintrustCliDiscovery[]; readonly install?: () => Promise; + readonly checkForUpdate?: ( + commandPath: string, + ) => Promise; readonly update?: (commandPath: string) => Promise; readonly status?: (commandPath: string) => Promise; readonly loginAndSwitch?: ( @@ -407,6 +411,8 @@ function createBraintrustCliStub( return Promise.resolve(discovery); }, install: args.install ?? (() => Promise.resolve()), + checkForUpdate: + args.checkForUpdate ?? (() => Promise.resolve({ upToDate: false })), update: args.update ?? (() => Promise.resolve()), status: args.status ?? (() => Promise.resolve({})), loginAndSwitch: args.loginAndSwitch ?? (() => Promise.resolve()), @@ -879,6 +885,82 @@ describe("runClackWizard", () => { expect(calls).toEqual([]); }); + it("skips the update question when the Braintrust CLI is already up to date", async () => { + const calls: string[] = []; + const { events } = createPrompts({ + selects: ["yes", "manual", "confirm", "understood"], + }); + const deps = buildDeps({ + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + ], + checkForUpdate: (commandPath) => { + calls.push(`check:${commandPath}`); + return Promise.resolve({ upToDate: true }); + }, + update: (commandPath) => { + calls.push(`update:${commandPath}`); + return Promise.resolve(); + }, + status: () => { + calls.push("status"); + return Promise.resolve({}); + }, + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect(events).not.toContain(`select:${CLI_UPDATE_MESSAGE}`); + expect(events).not.toContain("spinner.start:Updating Braintrust CLI..."); + expect( + events.some((event) => + event.startsWith("info:Braintrust CLI is up to date"), + ), + ).toBe(true); + expect(calls).toEqual(["check:/bin/bt", "status", "login"]); + }); + + it("asks to update when the update check fails", async () => { + const calls: string[] = []; + const { events } = createPrompts({ + selects: ["no", "yes", "manual", "confirm", "understood"], + }); + const deps = buildDeps({ + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + ], + checkForUpdate: () => Promise.reject(new Error("network down")), + status: () => { + calls.push("status"); + return Promise.resolve({}); + }, + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect( + events.some((event) => + event.startsWith( + "warn:Could not check for Braintrust CLI updates: network down", + ), + ), + ).toBe(true); + expect(events).toContain(`select:${CLI_UPDATE_MESSAGE}`); + expect(calls).toEqual(["status", "login"]); + }); + it("updates and configures an installed Braintrust CLI when accepted", async () => { const calls: string[] = []; const { events } = createPrompts({ From a4b018002d29ff28f303813c86f264e7d6fa7733 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Tue, 2 Jun 2026 11:45:09 +0200 Subject: [PATCH 2/3] clean --- .gitignore | 2 ++ packages/spark/src/clack-copy.ts | 5 ---- packages/spark/src/clack-wizard.ts | 19 +++----------- packages/spark/test/braintrust-cli.test.ts | 30 ++++++++++++++++++++++ packages/spark/test/clack-wizard.test.ts | 20 +++++++-------- 5 files changed, 46 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index c7512e8..ac735e9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ coverage/ .DS_Store .env .env.* +**/.env.braintrust +**/.braintrust.json diff --git a/packages/spark/src/clack-copy.ts b/packages/spark/src/clack-copy.ts index cc0958d..0d938c8 100644 --- a/packages/spark/src/clack-copy.ts +++ b/packages/spark/src/clack-copy.ts @@ -65,7 +65,6 @@ export const CLACK_WIZARD_COPY = { }, braintrustCli: { - installedVersionUnknown: "version unknown", installQuestion: "Install Braintrust CLI?", installChoices: { yes: { @@ -93,10 +92,6 @@ export const CLACK_WIZARD_COPY = { configuringContext: "Configuring Braintrust CLI context...", updateFailed: (message: string) => `Could not update Braintrust CLI: ${message}`, - updateCheckFailed: (message: string) => - `Could not check for Braintrust CLI updates: ${message}`, - upToDate: (installedLabel: string) => - `Braintrust CLI is up to date (${installedLabel}).`, installFailed: (message: string) => `Could not install Braintrust CLI: ${message}`, configureFailed: (message: string) => diff --git a/packages/spark/src/clack-wizard.ts b/packages/spark/src/clack-wizard.ts index cdc79b6..00905a8 100644 --- a/packages/spark/src/clack-wizard.ts +++ b/packages/spark/src/clack-wizard.ts @@ -395,26 +395,15 @@ async function handleBraintrustCliSetup( if (discovery.installed) { if (commandPath) { - const installedLabel = - discovery.version ?? - commandPath ?? - COPY.braintrustCli.installedVersionUnknown; - - let upToDate = false; + let upToDate: boolean; try { const check = await deps.braintrustCli.checkForUpdate(commandPath); upToDate = check.upToDate; - } catch (error) { - clack.log.warn( - COPY.braintrustCli.updateCheckFailed( - summarizeBraintrustCliError(error), - ), - ); + } catch { + upToDate = false; } - if (upToDate) { - clack.log.info(COPY.braintrustCli.upToDate(installedLabel)); - } else { + if (!upToDate) { const shouldUpdate = await selectBoolean({ message: COPY.braintrustCli.updateQuestion, choices: COPY.braintrustCli.updateChoices, diff --git a/packages/spark/test/braintrust-cli.test.ts b/packages/spark/test/braintrust-cli.test.ts index f539697..48d7d01 100644 --- a/packages/spark/test/braintrust-cli.test.ts +++ b/packages/spark/test/braintrust-cli.test.ts @@ -37,6 +37,36 @@ describe("Braintrust CLI runtime", () => { ]); }); + it("builds the update command", async () => { + const calls: Array<{ + readonly command: string; + readonly args: readonly string[]; + readonly env?: NodeJS.ProcessEnv; + }> = []; + const runtime = createBraintrustCliRuntime({ + env: { PATH: "/usr/bin" }, + exec: (spec) => { + calls.push(spec); + return Promise.resolve({ + exitCode: 0, + signal: null, + stdout: "", + stderr: "", + }); + }, + }); + + await runtime.update("/usr/local/bin/bt"); + + expect(calls).toEqual([ + { + command: "/usr/local/bin/bt", + args: ["self", "update"], + env: { PATH: "/usr/bin" }, + }, + ]); + }); + it("passes the API key only through env when configuring auth and context", async () => { const calls: Array<{ readonly command: string; diff --git a/packages/spark/test/clack-wizard.test.ts b/packages/spark/test/clack-wizard.test.ts index f9c5ec2..337d669 100644 --- a/packages/spark/test/clack-wizard.test.ts +++ b/packages/spark/test/clack-wizard.test.ts @@ -919,14 +919,12 @@ describe("runClackWizard", () => { expect(events).not.toContain(`select:${CLI_UPDATE_MESSAGE}`); expect(events).not.toContain("spinner.start:Updating Braintrust CLI..."); expect( - events.some((event) => - event.startsWith("info:Braintrust CLI is up to date"), - ), - ).toBe(true); + events.some((event) => event.includes("Braintrust CLI is up to date")), + ).toBe(false); expect(calls).toEqual(["check:/bin/bt", "status", "login"]); }); - it("asks to update when the update check fails", async () => { + it("updates when the update check fails and the user accepts", async () => { const calls: string[] = []; const { events } = createPrompts({ selects: ["no", "yes", "manual", "confirm", "understood"], @@ -937,6 +935,10 @@ describe("runClackWizard", () => { { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, ], checkForUpdate: () => Promise.reject(new Error("network down")), + update: (commandPath) => { + calls.push(`update:${commandPath}`); + return Promise.resolve(); + }, status: () => { calls.push("status"); return Promise.resolve({}); @@ -952,13 +954,11 @@ describe("runClackWizard", () => { expect( events.some((event) => - event.startsWith( - "warn:Could not check for Braintrust CLI updates: network down", - ), + event.includes("Could not check for Braintrust CLI updates"), ), - ).toBe(true); + ).toBe(false); expect(events).toContain(`select:${CLI_UPDATE_MESSAGE}`); - expect(calls).toEqual(["status", "login"]); + expect(calls).toEqual(["update:/bin/bt", "status", "login"]); }); it("updates and configures an installed Braintrust CLI when accepted", async () => { From 15786424ff371c96c24e6c1ae97e29c4e614a985 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Tue, 2 Jun 2026 11:47:40 +0200 Subject: [PATCH 3/3] wording --- packages/spark/src/clack-copy.ts | 2 +- packages/spark/test/clack-wizard.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/spark/src/clack-copy.ts b/packages/spark/src/clack-copy.ts index 0d938c8..27fe427 100644 --- a/packages/spark/src/clack-copy.ts +++ b/packages/spark/src/clack-copy.ts @@ -121,7 +121,7 @@ export const CLACK_WIZARD_COPY = { }, instrumentation: { - modeQuestion: "How do you want to add Braintrust instrumentation?", + modeQuestion: "How do you want to add Braintrust to your application?", modes: { builtIn: { label: "Use built-in coding agent", diff --git a/packages/spark/test/clack-wizard.test.ts b/packages/spark/test/clack-wizard.test.ts index 337d669..57babe3 100644 --- a/packages/spark/test/clack-wizard.test.ts +++ b/packages/spark/test/clack-wizard.test.ts @@ -196,7 +196,7 @@ const WIZARD_INTRO = "Braintrust Setup Wizard"; const WIZARD_CANCEL_MESSAGE = "Wizard cancelled."; const ACCOUNT_QUESTION = "Do you already have a Braintrust account?"; const INSTRUMENTATION_MODE_MESSAGE = - "How do you want to add Braintrust instrumentation?"; + "How do you want to add Braintrust to your application?"; const CLI_INSTALL_MESSAGE = "Install Braintrust CLI?"; const CLI_UPDATE_MESSAGE = "Update Braintrust CLI to the latest version?"; const TOOL_SELECT_MESSAGE = "Which coding agent should Braintrust Setup use?";