diff --git a/.gitignore b/.gitignore index c7512e8..ac735e9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ coverage/ .DS_Store .env .env.* +**/.env.braintrust +**/.braintrust.json diff --git a/packages/spark/src/braintrust-cli.ts b/packages/spark/src/braintrust-cli.ts index 1855c7c..d2aa55d 100644 --- a/packages/spark/src/braintrust-cli.ts +++ b/packages/spark/src/braintrust-cli.ts @@ -17,6 +17,10 @@ export type BraintrustCliContext = { readonly project?: string; }; +export type BraintrustCliUpdateCheck = { + readonly upToDate: boolean; +}; + export type BraintrustCliConfigureArgs = { readonly apiKey: string; readonly apiUrl: string; @@ -28,6 +32,9 @@ export type BraintrustCliConfigureArgs = { export type BraintrustCliRuntime = { readonly discover: () => Promise; readonly install: () => Promise; + readonly checkForUpdate: ( + commandPath: string, + ) => Promise; readonly update: (commandPath: string) => Promise; readonly status: (commandPath: string) => Promise; readonly loginAndSwitch: ( @@ -114,6 +121,20 @@ export function createBraintrustCliRuntime( await execChecked("Braintrust CLI install", spec, exec); }, + async checkForUpdate(commandPath) { + const result = await exec({ + command: commandPath, + args: ["self", "update", "--check", "--json"], + env, + }); + if (result.exitCode !== 0) { + throw new Error( + `Braintrust CLI update check failed with exit code ${result.exitCode}. ${summarizeCommandOutput(result)}`.trim(), + ); + } + return parseUpdateCheckJson(result.stdout); + }, + async update(commandPath) { await execChecked( "Braintrust CLI update", @@ -282,6 +303,25 @@ async function executableExists(path: string): Promise { ); } +function parseUpdateCheckJson(stdout: string): BraintrustCliUpdateCheck { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error("Braintrust CLI update check returned invalid JSON."); + } + if (!parsed || typeof parsed !== "object") { + throw new Error("Braintrust CLI update check returned invalid JSON."); + } + const upToDate = (parsed as Record)["up_to_date"]; + if (typeof upToDate !== "boolean") { + throw new Error( + "Braintrust CLI update check response was missing the `up_to_date` field.", + ); + } + return { upToDate }; +} + function parseStatusJson(stdout: string): BraintrustCliContext { let parsed: unknown; try { diff --git a/packages/spark/src/clack-copy.ts b/packages/spark/src/clack-copy.ts index b896875..27fe427 100644 --- a/packages/spark/src/clack-copy.ts +++ b/packages/spark/src/clack-copy.ts @@ -65,7 +65,6 @@ export const CLACK_WIZARD_COPY = { }, braintrustCli: { - installedVersionUnknown: "version unknown", installQuestion: "Install Braintrust CLI?", installChoices: { yes: { @@ -122,7 +121,7 @@ export const CLACK_WIZARD_COPY = { }, instrumentation: { - modeQuestion: "How do you want to add Braintrust instrumentation?", + modeQuestion: "How do you want to add Braintrust to your application?", modes: { builtIn: { label: "Use built-in coding agent", diff --git a/packages/spark/src/clack-wizard.ts b/packages/spark/src/clack-wizard.ts index 3e26953..00905a8 100644 --- a/packages/spark/src/clack-wizard.ts +++ b/packages/spark/src/clack-wizard.ts @@ -394,22 +394,36 @@ async function handleBraintrustCliSetup( let commandPath = discovery.commandPath; if (discovery.installed) { - const shouldUpdate = await selectBoolean({ - message: COPY.braintrustCli.updateQuestion, - choices: COPY.braintrustCli.updateChoices, - yesFirst: true, - }); - if (shouldUpdate && commandPath) { - spinner.update(COPY.braintrustCli.updating); + if (commandPath) { + let upToDate: boolean; try { - await deps.braintrustCli.update(commandPath); - discovery = await deps.braintrustCli.discover(); - commandPath = discovery.commandPath ?? commandPath; - } catch (error) { - spinner.clear(); - clack.log.warn( - COPY.braintrustCli.updateFailed(summarizeBraintrustCliError(error)), - ); + const check = await deps.braintrustCli.checkForUpdate(commandPath); + upToDate = check.upToDate; + } catch { + upToDate = false; + } + + if (!upToDate) { + const shouldUpdate = await selectBoolean({ + message: COPY.braintrustCli.updateQuestion, + choices: COPY.braintrustCli.updateChoices, + yesFirst: true, + }); + if (shouldUpdate) { + spinner.update(COPY.braintrustCli.updating); + try { + await deps.braintrustCli.update(commandPath); + discovery = await deps.braintrustCli.discover(); + commandPath = discovery.commandPath ?? commandPath; + } catch (error) { + spinner.clear(); + clack.log.warn( + COPY.braintrustCli.updateFailed( + summarizeBraintrustCliError(error), + ), + ); + } + } } } } else { diff --git a/packages/spark/test/braintrust-cli.test.ts b/packages/spark/test/braintrust-cli.test.ts index f539697..48d7d01 100644 --- a/packages/spark/test/braintrust-cli.test.ts +++ b/packages/spark/test/braintrust-cli.test.ts @@ -37,6 +37,36 @@ describe("Braintrust CLI runtime", () => { ]); }); + it("builds the update command", async () => { + const calls: Array<{ + readonly command: string; + readonly args: readonly string[]; + readonly env?: NodeJS.ProcessEnv; + }> = []; + const runtime = createBraintrustCliRuntime({ + env: { PATH: "/usr/bin" }, + exec: (spec) => { + calls.push(spec); + return Promise.resolve({ + exitCode: 0, + signal: null, + stdout: "", + stderr: "", + }); + }, + }); + + await runtime.update("/usr/local/bin/bt"); + + expect(calls).toEqual([ + { + command: "/usr/local/bin/bt", + args: ["self", "update"], + env: { PATH: "/usr/bin" }, + }, + ]); + }); + it("passes the API key only through env when configuring auth and context", async () => { const calls: Array<{ readonly command: string; diff --git a/packages/spark/test/clack-wizard.test.ts b/packages/spark/test/clack-wizard.test.ts index 5263918..57babe3 100644 --- a/packages/spark/test/clack-wizard.test.ts +++ b/packages/spark/test/clack-wizard.test.ts @@ -21,6 +21,7 @@ import { type BraintrustCliDiscovery, type BraintrustCliContext, type BraintrustCliRuntime, + type BraintrustCliUpdateCheck, } from "../src/braintrust-cli"; import { type CodingToolRuntime, @@ -195,7 +196,7 @@ const WIZARD_INTRO = "Braintrust Setup Wizard"; const WIZARD_CANCEL_MESSAGE = "Wizard cancelled."; const ACCOUNT_QUESTION = "Do you already have a Braintrust account?"; const INSTRUMENTATION_MODE_MESSAGE = - "How do you want to add Braintrust instrumentation?"; + "How do you want to add Braintrust to your application?"; const CLI_INSTALL_MESSAGE = "Install Braintrust CLI?"; const CLI_UPDATE_MESSAGE = "Update Braintrust CLI to the latest version?"; const TOOL_SELECT_MESSAGE = "Which coding agent should Braintrust Setup use?"; @@ -388,6 +389,9 @@ function createBraintrustCliStub( args: { readonly discoveries?: readonly BraintrustCliDiscovery[]; readonly install?: () => Promise; + readonly checkForUpdate?: ( + commandPath: string, + ) => Promise; readonly update?: (commandPath: string) => Promise; readonly status?: (commandPath: string) => Promise; readonly loginAndSwitch?: ( @@ -407,6 +411,8 @@ function createBraintrustCliStub( return Promise.resolve(discovery); }, install: args.install ?? (() => Promise.resolve()), + checkForUpdate: + args.checkForUpdate ?? (() => Promise.resolve({ upToDate: false })), update: args.update ?? (() => Promise.resolve()), status: args.status ?? (() => Promise.resolve({})), loginAndSwitch: args.loginAndSwitch ?? (() => Promise.resolve()), @@ -879,6 +885,82 @@ describe("runClackWizard", () => { expect(calls).toEqual([]); }); + it("skips the update question when the Braintrust CLI is already up to date", async () => { + const calls: string[] = []; + const { events } = createPrompts({ + selects: ["yes", "manual", "confirm", "understood"], + }); + const deps = buildDeps({ + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + ], + checkForUpdate: (commandPath) => { + calls.push(`check:${commandPath}`); + return Promise.resolve({ upToDate: true }); + }, + update: (commandPath) => { + calls.push(`update:${commandPath}`); + return Promise.resolve(); + }, + status: () => { + calls.push("status"); + return Promise.resolve({}); + }, + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect(events).not.toContain(`select:${CLI_UPDATE_MESSAGE}`); + expect(events).not.toContain("spinner.start:Updating Braintrust CLI..."); + expect( + events.some((event) => event.includes("Braintrust CLI is up to date")), + ).toBe(false); + expect(calls).toEqual(["check:/bin/bt", "status", "login"]); + }); + + it("updates when the update check fails and the user accepts", async () => { + const calls: string[] = []; + const { events } = createPrompts({ + selects: ["no", "yes", "manual", "confirm", "understood"], + }); + const deps = buildDeps({ + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + ], + checkForUpdate: () => Promise.reject(new Error("network down")), + update: (commandPath) => { + calls.push(`update:${commandPath}`); + return Promise.resolve(); + }, + status: () => { + calls.push("status"); + return Promise.resolve({}); + }, + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect( + events.some((event) => + event.includes("Could not check for Braintrust CLI updates"), + ), + ).toBe(false); + expect(events).toContain(`select:${CLI_UPDATE_MESSAGE}`); + expect(calls).toEqual(["update:/bin/bt", "status", "login"]); + }); + it("updates and configures an installed Braintrust CLI when accepted", async () => { const calls: string[] = []; const { events } = createPrompts({