diff --git a/packages/spark/src/clack-copy.ts b/packages/spark/src/clack-copy.ts index ca81670..b896875 100644 --- a/packages/spark/src/clack-copy.ts +++ b/packages/spark/src/clack-copy.ts @@ -1,9 +1,5 @@ import type { BraintrustCliContext } from "./braintrust-cli"; -const INSTRUMENTATION_DOCS_URL = - "https://www.braintrust.dev/docs/instrument/trace-llm-calls"; -const SETUP_PLAN = - "You'll sign in with Braintrust, choose an org and project, save an API key for local testing, set up the Braintrust CLI, then choose how to add instrumentation."; const BRAINTRUST_CLI_CONTEXT_FALLBACKS = { profile: "no profile", org: "no org", @@ -13,26 +9,22 @@ const BRAINTRUST_CLI_CONTEXT_FALLBACKS = { export const CLACK_WIZARD_COPY = { shared: { cancelMessage: "Wizard cancelled.", - instrumentationDocsUrl: INSTRUMENTATION_DOCS_URL, + instrumentationDocsUrl: + "https://www.braintrust.dev/docs/instrument/trace-llm-calls", }, welcome: { - intro: [ - "Welcome to the Braintrust setup wizard", - "", - "Setup plan:", - SETUP_PLAN, - ].join("\n"), + intro: "Braintrust Setup Wizard", }, gitRepository: { outsideRepoWarning: - "Heads up: this folder is not a git repository. The wizard may edit files; consider running it inside a checked-in repo.", + "Warning: You are running this wizard inside a folder that is not a git repository. The wizard may edit files.", continueOutsideRepoQuestion: "Continue without a git repository?", continueOutsideRepoChoices: { yes: { label: "Yes", - hint: "Continue setup", + hint: "Continue without git", }, no: { label: "No (recommended)", @@ -50,7 +42,7 @@ export const CLACK_WIZARD_COPY = { }, no: { label: "No", - hint: "Create account", + hint: "Sign up", }, }, browserLoginInfo: (args: { @@ -62,16 +54,14 @@ export const CLACK_WIZARD_COPY = { "", "If your browser didn't open automatically, open the link above to sign in.", `Verification code: ${args.verificationCode}`, - "", - "Choose the org and project you want to use; the wizard will resume here.", ].join("\n"), - waitingForBrowser: "Waiting for login in browser...", + waitingForBrowser: "Waiting for you to sign in via the browser...", browserSetupComplete: (args: { readonly orgName: string; readonly projectName: string; }) => `Browser setup complete. (org: ${args.orgName}, project: ${args.projectName})`, - browserSetupStopped: "Browser setup stopped.", + browserSetupStopped: "Browser setup cancelled.", }, braintrustCli: { @@ -80,15 +70,13 @@ export const CLACK_WIZARD_COPY = { installChoices: { yes: { label: "Yes (recommended)", - hint: "Install CLI", }, no: { label: "No", - hint: "Skip installation", + hint: "Skip CLI installation", }, }, - updateQuestion: (installedLabel: string) => - `Update Braintrust CLI? (${installedLabel} installed)`, + updateQuestion: "Update Braintrust CLI to the latest version?", updateChoices: { yes: { label: "Yes (recommended)", @@ -117,15 +105,15 @@ export const CLACK_WIZARD_COPY = { readonly currentContext: BraintrustCliContext; readonly targetContext: BraintrustCliContext; }) => - `Switch Braintrust CLI from ${formatBraintrustCliContext(args.currentContext)} to ${formatBraintrustCliContext(args.targetContext)}?`, + `Switch Braintrust CLI login profile from ${formatBraintrustCliContext(args.currentContext)} to ${formatBraintrustCliContext(args.targetContext)}?`, switchContextChoices: { yes: { label: "Yes (recommended)", - hint: "Use this project", + hint: "Use project selected in browser", }, no: { label: "No", - hint: "Keep existing context", + hint: "Keep current profile", }, }, contextFallbacks: { @@ -138,29 +126,29 @@ export const CLACK_WIZARD_COPY = { modes: { builtIn: { label: "Use built-in coding agent", - hint: "This wizard will launch a coding agent for you that will add instrumentation to your application (supports Claude Code and Codex). Careful: This will run the chosen tool in yolo mode (full permissions).", + hint: "Launch a locally installed coding agent", }, ownAgent: { label: "Use own coding agent", - hint: "You will receive a prompt to instrument your application with your own coding agent.", + hint: "Use a suggested prompt to pass to your own coding agent", }, manual: { label: "Set up manually", - hint: "Set up tracing for your application using instructions from the Braintrust docs.", + hint: "Use the Braintrust docs", }, }, builtIn: { - determiningAvailable: "Determining available coding agents...", + determiningAvailable: "Scanning for available coding agents...", running: (label: string) => `Running ${label}...`, proceedQuestion: - "This setup wizard will now invoke a coding agent. Proceed?", + "This setup wizard will now invoke a coding agent with full permissions. Proceed?", proceedChoices: { yes: { - label: "Yes, proceed", - hint: "Run the selected coding agent", + label: "Confirm", + hint: "Run the coding agent", }, no: { - label: "abort", + label: "Abort", hint: "Choose another setup path", }, }, @@ -183,7 +171,7 @@ export const CLACK_WIZARD_COPY = { toolExited: (toolLabel: string, exitCode: number) => `${toolLabel} exited with code ${exitCode}.`, codingToolExited: (exitCode: number) => - `Coding tool exited with code ${exitCode}.`, + `Coding agent exited with code ${exitCode}.`, incompleteRenderer: "Instrumentation incomplete.", incompleteWarning: "The coding tool reported incomplete instrumentation.", complete: "Instrumentation complete.", diff --git a/packages/spark/src/clack-wizard.ts b/packages/spark/src/clack-wizard.ts index 6bdc1fd..3e26953 100644 --- a/packages/spark/src/clack-wizard.ts +++ b/packages/spark/src/clack-wizard.ts @@ -394,12 +394,8 @@ async function handleBraintrustCliSetup( let commandPath = discovery.commandPath; if (discovery.installed) { - const installedLabel = - discovery.version ?? - commandPath ?? - COPY.braintrustCli.installedVersionUnknown; const shouldUpdate = await selectBoolean({ - message: COPY.braintrustCli.updateQuestion(installedLabel), + message: COPY.braintrustCli.updateQuestion, choices: COPY.braintrustCli.updateChoices, yesFirst: true, }); diff --git a/packages/spark/src/prompt.ts b/packages/spark/src/prompt.ts index 0784935..ec19cb1 100644 --- a/packages/spark/src/prompt.ts +++ b/packages/spark/src/prompt.ts @@ -13,7 +13,7 @@ export function renderPrompt(opts: { - In terms of instrumentation, always prefer adding auto-instrumentation over manual wrappers. - For the SDK initialization configure the project name "${opts.projectName || ""}". - Do not run application code, just do code changes to instrument the application. -- Also install the SDK or multiple SDKs if necessary. Always use the latest version. Do web research or web requests to look up the latest version. Make sure to use the right package manager that the project is already using. Also look upwards in the directory structure to check whether you're in a mono-repo or not. Ideally go to the root of the git repository if present to verify, but only instrument applications in or below the current working directory. Verify that the SDK has actually been installed. +- Also install the SDK or multiple SDKs if necessary. Always use the latest version. Do web research or web requests to look up the latest version and install that version - don't just pin to e.g. \`latest\`. Make sure to use the right package manager that the project is already using. Also look upwards in the directory structure to check whether you're in a mono-repo or not. Ideally go to the root of the git repository if present to verify, but only instrument applications in or below the current working directory. Verify that the SDK has actually been installed. - Be as concise and readable as possible with your code changes. - Do not break any application code. - Do not modify any application code in any meaningful way. diff --git a/packages/spark/src/tool-ui.ts b/packages/spark/src/tool-ui.ts index c0129f9..996770a 100644 --- a/packages/spark/src/tool-ui.ts +++ b/packages/spark/src/tool-ui.ts @@ -22,13 +22,22 @@ export class ClackToolRenderer { if (!this.output) { const title = `Running ${this.toolLabel} to instrument your application`; this.output = new TaskLogCodingAgentOutput(title); + setTimeout(() => { + void this.output?.message("Starting agent..."); + }, 10); } return this.output; } event(event: CodingToolEvent) { - if (event.type === "completed" || event.type === "failed") return; + if ( + event.type === "completed" || + event.type === "failed" || + event.type === "thinking" + ) { + return; + } const line = eventLine(event); if (line === this.lastLine) return; @@ -74,12 +83,11 @@ class TaskLogCodingAgentOutput implements CodingAgentOutput { } function eventLine(event: CodingToolEvent): string { - if (event.type === "thinking") return "thinking"; if (event.type === "editing") { const target = eventTarget(event) ?? toolInputValue(event.toolInput, ["file_path", "path", "notebook_path"]); - return actionLine("edit", target ?? event.message); + return actionLine("write", target ?? event.message); } if (event.type === "reading") { const target = @@ -102,7 +110,7 @@ function eventLine(event: CodingToolEvent): string { function actionLine(action: string, value: string | undefined): string { const text = value ? sanitizeText(value, 140) : ""; - return text ? `${action} ${text}` : action; + return text ? `${action}: ${text}` : action; } function eventTarget(event: CodingToolEvent): string | undefined { diff --git a/packages/spark/test/clack-wizard.test.ts b/packages/spark/test/clack-wizard.test.ts index d8a1b69..5263918 100644 --- a/packages/spark/test/clack-wizard.test.ts +++ b/packages/spark/test/clack-wizard.test.ts @@ -191,21 +191,17 @@ vi.mock("@clack/prompts", () => ({ }, })); -const WIZARD_INTRO = [ - "Welcome to the Braintrust setup wizard", - "", - "Setup plan:", - "You'll sign in with Braintrust, choose an org and project, save an API key for local testing, set up the Braintrust CLI, then choose how to add instrumentation.", -].join("\n"); +const WIZARD_INTRO = "Braintrust Setup Wizard"; const WIZARD_CANCEL_MESSAGE = "Wizard cancelled."; const ACCOUNT_QUESTION = "Do you already have a Braintrust account?"; const INSTRUMENTATION_MODE_MESSAGE = "How do you want to add Braintrust instrumentation?"; const CLI_INSTALL_MESSAGE = "Install Braintrust CLI?"; -const CLI_UPDATE_MESSAGE = "Update Braintrust CLI? (bt 0.10.0 installed)"; +const CLI_UPDATE_MESSAGE = "Update Braintrust CLI to the latest version?"; const TOOL_SELECT_MESSAGE = "Which coding agent should Braintrust Setup use?"; const CODING_AGENT_PROCEED_MESSAGE = - "This setup wizard will now invoke a coding agent. Proceed?"; + "This setup wizard will now invoke a coding agent with full permissions. Proceed?"; +const CODING_AGENT_SCAN_MESSAGE = "Scanning for available coding agents..."; const OWN_AGENT_DELIVERY_MESSAGE = "How should Braintrust Setup deliver the instrumentation prompt?"; const OWN_AGENT_COMPLETED_MESSAGE = @@ -362,6 +358,13 @@ function buildDeps( ); expect(prompt).toContain("Do not use the Braintrust CLI (`bt`)."); expect(prompt).not.toContain("Unattended mode (YOLO)"); + onEvent({ + type: "reading", + message: "Reading package.json", + target: "package.json", + toolInput: "file_path: package.json", + toolName: "Read", + }); onEvent({ type: "thinking", message: "Thinking..." }); onEvent({ type: "editing", @@ -429,11 +432,9 @@ describe("runClackWizard", () => { expect(events).toContain(`select:${CLI_INSTALL_MESSAGE}`); expect(events).toContain(`select:${INSTRUMENTATION_MODE_MESSAGE}`); expect(events).not.toContain(`select:${TOOL_SELECT_MESSAGE}`); - expect(events).toContain( - "spinner.start:Determining available coding agents...", - ); + expect(events).toContain(`spinner.start:${CODING_AGENT_SCAN_MESSAGE}`); const codingAgentSpinnerStart = events.indexOf( - "spinner.start:Determining available coding agents...", + `spinner.start:${CODING_AGENT_SCAN_MESSAGE}`, ); const instrumentationModePrompt = events.indexOf( `select:${INSTRUMENTATION_MODE_MESSAGE}`, @@ -471,7 +472,9 @@ describe("runClackWizard", () => { ), ), ).toBe(true); - expect(events).toContain("spinner.start:Waiting for login in browser..."); + expect(events).toContain( + "spinner.start:Waiting for you to sign in via the browser...", + ); expect( events.some((event) => event.startsWith("spinner.stop:Browser setup complete."), @@ -485,7 +488,9 @@ describe("runClackWizard", () => { expect(events).toContain( "taskLog:Running Claude Code to instrument your application:0:false", ); - expect(events).toContain("taskLog.message:edit package.json"); + expect(events).toContain("taskLog.message:read: package.json"); + expect(events).not.toContain("taskLog.message:thinking"); + expect(events).toContain("taskLog.message:write: package.json"); expect(events).toContain("taskLog.success:Instrumentation complete."); expect(readFileSync(join(deps.cwd, ".env.braintrust"), "utf8")).toBe( ENV_BRAINTRUST_FILE_CONTENT, @@ -573,7 +578,7 @@ describe("runClackWizard", () => { expect(maxActiveSmokeTests).toBe(2); expect(events).toContain(`select:${TOOL_SELECT_MESSAGE}`); const codingAgentSpinnerStart = events.indexOf( - "spinner.start:Determining available coding agents...", + `spinner.start:${CODING_AGENT_SCAN_MESSAGE}`, ); const instrumentationModePrompt = events.indexOf( `select:${INSTRUMENTATION_MODE_MESSAGE}`, @@ -599,7 +604,9 @@ describe("runClackWizard", () => { expect(events).toContain( "taskLog:Running Claude Code to instrument your application:0:false", ); - expect(events).toContain("taskLog.message:edit package.json"); + expect(events).toContain("taskLog.message:read: package.json"); + expect(events).not.toContain("taskLog.message:thinking"); + expect(events).toContain("taskLog.message:write: package.json"); expect(events).toContain("taskLog.success:Instrumentation complete."); }); @@ -782,9 +789,7 @@ describe("runClackWizard", () => { expect(events).toContain( "spinner.message:Configuring Braintrust CLI context...", ); - expect(events).toContain( - "spinner.message:Determining available coding agents...", - ); + expect(events).toContain(`spinner.message:${CODING_AGENT_SCAN_MESSAGE}`); expect(events).toContain("spinner.clear"); expect(events).not.toContain("spinner.stop:Installed Braintrust CLI."); expect(events).not.toContain("success:Configured Braintrust CLI."); @@ -792,15 +797,13 @@ describe("runClackWizard", () => { "spinner.message:Configuring Braintrust CLI context...", ); const codingAgentSpinnerIndex = events.indexOf( - "spinner.message:Determining available coding agents...", + `spinner.message:${CODING_AGENT_SCAN_MESSAGE}`, ); expect(codingAgentSpinnerIndex).toBeGreaterThan(configureSpinnerIndex); expect( events.slice(configureSpinnerIndex, codingAgentSpinnerIndex), ).not.toContain("spinner.clear"); - expect(events).not.toContain( - "spinner.start:Determining available coding agents...", - ); + expect(events).not.toContain(`spinner.start:${CODING_AGENT_SCAN_MESSAGE}`); expect( events.indexOf("spinner.message:Checking Braintrust CLI context..."), ).toBeLessThan(events.indexOf(`select:${INSTRUMENTATION_MODE_MESSAGE}`)); @@ -912,9 +915,7 @@ describe("runClackWizard", () => { expect(events).toContain( "spinner.message:Configuring Braintrust CLI context...", ); - expect(events).toContain( - "spinner.message:Determining available coding agents...", - ); + expect(events).toContain(`spinner.message:${CODING_AGENT_SCAN_MESSAGE}`); expect(events).toContain("spinner.clear"); expect(events).not.toContain("spinner.stop:Updated Braintrust CLI."); expect(events).not.toContain("success:Configured Braintrust CLI."); @@ -922,15 +923,13 @@ describe("runClackWizard", () => { "spinner.message:Configuring Braintrust CLI context...", ); const codingAgentSpinnerIndex = events.indexOf( - "spinner.message:Determining available coding agents...", + `spinner.message:${CODING_AGENT_SCAN_MESSAGE}`, ); expect(codingAgentSpinnerIndex).toBeGreaterThan(configureSpinnerIndex); expect( events.slice(configureSpinnerIndex, codingAgentSpinnerIndex), ).not.toContain("spinner.clear"); - expect(events).not.toContain( - "spinner.start:Determining available coding agents...", - ); + expect(events).not.toContain(`spinner.start:${CODING_AGENT_SCAN_MESSAGE}`); expect(calls).toEqual(["update:/bin/bt", "status:/bin/bt", "login"]); }); @@ -993,7 +992,7 @@ describe("runClackWizard", () => { expect(events).toContain(`select:${CLI_UPDATE_MESSAGE}`); expect( events.some((event) => - event.startsWith("select:Switch Braintrust CLI from"), + event.startsWith("select:Switch Braintrust CLI login profile from"), ), ).toBe(false); expect(events).toContain( @@ -1027,7 +1026,7 @@ describe("runClackWizard", () => { expect( events.some((event) => - event.startsWith("select:Switch Braintrust CLI from"), + event.startsWith("select:Switch Braintrust CLI login profile from"), ), ).toBe(false); expect(calls).toEqual(["login"]); @@ -1079,7 +1078,7 @@ describe("runClackWizard", () => { await runClackWizard(deps); expect(events).toContain( - "select:Switch Braintrust CLI from work (other/old) to acme (acme/demo)?", + "select:Switch Braintrust CLI login profile from work (other/old) to acme (acme/demo)?", ); expect(events).not.toContain( "info:Leaving existing Braintrust CLI context unchanged.", @@ -1247,7 +1246,13 @@ describe("runClackWizard", () => { const deps = buildDeps({ cwd: join(dir, "child") }); await runClackWizard(deps); - expect(events.some((e) => e.startsWith("warn:Heads up"))).toBe(true); + expect( + events.some((event) => + event.startsWith( + "warn:Warning: You are running this wizard inside a folder that is not a git repository.", + ), + ), + ).toBe(true); expect(events).toContain("select:Continue without a git repository?"); }); @@ -1403,9 +1408,7 @@ describe("runClackWizard", () => { await runClackWizard(deps); - expect(events).toContain( - "spinner.start:Determining available coding agents...", - ); + expect(events).toContain(`spinner.start:${CODING_AGENT_SCAN_MESSAGE}`); expect(events).toContain( `select.options:${INSTRUMENTATION_MODE_MESSAGE}:Use own coding agent|Set up manually`, );