From 38730fb954d7ecedf06534434f355e5d62ac70d3 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Thu, 26 Mar 2026 07:38:14 +0000 Subject: [PATCH 1/5] fix(core): accept string commands in workspace hook config Workspace hooks silently ignored string commands (e.g., `command: node scripts/setup.mjs`) because parseWorkspaceScriptConfig only accepted arrays. Now auto-splits string commands on whitespace to match user expectations. Closes #778 Co-Authored-By: Claude Opus 4.6 --- packages/core/src/evaluation/yaml-parser.ts | 46 +++++++++++++------ .../workspace-config-parsing.test.ts | 25 ++++++++++ 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index e8004bbc9..93fa4f9df 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -513,9 +513,33 @@ export async function loadTestById( /** @deprecated Use `loadTestById` instead */ export const loadEvalCaseById = loadTestById; +/** + * Build a WorkspaceScriptConfig from a parsed command array and raw YAML object. + */ +function buildScriptConfig( + commandArr: string[], + obj: Record, + evalFileDir: string, +): WorkspaceScriptConfig { + const timeoutMs = typeof obj.timeout_ms === 'number' ? obj.timeout_ms : undefined; + let cwd = typeof obj.cwd === 'string' ? obj.cwd : undefined; + + // Resolve relative cwd against eval file directory + if (cwd && !path.isAbsolute(cwd)) { + cwd = path.resolve(evalFileDir, cwd); + } + + const config: WorkspaceScriptConfig = { command: commandArr }; + if (timeoutMs !== undefined) { + return { ...config, timeout_ms: timeoutMs, ...(cwd !== undefined && { cwd }) }; + } + return cwd ? { ...config, cwd } : config; +} + /** * Parse a WorkspaceScriptConfig from raw YAML value. * Accepts both `command` (preferred) and `script` (deprecated alias). + * Command can be an array of strings or a single string (auto-split on whitespace). */ function parseWorkspaceScriptConfig( raw: unknown, @@ -528,23 +552,19 @@ function parseWorkspaceScriptConfig( logWarning("'script' is deprecated. Use 'command' instead."); } const commandSource = obj.command ?? obj.script; + + // Accept string commands by splitting on whitespace + if (typeof commandSource === 'string') { + const parts = commandSource.trim().split(/\s+/); + if (parts.length === 0 || parts[0] === '') return undefined; + return buildScriptConfig(parts, obj, evalFileDir); + } + if (!Array.isArray(commandSource) || commandSource.length === 0) return undefined; const commandArr = commandSource.filter((s): s is string => typeof s === 'string'); if (commandArr.length === 0) return undefined; - const timeoutMs = typeof obj.timeout_ms === 'number' ? obj.timeout_ms : undefined; - let cwd = typeof obj.cwd === 'string' ? obj.cwd : undefined; - - // Resolve relative cwd against eval file directory - if (cwd && !path.isAbsolute(cwd)) { - cwd = path.resolve(evalFileDir, cwd); - } - - const config: WorkspaceScriptConfig = { command: commandArr }; - if (timeoutMs !== undefined) { - return { ...config, timeout_ms: timeoutMs, ...(cwd !== undefined && { cwd }) }; - } - return cwd ? { ...config, cwd } : config; + return buildScriptConfig(commandArr, obj, evalFileDir); } function parseRepoSource(raw: unknown): RepoSource | undefined { diff --git a/packages/core/test/evaluation/workspace-config-parsing.test.ts b/packages/core/test/evaluation/workspace-config-parsing.test.ts index 8c215ed23..2f5fd5486 100644 --- a/packages/core/test/evaluation/workspace-config-parsing.test.ts +++ b/packages/core/test/evaluation/workspace-config-parsing.test.ts @@ -337,6 +337,31 @@ tests: await expect(loadTests(evalFile, testDir)).rejects.toThrow(/workspace\.pool has been removed/i); }); + it('should accept string command and auto-split on whitespace', async () => { + const evalFile = path.join(testDir, 'workspace-string-cmd.yaml'); + await writeFile( + evalFile, + ` +tests: + - id: test-string-cmd + input: "Do something" + criteria: "Should work" + workspace: + hooks: + before_all: + command: node scripts/setup.mjs + timeout_ms: 60000 +`, + ); + + const cases = await loadTests(evalFile, testDir); + expect(cases).toHaveLength(1); + expect(cases[0].workspace?.hooks?.before_all).toEqual({ + command: ['node', 'scripts/setup.mjs'], + timeout_ms: 60000, + }); + }); + it('should handle case with no workspace config', async () => { const evalFile = path.join(testDir, 'no-workspace.yaml'); await writeFile( From c3e73806a7cf899f211732e187aadadf33918812 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Thu, 26 Mar 2026 07:39:34 +0000 Subject: [PATCH 2/5] style: fix lint errors in setup.mjs Co-Authored-By: Claude Opus 4.6 --- .../workspace-template/scripts/setup.mjs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/evals/agentic-engineering/workspace-template/scripts/setup.mjs b/evals/agentic-engineering/workspace-template/scripts/setup.mjs index bfeddcf6e..afdd10da8 100644 --- a/evals/agentic-engineering/workspace-template/scripts/setup.mjs +++ b/evals/agentic-engineering/workspace-template/scripts/setup.mjs @@ -5,9 +5,9 @@ * Runs with cwd = eval file directory (which is inside the repo). */ -import { cpSync, mkdirSync, readdirSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; import { execSync } from 'node:child_process'; +import { cpSync, mkdirSync, readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; // Read workspace_path from stdin (provided by AgentV orchestrator) let workspacePath; @@ -32,10 +32,7 @@ console.log(`Workspace: ${workspacePath}`); console.log(`Repo root: ${repoRoot}`); // Copy to skill discovery directories in the workspace -const skillDirs = [ - join(workspacePath, '.agents', 'skills'), - join(workspacePath, '.pi', 'skills'), -]; +const skillDirs = [join(workspacePath, '.agents', 'skills'), join(workspacePath, '.pi', 'skills')]; for (const dir of skillDirs) { mkdirSync(dir, { recursive: true }); } From 7c1d30a42a67be6063345c582217a2538912cf63 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Thu, 26 Mar 2026 07:52:24 +0000 Subject: [PATCH 3/5] docs: add JSDoc note about naive whitespace splitting Co-Authored-By: Claude Opus 4.6 --- packages/core/src/evaluation/yaml-parser.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 93fa4f9df..917d11155 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -540,6 +540,8 @@ function buildScriptConfig( * Parse a WorkspaceScriptConfig from raw YAML value. * Accepts both `command` (preferred) and `script` (deprecated alias). * Command can be an array of strings or a single string (auto-split on whitespace). + * Note: string commands are split naively on whitespace. For arguments containing + * spaces, use the array form: command: ["node", "path with spaces/setup.mjs"] */ function parseWorkspaceScriptConfig( raw: unknown, From 622de1836a70088e6dc5527f9a58b7aa4f143846 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Thu, 26 Mar 2026 07:59:40 +0000 Subject: [PATCH 4/5] refactor: normalize input early instead of branching to shared helper Extract parseCommandArray to handle string/array normalization upfront, keeping parseWorkspaceScriptConfig as a single linear flow. Co-Authored-By: Claude Opus 4.6 --- packages/core/src/evaluation/yaml-parser.ts | 52 ++++++++++----------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 917d11155..8c2fc2393 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -514,26 +514,19 @@ export async function loadTestById( export const loadEvalCaseById = loadTestById; /** - * Build a WorkspaceScriptConfig from a parsed command array and raw YAML object. + * Normalize a command value from YAML into a string array. + * Accepts a string (split on whitespace) or an array of strings. */ -function buildScriptConfig( - commandArr: string[], - obj: Record, - evalFileDir: string, -): WorkspaceScriptConfig { - const timeoutMs = typeof obj.timeout_ms === 'number' ? obj.timeout_ms : undefined; - let cwd = typeof obj.cwd === 'string' ? obj.cwd : undefined; - - // Resolve relative cwd against eval file directory - if (cwd && !path.isAbsolute(cwd)) { - cwd = path.resolve(evalFileDir, cwd); +function parseCommandArray(source: unknown): string[] | undefined { + if (typeof source === 'string') { + const parts = source.trim().split(/\s+/); + return parts.length > 0 && parts[0] !== '' ? parts : undefined; } - - const config: WorkspaceScriptConfig = { command: commandArr }; - if (timeoutMs !== undefined) { - return { ...config, timeout_ms: timeoutMs, ...(cwd !== undefined && { cwd }) }; + if (Array.isArray(source)) { + const arr = source.filter((s): s is string => typeof s === 'string'); + return arr.length > 0 ? arr : undefined; } - return cwd ? { ...config, cwd } : config; + return undefined; } /** @@ -553,20 +546,23 @@ function parseWorkspaceScriptConfig( if (obj.script !== undefined && obj.command === undefined) { logWarning("'script' is deprecated. Use 'command' instead."); } - const commandSource = obj.command ?? obj.script; - // Accept string commands by splitting on whitespace - if (typeof commandSource === 'string') { - const parts = commandSource.trim().split(/\s+/); - if (parts.length === 0 || parts[0] === '') return undefined; - return buildScriptConfig(parts, obj, evalFileDir); - } + const command = parseCommandArray(obj.command ?? obj.script); + if (!command) return undefined; + + const timeoutMs = typeof obj.timeout_ms === 'number' ? obj.timeout_ms : undefined; + let cwd = typeof obj.cwd === 'string' ? obj.cwd : undefined; - if (!Array.isArray(commandSource) || commandSource.length === 0) return undefined; - const commandArr = commandSource.filter((s): s is string => typeof s === 'string'); - if (commandArr.length === 0) return undefined; + // Resolve relative cwd against eval file directory + if (cwd && !path.isAbsolute(cwd)) { + cwd = path.resolve(evalFileDir, cwd); + } - return buildScriptConfig(commandArr, obj, evalFileDir); + const config: WorkspaceScriptConfig = { command }; + if (timeoutMs !== undefined) { + return { ...config, timeout_ms: timeoutMs, ...(cwd !== undefined && { cwd }) }; + } + return cwd ? { ...config, cwd } : config; } function parseRepoSource(raw: unknown): RepoSource | undefined { From a3d2781bb6624dd6d50182fedffac533c8efbcf1 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Thu, 26 Mar 2026 08:01:22 +0000 Subject: [PATCH 5/5] test: replace deprecated script field with command in workspace tests Co-Authored-By: Claude Opus 4.6 --- .../test/evaluation/workspace-config-parsing.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/core/test/evaluation/workspace-config-parsing.test.ts b/packages/core/test/evaluation/workspace-config-parsing.test.ts index 2f5fd5486..5779b606d 100644 --- a/packages/core/test/evaluation/workspace-config-parsing.test.ts +++ b/packages/core/test/evaluation/workspace-config-parsing.test.ts @@ -29,10 +29,10 @@ tests: workspace: hooks: before_all: - script: ["bun", "run", "setup.ts"] + command: ["bun", "run", "setup.ts"] timeout_ms: 120000 after_each: - script: ["bun", "run", "teardown.ts"] + command: ["bun", "run", "teardown.ts"] timeout_ms: 30000 `, ); @@ -81,7 +81,7 @@ tests: workspace: hooks: before_all: - script: ["bun", "run", "default-setup.ts"] + command: ["bun", "run", "default-setup.ts"] tests: - id: case-1 @@ -112,7 +112,7 @@ tests: workspace: hooks: before_all: - script: ["bun", "run", "default-setup.ts"] + command: ["bun", "run", "default-setup.ts"] tests: - id: case-override @@ -121,7 +121,7 @@ tests: workspace: hooks: before_all: - script: ["bun", "run", "custom-setup.ts"] + command: ["bun", "run", "custom-setup.ts"] - id: case-default input: "Do something else" criteria: "Should work" @@ -158,7 +158,7 @@ tests: workspace: hooks: before_all: - script: ["bun", "run", "setup.ts"] + command: ["bun", "run", "setup.ts"] cwd: ./scripts `, );