From f122ad44936b39847bec4c52dca39c8262405aa5 Mon Sep 17 00:00:00 2001 From: Alex Ermolin Date: Tue, 7 Jul 2026 19:43:07 +0200 Subject: [PATCH 1/7] fix: isolate skill eval workers --- plugin/dist/build-manifest.json | 4 ++-- plugin/dist/package.json | 2 +- plugin/dist/skill-creator.js | 22 ++++++++++++---------- plugin/lib/run-eval.ts | 25 ++++++++++++++++--------- plugin/skill-creator.ts | 2 -- 5 files changed, 31 insertions(+), 24 deletions(-) diff --git a/plugin/dist/build-manifest.json b/plugin/dist/build-manifest.json index 8ab66d0..f8c4590 100644 --- a/plugin/dist/build-manifest.json +++ b/plugin/dist/build-manifest.json @@ -1,6 +1,6 @@ { "entrypoint": "skill-creator.ts", "runtimeEntrypoint": "runtime-entry.ts", - "sourceHash": "7ab254f2c22233b0ced8cc832878baacd73f3bd63fa45c81019e9149bf319605", - "builtAt": "2026-07-05T23:29:06.331Z" + "sourceHash": "bf781c7f2e63812356bd491f1211d59786b8d3669a1d861a93c78495b68382aa", + "builtAt": "2026-07-07T17:26:48.605Z" } diff --git a/plugin/dist/package.json b/plugin/dist/package.json index e069f31..6631248 100644 --- a/plugin/dist/package.json +++ b/plugin/dist/package.json @@ -1,6 +1,6 @@ { "name": "opencode-skill-creator", - "version": "0.2.21", + "version": "0.2.23", "description": "OpenCode plugin for skill creation — custom tools for validation, evaluation, description optimization, benchmarking, and review serving.", "type": "module", "main": "./dist/skill-creator.js", diff --git a/plugin/dist/skill-creator.js b/plugin/dist/skill-creator.js index 2013d18..0041966 100644 --- a/plugin/dist/skill-creator.js +++ b/plugin/dist/skill-creator.js @@ -12525,9 +12525,10 @@ function parseSkillMd(skillPath) { } // lib/run-eval.ts -import { existsSync as existsSync2, mkdirSync, rmSync, writeFileSync } from "fs"; +import { existsSync as existsSync2, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; import { dirname, join as join3, parse as parse5 } from "path"; import { randomBytes } from "crypto"; +import { tmpdir } from "os"; // lib/process.ts import { spawn } from "child_process"; @@ -12655,7 +12656,7 @@ function findSkillConflicts(stdoutText, skillName) { return []; } } -async function assertNoInstalledSkillConflict(skillName, projectRoot) { +async function assertNoInstalledSkillConflict2(skillName, projectRoot) { let result; try { result = await runProcess(["opencode", "debug", "skill"], { @@ -12687,15 +12688,17 @@ function findProjectRoot(cwd) { } return cwd ?? process.cwd(); } -async function runSingleQuery(query, skillName, skillDescription, timeout, projectRoot, agent, triggerOnly, model) { +async function runSingleQuery(query, skillName, skillDescription, timeout, _projectRoot, agent, triggerOnly, model) { if (!SKILL_NAME_RE.test(skillName)) { throw new Error(`Invalid skill name "${skillName}". Expected kebab-case (lowercase letters, numbers, and hyphens only).`); } const uniqueId = randomBytes(4).toString("hex"); const cleanName = `${skillName}-skill-${uniqueId}`; - const skillsDir = join3(projectRoot, ".opencode", "skills", cleanName); + const evalRoot = mkdtempSync(join3(tmpdir(), "opencode-skill-eval-")); + const skillsDir = join3(evalRoot, ".opencode", "skills", cleanName); const skillFile = join3(skillsDir, "SKILL.md"); try { + await assertNoInstalledSkillConflict2(skillName, evalRoot); mkdirSync(skillsDir, { recursive: true }); const indentedDesc = skillDescription.split(` `).join(` @@ -12755,7 +12758,7 @@ async function runSingleQuery(query, skillName, skillDescription, timeout, proje } }; const result = await runProcess(cmd, { - cwd: projectRoot, + cwd: evalRoot, env: { ...process.env }, timeoutMs, maxStderrChars, @@ -12775,8 +12778,8 @@ async function runSingleQuery(query, skillName, skillDescription, timeout, proje } return triggered; } finally { - if (existsSync2(skillsDir)) { - rmSync(skillsDir, { recursive: true, force: true }); + if (existsSync2(evalRoot)) { + rmSync(evalRoot, { recursive: true, force: true }); } } } @@ -12879,7 +12882,7 @@ async function runEval(opts) { // lib/improve-description.ts import { mkdirSync as mkdirSync2, unlinkSync, writeFileSync as writeFileSync2 } from "fs"; import { join as join4 } from "path"; -import { tmpdir } from "os"; +import { tmpdir as tmpdir2 } from "os"; import { randomBytes as randomBytes2 } from "crypto"; // lib/failure-taxonomy.ts @@ -12918,7 +12921,7 @@ function formatFailureDiagnostics(diagnostics) { // lib/improve-description.ts async function callOpenCode(prompt, model, timeout = 300) { - const tmpPath = join4(tmpdir(), `skill-creator-${randomBytes2(6).toString("hex")}.md`); + const tmpPath = join4(tmpdir2(), `skill-creator-${randomBytes2(6).toString("hex")}.md`); writeFileSync2(tmpPath, prompt); try { const cmd = ["opencode", "run", "--format", "json"]; @@ -15050,7 +15053,6 @@ var SkillCreatorPlugin = async (ctx) => { } const meta = parseSkillMd(args.skillPath); const projectRoot = findProjectRoot(); - await assertNoInstalledSkillConflict(meta.name, projectRoot); const result = await runEval({ evalSet, skillName: meta.name, diff --git a/plugin/lib/run-eval.ts b/plugin/lib/run-eval.ts index 6de4b97..4ed9694 100644 --- a/plugin/lib/run-eval.ts +++ b/plugin/lib/run-eval.ts @@ -10,9 +10,10 @@ * whether the skill was triggered. */ -import { existsSync, mkdirSync, rmSync, writeFileSync } from "fs" +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs" import { dirname, join, parse } from "path" import { randomBytes } from "crypto" +import { tmpdir } from "os" import { isFailedProcess, runProcess } from "./process" @@ -152,15 +153,19 @@ export function findProjectRoot(cwd?: string): string { } /** - * Run a single query against `opencode run` and return whether the - * temporary skill name appeared in the output. + * Run a single query against `opencode run` and return whether the temporary + * skill name appeared in the output. + * + * Each run gets its own temporary OpenCode project root. This keeps parallel + * workers from seeing one another's synthetic skills and producing false + * negatives when a sibling synthetic skill is selected. */ async function runSingleQuery( query: string, skillName: string, skillDescription: string, timeout: number, - projectRoot: string, + _projectRoot: string, agent: string, triggerOnly: boolean, model?: string, @@ -173,10 +178,12 @@ async function runSingleQuery( const uniqueId = randomBytes(4).toString("hex") const cleanName = `${skillName}-skill-${uniqueId}` - const skillsDir = join(projectRoot, ".opencode", "skills", cleanName) + const evalRoot = mkdtempSync(join(tmpdir(), "opencode-skill-eval-")) + const skillsDir = join(evalRoot, ".opencode", "skills", cleanName) const skillFile = join(skillsDir, "SKILL.md") try { + await assertNoInstalledSkillConflict(skillName, evalRoot) mkdirSync(skillsDir, { recursive: true }) // Use YAML block scalar to avoid breaking on quotes in description @@ -242,7 +249,7 @@ async function runSingleQuery( } const result = await runProcess(cmd, { - cwd: projectRoot, + cwd: evalRoot, env: { ...process.env }, timeoutMs, maxStderrChars, @@ -270,9 +277,9 @@ async function runSingleQuery( return triggered } finally { - // Clean up the temporary skill directory - if (existsSync(skillsDir)) { - rmSync(skillsDir, { recursive: true, force: true }) + // Clean up the isolated temporary project root. + if (existsSync(evalRoot)) { + rmSync(evalRoot, { recursive: true, force: true }) } } } diff --git a/plugin/skill-creator.ts b/plugin/skill-creator.ts index de177ad..8af46d4 100644 --- a/plugin/skill-creator.ts +++ b/plugin/skill-creator.ts @@ -22,7 +22,6 @@ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs" import { validateSkill } from "./lib/validate" import { parseSkillMd } from "./lib/utils" import { - assertNoInstalledSkillConflict, runEval, findProjectRoot, } from "./lib/run-eval" @@ -518,7 +517,6 @@ export const SkillCreatorPlugin: Plugin = async (ctx) => { const meta = parseSkillMd(args.skillPath) const projectRoot = findProjectRoot() - await assertNoInstalledSkillConflict(meta.name, projectRoot) const result = await runEval({ evalSet, From 18f16dacf43286dcda468b11581ce00724b4cff2 Mon Sep 17 00:00:00 2001 From: Alex Ermolin Date: Tue, 7 Jul 2026 19:52:56 +0200 Subject: [PATCH 2/7] chore: narrow skill eval isolation diff --- plugin/dist/build-manifest.json | 4 ++-- plugin/dist/package.json | 2 +- plugin/dist/skill-creator.js | 16 ++++++++-------- plugin/lib/run-eval.ts | 7 ++----- plugin/skill-creator.ts | 2 ++ 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/plugin/dist/build-manifest.json b/plugin/dist/build-manifest.json index f8c4590..c45d5e0 100644 --- a/plugin/dist/build-manifest.json +++ b/plugin/dist/build-manifest.json @@ -1,6 +1,6 @@ { "entrypoint": "skill-creator.ts", "runtimeEntrypoint": "runtime-entry.ts", - "sourceHash": "bf781c7f2e63812356bd491f1211d59786b8d3669a1d861a93c78495b68382aa", - "builtAt": "2026-07-07T17:26:48.605Z" + "sourceHash": "96e74ab246995ba68e457f119ad4567862ad17fe1323a45f9b0cedd7c7d05c68", + "builtAt": "2026-07-07T17:51:45.036Z" } diff --git a/plugin/dist/package.json b/plugin/dist/package.json index 6631248..e069f31 100644 --- a/plugin/dist/package.json +++ b/plugin/dist/package.json @@ -1,6 +1,6 @@ { "name": "opencode-skill-creator", - "version": "0.2.23", + "version": "0.2.21", "description": "OpenCode plugin for skill creation — custom tools for validation, evaluation, description optimization, benchmarking, and review serving.", "type": "module", "main": "./dist/skill-creator.js", diff --git a/plugin/dist/skill-creator.js b/plugin/dist/skill-creator.js index 0041966..3c098f6 100644 --- a/plugin/dist/skill-creator.js +++ b/plugin/dist/skill-creator.js @@ -12528,7 +12528,7 @@ function parseSkillMd(skillPath) { import { existsSync as existsSync2, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; import { dirname, join as join3, parse as parse5 } from "path"; import { randomBytes } from "crypto"; -import { tmpdir } from "os"; +import { tmpdir as osTmpdir } from "os"; // lib/process.ts import { spawn } from "child_process"; @@ -12656,7 +12656,7 @@ function findSkillConflicts(stdoutText, skillName) { return []; } } -async function assertNoInstalledSkillConflict2(skillName, projectRoot) { +async function assertNoInstalledSkillConflict(skillName, projectRoot) { let result; try { result = await runProcess(["opencode", "debug", "skill"], { @@ -12688,17 +12688,16 @@ function findProjectRoot(cwd) { } return cwd ?? process.cwd(); } -async function runSingleQuery(query, skillName, skillDescription, timeout, _projectRoot, agent, triggerOnly, model) { +async function runSingleQuery(query, skillName, skillDescription, timeout, agent, triggerOnly, model) { if (!SKILL_NAME_RE.test(skillName)) { throw new Error(`Invalid skill name "${skillName}". Expected kebab-case (lowercase letters, numbers, and hyphens only).`); } const uniqueId = randomBytes(4).toString("hex"); const cleanName = `${skillName}-skill-${uniqueId}`; - const evalRoot = mkdtempSync(join3(tmpdir(), "opencode-skill-eval-")); + const evalRoot = mkdtempSync(join3(osTmpdir(), "opencode-skill-eval-")); const skillsDir = join3(evalRoot, ".opencode", "skills", cleanName); const skillFile = join3(skillsDir, "SKILL.md"); try { - await assertNoInstalledSkillConflict2(skillName, evalRoot); mkdirSync(skillsDir, { recursive: true }); const indentedDesc = skillDescription.split(` `).join(` @@ -12811,7 +12810,7 @@ async function runEval(opts) { if (!job) break; try { - const triggered = await runSingleQuery(job.item.query, skillName, description, timeout, projectRoot, agent, triggerOnly, model); + const triggered = await runSingleQuery(job.item.query, skillName, description, timeout, agent, triggerOnly, model); jobResults.push({ query: job.item.query, triggered, @@ -12882,7 +12881,7 @@ async function runEval(opts) { // lib/improve-description.ts import { mkdirSync as mkdirSync2, unlinkSync, writeFileSync as writeFileSync2 } from "fs"; import { join as join4 } from "path"; -import { tmpdir as tmpdir2 } from "os"; +import { tmpdir } from "os"; import { randomBytes as randomBytes2 } from "crypto"; // lib/failure-taxonomy.ts @@ -12921,7 +12920,7 @@ function formatFailureDiagnostics(diagnostics) { // lib/improve-description.ts async function callOpenCode(prompt, model, timeout = 300) { - const tmpPath = join4(tmpdir2(), `skill-creator-${randomBytes2(6).toString("hex")}.md`); + const tmpPath = join4(tmpdir(), `skill-creator-${randomBytes2(6).toString("hex")}.md`); writeFileSync2(tmpPath, prompt); try { const cmd = ["opencode", "run", "--format", "json"]; @@ -15053,6 +15052,7 @@ var SkillCreatorPlugin = async (ctx) => { } const meta = parseSkillMd(args.skillPath); const projectRoot = findProjectRoot(); + await assertNoInstalledSkillConflict(meta.name, projectRoot); const result = await runEval({ evalSet, skillName: meta.name, diff --git a/plugin/lib/run-eval.ts b/plugin/lib/run-eval.ts index 4ed9694..ebd8c9c 100644 --- a/plugin/lib/run-eval.ts +++ b/plugin/lib/run-eval.ts @@ -13,7 +13,7 @@ import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs" import { dirname, join, parse } from "path" import { randomBytes } from "crypto" -import { tmpdir } from "os" +import { tmpdir as osTmpdir } from "os" import { isFailedProcess, runProcess } from "./process" @@ -165,7 +165,6 @@ async function runSingleQuery( skillName: string, skillDescription: string, timeout: number, - _projectRoot: string, agent: string, triggerOnly: boolean, model?: string, @@ -178,12 +177,11 @@ async function runSingleQuery( const uniqueId = randomBytes(4).toString("hex") const cleanName = `${skillName}-skill-${uniqueId}` - const evalRoot = mkdtempSync(join(tmpdir(), "opencode-skill-eval-")) + const evalRoot = mkdtempSync(join(osTmpdir(), "opencode-skill-eval-")) const skillsDir = join(evalRoot, ".opencode", "skills", cleanName) const skillFile = join(skillsDir, "SKILL.md") try { - await assertNoInstalledSkillConflict(skillName, evalRoot) mkdirSync(skillsDir, { recursive: true }) // Use YAML block scalar to avoid breaking on quotes in description @@ -351,7 +349,6 @@ export async function runEval(opts: RunEvalOptions): Promise { skillName, description, timeout, - projectRoot, agent, triggerOnly, model, diff --git a/plugin/skill-creator.ts b/plugin/skill-creator.ts index 8af46d4..de177ad 100644 --- a/plugin/skill-creator.ts +++ b/plugin/skill-creator.ts @@ -22,6 +22,7 @@ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs" import { validateSkill } from "./lib/validate" import { parseSkillMd } from "./lib/utils" import { + assertNoInstalledSkillConflict, runEval, findProjectRoot, } from "./lib/run-eval" @@ -517,6 +518,7 @@ export const SkillCreatorPlugin: Plugin = async (ctx) => { const meta = parseSkillMd(args.skillPath) const projectRoot = findProjectRoot() + await assertNoInstalledSkillConflict(meta.name, projectRoot) const result = await runEval({ evalSet, From c5d7f5226a2ea5bb1f4290c240eaea731e0b3699 Mon Sep 17 00:00:00 2001 From: Alex Ermolin Date: Tue, 7 Jul 2026 20:01:39 +0200 Subject: [PATCH 3/7] fix: preserve opencode config during eval isolation --- plugin/dist/build-manifest.json | 4 +-- plugin/dist/package.json | 2 +- plugin/dist/skill-creator.js | 55 ++++++++++++++++++++++++++------- plugin/lib/run-eval.ts | 40 +++++++++++++++++++++++- plugin/test/run-eval.test.ts | 23 ++++++++++++++ 5 files changed, 108 insertions(+), 16 deletions(-) diff --git a/plugin/dist/build-manifest.json b/plugin/dist/build-manifest.json index c45d5e0..51501ce 100644 --- a/plugin/dist/build-manifest.json +++ b/plugin/dist/build-manifest.json @@ -1,6 +1,6 @@ { "entrypoint": "skill-creator.ts", "runtimeEntrypoint": "runtime-entry.ts", - "sourceHash": "96e74ab246995ba68e457f119ad4567862ad17fe1323a45f9b0cedd7c7d05c68", - "builtAt": "2026-07-07T17:51:45.036Z" + "sourceHash": "a88d7c7ea1bb8e303a596571dd3deba4a60966ba90dffb8758d2fba7e9481db2", + "builtAt": "2026-07-07T18:01:18.620Z" } diff --git a/plugin/dist/package.json b/plugin/dist/package.json index e069f31..6631248 100644 --- a/plugin/dist/package.json +++ b/plugin/dist/package.json @@ -1,6 +1,6 @@ { "name": "opencode-skill-creator", - "version": "0.2.21", + "version": "0.2.23", "description": "OpenCode plugin for skill creation — custom tools for validation, evaluation, description optimization, benchmarking, and review serving.", "type": "module", "main": "./dist/skill-creator.js", diff --git a/plugin/dist/skill-creator.js b/plugin/dist/skill-creator.js index 3c098f6..d0e8f03 100644 --- a/plugin/dist/skill-creator.js +++ b/plugin/dist/skill-creator.js @@ -12525,7 +12525,15 @@ function parseSkillMd(skillPath) { } // lib/run-eval.ts -import { existsSync as existsSync2, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; +import { + existsSync as existsSync2, + mkdirSync, + mkdtempSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync +} from "fs"; import { dirname, join as join3, parse as parse5 } from "path"; import { randomBytes } from "crypto"; import { tmpdir as osTmpdir } from "os"; @@ -12688,7 +12696,29 @@ function findProjectRoot(cwd) { } return cwd ?? process.cwd(); } -async function runSingleQuery(query, skillName, skillDescription, timeout, agent, triggerOnly, model) { +function symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName) { + const sourceOpenCode = join3(projectRoot, ".opencode"); + if (!existsSync2(sourceOpenCode)) + return; + const targetOpenCode = join3(evalRoot, ".opencode"); + mkdirSync(targetOpenCode, { recursive: true }); + for (const entry of readdirSync(sourceOpenCode, { withFileTypes: true })) { + if (entry.name === "skills") + continue; + symlinkSync(join3(sourceOpenCode, entry.name), join3(targetOpenCode, entry.name)); + } + const sourceSkills = join3(sourceOpenCode, "skills"); + if (!existsSync2(sourceSkills)) + return; + const targetSkills = join3(targetOpenCode, "skills"); + mkdirSync(targetSkills, { recursive: true }); + for (const entry of readdirSync(sourceSkills, { withFileTypes: true })) { + if (entry.name === skillName) + continue; + symlinkSync(join3(sourceSkills, entry.name), join3(targetSkills, entry.name)); + } +} +async function runSingleQuery(query, skillName, skillDescription, timeout, projectRoot, agent, triggerOnly, model) { if (!SKILL_NAME_RE.test(skillName)) { throw new Error(`Invalid skill name "${skillName}". Expected kebab-case (lowercase letters, numbers, and hyphens only).`); } @@ -12698,6 +12728,7 @@ async function runSingleQuery(query, skillName, skillDescription, timeout, agent const skillsDir = join3(evalRoot, ".opencode", "skills", cleanName); const skillFile = join3(skillsDir, "SKILL.md"); try { + symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName); mkdirSync(skillsDir, { recursive: true }); const indentedDesc = skillDescription.split(` `).join(` @@ -12810,7 +12841,7 @@ async function runEval(opts) { if (!job) break; try { - const triggered = await runSingleQuery(job.item.query, skillName, description, timeout, agent, triggerOnly, model); + const triggered = await runSingleQuery(job.item.query, skillName, description, timeout, projectRoot, agent, triggerOnly, model); jobResults.push({ query: job.item.query, triggered, @@ -13656,7 +13687,7 @@ Exit reason: ${exitReason}`); } // lib/aggregate.ts -import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync, statSync } from "fs"; +import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync as readdirSync2, statSync } from "fs"; import { join as join5, basename } from "path"; function compareEvalIds(a, b) { const parseNumeric = (value) => { @@ -13725,7 +13756,7 @@ function calculateStats(values) { function sortedDirs(dir, pattern) { if (!existsSync3(dir)) return []; - return readdirSync(dir).filter((name) => { + return readdirSync2(dir).filter((name) => { const full = join5(dir, name); return statSync(full).isDirectory() && (!pattern || pattern.test(name)); }).sort().map((name) => join5(dir, name)); @@ -13966,7 +13997,7 @@ import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync4, - readdirSync as readdirSync2, + readdirSync as readdirSync3, statSync as statSync2, writeFileSync as writeFileSync5 } from "fs"; @@ -14090,7 +14121,7 @@ function findRunsRecursive(root, current, runs) { return; } const skip = new Set(["node_modules", ".git", "__pycache__", "skill", "inputs"]); - const entries = readdirSync2(current).sort(); + const entries = readdirSync3(current).sort(); for (const entry of entries) { const full = join6(current, entry); if (statSync2(full).isDirectory() && !skip.has(entry)) { @@ -14144,7 +14175,7 @@ function buildRun(root, runDir) { const outputsDir = join6(runDir, "outputs"); const outputFiles = []; if (existsSync4(outputsDir) && statSync2(outputsDir).isDirectory()) { - const files = readdirSync2(outputsDir).sort(); + const files = readdirSync3(outputsDir).sort(); for (const f of files) { const full = join6(outputsDir, f); if (statSync2(full).isFile() && !METADATA_FILES.has(f)) { @@ -14512,12 +14543,12 @@ function exportStaticReview(opts) { } // lib/workflow-guard.ts -import { existsSync as existsSync5, readdirSync as readdirSync3, statSync as statSync3 } from "fs"; +import { existsSync as existsSync5, readdirSync as readdirSync4, statSync as statSync3 } from "fs"; import { basename as basename3, join as join7 } from "path"; function sortedDirs2(dir, pattern) { if (!existsSync5(dir) || !statSync3(dir).isDirectory()) return []; - return readdirSync3(dir).map((name) => join7(dir, name)).filter((full) => statSync3(full).isDirectory()).filter((full) => pattern ? pattern.test(basename3(full)) : true).sort(); + return readdirSync4(dir).map((name) => join7(dir, name)).filter((full) => statSync3(full).isDirectory()).filter((full) => pattern ? pattern.test(basename3(full)) : true).sort(); } function hasAtLeastOneRun(configDir) { const runDirs = sortedDirs2(configDir, /^run-/); @@ -14684,7 +14715,7 @@ import { copyFileSync, existsSync as existsSync7, mkdirSync as mkdirSync5, - readdirSync as readdirSync4, + readdirSync as readdirSync5, readFileSync as readFileSync6, renameSync as renameSync2, rmSync as rmSync2, @@ -14697,7 +14728,7 @@ var LEGACY_SKILL_NAME = "skill-creator"; var INSTALL_VERSION_FILE = ".opencode-skill-creator-version"; function copyDirRecursive(src, dest) { mkdirSync5(dest, { recursive: true }); - for (const entry of readdirSync4(src)) { + for (const entry of readdirSync5(src)) { const srcPath = join9(src, entry); const destPath = join9(dest, entry); if (statSync4(srcPath).isDirectory()) { diff --git a/plugin/lib/run-eval.ts b/plugin/lib/run-eval.ts index ebd8c9c..54ccca6 100644 --- a/plugin/lib/run-eval.ts +++ b/plugin/lib/run-eval.ts @@ -10,7 +10,15 @@ * whether the skill was triggered. */ -import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs" +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "fs" import { dirname, join, parse } from "path" import { randomBytes } from "crypto" import { tmpdir as osTmpdir } from "os" @@ -152,6 +160,33 @@ export function findProjectRoot(cwd?: string): string { return cwd ?? process.cwd() } +export function symlinkProjectOpenCodeConfig( + projectRoot: string, + evalRoot: string, + skillName: string, +): void { + const sourceOpenCode = join(projectRoot, ".opencode") + if (!existsSync(sourceOpenCode)) return + + const targetOpenCode = join(evalRoot, ".opencode") + mkdirSync(targetOpenCode, { recursive: true }) + + for (const entry of readdirSync(sourceOpenCode, { withFileTypes: true })) { + if (entry.name === "skills") continue + symlinkSync(join(sourceOpenCode, entry.name), join(targetOpenCode, entry.name)) + } + + const sourceSkills = join(sourceOpenCode, "skills") + if (!existsSync(sourceSkills)) return + + const targetSkills = join(targetOpenCode, "skills") + mkdirSync(targetSkills, { recursive: true }) + for (const entry of readdirSync(sourceSkills, { withFileTypes: true })) { + if (entry.name === skillName) continue + symlinkSync(join(sourceSkills, entry.name), join(targetSkills, entry.name)) + } +} + /** * Run a single query against `opencode run` and return whether the temporary * skill name appeared in the output. @@ -165,6 +200,7 @@ async function runSingleQuery( skillName: string, skillDescription: string, timeout: number, + projectRoot: string, agent: string, triggerOnly: boolean, model?: string, @@ -182,6 +218,7 @@ async function runSingleQuery( const skillFile = join(skillsDir, "SKILL.md") try { + symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName) mkdirSync(skillsDir, { recursive: true }) // Use YAML block scalar to avoid breaking on quotes in description @@ -349,6 +386,7 @@ export async function runEval(opts: RunEvalOptions): Promise { skillName, description, timeout, + projectRoot, agent, triggerOnly, model, diff --git a/plugin/test/run-eval.test.ts b/plugin/test/run-eval.test.ts index ce476a5..dc67bab 100644 --- a/plugin/test/run-eval.test.ts +++ b/plugin/test/run-eval.test.ts @@ -1,8 +1,12 @@ import { expect, test } from "bun:test" +import { mkdtempSync, mkdirSync, readlinkSync, writeFileSync } from "fs" +import { tmpdir } from "os" +import { join } from "path" import { buildEvalWarnings, buildOpenCodeRunCommand, + symlinkProjectOpenCodeConfig, type EvalResultItem, } from "../lib/run-eval" @@ -78,3 +82,22 @@ test("buildEvalWarnings returns no warnings when any should-trigger query trigge ]), ).toEqual([]) }) + +test("symlinkProjectOpenCodeConfig preserves config and excludes tested skill", () => { + const projectRoot = mkdtempSync(join(tmpdir(), "skill-eval-project-")) + const evalRoot = mkdtempSync(join(tmpdir(), "skill-eval-root-")) + const sourceOpenCode = join(projectRoot, ".opencode") + mkdirSync(join(sourceOpenCode, "skills", "tested-skill"), { recursive: true }) + mkdirSync(join(sourceOpenCode, "skills", "other-skill"), { recursive: true }) + writeFileSync(join(sourceOpenCode, "opencode.json"), "{}") + + symlinkProjectOpenCodeConfig(projectRoot, evalRoot, "tested-skill") + + expect(readlinkSync(join(evalRoot, ".opencode", "opencode.json"))).toBe( + join(sourceOpenCode, "opencode.json"), + ) + expect(readlinkSync(join(evalRoot, ".opencode", "skills", "other-skill"))).toBe( + join(sourceOpenCode, "skills", "other-skill"), + ) + expect(() => readlinkSync(join(evalRoot, ".opencode", "skills", "tested-skill"))).toThrow() +}) From 335daeb1e38406410902ee01dce4a19438c56507 Mon Sep 17 00:00:00 2001 From: Alex Ermolin Date: Wed, 8 Jul 2026 01:13:29 +0200 Subject: [PATCH 4/7] fix: pin nested eval run PWD to evalRoot opencode resolves its project root (and thus project-scoped skills) from $PWD, not the spawn cwd. The trigger eval set cwd=evalRoot but leaked the caller's PWD, so nested runs loaded the real project skill under test and attributed its triggers to that skill instead of the synthetic one, producing false 0-trigger results. Pin PWD to evalRoot so only the synthetic skill (plus global skills) are in scope. --- plugin/dist/build-manifest.json | 4 ++-- plugin/dist/skill-creator.js | 2 +- plugin/lib/run-eval.ts | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/plugin/dist/build-manifest.json b/plugin/dist/build-manifest.json index 51501ce..4f83cf8 100644 --- a/plugin/dist/build-manifest.json +++ b/plugin/dist/build-manifest.json @@ -1,6 +1,6 @@ { "entrypoint": "skill-creator.ts", "runtimeEntrypoint": "runtime-entry.ts", - "sourceHash": "a88d7c7ea1bb8e303a596571dd3deba4a60966ba90dffb8758d2fba7e9481db2", - "builtAt": "2026-07-07T18:01:18.620Z" + "sourceHash": "e6fdf822992d11a7d7905421be668183b0d1f8d366efa2d9ff6f40beb2b10ccf", + "builtAt": "2026-07-07T22:25:55.270Z" } diff --git a/plugin/dist/skill-creator.js b/plugin/dist/skill-creator.js index d0e8f03..6cac521 100644 --- a/plugin/dist/skill-creator.js +++ b/plugin/dist/skill-creator.js @@ -12789,7 +12789,7 @@ async function runSingleQuery(query, skillName, skillDescription, timeout, proje }; const result = await runProcess(cmd, { cwd: evalRoot, - env: { ...process.env }, + env: { ...process.env, PWD: evalRoot }, timeoutMs, maxStderrChars, onStdoutChunk(chunk) { diff --git a/plugin/lib/run-eval.ts b/plugin/lib/run-eval.ts index 54ccca6..148c7f0 100644 --- a/plugin/lib/run-eval.ts +++ b/plugin/lib/run-eval.ts @@ -285,7 +285,12 @@ async function runSingleQuery( const result = await runProcess(cmd, { cwd: evalRoot, - env: { ...process.env }, + // opencode resolves its project root (and thus project-scoped skills) from + // $PWD, not the spawn cwd. Leaking the caller's PWD makes the nested run + // load the *real* project skill under test (base name), so its triggers are + // attributed to that skill instead of the synthetic one — a false 0. Pin PWD + // to evalRoot so only the synthetic skill (plus global skills) are in scope. + env: { ...process.env, PWD: evalRoot }, timeoutMs, maxStderrChars, onStdoutChunk(chunk) { From 6e1daedd934642e759374d2a0ccb753597c2fe1d Mon Sep 17 00:00:00 2001 From: Anton Gulin Date: Fri, 10 Jul 2026 21:51:20 -0700 Subject: [PATCH 5/7] fix(run-eval): make config isolation Windows-safe and clean up eval test - symlinkProjectOpenCodeConfig: pass an explicit symlink type and fall back to a recursive copy when symlinks are unavailable (Windows without Developer Mode rejects them with EPERM), so eval isolation works cross-platform. - run-eval.test: wrap the symlink test in try/finally to remove its temp dirs instead of leaking them into the system temp folder. Addresses Robin review on #35 (MEDIUM: symlink type; LOW: temp-dir leak). Co-Authored-By: Claude Opus 4.8 --- plugin/dist/build-manifest.json | 4 ++-- plugin/dist/skill-creator.js | 12 ++++++++++-- plugin/lib/run-eval.ts | 28 ++++++++++++++++++++++++++-- plugin/test/run-eval.test.ts | 31 ++++++++++++++++++------------- 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/plugin/dist/build-manifest.json b/plugin/dist/build-manifest.json index 4f83cf8..a64ee19 100644 --- a/plugin/dist/build-manifest.json +++ b/plugin/dist/build-manifest.json @@ -1,6 +1,6 @@ { "entrypoint": "skill-creator.ts", "runtimeEntrypoint": "runtime-entry.ts", - "sourceHash": "e6fdf822992d11a7d7905421be668183b0d1f8d366efa2d9ff6f40beb2b10ccf", - "builtAt": "2026-07-07T22:25:55.270Z" + "sourceHash": "c1caa98dc955f5413377f96803f836dd316ca6f621c30a5db668253019d25989", + "builtAt": "2026-07-11T04:50:52.786Z" } diff --git a/plugin/dist/skill-creator.js b/plugin/dist/skill-creator.js index 6cac521..104cf61 100644 --- a/plugin/dist/skill-creator.js +++ b/plugin/dist/skill-creator.js @@ -12526,6 +12526,7 @@ function parseSkillMd(skillPath) { // lib/run-eval.ts import { + cpSync, existsSync as existsSync2, mkdirSync, mkdtempSync, @@ -12696,6 +12697,13 @@ function findProjectRoot(cwd) { } return cwd ?? process.cwd(); } +function linkOrCopyConfigEntry(source, target, isDirectory) { + try { + symlinkSync(source, target, isDirectory ? "dir" : "file"); + } catch { + cpSync(source, target, { recursive: true }); + } +} function symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName) { const sourceOpenCode = join3(projectRoot, ".opencode"); if (!existsSync2(sourceOpenCode)) @@ -12705,7 +12713,7 @@ function symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName) { for (const entry of readdirSync(sourceOpenCode, { withFileTypes: true })) { if (entry.name === "skills") continue; - symlinkSync(join3(sourceOpenCode, entry.name), join3(targetOpenCode, entry.name)); + linkOrCopyConfigEntry(join3(sourceOpenCode, entry.name), join3(targetOpenCode, entry.name), entry.isDirectory()); } const sourceSkills = join3(sourceOpenCode, "skills"); if (!existsSync2(sourceSkills)) @@ -12715,7 +12723,7 @@ function symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName) { for (const entry of readdirSync(sourceSkills, { withFileTypes: true })) { if (entry.name === skillName) continue; - symlinkSync(join3(sourceSkills, entry.name), join3(targetSkills, entry.name)); + linkOrCopyConfigEntry(join3(sourceSkills, entry.name), join3(targetSkills, entry.name), entry.isDirectory()); } } async function runSingleQuery(query, skillName, skillDescription, timeout, projectRoot, agent, triggerOnly, model) { diff --git a/plugin/lib/run-eval.ts b/plugin/lib/run-eval.ts index 148c7f0..bd303e0 100644 --- a/plugin/lib/run-eval.ts +++ b/plugin/lib/run-eval.ts @@ -11,6 +11,7 @@ */ import { + cpSync, existsSync, mkdirSync, mkdtempSync, @@ -160,6 +161,21 @@ export function findProjectRoot(cwd?: string): string { return cwd ?? process.cwd() } +/** + * Mirror a project config entry into the eval root. Prefers a symlink (cheap, + * no copy), passing an explicit type so directory links resolve correctly. On + * platforms where symlinks are unavailable — Windows without admin/Developer + * Mode rejects them with EPERM — fall back to a recursive copy so eval + * isolation still works cross-platform. + */ +function linkOrCopyConfigEntry(source: string, target: string, isDirectory: boolean): void { + try { + symlinkSync(source, target, isDirectory ? "dir" : "file") + } catch { + cpSync(source, target, { recursive: true }) + } +} + export function symlinkProjectOpenCodeConfig( projectRoot: string, evalRoot: string, @@ -173,7 +189,11 @@ export function symlinkProjectOpenCodeConfig( for (const entry of readdirSync(sourceOpenCode, { withFileTypes: true })) { if (entry.name === "skills") continue - symlinkSync(join(sourceOpenCode, entry.name), join(targetOpenCode, entry.name)) + linkOrCopyConfigEntry( + join(sourceOpenCode, entry.name), + join(targetOpenCode, entry.name), + entry.isDirectory(), + ) } const sourceSkills = join(sourceOpenCode, "skills") @@ -183,7 +203,11 @@ export function symlinkProjectOpenCodeConfig( mkdirSync(targetSkills, { recursive: true }) for (const entry of readdirSync(sourceSkills, { withFileTypes: true })) { if (entry.name === skillName) continue - symlinkSync(join(sourceSkills, entry.name), join(targetSkills, entry.name)) + linkOrCopyConfigEntry( + join(sourceSkills, entry.name), + join(targetSkills, entry.name), + entry.isDirectory(), + ) } } diff --git a/plugin/test/run-eval.test.ts b/plugin/test/run-eval.test.ts index dc67bab..d574368 100644 --- a/plugin/test/run-eval.test.ts +++ b/plugin/test/run-eval.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test" -import { mkdtempSync, mkdirSync, readlinkSync, writeFileSync } from "fs" +import { mkdtempSync, mkdirSync, readlinkSync, rmSync, writeFileSync } from "fs" import { tmpdir } from "os" import { join } from "path" @@ -86,18 +86,23 @@ test("buildEvalWarnings returns no warnings when any should-trigger query trigge test("symlinkProjectOpenCodeConfig preserves config and excludes tested skill", () => { const projectRoot = mkdtempSync(join(tmpdir(), "skill-eval-project-")) const evalRoot = mkdtempSync(join(tmpdir(), "skill-eval-root-")) - const sourceOpenCode = join(projectRoot, ".opencode") - mkdirSync(join(sourceOpenCode, "skills", "tested-skill"), { recursive: true }) - mkdirSync(join(sourceOpenCode, "skills", "other-skill"), { recursive: true }) - writeFileSync(join(sourceOpenCode, "opencode.json"), "{}") + try { + const sourceOpenCode = join(projectRoot, ".opencode") + mkdirSync(join(sourceOpenCode, "skills", "tested-skill"), { recursive: true }) + mkdirSync(join(sourceOpenCode, "skills", "other-skill"), { recursive: true }) + writeFileSync(join(sourceOpenCode, "opencode.json"), "{}") - symlinkProjectOpenCodeConfig(projectRoot, evalRoot, "tested-skill") + symlinkProjectOpenCodeConfig(projectRoot, evalRoot, "tested-skill") - expect(readlinkSync(join(evalRoot, ".opencode", "opencode.json"))).toBe( - join(sourceOpenCode, "opencode.json"), - ) - expect(readlinkSync(join(evalRoot, ".opencode", "skills", "other-skill"))).toBe( - join(sourceOpenCode, "skills", "other-skill"), - ) - expect(() => readlinkSync(join(evalRoot, ".opencode", "skills", "tested-skill"))).toThrow() + expect(readlinkSync(join(evalRoot, ".opencode", "opencode.json"))).toBe( + join(sourceOpenCode, "opencode.json"), + ) + expect(readlinkSync(join(evalRoot, ".opencode", "skills", "other-skill"))).toBe( + join(sourceOpenCode, "skills", "other-skill"), + ) + expect(() => readlinkSync(join(evalRoot, ".opencode", "skills", "tested-skill"))).toThrow() + } finally { + rmSync(projectRoot, { recursive: true, force: true }) + rmSync(evalRoot, { recursive: true, force: true }) + } }) From 98c37e3e9651639969992a476c5dff2f45125ece Mon Sep 17 00:00:00 2001 From: Anton Gulin Date: Fri, 10 Jul 2026 21:56:27 -0700 Subject: [PATCH 6/7] test(run-eval): make config-isolation test cross-platform The Windows-safe copy fallback in symlinkProjectOpenCodeConfig means mirrored entries are copies (not symlinks) where symlinks are unavailable, so the test's readlinkSync assertions would throw on Windows CI. Assert presence in all cases and the link target only when the entry is actually a symlink. Also assert the tested skill is absent outright (existsSync === false), which is stricter than the previous readlinkSync-throws check. Addresses Robin re-review on #35 (test would fail on Windows). Co-Authored-By: Claude Opus 4.8 --- plugin/test/run-eval.test.ts | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/plugin/test/run-eval.test.ts b/plugin/test/run-eval.test.ts index d574368..014775d 100644 --- a/plugin/test/run-eval.test.ts +++ b/plugin/test/run-eval.test.ts @@ -1,5 +1,13 @@ import { expect, test } from "bun:test" -import { mkdtempSync, mkdirSync, readlinkSync, rmSync, writeFileSync } from "fs" +import { + existsSync, + lstatSync, + mkdtempSync, + mkdirSync, + readlinkSync, + rmSync, + writeFileSync, +} from "fs" import { tmpdir } from "os" import { join } from "path" @@ -94,13 +102,24 @@ test("symlinkProjectOpenCodeConfig preserves config and excludes tested skill", symlinkProjectOpenCodeConfig(projectRoot, evalRoot, "tested-skill") - expect(readlinkSync(join(evalRoot, ".opencode", "opencode.json"))).toBe( - join(sourceOpenCode, "opencode.json"), - ) - expect(readlinkSync(join(evalRoot, ".opencode", "skills", "other-skill"))).toBe( - join(sourceOpenCode, "skills", "other-skill"), - ) - expect(() => readlinkSync(join(evalRoot, ".opencode", "skills", "tested-skill"))).toThrow() + // Config and sibling skills are mirrored into the eval root. Where symlinks + // are supported the entries are links pointing at the source; where the copy + // fallback runs (e.g. Windows without Developer Mode) they are plain copies. + // Assert presence either way, and the exact link target when it is a symlink. + const opencodeJson = join(evalRoot, ".opencode", "opencode.json") + expect(existsSync(opencodeJson)).toBe(true) + if (lstatSync(opencodeJson).isSymbolicLink()) { + expect(readlinkSync(opencodeJson)).toBe(join(sourceOpenCode, "opencode.json")) + } + + const otherSkill = join(evalRoot, ".opencode", "skills", "other-skill") + expect(existsSync(otherSkill)).toBe(true) + if (lstatSync(otherSkill).isSymbolicLink()) { + expect(readlinkSync(otherSkill)).toBe(join(sourceOpenCode, "skills", "other-skill")) + } + + // The skill under test is excluded entirely so it cannot steal triggers. + expect(existsSync(join(evalRoot, ".opencode", "skills", "tested-skill"))).toBe(false) } finally { rmSync(projectRoot, { recursive: true, force: true }) rmSync(evalRoot, { recursive: true, force: true }) From 6a1c1847c8f59b3aceae7e66db681ee3d84ecc4e Mon Sep 17 00:00:00 2001 From: Anton Gulin Date: Fri, 10 Jul 2026 22:23:05 -0700 Subject: [PATCH 7/7] chore(release): bump version to 0.2.25 Ship the eval-worker isolation fix. Merging to main triggers publish.yml, which releases 0.2.25 to npm (0.2.24 is already published, so without this bump the fix would land unreleased). Co-Authored-By: Claude Opus 4.8 --- plugin/dist/build-manifest.json | 2 +- plugin/dist/package.json | 2 +- plugin/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin/dist/build-manifest.json b/plugin/dist/build-manifest.json index a64ee19..7a82121 100644 --- a/plugin/dist/build-manifest.json +++ b/plugin/dist/build-manifest.json @@ -2,5 +2,5 @@ "entrypoint": "skill-creator.ts", "runtimeEntrypoint": "runtime-entry.ts", "sourceHash": "c1caa98dc955f5413377f96803f836dd316ca6f621c30a5db668253019d25989", - "builtAt": "2026-07-11T04:50:52.786Z" + "builtAt": "2026-07-11T05:22:50.992Z" } diff --git a/plugin/dist/package.json b/plugin/dist/package.json index 6631248..aa13d25 100644 --- a/plugin/dist/package.json +++ b/plugin/dist/package.json @@ -1,6 +1,6 @@ { "name": "opencode-skill-creator", - "version": "0.2.23", + "version": "0.2.25", "description": "OpenCode plugin for skill creation — custom tools for validation, evaluation, description optimization, benchmarking, and review serving.", "type": "module", "main": "./dist/skill-creator.js", diff --git a/plugin/package.json b/plugin/package.json index 4b6a8aa..aa13d25 100644 --- a/plugin/package.json +++ b/plugin/package.json @@ -1,6 +1,6 @@ { "name": "opencode-skill-creator", - "version": "0.2.24", + "version": "0.2.25", "description": "OpenCode plugin for skill creation — custom tools for validation, evaluation, description optimization, benchmarking, and review serving.", "type": "module", "main": "./dist/skill-creator.js",