diff --git a/plugin/dist/build-manifest.json b/plugin/dist/build-manifest.json index 8ab66d0..7a82121 100644 --- a/plugin/dist/build-manifest.json +++ b/plugin/dist/build-manifest.json @@ -1,6 +1,6 @@ { "entrypoint": "skill-creator.ts", "runtimeEntrypoint": "runtime-entry.ts", - "sourceHash": "7ab254f2c22233b0ced8cc832878baacd73f3bd63fa45c81019e9149bf319605", - "builtAt": "2026-07-05T23:29:06.331Z" + "sourceHash": "c1caa98dc955f5413377f96803f836dd316ca6f621c30a5db668253019d25989", + "builtAt": "2026-07-11T05:22:50.992Z" } diff --git a/plugin/dist/package.json b/plugin/dist/package.json index e069f31..aa13d25 100644 --- a/plugin/dist/package.json +++ b/plugin/dist/package.json @@ -1,6 +1,6 @@ { "name": "opencode-skill-creator", - "version": "0.2.21", + "version": "0.2.25", "description": "OpenCode plugin for skill creation — custom tools for validation, evaluation, description optimization, benchmarking, and review serving.", "type": "module", "main": "./dist/skill-creator.js", diff --git a/plugin/dist/skill-creator.js b/plugin/dist/skill-creator.js index 2013d18..104cf61 100644 --- a/plugin/dist/skill-creator.js +++ b/plugin/dist/skill-creator.js @@ -12525,9 +12525,19 @@ function parseSkillMd(skillPath) { } // lib/run-eval.ts -import { existsSync as existsSync2, mkdirSync, rmSync, writeFileSync } from "fs"; +import { + cpSync, + existsSync as existsSync2, + mkdirSync, + mkdtempSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync +} from "fs"; import { dirname, join as join3, parse as parse5 } from "path"; import { randomBytes } from "crypto"; +import { tmpdir as osTmpdir } from "os"; // lib/process.ts import { spawn } from "child_process"; @@ -12687,15 +12697,46 @@ function findProjectRoot(cwd) { } return cwd ?? process.cwd(); } +function linkOrCopyConfigEntry(source, target, isDirectory) { + try { + symlinkSync(source, target, isDirectory ? "dir" : "file"); + } catch { + cpSync(source, target, { recursive: true }); + } +} +function symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName) { + const sourceOpenCode = join3(projectRoot, ".opencode"); + if (!existsSync2(sourceOpenCode)) + return; + const targetOpenCode = join3(evalRoot, ".opencode"); + mkdirSync(targetOpenCode, { recursive: true }); + for (const entry of readdirSync(sourceOpenCode, { withFileTypes: true })) { + if (entry.name === "skills") + continue; + linkOrCopyConfigEntry(join3(sourceOpenCode, entry.name), join3(targetOpenCode, entry.name), entry.isDirectory()); + } + const sourceSkills = join3(sourceOpenCode, "skills"); + if (!existsSync2(sourceSkills)) + return; + const targetSkills = join3(targetOpenCode, "skills"); + mkdirSync(targetSkills, { recursive: true }); + for (const entry of readdirSync(sourceSkills, { withFileTypes: true })) { + if (entry.name === skillName) + continue; + linkOrCopyConfigEntry(join3(sourceSkills, entry.name), join3(targetSkills, entry.name), entry.isDirectory()); + } +} async function runSingleQuery(query, skillName, skillDescription, timeout, projectRoot, agent, triggerOnly, model) { if (!SKILL_NAME_RE.test(skillName)) { throw new Error(`Invalid skill name "${skillName}". Expected kebab-case (lowercase letters, numbers, and hyphens only).`); } const uniqueId = randomBytes(4).toString("hex"); const cleanName = `${skillName}-skill-${uniqueId}`; - const skillsDir = join3(projectRoot, ".opencode", "skills", cleanName); + const evalRoot = mkdtempSync(join3(osTmpdir(), "opencode-skill-eval-")); + const skillsDir = join3(evalRoot, ".opencode", "skills", cleanName); const skillFile = join3(skillsDir, "SKILL.md"); try { + symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName); mkdirSync(skillsDir, { recursive: true }); const indentedDesc = skillDescription.split(` `).join(` @@ -12755,8 +12796,8 @@ async function runSingleQuery(query, skillName, skillDescription, timeout, proje } }; const result = await runProcess(cmd, { - cwd: projectRoot, - env: { ...process.env }, + cwd: evalRoot, + env: { ...process.env, PWD: evalRoot }, timeoutMs, maxStderrChars, onStdoutChunk(chunk) { @@ -12775,8 +12816,8 @@ async function runSingleQuery(query, skillName, skillDescription, timeout, proje } return triggered; } finally { - if (existsSync2(skillsDir)) { - rmSync(skillsDir, { recursive: true, force: true }); + if (existsSync2(evalRoot)) { + rmSync(evalRoot, { recursive: true, force: true }); } } } @@ -13654,7 +13695,7 @@ Exit reason: ${exitReason}`); } // lib/aggregate.ts -import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync, statSync } from "fs"; +import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync as readdirSync2, statSync } from "fs"; import { join as join5, basename } from "path"; function compareEvalIds(a, b) { const parseNumeric = (value) => { @@ -13723,7 +13764,7 @@ function calculateStats(values) { function sortedDirs(dir, pattern) { if (!existsSync3(dir)) return []; - return readdirSync(dir).filter((name) => { + return readdirSync2(dir).filter((name) => { const full = join5(dir, name); return statSync(full).isDirectory() && (!pattern || pattern.test(name)); }).sort().map((name) => join5(dir, name)); @@ -13964,7 +14005,7 @@ import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync4, - readdirSync as readdirSync2, + readdirSync as readdirSync3, statSync as statSync2, writeFileSync as writeFileSync5 } from "fs"; @@ -14088,7 +14129,7 @@ function findRunsRecursive(root, current, runs) { return; } const skip = new Set(["node_modules", ".git", "__pycache__", "skill", "inputs"]); - const entries = readdirSync2(current).sort(); + const entries = readdirSync3(current).sort(); for (const entry of entries) { const full = join6(current, entry); if (statSync2(full).isDirectory() && !skip.has(entry)) { @@ -14142,7 +14183,7 @@ function buildRun(root, runDir) { const outputsDir = join6(runDir, "outputs"); const outputFiles = []; if (existsSync4(outputsDir) && statSync2(outputsDir).isDirectory()) { - const files = readdirSync2(outputsDir).sort(); + const files = readdirSync3(outputsDir).sort(); for (const f of files) { const full = join6(outputsDir, f); if (statSync2(full).isFile() && !METADATA_FILES.has(f)) { @@ -14510,12 +14551,12 @@ function exportStaticReview(opts) { } // lib/workflow-guard.ts -import { existsSync as existsSync5, readdirSync as readdirSync3, statSync as statSync3 } from "fs"; +import { existsSync as existsSync5, readdirSync as readdirSync4, statSync as statSync3 } from "fs"; import { basename as basename3, join as join7 } from "path"; function sortedDirs2(dir, pattern) { if (!existsSync5(dir) || !statSync3(dir).isDirectory()) return []; - return readdirSync3(dir).map((name) => join7(dir, name)).filter((full) => statSync3(full).isDirectory()).filter((full) => pattern ? pattern.test(basename3(full)) : true).sort(); + return readdirSync4(dir).map((name) => join7(dir, name)).filter((full) => statSync3(full).isDirectory()).filter((full) => pattern ? pattern.test(basename3(full)) : true).sort(); } function hasAtLeastOneRun(configDir) { const runDirs = sortedDirs2(configDir, /^run-/); @@ -14682,7 +14723,7 @@ import { copyFileSync, existsSync as existsSync7, mkdirSync as mkdirSync5, - readdirSync as readdirSync4, + readdirSync as readdirSync5, readFileSync as readFileSync6, renameSync as renameSync2, rmSync as rmSync2, @@ -14695,7 +14736,7 @@ var LEGACY_SKILL_NAME = "skill-creator"; var INSTALL_VERSION_FILE = ".opencode-skill-creator-version"; function copyDirRecursive(src, dest) { mkdirSync5(dest, { recursive: true }); - for (const entry of readdirSync4(src)) { + for (const entry of readdirSync5(src)) { const srcPath = join9(src, entry); const destPath = join9(dest, entry); if (statSync4(srcPath).isDirectory()) { diff --git a/plugin/lib/run-eval.ts b/plugin/lib/run-eval.ts index 6de4b97..bd303e0 100644 --- a/plugin/lib/run-eval.ts +++ b/plugin/lib/run-eval.ts @@ -10,9 +10,19 @@ * whether the skill was triggered. */ -import { existsSync, mkdirSync, rmSync, writeFileSync } from "fs" +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "fs" import { dirname, join, parse } from "path" import { randomBytes } from "crypto" +import { tmpdir as osTmpdir } from "os" import { isFailedProcess, runProcess } from "./process" @@ -152,8 +162,62 @@ export function findProjectRoot(cwd?: string): string { } /** - * Run a single query against `opencode run` and return whether the - * temporary skill name appeared in the output. + * Mirror a project config entry into the eval root. Prefers a symlink (cheap, + * no copy), passing an explicit type so directory links resolve correctly. On + * platforms where symlinks are unavailable — Windows without admin/Developer + * Mode rejects them with EPERM — fall back to a recursive copy so eval + * isolation still works cross-platform. + */ +function linkOrCopyConfigEntry(source: string, target: string, isDirectory: boolean): void { + try { + symlinkSync(source, target, isDirectory ? "dir" : "file") + } catch { + cpSync(source, target, { recursive: true }) + } +} + +export function symlinkProjectOpenCodeConfig( + projectRoot: string, + evalRoot: string, + skillName: string, +): void { + const sourceOpenCode = join(projectRoot, ".opencode") + if (!existsSync(sourceOpenCode)) return + + const targetOpenCode = join(evalRoot, ".opencode") + mkdirSync(targetOpenCode, { recursive: true }) + + for (const entry of readdirSync(sourceOpenCode, { withFileTypes: true })) { + if (entry.name === "skills") continue + linkOrCopyConfigEntry( + join(sourceOpenCode, entry.name), + join(targetOpenCode, entry.name), + entry.isDirectory(), + ) + } + + const sourceSkills = join(sourceOpenCode, "skills") + if (!existsSync(sourceSkills)) return + + const targetSkills = join(targetOpenCode, "skills") + mkdirSync(targetSkills, { recursive: true }) + for (const entry of readdirSync(sourceSkills, { withFileTypes: true })) { + if (entry.name === skillName) continue + linkOrCopyConfigEntry( + join(sourceSkills, entry.name), + join(targetSkills, entry.name), + entry.isDirectory(), + ) + } +} + +/** + * Run a single query against `opencode run` and return whether the temporary + * skill name appeared in the output. + * + * Each run gets its own temporary OpenCode project root. This keeps parallel + * workers from seeing one another's synthetic skills and producing false + * negatives when a sibling synthetic skill is selected. */ async function runSingleQuery( query: string, @@ -173,10 +237,12 @@ async function runSingleQuery( const uniqueId = randomBytes(4).toString("hex") const cleanName = `${skillName}-skill-${uniqueId}` - const skillsDir = join(projectRoot, ".opencode", "skills", cleanName) + const evalRoot = mkdtempSync(join(osTmpdir(), "opencode-skill-eval-")) + const skillsDir = join(evalRoot, ".opencode", "skills", cleanName) const skillFile = join(skillsDir, "SKILL.md") try { + symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName) mkdirSync(skillsDir, { recursive: true }) // Use YAML block scalar to avoid breaking on quotes in description @@ -242,8 +308,13 @@ async function runSingleQuery( } const result = await runProcess(cmd, { - cwd: projectRoot, - env: { ...process.env }, + cwd: evalRoot, + // opencode resolves its project root (and thus project-scoped skills) from + // $PWD, not the spawn cwd. Leaking the caller's PWD makes the nested run + // load the *real* project skill under test (base name), so its triggers are + // attributed to that skill instead of the synthetic one — a false 0. Pin PWD + // to evalRoot so only the synthetic skill (plus global skills) are in scope. + env: { ...process.env, PWD: evalRoot }, timeoutMs, maxStderrChars, onStdoutChunk(chunk) { @@ -270,9 +341,9 @@ async function runSingleQuery( return triggered } finally { - // Clean up the temporary skill directory - if (existsSync(skillsDir)) { - rmSync(skillsDir, { recursive: true, force: true }) + // Clean up the isolated temporary project root. + if (existsSync(evalRoot)) { + rmSync(evalRoot, { recursive: true, force: true }) } } } diff --git a/plugin/package.json b/plugin/package.json index 4b6a8aa..aa13d25 100644 --- a/plugin/package.json +++ b/plugin/package.json @@ -1,6 +1,6 @@ { "name": "opencode-skill-creator", - "version": "0.2.24", + "version": "0.2.25", "description": "OpenCode plugin for skill creation — custom tools for validation, evaluation, description optimization, benchmarking, and review serving.", "type": "module", "main": "./dist/skill-creator.js", diff --git a/plugin/test/run-eval.test.ts b/plugin/test/run-eval.test.ts index ce476a5..014775d 100644 --- a/plugin/test/run-eval.test.ts +++ b/plugin/test/run-eval.test.ts @@ -1,8 +1,20 @@ import { expect, test } from "bun:test" +import { + existsSync, + lstatSync, + mkdtempSync, + mkdirSync, + readlinkSync, + rmSync, + writeFileSync, +} from "fs" +import { tmpdir } from "os" +import { join } from "path" import { buildEvalWarnings, buildOpenCodeRunCommand, + symlinkProjectOpenCodeConfig, type EvalResultItem, } from "../lib/run-eval" @@ -78,3 +90,38 @@ test("buildEvalWarnings returns no warnings when any should-trigger query trigge ]), ).toEqual([]) }) + +test("symlinkProjectOpenCodeConfig preserves config and excludes tested skill", () => { + const projectRoot = mkdtempSync(join(tmpdir(), "skill-eval-project-")) + const evalRoot = mkdtempSync(join(tmpdir(), "skill-eval-root-")) + try { + const sourceOpenCode = join(projectRoot, ".opencode") + mkdirSync(join(sourceOpenCode, "skills", "tested-skill"), { recursive: true }) + mkdirSync(join(sourceOpenCode, "skills", "other-skill"), { recursive: true }) + writeFileSync(join(sourceOpenCode, "opencode.json"), "{}") + + symlinkProjectOpenCodeConfig(projectRoot, evalRoot, "tested-skill") + + // Config and sibling skills are mirrored into the eval root. Where symlinks + // are supported the entries are links pointing at the source; where the copy + // fallback runs (e.g. Windows without Developer Mode) they are plain copies. + // Assert presence either way, and the exact link target when it is a symlink. + const opencodeJson = join(evalRoot, ".opencode", "opencode.json") + expect(existsSync(opencodeJson)).toBe(true) + if (lstatSync(opencodeJson).isSymbolicLink()) { + expect(readlinkSync(opencodeJson)).toBe(join(sourceOpenCode, "opencode.json")) + } + + const otherSkill = join(evalRoot, ".opencode", "skills", "other-skill") + expect(existsSync(otherSkill)).toBe(true) + if (lstatSync(otherSkill).isSymbolicLink()) { + expect(readlinkSync(otherSkill)).toBe(join(sourceOpenCode, "skills", "other-skill")) + } + + // The skill under test is excluded entirely so it cannot steal triggers. + expect(existsSync(join(evalRoot, ".opencode", "skills", "tested-skill"))).toBe(false) + } finally { + rmSync(projectRoot, { recursive: true, force: true }) + rmSync(evalRoot, { recursive: true, force: true }) + } +})