fix: isolate skill eval workers

plugin/dist/build-manifest.json

@@ -1,6 +1,6 @@
 {
   "entrypoint": "skill-creator.ts",
   "runtimeEntrypoint": "runtime-entry.ts",
-  "sourceHash": "7ab254f2c22233b0ced8cc832878baacd73f3bd63fa45c81019e9149bf319605",
-  "builtAt": "2026-07-05T23:29:06.331Z"
+  "sourceHash": "c1caa98dc955f5413377f96803f836dd316ca6f621c30a5db668253019d25989",
+  "builtAt": "2026-07-11T05:22:50.992Z"
 }

plugin/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-skill-creator",
-  "version": "0.2.21",
+  "version": "0.2.25",
   "description": "OpenCode plugin for skill creation — custom tools for validation, evaluation, description optimization, benchmarking, and review serving.",
   "type": "module",
   "main": "./dist/skill-creator.js",

plugin/dist/skill-creator.js

@@ -12525,9 +12525,19 @@ function parseSkillMd(skillPath) {
 }
 
 // lib/run-eval.ts
-import { existsSync as existsSync2, mkdirSync, rmSync, writeFileSync } from "fs";
+import {
+  cpSync,
+  existsSync as existsSync2,
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync
+} from "fs";
 import { dirname, join as join3, parse as parse5 } from "path";
 import { randomBytes } from "crypto";
+import { tmpdir as osTmpdir } from "os";
 
 // lib/process.ts
 import { spawn } from "child_process";
@@ -12687,15 +12697,46 @@ function findProjectRoot(cwd) {
   }
   return cwd ?? process.cwd();
 }
+function linkOrCopyConfigEntry(source, target, isDirectory) {
+  try {
+    symlinkSync(source, target, isDirectory ? "dir" : "file");
+  } catch {
+    cpSync(source, target, { recursive: true });
+  }
+}
+function symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName) {
+  const sourceOpenCode = join3(projectRoot, ".opencode");
+  if (!existsSync2(sourceOpenCode))
+    return;
+  const targetOpenCode = join3(evalRoot, ".opencode");
+  mkdirSync(targetOpenCode, { recursive: true });
+  for (const entry of readdirSync(sourceOpenCode, { withFileTypes: true })) {
+    if (entry.name === "skills")
+      continue;
+    linkOrCopyConfigEntry(join3(sourceOpenCode, entry.name), join3(targetOpenCode, entry.name), entry.isDirectory());
+  }
+  const sourceSkills = join3(sourceOpenCode, "skills");
+  if (!existsSync2(sourceSkills))
+    return;
+  const targetSkills = join3(targetOpenCode, "skills");
+  mkdirSync(targetSkills, { recursive: true });
+  for (const entry of readdirSync(sourceSkills, { withFileTypes: true })) {
+    if (entry.name === skillName)
+      continue;
+    linkOrCopyConfigEntry(join3(sourceSkills, entry.name), join3(targetSkills, entry.name), entry.isDirectory());
+  }
+}
 async function runSingleQuery(query, skillName, skillDescription, timeout, projectRoot, agent, triggerOnly, model) {
   if (!SKILL_NAME_RE.test(skillName)) {
     throw new Error(`Invalid skill name "${skillName}". Expected kebab-case (lowercase letters, numbers, and hyphens only).`);
   }
   const uniqueId = randomBytes(4).toString("hex");
   const cleanName = `${skillName}-skill-${uniqueId}`;
-  const skillsDir = join3(projectRoot, ".opencode", "skills", cleanName);
+  const evalRoot = mkdtempSync(join3(osTmpdir(), "opencode-skill-eval-"));
+  const skillsDir = join3(evalRoot, ".opencode", "skills", cleanName);
   const skillFile = join3(skillsDir, "SKILL.md");
   try {
+    symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName);
     mkdirSync(skillsDir, { recursive: true });
     const indentedDesc = skillDescription.split(`
 `).join(`
@@ -12755,8 +12796,8 @@ async function runSingleQuery(query, skillName, skillDescription, timeout, proje
       }
     };
     const result = await runProcess(cmd, {
-      cwd: projectRoot,
-      env: { ...process.env },
+      cwd: evalRoot,
+      env: { ...process.env, PWD: evalRoot },
       timeoutMs,
       maxStderrChars,
       onStdoutChunk(chunk) {
@@ -12775,8 +12816,8 @@ async function runSingleQuery(query, skillName, skillDescription, timeout, proje
     }
     return triggered;
   } finally {
-    if (existsSync2(skillsDir)) {
-      rmSync(skillsDir, { recursive: true, force: true });
+    if (existsSync2(evalRoot)) {
+      rmSync(evalRoot, { recursive: true, force: true });
     }
   }
 }
@@ -13654,7 +13695,7 @@ Exit reason: ${exitReason}`);
 }
 
 // lib/aggregate.ts
-import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync, statSync } from "fs";
+import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync as readdirSync2, statSync } from "fs";
 import { join as join5, basename } from "path";
 function compareEvalIds(a, b) {
   const parseNumeric = (value) => {
@@ -13723,7 +13764,7 @@ function calculateStats(values) {
 function sortedDirs(dir, pattern) {
   if (!existsSync3(dir))
     return [];
-  return readdirSync(dir).filter((name) => {
+  return readdirSync2(dir).filter((name) => {
     const full = join5(dir, name);
     return statSync(full).isDirectory() && (!pattern || pattern.test(name));
   }).sort().map((name) => join5(dir, name));
@@ -13964,7 +14005,7 @@ import {
   existsSync as existsSync4,
   mkdirSync as mkdirSync3,
   readFileSync as readFileSync4,
-  readdirSync as readdirSync2,
+  readdirSync as readdirSync3,
   statSync as statSync2,
   writeFileSync as writeFileSync5
 } from "fs";
@@ -14088,7 +14129,7 @@ function findRunsRecursive(root, current, runs) {
     return;
   }
   const skip = new Set(["node_modules", ".git", "__pycache__", "skill", "inputs"]);
-  const entries = readdirSync2(current).sort();
+  const entries = readdirSync3(current).sort();
   for (const entry of entries) {
     const full = join6(current, entry);
     if (statSync2(full).isDirectory() && !skip.has(entry)) {
@@ -14142,7 +14183,7 @@ function buildRun(root, runDir) {
   const outputsDir = join6(runDir, "outputs");
   const outputFiles = [];
   if (existsSync4(outputsDir) && statSync2(outputsDir).isDirectory()) {
-    const files = readdirSync2(outputsDir).sort();
+    const files = readdirSync3(outputsDir).sort();
     for (const f of files) {
       const full = join6(outputsDir, f);
       if (statSync2(full).isFile() && !METADATA_FILES.has(f)) {
@@ -14510,12 +14551,12 @@ function exportStaticReview(opts) {
 }
 
 // lib/workflow-guard.ts
-import { existsSync as existsSync5, readdirSync as readdirSync3, statSync as statSync3 } from "fs";
+import { existsSync as existsSync5, readdirSync as readdirSync4, statSync as statSync3 } from "fs";
 import { basename as basename3, join as join7 } from "path";
 function sortedDirs2(dir, pattern) {
   if (!existsSync5(dir) || !statSync3(dir).isDirectory())
     return [];
-  return readdirSync3(dir).map((name) => join7(dir, name)).filter((full) => statSync3(full).isDirectory()).filter((full) => pattern ? pattern.test(basename3(full)) : true).sort();
+  return readdirSync4(dir).map((name) => join7(dir, name)).filter((full) => statSync3(full).isDirectory()).filter((full) => pattern ? pattern.test(basename3(full)) : true).sort();
 }
 function hasAtLeastOneRun(configDir) {
   const runDirs = sortedDirs2(configDir, /^run-/);
@@ -14682,7 +14723,7 @@ import {
   copyFileSync,
   existsSync as existsSync7,
   mkdirSync as mkdirSync5,
-  readdirSync as readdirSync4,
+  readdirSync as readdirSync5,
   readFileSync as readFileSync6,
   renameSync as renameSync2,
   rmSync as rmSync2,
@@ -14695,7 +14736,7 @@ var LEGACY_SKILL_NAME = "skill-creator";
 var INSTALL_VERSION_FILE = ".opencode-skill-creator-version";
 function copyDirRecursive(src, dest) {
   mkdirSync5(dest, { recursive: true });
-  for (const entry of readdirSync4(src)) {
+  for (const entry of readdirSync5(src)) {
     const srcPath = join9(src, entry);
     const destPath = join9(dest, entry);
     if (statSync4(srcPath).isDirectory()) {

plugin/lib/run-eval.ts

@@ -10,9 +10,19 @@
  * whether the skill was triggered.
  */
 
-import { existsSync, mkdirSync, rmSync, writeFileSync } from "fs"
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from "fs"
 import { dirname, join, parse } from "path"
 import { randomBytes } from "crypto"
+import { tmpdir as osTmpdir } from "os"
 
 import { isFailedProcess, runProcess } from "./process"
 
@@ -152,8 +162,62 @@ export function findProjectRoot(cwd?: string): string {
 }
 
 /**
- * Run a single query against `opencode run` and return whether the
- * temporary skill name appeared in the output.
+ * Mirror a project config entry into the eval root. Prefers a symlink (cheap,
+ * no copy), passing an explicit type so directory links resolve correctly. On
+ * platforms where symlinks are unavailable — Windows without admin/Developer
+ * Mode rejects them with EPERM — fall back to a recursive copy so eval
+ * isolation still works cross-platform.
+ */
+function linkOrCopyConfigEntry(source: string, target: string, isDirectory: boolean): void {
+  try {
+    symlinkSync(source, target, isDirectory ? "dir" : "file")
+  } catch {
+    cpSync(source, target, { recursive: true })
+  }
+}
+
+export function symlinkProjectOpenCodeConfig(
+  projectRoot: string,
+  evalRoot: string,
+  skillName: string,
+): void {
+  const sourceOpenCode = join(projectRoot, ".opencode")
+  if (!existsSync(sourceOpenCode)) return
+
+  const targetOpenCode = join(evalRoot, ".opencode")
+  mkdirSync(targetOpenCode, { recursive: true })
+
+  for (const entry of readdirSync(sourceOpenCode, { withFileTypes: true })) {
+    if (entry.name === "skills") continue
+    linkOrCopyConfigEntry(
+      join(sourceOpenCode, entry.name),
+      join(targetOpenCode, entry.name),
+      entry.isDirectory(),
+    )
+  }
+
+  const sourceSkills = join(sourceOpenCode, "skills")
+  if (!existsSync(sourceSkills)) return
+
+  const targetSkills = join(targetOpenCode, "skills")
+  mkdirSync(targetSkills, { recursive: true })
+  for (const entry of readdirSync(sourceSkills, { withFileTypes: true })) {
+    if (entry.name === skillName) continue
+    linkOrCopyConfigEntry(
+      join(sourceSkills, entry.name),
+      join(targetSkills, entry.name),
+      entry.isDirectory(),
+    )
+  }
+}
+
+/**
+ * Run a single query against `opencode run` and return whether the temporary
+ * skill name appeared in the output.
+ *
+ * Each run gets its own temporary OpenCode project root. This keeps parallel
+ * workers from seeing one another's synthetic skills and producing false
+ * negatives when a sibling synthetic skill is selected.
  */
 async function runSingleQuery(
   query: string,
@@ -173,10 +237,12 @@ async function runSingleQuery(
 
   const uniqueId = randomBytes(4).toString("hex")
   const cleanName = `${skillName}-skill-${uniqueId}`
-  const skillsDir = join(projectRoot, ".opencode", "skills", cleanName)
+  const evalRoot = mkdtempSync(join(osTmpdir(), "opencode-skill-eval-"))
+  const skillsDir = join(evalRoot, ".opencode", "skills", cleanName)
   const skillFile = join(skillsDir, "SKILL.md")
 
   try {
+    symlinkProjectOpenCodeConfig(projectRoot, evalRoot, skillName)
     mkdirSync(skillsDir, { recursive: true })
 
     // Use YAML block scalar to avoid breaking on quotes in description
@@ -242,8 +308,13 @@ async function runSingleQuery(
     }
 
     const result = await runProcess(cmd, {
-      cwd: projectRoot,
-      env: { ...process.env },
+      cwd: evalRoot,
+      // opencode resolves its project root (and thus project-scoped skills) from
+      // $PWD, not the spawn cwd. Leaking the caller's PWD makes the nested run
+      // load the *real* project skill under test (base name), so its triggers are
+      // attributed to that skill instead of the synthetic one — a false 0. Pin PWD
+      // to evalRoot so only the synthetic skill (plus global skills) are in scope.
+      env: { ...process.env, PWD: evalRoot },
       timeoutMs,
       maxStderrChars,
       onStdoutChunk(chunk) {
@@ -270,9 +341,9 @@ async function runSingleQuery(
 
     return triggered
   } finally {
-    // Clean up the temporary skill directory
-    if (existsSync(skillsDir)) {
-      rmSync(skillsDir, { recursive: true, force: true })
+    // Clean up the isolated temporary project root.
+    if (existsSync(evalRoot)) {
+      rmSync(evalRoot, { recursive: true, force: true })
     }
   }
 }

plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-skill-creator",
-  "version": "0.2.24",
+  "version": "0.2.25",
   "description": "OpenCode plugin for skill creation — custom tools for validation, evaluation, description optimization, benchmarking, and review serving.",
   "type": "module",
   "main": "./dist/skill-creator.js",

plugin/test/run-eval.test.ts

@@ -1,8 +1,20 @@
 import { expect, test } from "bun:test"
+import {
+  existsSync,
+  lstatSync,
+  mkdtempSync,
+  mkdirSync,
+  readlinkSync,
+  rmSync,
+  writeFileSync,
+} from "fs"
+import { tmpdir } from "os"
+import { join } from "path"
 
 import {
   buildEvalWarnings,
   buildOpenCodeRunCommand,
+  symlinkProjectOpenCodeConfig,
   type EvalResultItem,
 } from "../lib/run-eval"
 
@@ -78,3 +90,38 @@ test("buildEvalWarnings returns no warnings when any should-trigger query trigge
     ]),
   ).toEqual([])
 })
+
+test("symlinkProjectOpenCodeConfig preserves config and excludes tested skill", () => {
+  const projectRoot = mkdtempSync(join(tmpdir(), "skill-eval-project-"))
+  const evalRoot = mkdtempSync(join(tmpdir(), "skill-eval-root-"))
+  try {
+    const sourceOpenCode = join(projectRoot, ".opencode")
+    mkdirSync(join(sourceOpenCode, "skills", "tested-skill"), { recursive: true })
+    mkdirSync(join(sourceOpenCode, "skills", "other-skill"), { recursive: true })
+    writeFileSync(join(sourceOpenCode, "opencode.json"), "{}")
+
+    symlinkProjectOpenCodeConfig(projectRoot, evalRoot, "tested-skill")
+
+    // Config and sibling skills are mirrored into the eval root. Where symlinks
+    // are supported the entries are links pointing at the source; where the copy
+    // fallback runs (e.g. Windows without Developer Mode) they are plain copies.
+    // Assert presence either way, and the exact link target when it is a symlink.
+    const opencodeJson = join(evalRoot, ".opencode", "opencode.json")
+    expect(existsSync(opencodeJson)).toBe(true)
+    if (lstatSync(opencodeJson).isSymbolicLink()) {
+      expect(readlinkSync(opencodeJson)).toBe(join(sourceOpenCode, "opencode.json"))
+    }
+
+    const otherSkill = join(evalRoot, ".opencode", "skills", "other-skill")
+    expect(existsSync(otherSkill)).toBe(true)
+    if (lstatSync(otherSkill).isSymbolicLink()) {
+      expect(readlinkSync(otherSkill)).toBe(join(sourceOpenCode, "skills", "other-skill"))
+    }
+
+    // The skill under test is excluded entirely so it cannot steal triggers.
+    expect(existsSync(join(evalRoot, ".opencode", "skills", "tested-skill"))).toBe(false)
+  } finally {
+    rmSync(projectRoot, { recursive: true, force: true })
+    rmSync(evalRoot, { recursive: true, force: true })
+  }
+})