diff --git a/src/skill.mjs b/src/skill.mjs index 8dd7bd6..37030dd 100644 Binary files a/src/skill.mjs and b/src/skill.mjs differ diff --git a/support/marketplace-watch.mjs b/support/marketplace-watch.mjs index bffeb4b..b2c4fb3 100644 --- a/support/marketplace-watch.mjs +++ b/support/marketplace-watch.mjs @@ -16,16 +16,41 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { scan, skillHash, discoverMarketplaceSkills } from '../src/index.mjs'; +import { scan, scanSkill, skillHash, discoverMarketplaceSkills } from '../src/index.mjs'; const ADVISORY_ROWS_SHOWN = 80; // WATCH.md stays readable; results.json has every row // Reviewed-benign findings, accepted with truecopy's `--force` semantics: each // entry accepts a skill's findings for EXACTLY the bytes reviewed (keyed by // skill hash). Any drift — or new findings on other skills — flags as usual. +// High-churn vendor skills can opt into per-file granularity (#68) with +// `"granularity": "finding-files"` + `"files": { : , … }`: the +// acceptance is keyed to the reviewed finding-bearing files instead of the +// whole-skill hash, so an unrelated upstream docs release no longer lapses it. let accepted = {}; try { accepted = JSON.parse(fs.readFileSync(fileURLToPath(new URL('watch-accepted.json', import.meta.url)), 'utf8')); } catch { /* no accept file = accept nothing */ } +// Does an accept entry still cover this scanned skill? +// whole-skill (default) — `hash` must equal today's skill hash: any byte +// anywhere re-flags. Fail-closed and churn-prone by design. +// finding-files — every file listed in `files` that is still present at its +// reviewed bytes is EXCLUDED, and the REMAINDER of the skill must scan clean +// on the same detection pipeline. Since we only get here when the full skill +// flagged, a clean remainder proves every current finding lives in a reviewed, +// byte-identical file: a reviewed file that drifts rejoins the scan (its +// fixtures re-flag), and a new finding in a new or changed file flags on its +// own. An entry with no usable `files` map excludes nothing — the remainder is +// the whole flagged skill, so it fails closed. +function covers(a, skill) { + if (a.granularity !== 'finding-files') return a.hash === skillHash(skill); + const reviewed = (a.files && typeof a.files === 'object') ? a.files : {}; + const hashOf = Object.fromEntries((skill.files || []).map((f) => [f.path, f.hash])); + const rest = (skill.scanPieces || []).filter((p) => hashOf[p.path] !== reviewed[p.path]); + if (!rest.length) return true; + const s = scanSkill({ kind: 'skill', name: skill.name, scanTargets: [{ name: skill.name, description: rest.map((p) => p.text).join('\n') }] }); + return s.verdict === 'clean'; +} + const [rootArg, outDir] = process.argv.slice(2); if (!rootArg || !outDir) { console.error('usage: marketplace-watch.mjs '); @@ -86,7 +111,7 @@ for (const s of skills) { if (r.verdict !== 'clean') { const findings = r.findings.map((f) => `${f.tool}: ${f.flags.join('; ')}`); const a = accepted[s.name]; - if (a && a.hash === skillHash(r.skill)) acceptedRows.push({ name: s.name, findings, class: a.class, note: a.note }); + if (a && covers(a, r.skill)) acceptedRows.push({ name: s.name, findings, class: a.class, note: a.note, ...(a.granularity ? { granularity: a.granularity } : {}) }); else flaggedRows.push({ name: s.name, verdict: r.verdict, findings }); } else if (advisories.length) { advisoryRows.push({ name: s.name, advisories }); @@ -134,10 +159,10 @@ if (poisoned) { if (acceptedRows.length) { md.push('## Accepted findings (reviewed benign)'); md.push(''); - md.push('Skills whose findings were manually reviewed and accepted for **exactly these bytes** ([watch-accepted.json](https://github.com/askalf/truecopy/blob/master/support/watch-accepted.json), truecopy\'s `--force` semantics) — any content change re-flags them.'); + md.push('Skills whose findings were manually reviewed and accepted for **exactly these bytes** ([watch-accepted.json](https://github.com/askalf/truecopy/blob/master/support/watch-accepted.json), truecopy\'s `--force` semantics) — any content change re-flags them. Entries marked *per-file* key the acceptance to the reviewed finding-bearing files instead: those files changing re-flags, and everything else in the skill must still scan clean, but unrelated upstream churn no longer lapses the review.'); md.push(''); for (const r of acceptedRows) { - md.push(`- **${r.name}** — ${r.findings.join(' · ')} — *${r.class}${r.note ? `: ${r.note}` : ''}*`); + md.push(`- **${r.name}** — ${r.findings.join(' · ')} — *${r.class}${r.note ? `: ${r.note}` : ''}*${r.granularity === 'finding-files' ? ' *(per-file)*' : ''}`); } md.push(''); } diff --git a/support/watch-accept.mjs b/support/watch-accept.mjs new file mode 100644 index 0000000..51c9cec --- /dev/null +++ b/support/watch-accept.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +// Author a watch-accepted.json entry for a skill you just hand-reviewed — +// prints the entry JSON to paste under its catalog name (fill in class/note). +// +// node support/watch-accept.mjs # whole-skill hash entry +// node support/watch-accept.mjs --files # per-file granularity (#68) +// +// --files keys the acceptance to the finding-bearing files: each file is scanned +// alone to attribute the findings, then the attribution is verified with exactly +// the predicate the watch uses (everything OUTSIDE the recorded files must scan +// clean). If that remainder still flags — e.g. a finding only matches across a +// file boundary — the helper refuses per-file mode rather than emit an entry +// that silences something no single file carries. +import { scan, scanSkill, skillHash } from '../src/index.mjs'; + +const args = process.argv.slice(2); +const wantFiles = args.includes('--files'); +const dir = args.find((a) => a !== '--files'); +if (!dir) { + console.error('usage: watch-accept.mjs [--files]'); + process.exit(2); +} + +const r = scan(dir); +if (r.verdict === 'clean') { + console.error(`${dir}: scans clean — nothing to accept`); + process.exit(2); +} +console.error(`${r.skill.name}: ${r.findings.length} finding(s)`); +for (const f of r.findings) console.error(` ${f.tool}: ${f.flags.join('; ')}`); + +const reviewed = new Date().toISOString().slice(0, 10); +const entry = { class: 'FILL ME IN', note: 'FILL ME IN', reviewed }; + +if (!wantFiles) { + console.log(JSON.stringify({ hash: skillHash(r.skill), ...entry }, null, 2)); + process.exit(0); +} + +const pieces = r.skill.scanPieces || []; +if (!pieces.length) { + console.error(`${dir}: not a skill directory — per-file granularity needs one`); + process.exit(2); +} +const scanOne = (ps) => scanSkill({ kind: 'skill', name: r.skill.name, scanTargets: [{ name: r.skill.name, description: ps.map((p) => p.text).join('\n') }] }); +const bearing = pieces.filter((p) => scanOne([p]).verdict !== 'clean'); +if (scanOne(pieces.filter((p) => !bearing.includes(p))).verdict !== 'clean') { + console.error(`${dir}: findings not attributable to individual files — use the whole-skill hash entry`); + process.exit(2); +} +const hashOf = Object.fromEntries(r.skill.files.map((f) => [f.path, f.hash])); +const files = Object.fromEntries(bearing.map((p) => [p.path, hashOf[p.path]])); +console.error(`finding-bearing files: ${Object.keys(files).join(', ') || '(none)'}`); +console.log(JSON.stringify({ granularity: 'finding-files', files, ...entry }, null, 2)); diff --git a/support/watch-accepted.json b/support/watch-accepted.json index f4bfdb7..ca53233 100644 --- a/support/watch-accepted.json +++ b/support/watch-accepted.json @@ -1,20 +1,33 @@ { "agentforce-adlc:agentforce-generate": { - "hash": "bddf57157b63206f7c5ac2ea68321a71b316d7b259ffa9041eb3aa8ac762a833", + "granularity": "finding-files", + "files": { + "references/safety-review-reference.md": "af86e02ad86aa84deec037bbb512ce04f11b66c8fecc4edc128ba836458e61ac" + }, "class": "security-testing fixtures", - "note": "Salesforce Agentforce docs ship attack strings as test payloads — re-accepted 2026-07-17 after a second upstream drift this week: v0.9 adds an agent-optimization workflow (4 new pattern reference docs, hand-read in full); fixture content unchanged", + "note": "Salesforce Agentforce docs ship attack strings as test payloads — per-file acceptance (#68) after two whole-skill lapses in one week (#65, #67), both re-reviews finding only unrelated docs churn; the fixture file is the reviewed bytes from those reads", "reviewed": "2026-07-17" }, "agentforce-adlc:agentforce-secure": { - "hash": "2f2bf93f5f3efcb07e6e404cffbd295195267fb441c88857238c72b315e60723", + "granularity": "finding-files", + "files": { + "SKILL.md": "5f1a1485c36043656047794d08e4f78f3d274ed1e56b2bb0d969349c5ee75985", + "assets/payloads/prompt-injection.yaml": "a42cebb9dbf7633f1f0ad1ebf6ba0a221f9e5817be49dc6dc42786d8fe4094d4", + "assets/payloads/system-prompt-leakage.yaml": "3a156fc7815c1ca1ed7b2535929f0d6a59fce4d009bf3aef8c01073b907869d6", + "references/owasp-categories.md": "98fb14d493183edb965932ccd69bc4d2050ef334ad35b74aa4443e4a4d75ec0e", + "references/remediation-guide.md": "529cfc5533688473735f5b082af5525820dca1c4c401051deccb3b817f4daa7a" + }, "class": "security-testing fixtures", - "note": "Salesforce Agentforce docs ship attack strings as test payloads", + "note": "Salesforce Agentforce docs ship attack strings as test payloads — per-file acceptance (#68), fixture bytes identical to the 2026-07-09 review", "reviewed": "2026-07-09" }, "agentforce-adlc:agentforce-test": { - "hash": "8a23ff1c327b162c024bc79d348f91c3644cebf4bd04bcfbfb58dbb26ffed989", + "granularity": "finding-files", + "files": { + "references/preview-testing.md": "d3a51189f07a524f4eb5c41fb96c4ac7adbfcbc2c858241bd9a4d426af1cf7c5" + }, "class": "security-testing fixtures", - "note": "Salesforce Agentforce docs ship attack strings as test payloads", + "note": "Salesforce Agentforce docs ship attack strings as test payloads — per-file acceptance (#68), fixture bytes identical to the 2026-07-09 review", "reviewed": "2026-07-09" }, "aws-data-analytics:finding-data-lake-assets": { diff --git a/test/marketplace-watch.test.mjs b/test/marketplace-watch.test.mjs index 91e165d..f6315dc 100644 --- a/test/marketplace-watch.test.mjs +++ b/test/marketplace-watch.test.mjs @@ -176,6 +176,105 @@ test('accepted findings pass for exactly those bytes and re-flag on drift', asyn assert.equal(s2.accepted, 0); }); +// ── per-file acceptance granularity (#68): reviewed files pin, the rest may churn ── + +// Stage a watch script copy next to a bespoke accept file (the script resolves +// watch-accepted.json and ../src relative to itself). src/node_modules land in +// baseDir once, shared by every stage. +function stageWatch(name, acceptedMap) { + const stage = path.join(baseDir, name); + fs.mkdirSync(stage, { recursive: true }); + const staged = path.join(stage, 'marketplace-watch.mjs'); + fs.copyFileSync(WATCH, staged); + if (!fs.existsSync(path.join(baseDir, 'src'))) fs.cpSync(fileURLToPath(new URL('../src', import.meta.url)), path.join(baseDir, 'src'), { recursive: true }); + if (!fs.existsSync(path.join(baseDir, 'node_modules'))) fs.cpSync(fileURLToPath(new URL('../node_modules', import.meta.url)), path.join(baseDir, 'node_modules'), { recursive: true }); + fs.writeFileSync(path.join(stage, 'watch-accepted.json'), JSON.stringify(acceptedMap)); + return staged; +} + +test('per-file acceptance: unrelated churn holds, new findings and reviewed-file drift lapse', async () => { + const { scan } = await import('../src/index.mjs'); + const dir = path.join(baseDir, 'p-perfile'); + const skillDir = path.join(dir, 'skills', 'sneaky'); + put(path.join(skillDir, 'SKILL.md'), POISON); + put(path.join(skillDir, 'docs', 'notes.md'), CLEAN); + const corpus = mkCorpus('corpus-perfile', [{ name: 'p-perfile', kind: 'local', dir, status: 'ok' }]); + + // accept keyed to the finding-bearing file only + const hashOf = Object.fromEntries(scan(skillDir).skill.files.map((f) => [f.path, f.hash])); + const staged = stageWatch('stage-perfile', { + 'p-perfile:sneaky': { granularity: 'finding-files', files: { 'SKILL.md': hashOf['SKILL.md'] }, class: 'test fixture', note: 'per-file acceptance' }, + }); + const run = (out) => spawnSync(process.execPath, [staged, corpus, path.join(baseDir, out)], { encoding: 'utf8' }); + + // baseline: accepted, and reported as per-file + const r1 = run('out-perfile-1'); + assert.equal(r1.status, 0, r1.stdout + r1.stderr); + assert.equal(JSON.parse(r1.stdout).accepted, 1); + const results1 = JSON.parse(fs.readFileSync(path.join(baseDir, 'out-perfile-1', 'results.json'), 'utf8')); + assert.equal(results1.acceptedDetail[0].granularity, 'finding-files'); + assert.match(fs.readFileSync(path.join(baseDir, 'out-perfile-1', 'WATCH.md'), 'utf8'), /\(per-file\)/); + + // unrelated churn — a new doc and a changed doc — HOLDS (the whole point of #68) + put(path.join(skillDir, 'docs', 'notes.md'), CLEAN + 'v2: more notes\n'); + put(path.join(skillDir, 'docs', 'changelog.md'), CLEAN); + const r2 = run('out-perfile-2'); + assert.equal(r2.status, 0, r2.stdout + r2.stderr); + const s2 = JSON.parse(r2.stdout); + assert.equal(s2.poisoned, 0); + assert.equal(s2.accepted, 1); + + // a NEW finding in a new file lapses it — detection still runs on the full skill + put(path.join(skillDir, 'docs', 'extra.md'), POISON); + const r3 = run('out-perfile-3'); + assert.equal(r3.status, 1, r3.stdout + r3.stderr); + assert.equal(JSON.parse(r3.stdout).poisoned, 1); + fs.rmSync(path.join(skillDir, 'docs', 'extra.md')); + + // the reviewed file itself drifting lapses it — that guarantee is preserved + put(path.join(skillDir, 'SKILL.md'), POISON + 'now with different bytes\n'); + const r4 = run('out-perfile-4'); + assert.equal(r4.status, 1, r4.stdout + r4.stderr); + const s4 = JSON.parse(r4.stdout); + assert.equal(s4.poisoned, 1); + assert.equal(s4.accepted, 0); +}); + +test('per-file acceptance: an entry with no files map fails closed', () => { + const dir = path.join(baseDir, 'p-nofiles'); + put(path.join(dir, 'skills', 'sneaky', 'SKILL.md'), POISON); + const corpus = mkCorpus('corpus-nofiles', [{ name: 'p-nofiles', kind: 'local', dir, status: 'ok' }]); + const staged = stageWatch('stage-nofiles', { + 'p-nofiles:sneaky': { granularity: 'finding-files', class: 'test fixture', note: 'no files listed' }, + }); + const r = spawnSync(process.execPath, [staged, corpus, path.join(baseDir, 'out-nofiles')], { encoding: 'utf8' }); + assert.equal(r.status, 1, r.stdout + r.stderr); + assert.equal(JSON.parse(r.stdout).poisoned, 1); +}); + +test('watch-accept --files emits an entry the watch accepts, keyed to the finding-bearing file', async () => { + const { scan } = await import('../src/index.mjs'); + const ACCEPT = fileURLToPath(new URL('../support/watch-accept.mjs', import.meta.url)); + const dir = path.join(baseDir, 'p-author'); + const skillDir = path.join(dir, 'skills', 'sneaky'); + put(path.join(skillDir, 'SKILL.md'), POISON); + put(path.join(skillDir, 'docs', 'notes.md'), CLEAN); + + const r = spawnSync(process.execPath, [ACCEPT, skillDir, '--files'], { encoding: 'utf8' }); + assert.equal(r.status, 0, r.stdout + r.stderr); + const entry = JSON.parse(r.stdout); + assert.equal(entry.granularity, 'finding-files'); + const hashOf = Object.fromEntries(scan(skillDir).skill.files.map((f) => [f.path, f.hash])); + assert.deepEqual(entry.files, { 'SKILL.md': hashOf['SKILL.md'] }); + + // the emitted entry round-trips through the watch + const corpus = mkCorpus('corpus-author', [{ name: 'p-author', kind: 'local', dir, status: 'ok' }]); + const staged = stageWatch('stage-author', { 'p-author:sneaky': { ...entry, class: 'test fixture', note: 'authored' } }); + const w = spawnSync(process.execPath, [staged, corpus, path.join(baseDir, 'out-author')], { encoding: 'utf8' }); + assert.equal(w.status, 0, w.stdout + w.stderr); + assert.equal(JSON.parse(w.stdout).accepted, 1); +}); + // ── legacy mode: a plain marketplace clone still works ── test('legacy mode: a plugins/-tree clone scans in place with a plugin count', () => {