From 2f48721cea4a6fd9b32574c4e41fdb06fe33230b Mon Sep 17 00:00:00 2001 From: rNoz Date: Sun, 19 Jul 2026 23:14:55 +0200 Subject: [PATCH 1/2] feat(annotate): add strict atomic result output --- AGENTS.md | 6 + apps/hook/server/annotate-command.test.ts | 178 +++++++++++++++ apps/hook/server/annotate-command.ts | 59 +++++ apps/hook/server/cli.test.ts | 130 +++++++++++ apps/hook/server/cli.ts | 67 +++++- apps/hook/server/index.ts | 56 +++-- .../server/strict-annotate-result.test.ts | 211 ++++++++++++++++++ apps/hook/server/strict-annotate-result.ts | 111 +++++++++ .../annotate-gates-and-json-responses.md | 24 +- 9 files changed, 824 insertions(+), 18 deletions(-) create mode 100644 apps/hook/server/annotate-command.test.ts create mode 100644 apps/hook/server/annotate-command.ts create mode 100644 apps/hook/server/strict-annotate-result.test.ts create mode 100644 apps/hook/server/strict-annotate-result.ts diff --git a/AGENTS.md b/AGENTS.md index edfd5c293..2ea9e69fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -268,6 +268,12 @@ User annotates content, provides feedback Send Annotations → feedback sent to agent session ``` +### Strict direct annotate results + +Direct `plannotator annotate` invocations may add `--require-approval` and/or `--result-file ` only with `--gate --json`; both reject `--hook` and are not shared with OpenCode/Pi slash-command parsing. Legacy plaintext, JSON, hook, and exit behavior remains unchanged when neither strict option is present. + +Strict decisions use one newline-terminated JSON record on stdout and, when requested, identical bytes in the result file. Approval exits `0`; with `--require-approval`, annotated and dismissed decisions are published before a nonzero exit. Result paths resolve from the invocation working directory, require an existing parent and absent destination, and publish via a flushed/closed `0600` same-directory temporary file plus an atomic no-clobber hard link—never copy or overwrite fallback. Keep reviewed sources at stable project paths; unique result and diagnostic log files may use a narrow temporary directory. Explicit Close emits `dismissed`; missing results or process/browser failures are recovery cases, never approval. + ## Archive Flow ``` diff --git a/apps/hook/server/annotate-command.test.ts b/apps/hook/server/annotate-command.test.ts new file mode 100644 index 000000000..2bbe81ba2 --- /dev/null +++ b/apps/hook/server/annotate-command.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, test } from "bun:test"; +import { completeAnnotateCommand } from "./annotate-command"; +import type { AnnotateOutcome } from "./strict-annotate-result"; + +interface RunResult { + events: string[]; + stdout: string[]; + resultBytes: string[]; + legacy: AnnotateOutcome[]; +} + +async function runCompletion( + outcome: AnnotateOutcome, + options: { + requireApproval?: boolean; + resultFile?: string; + resultWriter?: (path: string, serialized: string) => Promise; + } = {}, +): Promise { + const events: string[] = []; + const stdout: string[] = []; + const resultBytes: string[] = []; + const legacy: AnnotateOutcome[] = []; + + await completeAnnotateCommand({ + waitForDecision: async () => { + events.push("decision"); + return outcome; + }, + settleAfterDecision: async () => { + events.push("settle"); + }, + stopServer: () => { + events.push("stop"); + }, + requireApproval: options.requireApproval ?? false, + resultFile: options.resultFile, + writeResultFile: + options.resultWriter ?? + (async (_path, serialized) => { + events.push("result"); + resultBytes.push(`${serialized}\n`); + }), + writeStdout: async (bytes) => { + events.push("stdout"); + stdout.push(bytes); + }, + emitLegacyOutcome: (result) => { + events.push("legacy"); + legacy.push(result); + }, + exit: (code) => { + events.push(`exit:${code}`); + }, + }); + + return { events, stdout, resultBytes, legacy }; +} + +describe("completeAnnotateCommand", () => { + test("publishes approved feedback to matching stdout and result bytes", async () => { + const result = await runCompletion( + { + approved: true, + feedback: "Keep the cache bounded.", + }, + { + requireApproval: true, + resultFile: "/result.json", + }, + ); + + const expected = + '{"decision":"approved","feedback":"Keep the cache bounded."}\n'; + expect(result.resultBytes).toEqual([expected]); + expect(result.stdout).toEqual([expected]); + expect(result.events).toEqual([ + "decision", + "settle", + "stop", + "result", + "stdout", + "exit:0", + ]); + }); + + test("publishes annotated and dismissed decisions before nonzero exit", async () => { + const annotated = await runCompletion( + { approved: false, exit: false, feedback: "revise" }, + { requireApproval: true, resultFile: "/annotated.json" }, + ); + const dismissed = await runCompletion( + { exit: true, feedback: "" }, + { requireApproval: true, resultFile: "/dismissed.json" }, + ); + + expect(annotated.stdout).toEqual([ + '{"decision":"annotated","feedback":"revise"}\n', + ]); + expect(annotated.resultBytes).toEqual(annotated.stdout); + expect(annotated.events.slice(-3)).toEqual([ + "result", + "stdout", + "exit:1", + ]); + expect(dismissed.stdout).toEqual([ + '{"decision":"dismissed"}\n', + ]); + expect(dismissed.resultBytes).toEqual(dismissed.stdout); + expect(dismissed.events.slice(-3)).toEqual([ + "result", + "stdout", + "exit:1", + ]); + }); + + test("delegates legacy output unchanged and exits zero", async () => { + const outcome = { exit: false, feedback: "legacy feedback" }; + const result = await runCompletion(outcome); + + expect(result.legacy).toEqual([outcome]); + expect(result.stdout).toEqual([]); + expect(result.resultBytes).toEqual([]); + expect(result.events.slice(-2)).toEqual(["legacy", "exit:0"]); + }); + + test("supports each strict option independently", async () => { + const resultFileOnly = await runCompletion( + { approved: false, feedback: "revise" }, + { resultFile: "/result.json" }, + ); + const approvalOnly = await runCompletion( + { exit: true, feedback: "" }, + { requireApproval: true }, + ); + + expect(resultFileOnly.events.slice(-3)).toEqual([ + "result", + "stdout", + "exit:0", + ]); + expect(resultFileOnly.resultBytes).toEqual(resultFileOnly.stdout); + expect(approvalOnly.events.slice(-2)).toEqual(["stdout", "exit:1"]); + expect(approvalOnly.resultBytes).toEqual([]); + }); + + test("does not emit stdout or exit when result publication fails", async () => { + const events: string[] = []; + + await expect( + completeAnnotateCommand({ + waitForDecision: async () => ({ + approved: false, + feedback: "revise", + }), + settleAfterDecision: async () => {}, + stopServer: () => {}, + requireApproval: true, + resultFile: "/raced.json", + writeResultFile: async () => { + events.push("result"); + throw new Error("destination appeared"); + }, + writeStdout: async () => { + events.push("stdout"); + }, + emitLegacyOutcome: () => { + events.push("legacy"); + }, + exit: (code) => { + events.push(`exit:${code}`); + }, + }), + ).rejects.toThrow("destination appeared"); + + expect(events).toEqual(["result"]); + }); +}); diff --git a/apps/hook/server/annotate-command.ts b/apps/hook/server/annotate-command.ts new file mode 100644 index 000000000..9946178e4 --- /dev/null +++ b/apps/hook/server/annotate-command.ts @@ -0,0 +1,59 @@ +import { + annotateOutcomeExitCode, + serializeStrictAnnotateResult, + writeAnnotateResultFile, + type AnnotateOutcome, +} from "./strict-annotate-result"; + +export interface CompleteAnnotateCommandOptions { + waitForDecision: () => Promise; + settleAfterDecision: () => Promise; + stopServer: () => void; + requireApproval: boolean; + resultFile?: string; + writeResultFile?: ( + resultFile: string, + serialized: string, + ) => Promise; + writeStdout?: (output: string) => Promise; + emitLegacyOutcome: (result: AnnotateOutcome) => void; + exit?: (code: number) => void; +} + +export function writeStdout(output: string): Promise { + return new Promise((resolve, reject) => { + process.stdout.write(output, (error) => { + if (error) reject(error); + else resolve(); + }); + }); +} + +export async function completeAnnotateCommand({ + waitForDecision, + settleAfterDecision, + stopServer, + requireApproval, + resultFile, + writeResultFile = writeAnnotateResultFile, + writeStdout: outputWriter = writeStdout, + emitLegacyOutcome, + exit = process.exit, +}: CompleteAnnotateCommandOptions): Promise { + const result = await waitForDecision(); + await settleAfterDecision(); + stopServer(); + + if (requireApproval || resultFile) { + const serialized = serializeStrictAnnotateResult(result); + if (resultFile) { + await writeResultFile(resultFile, serialized); + } + await outputWriter(`${serialized}\n`); + exit(annotateOutcomeExitCode(result, requireApproval)); + return; + } + + emitLegacyOutcome(result); + exit(0); +} diff --git a/apps/hook/server/cli.test.ts b/apps/hook/server/cli.test.ts index 48803220d..f180dc99d 100644 --- a/apps/hook/server/cli.test.ts +++ b/apps/hook/server/cli.test.ts @@ -9,6 +9,7 @@ import { isSubcommandHelpInvocation, isTopLevelHelpInvocation, isVersionInvocation, + parseStrictAnnotateOptions, } from "./cli"; describe("CLI top-level help", () => { @@ -100,12 +101,141 @@ describe("CLI subcommand help", () => { expect(formatSubcommandHelp("review")).toContain("--gitbutler"); expect(formatSubcommandHelp("review")).toContain("PR_URL"); expect(formatSubcommandHelp("annotate")).toContain("--no-jina"); + expect(formatSubcommandHelp("annotate")).toContain("--require-approval"); + expect(formatSubcommandHelp("annotate")).toContain("--result-file "); + expect(formatSubcommandHelp("annotate-last")).not.toContain( + "--require-approval", + ); expect(formatSubcommandHelp("sessions")).toContain("--open [N]"); // unknown key falls back to top-level help expect(formatSubcommandHelp("nope")).toBe(formatTopLevelHelp()); }); }); +describe("strict annotate CLI options", () => { + test("extracts strict options before or after the target path", () => { + const strictOrderings = [ + ["plan.md", "--require-approval", "--result-file", "result.json"], + ["plan.md", "--result-file", "result.json", "--require-approval"], + ["--require-approval", "plan.md", "--result-file", "result.json"], + ["--require-approval", "--result-file", "result.json", "plan.md"], + ["--result-file", "result.json", "plan.md", "--require-approval"], + ["--result-file", "result.json", "--require-approval", "plan.md"], + ]; + + for (const ordering of strictOrderings) { + expect( + parseStrictAnnotateOptions([ + "annotate", + ...ordering, + "--gate", + "--json", + ]), + ).toEqual({ + requireApproval: true, + resultFile: "result.json", + remainingArgs: ["annotate", "plan.md", "--gate", "--json"], + }); + } + }); + + test("allows either strict option independently", () => { + expect( + parseStrictAnnotateOptions([ + "annotate", + "plan.md", + "--gate", + "--json", + "--require-approval", + ]), + ).toEqual({ + requireApproval: true, + remainingArgs: ["annotate", "plan.md", "--gate", "--json"], + }); + expect( + parseStrictAnnotateOptions([ + "annotate", + "--result-file", + "result.json", + "plan.md", + "--gate", + "--json", + ]), + ).toEqual({ + requireApproval: false, + resultFile: "result.json", + remainingArgs: ["annotate", "plan.md", "--gate", "--json"], + }); + }); + + test("leaves ordinary direct arguments unchanged", () => { + const args = [ + "annotate", + "plan.md", + "--gate", + "--json", + "--markdown", + ]; + expect(parseStrictAnnotateOptions(args)).toEqual({ + requireApproval: false, + remainingArgs: args, + }); + }); + + test("requires annotate --gate --json without --hook", () => { + for (const args of [ + ["review", "--gate", "--json", "--require-approval"], + ["annotate-last", "--gate", "--json", "--require-approval"], + ["annotate", "plan.md", "--json", "--require-approval"], + ["annotate", "plan.md", "--gate", "--require-approval"], + [ + "annotate", + "plan.md", + "--gate", + "--json", + "--hook", + "--require-approval", + ], + ]) { + expect(() => parseStrictAnnotateOptions(args)).toThrow(); + } + }); + + test("rejects missing and duplicate strict option values", () => { + expect(() => + parseStrictAnnotateOptions([ + "annotate", + "plan.md", + "--gate", + "--json", + "--result-file", + ]), + ).toThrow("Missing value for --result-file"); + expect(() => + parseStrictAnnotateOptions([ + "annotate", + "plan.md", + "--gate", + "--json", + "--result-file", + "first.json", + "--result-file", + "second.json", + ]), + ).toThrow("--result-file may only be specified once"); + expect(() => + parseStrictAnnotateOptions([ + "annotate", + "plan.md", + "--gate", + "--json", + "--require-approval", + "--require-approval", + ]), + ).toThrow("--require-approval may only be specified once"); + }); +}); + describe("CLI --version", () => { test("recognizes --version and -v", () => { expect(isVersionInvocation(["--version"])).toBe(true); diff --git a/apps/hook/server/cli.ts b/apps/hook/server/cli.ts index 00d7dafa5..c38e0e97d 100644 --- a/apps/hook/server/cli.ts +++ b/apps/hook/server/cli.ts @@ -1,5 +1,64 @@ const HELP_FLAGS = new Set(["--help", "-h"]); +export interface ParsedStrictAnnotateOptions { + requireApproval: boolean; + resultFile?: string; + remainingArgs: string[]; +} + +export function parseStrictAnnotateOptions( + args: string[], +): ParsedStrictAnnotateOptions { + let requireApproval = false; + let resultFile: string | undefined; + const remainingArgs: string[] = []; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--require-approval") { + if (requireApproval) { + throw new Error("--require-approval may only be specified once"); + } + requireApproval = true; + continue; + } + if (arg === "--result-file") { + if (resultFile !== undefined) { + throw new Error("--result-file may only be specified once"); + } + const value = args[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error("Missing value for --result-file"); + } + resultFile = value; + index += 1; + continue; + } + remainingArgs.push(arg); + } + + if (!requireApproval && resultFile === undefined) { + return { requireApproval: false, remainingArgs }; + } + if (remainingArgs[0] !== "annotate") { + throw new Error( + "--require-approval and --result-file are only valid with annotate", + ); + } + if (!remainingArgs.includes("--gate") || !remainingArgs.includes("--json")) { + throw new Error( + "--require-approval and --result-file require --gate --json", + ); + } + if (remainingArgs.includes("--hook")) { + throw new Error( + "--require-approval and --result-file cannot be used with --hook", + ); + } + + return { requireApproval, resultFile, remainingArgs }; +} + /** True when any token is a help flag (`--help` / `-h`). */ export function hasHelpFlag(args: string[]): boolean { return args.some((arg) => HELP_FLAGS.has(arg)); @@ -33,7 +92,7 @@ export function formatTopLevelHelp(): string { " plannotator --version, -v", " plannotator [--browser ]", " plannotator review [--git | --gitbutler] [PR_URL]", - " plannotator annotate [--markdown] [--no-jina] [--gate] [--json] [--hook]", + " plannotator annotate [--markdown] [--no-jina] [--gate] [--json] [--hook] [--require-approval] [--result-file ]", " plannotator annotate-last [--stdin] [--gate] [--json] [--hook]", " plannotator setup-goal [--json]", " plannotator last", @@ -76,7 +135,7 @@ const SUBCOMMAND_HELP: Record = { ].join("\n"), annotate: [ "Usage:", - " plannotator annotate [--markdown] [--no-jina] [--gate] [--json] [--hook]", + " plannotator annotate [--markdown] [--no-jina] [--gate] [--json] [--hook] [--require-approval] [--result-file ]", "", "Open a markdown/text/HTML file, a URL, or a folder of documents in the annotation UI.", "", @@ -86,6 +145,10 @@ const SUBCOMMAND_HELP: Record = { " --gate Add an Approve button (review-gate UX)", " --json Emit a structured decision JSON on stdout", " --hook Emit hook-native JSON (block/pass) for PostToolUse/Stop hooks", + " --require-approval", + " Exit nonzero unless the reviewer approves (requires --gate --json)", + " --result-file ", + " Atomically publish the stdout JSON (requires --gate --json)", ].join("\n"), "annotate-last": [ "Usage:", diff --git a/apps/hook/server/index.ts b/apps/hook/server/index.ts index 2df24c7cc..8ac62c20e 100644 --- a/apps/hook/server/index.ts +++ b/apps/hook/server/index.ts @@ -142,7 +142,13 @@ import { isSubcommandHelpInvocation, isTopLevelHelpInvocation, isVersionInvocation, + parseStrictAnnotateOptions, } from "./cli"; +import { completeAnnotateCommand } from "./annotate-command"; +import { + assertResultPathAvailable, + resolveResultFilePath, +} from "./strict-annotate-result"; import path from "path"; import { tmpdir } from "os"; import { buildLocalWorkspaceReview, type WorkspaceDiffType } from "@plannotator/server/review-workspace"; @@ -157,7 +163,24 @@ import reviewHtml from "../dist/review.html" with { type: "text" }; const reviewHtmlContent = reviewHtml as unknown as string; // Check for subcommand -const args = process.argv.slice(2); +let parsedStrictAnnotateOptions; +try { + parsedStrictAnnotateOptions = parseStrictAnnotateOptions( + process.argv.slice(2), + ); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} +const args = parsedStrictAnnotateOptions.remainingArgs; +const requireApprovalFlag = + parsedStrictAnnotateOptions.requireApproval; +const resultFile = parsedStrictAnnotateOptions.resultFile + ? resolveResultFilePath( + parsedStrictAnnotateOptions.resultFile, + process.env.PLANNOTATOR_CWD || process.cwd(), + ) + : undefined; // Global flag: --browser const browserIdx = args.indexOf("--browser"); @@ -901,7 +924,7 @@ if (args[0] === "sessions") { const rawFilePath = args[1]; if (!rawFilePath) { - console.error("Usage: plannotator annotate [--markdown] [--no-jina] [--gate] [--json] [--hook]"); + console.error("Usage: plannotator annotate [--markdown] [--no-jina] [--gate] [--json] [--hook] [--require-approval] [--result-file ]"); process.exit(1); } @@ -913,6 +936,15 @@ if (args[0] === "sessions") { // Use PLANNOTATOR_CWD if set (original working directory before script cd'd) const projectRoot = process.env.PLANNOTATOR_CWD || process.cwd(); + if (resultFile) { + try { + await assertResultPathAvailable(resultFile); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } + } + if (process.env.PLANNOTATOR_DEBUG) { console.error(`[DEBUG] Project root: ${projectRoot}`); console.error(`[DEBUG] File path arg: ${filePath}`); @@ -1079,18 +1111,14 @@ if (args[0] === "sessions") { : `annotate-${isUrl ? hostnameOrFallback(absolutePath) : path.basename(absolutePath)}`, }); - // Wait for user feedback - const result = await server.waitForDecision(); - - // Give browser time to receive response and update UI - await Bun.sleep(1500); - - // Cleanup - server.stop(); - - // Output feedback (captured by slash command) - emitAnnotateOutcome(result); - process.exit(0); + await completeAnnotateCommand({ + waitForDecision: server.waitForDecision, + settleAfterDecision: () => Bun.sleep(1500), + stopServer: server.stop, + requireApproval: requireApprovalFlag, + resultFile, + emitLegacyOutcome: emitAnnotateOutcome, + }); } else if (args[0] === "annotate-last" || args[0] === "last") { // ============================================ diff --git a/apps/hook/server/strict-annotate-result.test.ts b/apps/hook/server/strict-annotate-result.test.ts new file mode 100644 index 000000000..d71b703f8 --- /dev/null +++ b/apps/hook/server/strict-annotate-result.test.ts @@ -0,0 +1,211 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + link, + mkdtemp, + open, + readFile, + readdir, + rm, + stat, + symlink, + unlink, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + annotateOutcomeExitCode, + assertResultPathAvailable, + resolveResultFilePath, + serializeStrictAnnotateResult, + writeAnnotateResultFile, +} from "./strict-annotate-result"; + +const temporaryDirectories: string[] = []; + +async function makeTemporaryDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "plannotator-result-")); + temporaryDirectories.push(directory); + return directory; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => + rm(directory, { recursive: true, force: true }), + ), + ); +}); + +describe("strict annotate result serialization", () => { + test("serializes approval without feedback", () => { + expect( + serializeStrictAnnotateResult({ approved: true, feedback: "" }), + ).toBe('{"decision":"approved"}'); + }); + + test("serializes approval with feedback", () => { + expect( + serializeStrictAnnotateResult({ + approved: true, + feedback: "Keep the cache bounded.", + }), + ).toBe( + '{"decision":"approved","feedback":"Keep the cache bounded."}', + ); + }); + + test("serializes annotated and dismissed decisions", () => { + expect( + serializeStrictAnnotateResult({ + approved: false, + exit: false, + feedback: "revise", + }), + ).toBe('{"decision":"annotated","feedback":"revise"}'); + expect( + serializeStrictAnnotateResult({ exit: true, feedback: "" }), + ).toBe('{"decision":"dismissed"}'); + }); +}); + +describe("strict annotate exit policy", () => { + test("requires approval when requested", () => { + expect( + annotateOutcomeExitCode( + { approved: false, exit: false, feedback: "revise" }, + true, + ), + ).toBe(1); + expect( + annotateOutcomeExitCode({ approved: true, feedback: "" }, true), + ).toBe(0); + }); + + test("keeps legacy outcomes successful", () => { + expect( + annotateOutcomeExitCode({ exit: true, feedback: "" }, false), + ).toBe(0); + }); +}); + +describe("atomic annotate result publication", () => { + test("resolves relative result paths from the invocation directory", () => { + expect( + resolveResultFilePath("results/review.json", "/workspace/project"), + ).toBe("/workspace/project/results/review.json"); + expect( + resolveResultFilePath("/var/tmp/review.json", "/workspace/project"), + ).toBe("/var/tmp/review.json"); + }); + + test("publishes one complete private newline-terminated record", async () => { + const directory = await makeTemporaryDirectory(); + const resultFile = join(directory, "result.json"); + + await assertResultPathAvailable(resultFile); + await writeAnnotateResultFile( + resultFile, + '{"decision":"approved"}', + ); + + expect(await readFile(resultFile, "utf8")).toBe( + '{"decision":"approved"}\n', + ); + if (process.platform !== "win32") { + expect((await stat(resultFile)).mode & 0o077).toBe(0); + } + expect(await readdir(directory)).toEqual(["result.json"]); + }); + + test("rejects a missing parent and an existing destination", async () => { + const directory = await makeTemporaryDirectory(); + const missingParentResult = join(directory, "missing", "result.json"); + const existingResult = join(directory, "existing.json"); + await writeFile(existingResult, "existing", "utf8"); + + await expect( + assertResultPathAvailable(missingParentResult), + ).rejects.toThrow( + `Result file parent does not exist: ${join(directory, "missing")}`, + ); + await expect( + assertResultPathAvailable(existingResult), + ).rejects.toThrow(`Result file already exists: ${existingResult}`); + expect(await readdir(directory)).toEqual(["existing.json"]); + }); + + test.skipIf(process.platform === "win32")( + "rejects a dangling destination symlink before startup", + async () => { + const directory = await makeTemporaryDirectory(); + const resultFile = join(directory, "result.json"); + await symlink(join(directory, "missing-target"), resultFile); + + await expect( + assertResultPathAvailable(resultFile), + ).rejects.toThrow(`Result file already exists: ${resultFile}`); + }, + ); + + test("never overwrites a destination created after validation", async () => { + const directory = await makeTemporaryDirectory(); + const resultFile = join(directory, "result.json"); + await assertResultPathAvailable(resultFile); + await writeFile(resultFile, "raced", { mode: 0o600 }); + + await expect( + writeAnnotateResultFile( + resultFile, + '{"decision":"approved"}', + ), + ).rejects.toThrow(); + + expect(await readFile(resultFile, "utf8")).toBe("raced"); + expect(await readdir(directory)).toEqual(["result.json"]); + }); + + test("removes the temporary file when writing fails", async () => { + const directory = await makeTemporaryDirectory(); + const resultFile = join(directory, "result.json"); + + await expect( + writeAnnotateResultFile( + resultFile, + '{"decision":"approved"}', + { + open, + link, + unlink, + write: async () => { + throw new Error("write failed"); + }, + }, + ), + ).rejects.toThrow("write failed"); + + expect(await readdir(directory)).toEqual([]); + }); + + test("fails closed when hard-link publication is unavailable", async () => { + const directory = await makeTemporaryDirectory(); + const resultFile = join(directory, "result.json"); + + await expect( + writeAnnotateResultFile( + resultFile, + '{"decision":"approved"}', + { + open, + link: async () => { + throw new Error("hard links unavailable"); + }, + unlink, + write: (handle, contents) => handle.writeFile(contents, "utf8"), + }, + ), + ).rejects.toThrow("hard links unavailable"); + + expect(await readdir(directory)).toEqual([]); + }); +}); diff --git a/apps/hook/server/strict-annotate-result.ts b/apps/hook/server/strict-annotate-result.ts new file mode 100644 index 000000000..28ff86376 --- /dev/null +++ b/apps/hook/server/strict-annotate-result.ts @@ -0,0 +1,111 @@ +import { existsSync, lstatSync, statSync } from "node:fs"; +import { + link, + open, + unlink, + type FileHandle, +} from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { basename, dirname, join, resolve } from "node:path"; + +export interface AnnotateOutcome { + feedback: string; + exit?: boolean; + approved?: boolean; +} + +export function serializeStrictAnnotateResult( + result: AnnotateOutcome, +): string { + if (result.approved) { + return JSON.stringify({ + decision: "approved", + ...(result.feedback ? { feedback: result.feedback } : {}), + }); + } + if (result.exit) return JSON.stringify({ decision: "dismissed" }); + return JSON.stringify({ + decision: "annotated", + feedback: result.feedback || "", + }); +} + +export function annotateOutcomeExitCode( + result: AnnotateOutcome, + requireApproval: boolean, +): number { + return requireApproval && !result.approved ? 1 : 0; +} + +export function resolveResultFilePath( + resultFile: string, + invocationCwd: string, +): string { + return resolve(invocationCwd, resultFile); +} + +export async function assertResultPathAvailable( + resultFile: string, +): Promise { + let destinationExists = existsSync(resultFile); + if (!destinationExists) { + try { + lstatSync(resultFile); + destinationExists = true; + } catch (error) { + if ( + !(error instanceof Error) || + !("code" in error) || + error.code !== "ENOENT" + ) { + throw error; + } + } + } + if (destinationExists) { + throw new Error(`Result file already exists: ${resultFile}`); + } + const parent = dirname(resultFile); + if (!existsSync(parent) || !statSync(parent).isDirectory()) { + throw new Error(`Result file parent does not exist: ${parent}`); + } +} + +interface ResultFileOperations { + open: typeof open; + link: typeof link; + unlink: typeof unlink; + write: (handle: FileHandle, contents: string) => Promise; +} + +const defaultResultFileOperations: ResultFileOperations = { + open, + link, + unlink, + write: (handle, contents) => handle.writeFile(contents, "utf8"), +}; + +export async function writeAnnotateResultFile( + resultFile: string, + serialized: string, + operations: ResultFileOperations = defaultResultFileOperations, +): Promise { + const temporary = join( + dirname(resultFile), + `.${basename(resultFile)}.${process.pid}.${randomUUID()}.tmp`, + ); + let handle: FileHandle | null = null; + try { + handle = await operations.open(temporary, "wx", 0o600); + await operations.write(handle, `${serialized}\n`); + await handle.sync(); + await handle.close(); + handle = null; + await operations.link(temporary, resultFile); + await operations.unlink(temporary); + } catch (error) { + await handle?.close().catch(() => {}); + await operations.unlink(temporary).catch(() => {}); + throw error; + } +} diff --git a/apps/marketing/src/content/docs/guides/annotate-gates-and-json-responses.md b/apps/marketing/src/content/docs/guides/annotate-gates-and-json-responses.md index 9ece15c47..8a94cc6f8 100644 --- a/apps/marketing/src/content/docs/guides/annotate-gates-and-json-responses.md +++ b/apps/marketing/src/content/docs/guides/annotate-gates-and-json-responses.md @@ -6,7 +6,7 @@ sidebar: section: "Guides" --- -`plannotator annotate` and `plannotator annotate-last` accept three flags that turn markdown annotation into a full review gate with structured output. +`plannotator annotate` and `plannotator annotate-last` accept three flags that turn markdown annotation into a full review gate with structured output. Direct `plannotator annotate` invocations also support strict automation with `--require-approval` and `--result-file`. ## Capabilities @@ -94,6 +94,26 @@ This is the recommended approach for hook integrations. The `{"decision":"block" The flag is accepted silently on OpenCode and Pi for the same reason `--json` is: those harnesses don't use stdout as the signal channel. +## Strict direct gates + +For a fail-closed direct CLI gate, add either or both strict options: + +```sh +plannotator annotate docs/plan.md --gate --json \ + --require-approval \ + --result-file .tmp/plan-review-result.json +``` + +- **`--require-approval`** exits `0` only for `approved`. `annotated` and `dismissed` still publish their valid JSON decision, then exit nonzero. +- **`--result-file `** atomically publishes the same newline-terminated JSON bytes written to stdout. The path is resolved from the invocation working directory. +- Both options require `--gate --json`, are available only on direct `annotate` invocations, and cannot be combined with `--hook`. Hook output and exit behavior are unchanged. + +The result-file parent directory must already exist and the destination must not. Plannotator writes a private `0600` temporary file in the same directory, flushes and closes it, then publishes it with an atomic no-clobber hard link. It never overwrites an existing destination or falls back to a non-atomic copy. Use a unique result path for every invocation. + +Keep the reviewed source at a stable project path so revisions and version history continue to refer to the same artifact. Result and diagnostic log files can instead live in a narrowly scoped temporary directory. + +Clicking Close publishes `{"decision":"dismissed"}`. Closing or crashing the browser outside that explicit action is not guaranteed to produce a decision; callers should treat a missing result or failed process as a recovery case, never as approval. + ## Primary use cases ### Spec-driven development frameworks @@ -116,4 +136,4 @@ See [Hook Integration](/docs/guides/hook-integration/) for copy-paste recipes th ## Exit codes -Every decision exits `0`. Signals live on stdout. This keeps Plannotator composable with harnesses that use exit codes for their own purposes. +By default, every decision exits `0`; existing plaintext, JSON, and hook integrations are unchanged. With `--require-approval`, only `approved` exits `0`. `annotated` and `dismissed` publish their JSON result before exiting nonzero. From 074899c1976090431f82a281cac38ea486274cac Mon Sep 17 00:00:00 2001 From: Michael Ramos Date: Mon, 20 Jul 2026 13:35:54 -0700 Subject: [PATCH 2/2] feat(annotate): exit 2 for strict-gate usage and publication errors Adopt the grep convention for the strict annotate gate's exit codes: 0 = approved, 1 = negative human outcome (annotated/dismissed under --require-approval), 2 = the gate itself was misconfigured or could not start/deliver a decision. Previously all usage/startup/validation failures shared exit 1 with "reviewer did not approve", so callers could not tell a denied review from a broken gate. - parseStrictAnnotateOptions failures (bad flag combos, strict flags outside annotate --gate --json) now exit 2 - --result-file preflight failures (missing parent, pre-existing or dangling-symlink destination) now exit 2 - post-decision publication failures (destination raced into existence, hard links unavailable, stdout write failure) now exit 2: they deliver no decision record at all, so the code's own fail-closed handling presents them as environment errors, never as a reviewer outcome -- and never approval, since only 0 means approved - decision outcomes keep 0/1 exactly as before; signal deaths keep 128+n - document the contract in AGENTS.md and the annotate-gates guide Claude-Session: https://claude.ai/code/session_01YXkgsNucxDwAL4GdR4XYRk --- AGENTS.md | 2 +- apps/hook/server/annotate-command.test.ts | 91 +++++++++++++------ apps/hook/server/annotate-command.ts | 19 +++- apps/hook/server/cli.ts | 3 +- apps/hook/server/index.ts | 7 +- .../server/strict-annotate-result.test.ts | 10 ++ apps/hook/server/strict-annotate-result.ts | 9 ++ .../annotate-gates-and-json-responses.md | 2 +- 8 files changed, 109 insertions(+), 34 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2ea9e69fa..06eba9b5b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -272,7 +272,7 @@ Send Annotations → feedback sent to agent session Direct `plannotator annotate` invocations may add `--require-approval` and/or `--result-file ` only with `--gate --json`; both reject `--hook` and are not shared with OpenCode/Pi slash-command parsing. Legacy plaintext, JSON, hook, and exit behavior remains unchanged when neither strict option is present. -Strict decisions use one newline-terminated JSON record on stdout and, when requested, identical bytes in the result file. Approval exits `0`; with `--require-approval`, annotated and dismissed decisions are published before a nonzero exit. Result paths resolve from the invocation working directory, require an existing parent and absent destination, and publish via a flushed/closed `0600` same-directory temporary file plus an atomic no-clobber hard link—never copy or overwrite fallback. Keep reviewed sources at stable project paths; unique result and diagnostic log files may use a narrow temporary directory. Explicit Close emits `dismissed`; missing results or process/browser failures are recovery cases, never approval. +Strict decisions use one newline-terminated JSON record on stdout and, when requested, identical bytes in the result file. Exit codes follow the grep convention: approval exits `0`; with `--require-approval`, annotated and dismissed decisions are published before exiting `1` (negative human outcome); usage/startup/validation failures — bad flag combinations, strict flags outside `annotate --gate --json`, a missing `--result-file` parent, a pre-existing or dangling-symlink destination — exit `2` (the gate itself was misconfigured or could not start). Post-decision publication failures (destination appears between validation and publish, hard links unavailable, stdout write failure) also exit `2`: they deliver no decision record at all, so they present as environment errors — "the gate could not deliver a decision" — never as a reviewer outcome, and never as approval (still fail-closed, since only `0` means approved). Signal deaths keep `128+n`. Result paths resolve from the invocation working directory, require an existing parent and absent destination, and publish via a flushed/closed `0600` same-directory temporary file plus an atomic no-clobber hard link—never copy or overwrite fallback. Keep reviewed sources at stable project paths; unique result and diagnostic log files may use a narrow temporary directory. Explicit Close emits `dismissed`; missing results or process/browser failures are recovery cases, never approval. ## Archive Flow diff --git a/apps/hook/server/annotate-command.test.ts b/apps/hook/server/annotate-command.test.ts index 2bbe81ba2..902efd5fa 100644 --- a/apps/hook/server/annotate-command.test.ts +++ b/apps/hook/server/annotate-command.test.ts @@ -144,35 +144,74 @@ describe("completeAnnotateCommand", () => { expect(approvalOnly.resultBytes).toEqual([]); }); - test("does not emit stdout or exit when result publication fails", async () => { + test("exits 2 without emitting stdout when result publication fails", async () => { const events: string[] = []; + const errors: string[] = []; - await expect( - completeAnnotateCommand({ - waitForDecision: async () => ({ - approved: false, - feedback: "revise", - }), - settleAfterDecision: async () => {}, - stopServer: () => {}, - requireApproval: true, - resultFile: "/raced.json", - writeResultFile: async () => { - events.push("result"); - throw new Error("destination appeared"); - }, - writeStdout: async () => { - events.push("stdout"); - }, - emitLegacyOutcome: () => { - events.push("legacy"); - }, - exit: (code) => { - events.push(`exit:${code}`); - }, + await completeAnnotateCommand({ + waitForDecision: async () => ({ + approved: false, + feedback: "revise", + }), + settleAfterDecision: async () => {}, + stopServer: () => {}, + requireApproval: true, + resultFile: "/raced.json", + writeResultFile: async () => { + events.push("result"); + throw new Error("destination appeared"); + }, + writeStdout: async () => { + events.push("stdout"); + }, + emitLegacyOutcome: () => { + events.push("legacy"); + }, + exit: (code) => { + events.push(`exit:${code}`); + }, + logError: (message) => { + errors.push(message); + }, + }); + + // No decision record was delivered, so this is an environment error + // (exit 2), never a reviewer outcome — and never approval. + expect(events).toEqual(["result", "exit:2"]); + expect(errors).toEqual(["destination appeared"]); + }); + + test("exits 2 when the stdout record cannot be written", async () => { + const events: string[] = []; + const errors: string[] = []; + + await completeAnnotateCommand({ + waitForDecision: async () => ({ + approved: true, + feedback: "", }), - ).rejects.toThrow("destination appeared"); + settleAfterDecision: async () => {}, + stopServer: () => {}, + requireApproval: true, + writeResultFile: async () => { + events.push("result"); + }, + writeStdout: async () => { + events.push("stdout"); + throw new Error("stdout closed"); + }, + emitLegacyOutcome: () => { + events.push("legacy"); + }, + exit: (code) => { + events.push(`exit:${code}`); + }, + logError: (message) => { + errors.push(message); + }, + }); - expect(events).toEqual(["result"]); + expect(events).toEqual(["stdout", "exit:2"]); + expect(errors).toEqual(["stdout closed"]); }); }); diff --git a/apps/hook/server/annotate-command.ts b/apps/hook/server/annotate-command.ts index 9946178e4..c0acdff6c 100644 --- a/apps/hook/server/annotate-command.ts +++ b/apps/hook/server/annotate-command.ts @@ -1,6 +1,7 @@ import { annotateOutcomeExitCode, serializeStrictAnnotateResult, + STRICT_GATE_ERROR_EXIT_CODE, writeAnnotateResultFile, type AnnotateOutcome, } from "./strict-annotate-result"; @@ -18,6 +19,7 @@ export interface CompleteAnnotateCommandOptions { writeStdout?: (output: string) => Promise; emitLegacyOutcome: (result: AnnotateOutcome) => void; exit?: (code: number) => void; + logError?: (message: string) => void; } export function writeStdout(output: string): Promise { @@ -39,6 +41,7 @@ export async function completeAnnotateCommand({ writeStdout: outputWriter = writeStdout, emitLegacyOutcome, exit = process.exit, + logError = (message) => console.error(message), }: CompleteAnnotateCommandOptions): Promise { const result = await waitForDecision(); await settleAfterDecision(); @@ -46,10 +49,20 @@ export async function completeAnnotateCommand({ if (requireApproval || resultFile) { const serialized = serializeStrictAnnotateResult(result); - if (resultFile) { - await writeResultFile(resultFile, serialized); + try { + if (resultFile) { + await writeResultFile(resultFile, serialized); + } + await outputWriter(`${serialized}\n`); + } catch (error) { + // Publication failed: no decision record was delivered, so this is an + // environment error ("the gate could not deliver a decision"), not a + // reviewer outcome. Exit 2 — fail-closed, but distinct from exit 1's + // "gate ran and the reviewer did not approve". + logError(error instanceof Error ? error.message : String(error)); + exit(STRICT_GATE_ERROR_EXIT_CODE); + return; } - await outputWriter(`${serialized}\n`); exit(annotateOutcomeExitCode(result, requireApproval)); return; } diff --git a/apps/hook/server/cli.ts b/apps/hook/server/cli.ts index c38e0e97d..4ade5d4ab 100644 --- a/apps/hook/server/cli.ts +++ b/apps/hook/server/cli.ts @@ -146,7 +146,8 @@ const SUBCOMMAND_HELP: Record = { " --json Emit a structured decision JSON on stdout", " --hook Emit hook-native JSON (block/pass) for PostToolUse/Stop hooks", " --require-approval", - " Exit nonzero unless the reviewer approves (requires --gate --json)", + " Exit 1 unless the reviewer approves (requires --gate --json;", + " usage/startup errors exit 2)", " --result-file ", " Atomically publish the stdout JSON (requires --gate --json)", ].join("\n"), diff --git a/apps/hook/server/index.ts b/apps/hook/server/index.ts index 8ac62c20e..9d8cf3573 100644 --- a/apps/hook/server/index.ts +++ b/apps/hook/server/index.ts @@ -148,6 +148,7 @@ import { completeAnnotateCommand } from "./annotate-command"; import { assertResultPathAvailable, resolveResultFilePath, + STRICT_GATE_ERROR_EXIT_CODE, } from "./strict-annotate-result"; import path from "path"; import { tmpdir } from "os"; @@ -170,7 +171,8 @@ try { ); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + // Usage error: the gate was misconfigured, not a reviewer decision. + process.exit(STRICT_GATE_ERROR_EXIT_CODE); } const args = parsedStrictAnnotateOptions.remainingArgs; const requireApprovalFlag = @@ -941,7 +943,8 @@ if (args[0] === "sessions") { await assertResultPathAvailable(resultFile); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + // Startup validation error: the gate could not start. + process.exit(STRICT_GATE_ERROR_EXIT_CODE); } } diff --git a/apps/hook/server/strict-annotate-result.test.ts b/apps/hook/server/strict-annotate-result.test.ts index d71b703f8..f0f14cee5 100644 --- a/apps/hook/server/strict-annotate-result.test.ts +++ b/apps/hook/server/strict-annotate-result.test.ts @@ -18,6 +18,7 @@ import { assertResultPathAvailable, resolveResultFilePath, serializeStrictAnnotateResult, + STRICT_GATE_ERROR_EXIT_CODE, writeAnnotateResultFile, } from "./strict-annotate-result"; @@ -87,6 +88,15 @@ describe("strict annotate exit policy", () => { annotateOutcomeExitCode({ exit: true, feedback: "" }, false), ).toBe(0); }); + + test("reserves exit 2 for gate errors, distinct from decision outcomes", () => { + // grep convention: 0 = approved, 1 = negative human outcome, + // 2 = the gate itself was misconfigured or could not start. + expect(STRICT_GATE_ERROR_EXIT_CODE).toBe(2); + expect( + annotateOutcomeExitCode({ exit: true, feedback: "" }, true), + ).toBe(1); + }); }); describe("atomic annotate result publication", () => { diff --git a/apps/hook/server/strict-annotate-result.ts b/apps/hook/server/strict-annotate-result.ts index 28ff86376..b2ccf4bd1 100644 --- a/apps/hook/server/strict-annotate-result.ts +++ b/apps/hook/server/strict-annotate-result.ts @@ -14,6 +14,15 @@ export interface AnnotateOutcome { approved?: boolean; } +/** + * Exit code for gate errors, following the grep convention: + * `0` = approved, `1` = negative human outcome (annotated/dismissed under + * `--require-approval`), `2` = the gate itself was misconfigured or could not + * start/deliver a decision (usage, startup, validation, and publication + * failures). A gate error never publishes a decision record. + */ +export const STRICT_GATE_ERROR_EXIT_CODE = 2; + export function serializeStrictAnnotateResult( result: AnnotateOutcome, ): string { diff --git a/apps/marketing/src/content/docs/guides/annotate-gates-and-json-responses.md b/apps/marketing/src/content/docs/guides/annotate-gates-and-json-responses.md index 8a94cc6f8..6a98cf639 100644 --- a/apps/marketing/src/content/docs/guides/annotate-gates-and-json-responses.md +++ b/apps/marketing/src/content/docs/guides/annotate-gates-and-json-responses.md @@ -136,4 +136,4 @@ See [Hook Integration](/docs/guides/hook-integration/) for copy-paste recipes th ## Exit codes -By default, every decision exits `0`; existing plaintext, JSON, and hook integrations are unchanged. With `--require-approval`, only `approved` exits `0`. `annotated` and `dismissed` publish their JSON result before exiting nonzero. +By default, every decision exits `0`; existing plaintext, JSON, and hook integrations are unchanged. With `--require-approval`, only `approved` exits `0`; `annotated` and `dismissed` publish their JSON result before exiting `1`. Strict invocations that are misconfigured or cannot start or deliver a decision — bad flag combinations, an invalid `--result-file` destination, or a failed atomic publish — exit `2` without publishing a decision, following the grep convention (`0` approved, `1` not approved, `2` the gate itself errored).